SundayHK
监控方案一: 部署redis-exporter及serviceMonitor
[root@harbor ~]# helm repo add bitnami https://charts.bitnami.com/bitnami
"bitnami" has been added to your repositories
[root@harbor ~]# helm repo update[root@harbor ~]# helm search repo redis
NAME CHART VERSION APP VERSION DESCRIPTION
bitnami/redis 19.6.4 7.2.5 Redis(R) is an open source, advanced key-value ...
bitnami/redis-cluster 10.3.0 7.2.5 Redis(R) is an open source, scalable, distribut...
prometheus-community/prometheus-redis-exporter 6.3.0 v1.61.0 Prometheus exporter for Redis metrics 提前准备动态存储
[root@harbor ~]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-client (default) cluster.local/nfs-subdir-external-provisioner Delete Immediate true 部署Redis
[root@harbor ~]# helm install redis bitnami/redis --set global.storageClass=nfs-client --set global.redis.password=root --set architecture=standalone --version 19.6.4 -n redis --create-namespace --debug
[root@harbor ~]# helm list -n redis
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
redis redis 1 2024-08-05 12:38:05.313761625 -0400 EDT deployed redis-19.6.4 7.2.5 ``
[root@harbor ~]# kubectl get pod -n redis
NAME READY STATUS RESTARTS AGE
redis-master-0 1/1 Running 0 62s[root@k8s-master01 sunday]# dig -t a redis-headless.redis.svc.cluster.local @10.96.0.10
; <<>> DiG 9.16.23-RH <<>> -t a redis-headless.redis.svc.cluster.local @10.96.0.10
;; global options: +cmd
;; Got answer:
;; WARNING: .local is reserved for Multicast DNS
;; You are currently testing what happens when an mDNS query is leaked to DNS
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 9620
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
; COOKIE: 9acef610437cf2a1 (echoed)
;; QUESTION SECTION:
;redis-headless.redis.svc.cluster.local. IN A
;; ANSWER SECTION:
redis-headless.redis.svc.cluster.local. 5 IN A 10.243.58.216
;; Query time: 3 msec
;; SERVER: 10.96.0.10#53(10.96.0.10)
;; WHEN: Mon Aug 05 12:43:04 EDT 2024
;; MSG SIZE rcvd: 133
[root@harbor ~]# kubectl get pod -n redis -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
redis-master-0 1/1 Running 0 5m58s 10.243.58.216 k8s-node02 <none> <none>部署redis-exporter
[root@harbor monitoring]# cat redis-exporter.yaml
apiVersion: v1
kind: Service
metadata:
name: redis-exporter
namespace: monitoring
labels:
k8s-app: redis-exporter
spec:
ports:
- port: 9121
protocol: TCP
name: http
selector:
k8s-app: redis-exporter
type: ClusterIP
---
apiVersion: apps/v1 # 版本
kind: Deployment # 资源
metadata:
labels:
app: redis-exporter # 标签
name: redis-exporter # deploy的名称
namespace: monitoring # 命名空间
spec:
replicas: 1 # 副本
selector:
matchLabels:
k8s-app: redis-exporter # po标签
strategy: {}
template:
metadata:
labels:
k8s-app: redis-exporter # po标签
spec:
containers:
#- image: oliver006/redis_exporter:latest
- image: bitnami/redis-exporter:latest
name: redis-exporter
args: ["-redis.addr", "redis://redis-headless.redis.svc.cluster.local:6379", "-redis.password", "root"] # 注意名称空间
ports:
- containerPort: 9121 # 默认暴露的端口号
name: http[root@harbor monitoring]# kubectl apply -f redis-exporter.yaml [root@harbor monitoring]# kubectl get deployment -n monitoring
NAME READY UP-TO-DATE AVAILABLE AGE
kps-grafana 1/1 1 1 37h
kps-kube-prometheus-stack-operator 1/1 1 1 37h
kps-kube-state-metrics 1/1 1 1 37h
redis-exporter 1/1 1 1 86s
[root@harbor monitoring]# kubectl get svc -n monitoring | grep redis
redis-exporter ClusterIP 10.96.32.37 <none> 9121/TCP 36s这里需要在K8S节点测试
[root@k8s-master01 ~]# curl 10.96.32.37:9121/metrics | tail -n3
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 7000 0 7000 0 0 976k 0 --:--:-- --:--:-- --:--:-- 976k
# HELP redis_up Information about the Redis instance
# TYPE redis_up gauge
redis_up 0prometheus ServiceMonitor
[root@harbor monitoring]# cat redis-exporter-service-monitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: redis-exporter
namespace: monitoring
labels:
k8s-app: redis-exporter
spec:
endpoints:
- interval: 10s
port: http # 这个是port 对应 Service.spec.ports.name
scheme: http
jobLabel: k8s-app
selector:
matchLabels:
k8s-app: redis-exporter # 跟 svc 的 lables 保持一致
namespaceSelector:
matchNames:
- monitoring # svc的命名空间[root@harbor monitoring]# kubectl apply -f redis-exporter-service-monitor.yaml [root@harbor monitoring]# kubectl get servicemonitor -n monitoring | grep redis
redis-exporter 75sprometheus
grafana
导入dashboard
监控方案二：使用helm部署prometheus redis exporter监控手动部署的redis
[root@harbor monitoring]# cat redis.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: redis
namespace: redis
data:
redis.conf: |+
requirepass redis@passwd
maxmemory 268435456
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: redis
labels:
app: redis
spec:
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
annotations:
version/date: "20240720"
spec:
containers:
- name: redis
image: redis
imagePullPolicy: Always
command: ["redis-server","/etc/redis/redis.conf"]
ports:
- containerPort: 6379
volumeMounts:
- name: redis-config
mountPath: /etc/redis/redis.conf
subPath: redis.conf
volumes:
- name: redis-config
configMap:
name: redis
items:
- key: redis.conf
path: redis.conf
---
kind: Service
apiVersion: v1
metadata:
name: redis
namespace: redis
spec:
selector:
app: redis
ports:
- port: 6379
targetPort: 6379[root@harbor monitoring]# kubectl apply -f redis.yaml [root@harbor monitoring]# kubectl get svc -n redis
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
redis ClusterIP 10.108.62.50 <none> 6379/TCP 23s
[root@harbor monitoring]# kubectl get pod -n redis -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
redis-64f9457c95-g5q8w 1/1 Running 0 26s 10.243.58.232 k8s-node02 <none> <none>[root@harbor prometheus-redis-exporter]# vim values.yaml
# 54行 修改redis svc
redisAddress: redis://redis.redis.svc.cluster.local:6379
...
# 58行 添加注解
annotations:
prometheus.io/path: /metrics
prometheus.io/port: "9121"
prometheus.io/scrape: "true"
labels: {}
...
# 110行
serviceMonitor:
# When set true then use a ServiceMonitor to configure scraping
enabled: true # 开启
multipleTarget: false
targets:
# for every targets, url and name must be set,
# an individual additionalRelabeling can be set for every target
- url: "redis://redis.redis.svc.cluster.local:6379" # 修改
name: "my-redis"
# - url: "redis://my-redis-cluster:6379"
...
# 197行
auth:
# Use password authentication
enabled: true # 开启
# Use existing secret (ignores redisPassword)
secret:
name: ""
key: ""
# Redis password (when not stored in a secret)
redisPassword: "redis@passwd" # 密码要与redis配置文件中密码保持一致即可
# Redis user (version 6.X and above)
redisUser: ""
...[root@harbor prometheus-redis-exporter]# helm install redis-exporter ./ -f values.yaml -n monitoring[root@harbor prometheus-redis-exporter]# kubectl get pod -n monitoring -owide | grep redis-exporter
redis-exporter-prometheus-redis-exporter-7bcb4cd6d-4zg5t 1/1 Running 0 40s 10.243.85.198 k8s-node01 <none> <none>
监控方案三：helm部署prometheus redis exporter监控helm部署的redis集群
Redis部署
https://artifacthub.io/packages/helm/bitnami/redis
默认一主三从
helm install redis bitnami/redis --set global.storageClass=nfs-client --set global.redis.password=root --set architecture=replication --version 19.6.4 -n redis --create-namespace --debug[root@harbor prometheus-redis-exporter]# kubectl get svc -n redis
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
redis-headless ClusterIP None <none> 6379/TCP 99s
redis-master ClusterIP 10.103.209.47 <none> 6379/TCP 99s
redis-replicas ClusterIP 10.103.116.168 <none> 6379/TCP 99s
[root@harbor prometheus-redis-exporter]# kubectl get pod -n redis -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
redis-master-0 1/1 Running 0 116s 10.243.58.226 k8s-node02 <none> <none>
redis-replicas-0 1/1 Running 0 116s 10.243.58.223 k8s-node02 <none> <none>
redis-replicas-1 1/1 Running 0 84s 10.243.85.201 k8s-node01 <none> <none>
redis-replicas-2 1/1 Running 0 58s 10.243.58.228 k8s-node02 <none> <none>[root@k8s-master01 ~]# dig -t A redis-headless.redis.svc.cluster.local @10.96.0.10
; <<>> DiG 9.16.23-RH <<>> -t A redis-headless.redis.svc.cluster.local @10.96.0.10
;; global options: +cmd
;; Got answer:
;; WARNING: .local is reserved for Multicast DNS
;; You are currently testing what happens when an mDNS query is leaked to DNS
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 5322
;; flags: qr aa rd; QUERY: 1, ANSWER: 4, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
; COOKIE: 489254ab7b4dccbe (echoed)
;; QUESTION SECTION:
;redis-headless.redis.svc.cluster.local. IN A
;; ANSWER SECTION:
redis-headless.redis.svc.cluster.local. 5 IN A 10.243.58.228
redis-headless.redis.svc.cluster.local. 5 IN A 10.243.58.223
redis-headless.redis.svc.cluster.local. 5 IN A 10.243.58.226
redis-headless.redis.svc.cluster.local. 5 IN A 10.243.85.201
;; Query time: 1 msec
;; SERVER: 10.96.0.10#53(10.96.0.10)
;; WHEN: Sat Aug 10 03:29:44 EDT 2024
;; MSG SIZE rcvd: 295Redis-exporter部署
[root@harbor monitoring]# helm search repo redis-exporter
NAME CHART VERSION APP VERSION DESCRIPTION
prometheus-community/prometheus-redis-exporter 6.3.0 v1.61.0 Prometheus exporter for Redis metrics[root@harbor monitoring]# helm fetch prometheus-community/prometheus-redis-exporter
[root@harbor monitoring]# tar xf prometheus-redis-exporter-6.3.0.tgz
[root@harbor monitoring]# cd prometheus-redis-exporter[root@harbor prometheus-redis-exporter]# vim values.yaml
# 54行 修改redis svc
redisAddress: redis://redis-headless.redis.svc.cluster.local:6379
...
# 58行 添加注解
annotations:
prometheus.io/path: /metrics
prometheus.io/port: "9121"
prometheus.io/scrape: "true"
labels: {}
...
# 110行
serviceMonitor:
# When set true then use a ServiceMonitor to configure scraping
enabled: true # 开启
multipleTarget: false
targets:
# for every targets, url and name must be set,
# an individual additionalRelabeling can be set for every target
- url: "redis://redis-headless.redis.svc.cluster.local:6379" # 修改
name: "my-redis"
# - url: "redis://my-redis-cluster:6379"
...
# 197行
auth:
# Use password authentication
enabled: true # 开启
# Use existing secret (ignores redisPassword)
secret:
name: ""
key: ""
# Redis password (when not stored in a secret)
redisPassword: "root" # 密码
# Redis user (version 6.X and above)
redisUser: ""
...[root@harbor prometheus-redis-exporter]# helm install redis-exporter ./ -f values.yaml -n monitoring[root@harbor monitoring]# kubectl get pod -n monitoring -owide | grep redis-exporter
redis-exporter-prometheus-redis-exporter-85544d8544-dmbzr 1/1 Running 0 3m51s 10.243.85.194 k8s-node01 <none> <none>
