k8s部署redis一主两从三哨兵(集群内访问)

本文主要是介绍k8s部署redis一主两从三哨兵(集群内访问)，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

一、配置清单
1.基于K8s搭建部署1主2从3哨兵的Redis集群哨兵模式(集群内访问)

2.持久化数据选择用storageclass，动态创建pv存储，动态存储使用longhorn

创建redis配置文件

#注意内容中注释

apiVersion: v1
kind: Namespace
metadata:name: prod---
apiVersion: v1
kind: ConfigMap
metadata:name: redis5-configmapnamespace: prodlabels:app: redis
data:redis.conf: |dir "/data"maxmemory 0maxmemory-policy volatile-lrumin-slaves-max-lag 5min-slaves-to-write 1rdbchecksum yesrdbcompression yesrepl-diskless-sync yessave 900 1sentinel.conf: |dir "/data"sentinel down-after-milliseconds mymaster 10000sentinel failover-timeout mymaster 180000sentinel parallel-syncs mymaster 5init.sh: |HOSTNAME="$(hostname)"INDEX="${HOSTNAME##*-}"#redis5-sentinel-svc和service名字相关MASTER="$(redis-cli -h redis5-sentinel-svc -p 26379 sentinel get-master-addr-by-name mymaster | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"MASTER_GROUP="mymaster"QUORUM="2"REDIS_CONF=/data/conf/redis.confREDIS_PORT=6379SENTINEL_CONF=/data/conf/sentinel.confSENTINEL_PORT=26379#需要修改为自己service名字无头服务的名字SERVICE=redis5-sentinel-headlessset -eusentinel_update() {echo "Updating sentinel config"eval MY_SENTINEL_ID="\${SENTINEL_ID_$INDEX}"sed -i "1s/^/sentinel myid $MY_SENTINEL_ID\\n/" "$SENTINEL_CONF"sed -i "2s/^/sentinel monitor $MASTER_GROUP $1 $REDIS_PORT $QUORUM \\n/" "$SENTINEL_CONF"echo "sentinel announce-ip $ANNOUNCE_IP" >> $SENTINEL_CONFecho "sentinel announce-port $SENTINEL_PORT" >> $SENTINEL_CONF}redis_update() {echo "Updating redis config"echo "slaveof $1 $REDIS_PORT" >> "$REDIS_CONF"echo "slave-announce-ip $ANNOUNCE_IP" >> $REDIS_CONFecho "slave-announce-port $REDIS_PORT" >> $REDIS_CONF}copy_config() {cp /readonly-config/redis.conf "$REDIS_CONF"cp /readonly-config/sentinel.conf "$SENTINEL_CONF"}setup_defaults() {echo "Setting up defaults"if [ "$INDEX" = "0" ]; thenecho "Setting this pod as the default master"redis_update "$ANNOUNCE_IP"sentinel_update "$ANNOUNCE_IP"sed -i "s/^.*slaveof.*//" "$REDIS_CONF"else#  "redis-0.$SERVICE" 和statefulset的名字相关DEFAULT_MASTER="$(getent hosts "redis5-sentinel-0.$SERVICE" | awk '{ print $1 }')"if [ -z "$DEFAULT_MASTER" ]; thenecho "Unable to resolve host"exit 1fiecho "Setting default slave config.."redis_update "$DEFAULT_MASTER"sentinel_update "$DEFAULT_MASTER"fi}find_master() {echo "Attempting to find master"if [ "$(redis-cli -h "$MASTER" ping)" != "PONG" ]; thenecho "Can't ping master, attempting to force failover"if redis-cli -h "$SERVICE" -p "$SENTINEL_PORT" sentinel failover "$MASTER_GROUP" | grep -q 'NOGOODSLAVE' ; then setup_defaultsreturn 0fisleep 10MASTER="$(redis-cli -h $SERVICE -p $SENTINEL_PORT sentinel get-master-addr-by-name $MASTER_GROUP | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"if [ "$MASTER" ]; thensentinel_update "$MASTER"redis_update "$MASTER"elseecho "Could not failover, exiting..."exit 1fielseecho "Found reachable master, updating config"sentinel_update "$MASTER"redis_update "$MASTER"fi}mkdir -p /data/conf/echo "Initializing config.."copy_config#  "redis-$INDEX.$SERVICE" 和statefulset的名字相关ANNOUNCE_IP=$(getent hosts "redis5-sentinel-$INDEX.$SERVICE" | awk '{ print $1 }')if [ -z "$ANNOUNCE_IP" ]; then"Could not resolve the announce ip for this pod"exit 1elif [ "$MASTER" ]; thenfind_masterelsesetup_defaultsfiif [ "${AUTH:-}" ]; thenecho "Setting auth values"ESCAPED_AUTH=$(echo "$AUTH" | sed -e 's/[\/&]/\\&/g');sed -i "s/replace-default-auth/${ESCAPED_AUTH}/" "$REDIS_CONF" "$SENTINEL_CONF"fiecho "Ready..."---
apiVersion: v1
kind: ConfigMap
metadata:name: redis-probesnamespace: prodlabels:app: redis
data:check-quorum.sh: |#!/bin/shset -euMASTER_GROUP="mymaster"SENTINEL_PORT=26379REDIS_PORT=6379NUM_SLAVES=$(redis-cli -p "$SENTINEL_PORT" sentinel master mymaster | awk '/num-slaves/{getline; print}')MIN_SLAVES=1if [ "$1" = "$SENTINEL_PORT" ]; thenif redis-cli -p "$SENTINEL_PORT" sentinel ckquorum "$MASTER_GROUP" | grep -q NOQUORUM ; thenecho "ERROR: NOQUORUM. Sentinel quorum check failed, not enough sentinels found"exit 1fielif [ "$1" = "$REDIS_PORT" ]; thenif [ "$MIN_SLAVES" -gt "$NUM_SLAVES" ]; thenecho "Could not find enough replicating slaves. Needed $MIN_SLAVES but found $NUM_SLAVES"exit 1fifish /probes/readiness.sh "$1"readiness.sh: |#!/bin/shset -euCHECK_SERVER="$(redis-cli -p "$1" ping)"if [ "$CHECK_SERVER" != "PONG" ]; thenecho "Server check failed with: $CHECK_SERVER"exit 1fi---
apiVersion: v1
kind: ServiceAccount
metadata:name: redisnamespace: prodlabels:app: redis---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:name: redisnamespace: prodlabels:app: redis
rules:
- apiGroups:- ""resources:- endpointsverbs:- get---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: redisnamespace: prodlabels:app: redis
subjects:
- kind: ServiceAccountname: redis
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: redis---
apiVersion: v1
kind: Service
metadata:name: redis5-sentinel-headlessnamespace: prodlabels:app: redis-haannotations:service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:publishNotReadyAddresses: truetype: ClusterIPclusterIP: Noneports:- name: serverport: 6379protocol: TCPtargetPort: redis- name: sentinelport: 26379protocol: TCPtargetPort: sentinelselector:app: redis-ha---
apiVersion: v1
kind: Service
metadata:name: redis5-sentinel-svcnamespace: prodlabels:app: redis-haannotations:
spec:type: ClusterIPports:- name: serverport: 6379protocol: TCPtargetPort: redis- name: sentinelport: 26379protocol: TCPtargetPort: sentinelselector:app: redis-ha---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: redis5-sentinelnamespace: prodlabels:app: redis-ha
spec:selector:matchLabels:app: redis-haserviceName: redis5-sentinel-headlessreplicas: 3podManagementPolicy: OrderedReadyupdateStrategy:type: RollingUpdatetemplate:metadata:labels:app: redis-haspec:affinity:podAntiAffinity:requiredDuringSchedulingIgnoredDuringExecution:- labelSelector:matchLabels:app: redis-hatopologyKey: kubernetes.io/hostnamepreferredDuringSchedulingIgnoredDuringExecution:- weight: 100podAffinityTerm:labelSelector:matchLabels:app:  redis-hatopologyKey: failure-domain.beta.kubernetes.io/zonesecurityContext:fsGroup: 1000runAsNonRoot: truerunAsUser: 1000serviceAccountName: redisinitContainers:- name: config-initimage: redis:5.0.3imagePullPolicy: IfNotPresentresources:{}command:- shargs:- /readonly-config/init.shenv:- name: SENTINEL_ID_0value: 0c09a3866dba0f3b43ef2e383b5dc05980900fd8- name: SENTINEL_ID_1value: e6be0f70406122877338f7c814b17a7c7b648d82- name: SENTINEL_ID_2value: 31f8f52b34feaddcabdd6bf1827aeb02be44d2e3volumeMounts:- name: configmountPath: /readonly-configreadOnly: true- name: datamountPath: /datacontainers:- name: redisimage: redis:5.0.3imagePullPolicy: IfNotPresentcommand:- redis-serverargs:- /data/conf/redis.conflivenessProbe:exec:command: [ "sh", "/probes/readiness.sh", "6379"]initialDelaySeconds: 15periodSeconds: 5readinessProbe:exec:command: ["sh", "/probes/readiness.sh", "6379"]initialDelaySeconds: 15periodSeconds: 5resources:{}ports:- name: rediscontainerPort: 6379volumeMounts:- mountPath: /dataname: data- mountPath: /probesname: probes- name: sentinelimage: redis:5.0.3imagePullPolicy: IfNotPresentcommand:- redis-sentinelargs:- /data/conf/sentinel.conflivenessProbe:exec:command: [ "sh", "/probes/readiness.sh", "26379"]initialDelaySeconds: 15periodSeconds: 5readinessProbe:exec:command: ["sh", "/probes/readiness.sh", "26379"]initialDelaySeconds: 15periodSeconds: 5resources:{}ports:- name: sentinelcontainerPort: 26379volumeMounts:- mountPath: /dataname: data- mountPath: /probesname: probesvolumes:- name: configconfigMap:name: redis5-configmap- name: probesconfigMap:name: redis-probesvolumeClaimTemplates:- metadata:name: dataannotations:spec:accessModes:- "ReadWriteMany"resources:requests:storage: "10Gi"#修改成自己的存储类storageClassName: longhorn

三、执行配置文件

 kubectl apply -f   redis-cluster-sts.yaml

四、查看节点集群状态，主从状态

[root@xm-nano-k8s-master-113-59 ]# kubectl exec -it -n prod redis5-sentinel-0 -- sh -c redis-cli
Defaulted container "redis" out of: redis, sentinel, config-init (init)
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
min_slaves_good_slaves:2
slave0:ip=10.42.1.155,port=6379,state=online,offset=154978,lag=0
slave1:ip=10.42.7.26,port=6379,state=online,offset=154978,lag=0
master_replid:adecf662565b1dc90eca2af58e3ebbe3ce899b45
master_replid2:7e486d561259c04f0785e1250ef089683bdc4241
master_repl_offset:155114
second_repl_offset:84363
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:74662
repl_backlog_histlen:80453

验证：master节点删除后重启，查看master是否自动漂移。Pod被删除后会自动启动，且启动成功后会自动变为Slave节点.

集群内部程序连接哨兵的地址：
创建一个Headless Service（无头服务）来管理哨兵Pod，但不提供负载均衡和ClusterIP。Headless Service会为每个哨兵Pod分配一个DNS条目，格式为<pod-name>.<service-name>.<namespace>.svc.cluster.local。

例如：redis5-sentinel-2.redis5-sentinel-headless.prod.svc.cluster.local.:26379

集群外部访问K8s的redis哨兵(gpt版本)

在Kubernetes（k8s）环境中，由于Pod的IP地址是动态分配的，并且会随着Pod的重启或重新调度而改变，因此直接在外部程序的连接配置中硬编码哨兵节点的IP地址和端口是不可行的。然而，如果你的外部程序确实需要这种配置（尽管这通常不是最佳实践），你可以采取以下几种策略：

1. 使用NodePort或LoadBalancer类型的Service

虽然这不是直接提供哨兵节点IP地址的方法，但你可以为哨兵节点创建一个或多个NodePort或LoadBalancer类型的Service。然而，请注意，这通常不是哨兵节点的标准用法，因为哨兵的主要目的是在Redis主节点故障时自动进行故障转移，而不是直接对外提供服务。

不过，如果你确实需要这样做，你可以为每个哨兵节点（或一组哨兵节点）创建一个Service，并配置为NodePort或LoadBalancer类型。然后，你可以将Service的外部IP地址（对于LoadBalancer类型）或节点的IP地址加上NodePort端口（对于NodePort类型）提供给外部程序。但是，请注意，这将为每个哨兵节点提供一个可访问的端点，而不是一个统一的哨兵服务入口。