之前因为某些需要，写了一个运行于 K8S 的 MySQL 双主复制高可用的编排。
核心使用 StatefulSet，通过 initContainers 初始化的方式完成初次启动配置及同步， 完全自动化。
当然，这只是用于测试环境的一个编排，因为缺少存储部分，因此不推荐用于生产。
编排代码如下：
mysql-mm-rbac.yml
RBAC 权限定义。
1apiVersion: v1
2kind: Namespace
3metadata:
4 name: mysql-ha
5
6---
7apiVersion: v1
8kind: ServiceAccount
9metadata:
10 name: cluster-helper
11 namespace: mysql-ha
12
13---
14kind: Role
15apiVersion: rbac.authorization.k8s.io/v1
16metadata:
17 name: cluster-helper
18 namespace: mysql-ha
19rules:
20 - apiGroups: ["apps"]
21 resources: ["statefulsets"]
22 verbs: ["get", "list"]
23
24---
25kind: RoleBinding
26apiVersion: rbac.authorization.k8s.io/v1
27metadata:
28 name: cluster-helper
29 namespace: mysql-ha
30subjects:
31 - kind: ServiceAccount
32 name: cluster-helper
33roleRef:
34 kind: Role
35 name: cluster-helper
36 apiGroup: rbac.authorization.k8s.iomysql-mm.yml
核心编排定义。
1---
2apiVersion: v1
3kind: ConfigMap
4metadata:
5 name: mysql-scripts
6 namespace: mysql-ha
7data:
8 common.sh: |
9 #!/bin/bash
10 update_data() {
11 local svc=${HOSTNAME%-*}
12 local token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
13 local ns=${POD_NAMESPACE}
14 local url=https://${KUBERNETES_SERVICE_HOST}/apis/apps/v1/namespaces/${ns}/statefulsets/${svc}
15 curl -kfsSL -H "Authorization: Bearer ${token}" ${url} > /ha-info/state.json
16 }
17 get_data() {
18 if [ ! -f /ha-info/state.json ]; then
19 return
20 fi
21 cat /ha-info/state.json
22 }
23 get_replicas() {
24 get_data | grep '"replicas":' | head -n 1 | awk '{print $2}' | tr -d ","
25 }
26 get_ready_replicas() {
27 get_data | grep '"readyReplicas":' | awk '{print $2}' | tr -d ","
28 }
29 get_current_replicas() {
30 get_data | grep '"currentReplicas":' | awk '{print $2}' | tr -d ","
31 }
32 sync_done() {
33 echo "Sync has been done."
34 # Infinite blocking
35 while true; do
36 sleep 3600
37 done
38 }
39 init-cluster.sh: |
40 #!/bin/sh
41 if [[ -f /var/lib/mysql/CLUSTER_WAS_INIT ]]; then
42 exit 0
43 fi
44 echo Initialize cluster configuration...
45 source /scripts/common.sh
46 SERVER_ID=$(hostname | grep -oE '[^-]+$')
47 cat > /etc/mysql/conf.d/master-to-master.cnf <<EOF
48 [mysql]
49 no-auto-rehash
50 [mysqld]
51 skip-host-cache
52 skip-name-resolve
53 server_id=1${SERVER_ID}
54 log-bin=mysql-bin
55 auto_increment_increment=2
56 auto_increment_offset=$((${SERVER_ID}+1))
57 EOF
58 echo Initialize cluster configuration successful.
59 sync-ha.sh: |
60 #!/bin/bash
61 set -e
62 if [ -f /var/lib/mysql/CLUSTER_WAS_INIT ]; then
63 sync_done
64 fi
65 source /scripts/common.sh
66 echo "Waiting all replicas up..."
67 while [ `get_current_replicas` -ne `get_ready_replicas` ]
68 do
69 sleep 5
70 done
71 get_another_host() {
72 local svc=${HOSTNAME%-*}
73 local server_id=$(hostname | grep -oE '[^-]+$')
74 local another_id
75 if [[ $server_id == 0 ]]; then
76 another_id=1
77 else
78 another_id=0
79 fi
80 echo "${svc}-${another_id}"
81 }
82 get_binlog_file() {
83 local host=$1
84 mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'show master status\G;' | grep 'File' | awk '{print $2}'
85 }
86 get_binlog_pos() {
87 local host=$1
88 mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'show master status\G;' | grep 'Position' | awk '{print $2}'
89 }
90 ensure_db() {
91 while ! mysql -p"${MYSQL_ROOT_PASSWORD}" -e 'SELECT 1' > /dev/null; do
92 sleep 5
93 done
94 mysql -p"${MYSQL_ROOT_PASSWORD}" -e "GRANT RELOAD, SUPER, REPLICATION SLAVE ON *.* TO 'root'@'%' identified by '${MYSQL_ROOT_PASSWORD}'"
95 }
96 wait_db() {
97 local host=$1
98 while ! mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'SELECT 1' > /dev/null; do
99 sleep 5
100 done
101 }
102 sync_core() {
103 local mainHost=$1
104 local targetHost=$2
105 local file=$3
106 local pos=$4
107 echo Synchionize "${mainHost}" "file=${file}" "pos=${pos}"
108 mysql -h "${mainHost}" -p"${MYSQL_ROOT_PASSWORD}" -e "CHANGE MASTER TO MASTER_HOST='${targetHost}',
109 MASTER_USER='root',
110 MASTER_PASSWORD='${MYSQL_ROOT_PASSWORD}',
111 MASTER_LOG_FILE='${file}',
112 MASTER_LOG_POS=${pos};"
113 mysql -h "${mainHost}" -p"${MYSQL_ROOT_PASSWORD}" -e 'start slave;'
114 }
115 sync_db() {
116 local svc=${HOSTNAME%-*}
117 local right_fqdn=`get_another_host`.${svc}.${POD_NAMESPACE}.svc.cluster.local
118 local left_fqdn=${HOSTNAME}.${svc}.${POD_NAMESPACE}.svc.cluster.local
119 echo "Wait db $right_fqdn ..."
120 wait_db $right_fqdn
121 local left_file=`get_binlog_file ${left_fqdn}`
122 local left_pos=`get_binlog_pos ${left_fqdn}`
123 local right_file=`get_binlog_file ${right_fqdn}`
124 local right_pos=`get_binlog_pos ${right_fqdn}`
125 echo Sync right -> left ...
126 sync_core "$left_fqdn" "$right_fqdn" "$right_file" "$right_pos"
127 echo Sync left -> right ...
128 sync_core "$right_fqdn" "$left_fqdn" "$left_file" "$left_pos"
129 }
130 echo "Ensuring db..."
131 ensure_db
132 server_id=$(hostname | grep -oE '[^-]+$')
133 if [[ $server_id -eq 0 ]]; then
134 sync_db
135 echo "master-$server_id synchionize suffessful."
136 else
137 echo "master-$server_id are ready."
138 fi
139 touch /var/lib/mysql/CLUSTER_WAS_INIT
140 sync_done
141 sync-ha-metadata.sh: |
142 #!/bin/sh
143 if [ -f /var/lib/mysql/CLUSTER_WAS_INIT ]; then
144 sync_done
145 fi
146 apk add curl
147 source /scripts/common.sh
148 while [ ! -f /var/lib/mysql/CLUSTER_WAS_INIT ]
149 do
150 update_data
151 sleep 3
152 done
153 sync_done
154
155---
156apiVersion: apps/v1beta2
157kind: StatefulSet
158metadata:
159 name: mysql-mm
160 namespace: mysql-ha
161spec:
162 serviceName: mysql-mm
163 replicas: 2
164 selector:
165 matchLabels:
166 app: mysql-mm
167 template:
168 metadata:
169 labels:
170 app: mysql-mm
171 spec:
172 initContainers:
173 - name: init-cluster
174 image: alpine
175 command:
176 - /scripts/init-cluster.sh
177 volumeMounts:
178 - name: scripts
179 mountPath: /scripts
180 readOnly: true
181 - name: mysql-conf
182 mountPath: /etc/mysql/conf.d
183 - name: ha-metadata
184 mountPath: /ha-info
185 readOnly: true
186 containers:
187 - name: mysql
188 image: mysql:5.7
189 env:
190 - name: "MYSQL_ROOT_PASSWORD"
191 value: "123456!!"
192 - name: "MYSQL_ROOT_HOST"
193 value: "%"
194 volumeMounts:
195 - name: scripts
196 mountPath: /scripts
197 readOnly: true
198 - name: mysql-conf
199 mountPath: /etc/mysql/conf.d
200 - name: mysql-sock
201 mountPath: /var/run/mysqld
202 - name: mysql-data
203 mountPath: /var/lib/mysql
204 ports:
205 - containerPort: 3306
206 - name: sync-ha
207 image: mysql:5.7
208 command:
209 - /scripts/sync-ha.sh
210 env:
211 - name: "MYSQL_ROOT_PASSWORD"
212 value: "123456!!"
213 - name: POD_NAMESPACE
214 valueFrom:
215 fieldRef:
216 fieldPath: metadata.namespace
217 volumeMounts:
218 - name: scripts
219 mountPath: /scripts
220 readOnly: true
221 - name: mysql-sock
222 mountPath: /var/run/mysqld
223 - name: ha-metadata
224 mountPath: /ha-info
225 - name: mysql-data
226 mountPath: /var/lib/mysql
227 - name: sync-ha-metadata
228 image: alpine
229 command:
230 - /scripts/sync-ha-metadata.sh
231 env:
232 - name: POD_NAMESPACE
233 valueFrom:
234 fieldRef:
235 fieldPath: metadata.namespace
236 volumeMounts:
237 - name: scripts
238 mountPath: /scripts
239 readOnly: true
240 - name: ha-metadata
241 mountPath: /ha-info
242 - name: mysql-data
243 mountPath: /var/lib/mysql
244 volumes:
245 - name: scripts
246 configMap:
247 name: mysql-scripts
248 defaultMode: 0755
249 - name: ha-metadata
250 emptyDir: {}
251 - name: mysql-conf
252 emptyDir: {}
253 - name: mysql-sock
254 emptyDir: {}
255 - name: mysql-data
256 emptyDir: {}
257 serviceAccountName: cluster-helper
258 # volumeClaimTemplates:
259 # - metadata:
260 # name: pvc-mysql-data
261 # spec:
262 # accessModes: [ "ReadWriteOnce" ]
263 # resources:
264 # requests:
265 # storage: 10Gi
266
267---
268# Service
269apiVersion: v1
270kind: Service
271metadata:
272 name: mysql-mm
273 namespace: mysql-ha
274spec:
275 ports:
276 - port: 3306
277 targetPort: 3306
278 selector:
279 app: mysql-mm