Ensuring High Availability for MongoDB on Kubernetes

1
apiVersion: v1
2
kind: PersistentVolume
3
metadata:
4
  name: mongodb-pv
5
spec:
6
  capacity:
7
    storage: 5Gi
8
  accessModes:
9
    - ReadWriteOnce
10
  hostPath:
11
    path: /data/mongodb
12
---
13
apiVersion: v1
14
kind: PersistentVolumeClaim
15
metadata:
16
  name: mongodb-pvc
17
spec:
18
  storageClassName: ""
19
  accessModes:
20
    - ReadWriteOnce
21
  resources:
22
    requests:
23
      storage: 5Gi

1
kubectl apply -f pv-pvc.yaml

1
kubectl get pv
2
kubectl get pvc

1
apiVersion: v1
2
kind: Service
3
metadata:
4
  name: mongodb-service
5
spec:
6
  clusterIP: None
7
  selector:
8
    app: mongodb
9
  ports:
10
    - port: 27017

1
kubectl apply -f headless-service.yaml

1
kubectl get svc

1
sudo bash -c "openssl rand -base64 756 > mongodb-keyfile"
2
sudo chmod 400 mongodb-keyfile

1
kubectl create secret generic mongodb-keyfile --from-file=mongodb-keyfile

1
apiVersion: apps/v1
2
kind: StatefulSet
3
metadata:
4
  name: mongodb # Specifies the name of the statefulset
5
spec:
6
  serviceName: "mongodb-service" # Specifies the service to use
7
  replicas: 3
8
  selector:
9
    matchLabels:
10
      app: mongodb
11
  template:
12
    metadata:
13
      labels:
14
        app: mongodb
15
    spec:
16
      containers:
17
      - name: mongodb
18
        image: mongo:latest
19
        command:
20
        - mongod
21
        - "--replSet"
22
        - rs0
23
        - "--bind_ip_all"
24
        ports:
25
        - containerPort: 27017
26
        volumeMounts:
27
        - name: mongodb-storage
28
          mountPath: /data/db
29
        - name: keyfile
30
          mountPath: /etc/mongodb-keyfile
31
          readOnly: true
32
        resources:
33
          requests:
34
            cpu: "100m"
35
            memory: "256Mi"
36
          limits:
37
            cpu: "500m"
38
            memory: "512Mi"
39
      volumes:
40
      - name: keyfile
41
        secret:
42
          secretName: mongodb-keyfile
43
          defaultMode: 0400  
44
  volumeClaimTemplates:
45
  - metadata:
46
      name: mongodb-storage
47
    spec:
48
      accessModes: ["ReadWriteOnce"]
49
      resources:
50
        requests:
51
          storage: 5Gi

1
kubectl apply -f statefulset.yaml

1
kubectl get statefulsets
2
kubectl get pods

1
kubectl exec -it mongodb-0 -- mongosh
2
3
rs.initiate({
4
  _id: "rs0",
5
  members: [
6
    { _id: 0, host: "mongodb-0.mongodb-service:27017", priority: 2 },
7
    { _id: 1, host: "mongodb-1.mongodb-service:27017", priority: 1 },
8
    { _id: 2, host: "mongodb-2.mongodb-service:27017", priority: 1 }
9
  ]
10
})

1
test> rs.initiate({
2
...   _id: "rs0",
3
...   members: [
4
...     { _id: 0, host: "mongodb-0.mongodb-service:27017", priority: 2 },
5
...     { _id: 1, host: "mongodb-1.mongodb-service:27017", priority: 1 },
6
...     { _id: 2, host: "mongodb-2.mongodb-service:27017", priority: 1 }
7
...   ]
8
... })
9
{
10
  ok: 1,
11
  '$clusterTime': {
12
    clusterTime: Timestamp({ t: 1718718451, i: 1 }),
13
    signature: {
14
      hash: Binary.createFromBase64('AAAAAAAAAAAAAAAAAAAAAAAAAAA=', 0),
15
      keyId: Long('0')
16
    }
17
  },
18
  operationTime: Timestamp({ t: 1718718451, i: 1 })
19
}
20
rs0 [direct: secondary] test>

1
rs.status()

1
use users
2
3
db.femaleusers.insertOne({ name: "Jane Doe", age: 30, email: "jane.doe@example.com" })
4
db.femaleusers.insertOne({ name: "Mary Smith", age: 25, email: "mary.smith@example.com" })
5
db.femaleusers.insertOne({ name: "Alice Johnson", age: 28, email: "alice.johnson@example.com" })
6
db.femaleusers.insertOne({ name: "Emily Davis", age: 32, email: "emily.davis@example.com" })
7
db.femaleusers.insertOne({ name: "Sarah Brown", age: 27, email: "sarah.brown@example.com" })
8
9
db.femaleusers.find()
10
exit

1
apiVersion: v1
2
kind: PersistentVolume
3
metadata:
4
  name: backup-pv
5
spec:
6
  capacity:
7
    storage: 5Gi
8
  accessModes:
9
    - ReadWriteOnce
10
  hostPath:
11
    path: /mnt/backup
12
---
13
apiVersion: v1
14
kind: PersistentVolumeClaim
15
metadata:
16
  name: backup-pvc
17
spec:
18
  accessModes:
19
    - ReadWriteOnce
20
  resources:
21
    requests:
22
      storage: 5Gi

1
kubectl apply -f backup-pv-pvc.yaml

1
kubectl get pv
2
kubectl get pvc

1
apiVersion: batch/v1
2
kind: CronJob
3
metadata:
4
  name: mongodb-backup
5
spec:
6
  schedule: "*/5 * * * *" # Runs backup every five minutes
7
  jobTemplate:
8
    spec:
9
      template:
10
        spec:
11
          containers:
12
          - name: mongodump
13
            image: mongo:latest
14
            command:
15
            - sh
16
            - -c
17
            - |
18
              # Perform backup
19
              mongodump --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
20
              # Remove backups older than 7 days
21
              find /backup -type d -mtime +7 -exec rm -rf {} +
22
            volumeMounts:
23
            - name: backup-storage
24
              mountPath: /backup
25
          restartPolicy: Never
26
          volumes:
27
          - name: backup-storage
28
            persistentVolumeClaim:
29
              claimName: backup-pvc

1
kubectl apply -f mongodb-backup-cronjob.yaml
2
kubectl get cronjob

1
kubectl get pods

1
kubectl logs mongodb-backup-28635290-jfsjn

1
apiVersion: v1
2
kind: Pod
3
metadata:
4
  name: backup-access
5
spec:
6
  containers:
7
  - name: busybox
8
    image: busybox
9
    command: ["sh", "-c", "sleep 3600"]
10
    volumeMounts:
11
    - name: backup-storage
12
      mountPath: /backup
13
  volumes:
14
  - name: backup-storage
15
    persistentVolumeClaim:
16
      claimName: backup-pvc 

1
kubectl apply -f backup-access.yaml
2
kubectl exec -it backup-access -- sh

1
cd /backup
2
ls
3
cd <file>
4
ls
5
cd users

1
kubectl exec -it mongodb-0 -- mongosh --eval "rs.status()"

1
{
2
  "_id" : 0,
3
  "name" : "mongodb-0.mongodb-service:27017",
4
  "stateStr" : "PRIMARY",
5
  ...
6
}

1
kubectl delete pod mongodb-0

1
kubectl exec -it mongodb-1 -- mongosh --eval "rs.status()"

1
{
2
  "_id" : 1,
3
  "name" : "mongodb-1.mongodb-service:27017",
4
  "stateStr" : "PRIMARY",
5
  ...
6
}

1
kubectl get pods

1
NAME                       READY   STATUS     RESTARTS     	AGE
2
mongodb-0                       1/1     Running     0         		 70s
3
mongodb-1                       1/1     Running     0          		77m
4
mongodb-2                       1/1     Running     0         		 73m

1
kubectl exec -it mongodb-0 -- mongosh --eval "rs.status()"

1
{
2
  "_id" : 0,
3
  "name" : "mongodb-0.mongodb-service:27017",
4
  "stateStr" : "PRIMARY",
5
  ...
6
}

1
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

1
~$ kubectl get pods -n kube-system
2
NAME                                       READY   STATUS    RESTARTS   AGE
3
...
4
metrics-server-6d94bc8694-mkdrb            0/1     Running   0          60s

1
kubectl edit deployment metrics-server -n kube-system

1
spec:
2
  containers:
3
  - args:
4
    ...
5
    command:
6
    - /metrics-server
7
    - --kubelet-insecure-tls
8
    - --kubelet-preferred-address-types=InternalIP

1
kubectl get pods -n kube-system

1
~$ kubectl get pods -n kube-system
2
NAME                                       READY   STATUS    RESTARTS   AGEcalico-kube-controllers-67dfb8c8c9-spmcs   1/1     Running   0          16m
3
...
4
metrics-server-777dff589b-b56lg            1/1     Running   0          117s

1
kubectl top pods

1
kubectl autoscale statefulset mongodb --min=3 --max=10 --cpu-percent=50

1
horizontalpodautoscaler.autoscaling/mongodb autoscaled

1
kubectl get hpa