Deploying MongoDB Across Multiple Kubernetes Clusters With MongoDBMulti

1
CLUSTER_NAMES=(mdb-cluster-1 mdb-cluster-2 mdb-cluster-3)
2
ZONES=(us-west2-a us-central1-a us-east1-b)
3
4
for ((i=0; i<${#CLUSTER_NAMES[@]:0:1}; i++)); do
5
  gcloud container clusters create "${CLUSTER_NAMES[$i]}" \
6
    --zone "${ZONES[$i]}" \
7
    --machine-type n2-standard-2 --cluster-version="${K8S_VERSION}" \
8
    --disk-type=pd-standard --num-nodes 1
9
done

1
for ((i=0; i<${#CLUSTER_NAMES[@]:0:1}; i++)); do
2
  gcloud container clusters get-credentials "${CLUSTER_NAMES[$i]}" \
3
    --zone "${ZONES[$i]}"
4
done

1
kubectx

1
gke_lustrous-spirit-371620_us-central1-a_mdb-cluster-2
2
gke_lustrous-spirit-371620_us-east1-b_mdb-cluster-3
3
gke_lustrous-spirit-371620_us-south1-a_master-operator
4
gke_lustrous-spirit-371620_us-west2-a_mdb-cluster-1

1
KUBECTX_OUTPUT=($(kubectx))
2
CLUSTER_NUMBER=0
3
for context in  "${KUBECTX_OUTPUT[@]}"; do
4
 if [[ $context == *"master"* ]]; then
5
    MASTER="$context"
6
 else
7
    CLUSTER_NUMBER=$((CLUSTER_NUMBER+1))
8
    eval  "MDB_$CLUSTER_NUMBER=$context"
9
 fi
10
done

1
REPO_URL="https://github.com/mongodb/mongodb-enterprise-kubernetes.git"
2
SUBDIR_PATH="mongodb-enterprise-kubernetes/tools/multicluster"
3
SCRIPT_NAME="install_istio_separate_network.sh"
4
ISTIO_VERSION="1.16.1"
5
git clone "$REPO_URL"
6
for ((i = 1; i <= ${#CLUSTER_NAMES[@]}; i++)); do
7
  eval mdb="\$MDB_${i}"
8
  eval k8s="CTX_CLUSTER${i}"
9
  sed -i'' -e "s/export ${k8s}=.*/export CTX_CLUSTER${i}=${mdb}/"  "$SUBDIR_PATH/$SCRIPT_NAME"
10
done
11
sed -i'' -e "s/export VERSION=.*/export VERSION=${ISTIO_VERSION}/"  "$SUBDIR_PATH/$SCRIPT_NAME"

1
yes | "$SUBDIR_PATH/$SCRIPT_NAME"

1
CLUSTERS=$MDB_1,$MDB_2,$MDB_3
2
cd  "$SUBDIR_PATH"
3
go run main.go setup \
4
  -central-cluster="${MASTER}" \
5
  -member-clusters="${CLUSTERS}" \
6
  -member-cluster-namespace="mongodb" \
7
  -central-cluster-namespace="mongodb"

1
kubectx $MDB_2

1
Switched to context "gke_lustrous-spirit-371620_us-central1-a_mdb-cluster-2"

1
kubectl get ns
2
3
NAME              STATUS   AGE
4
default           Active   62m
5
istio-system      Active   7m45s
6
kube-node-lease   Active   62m
7
kube-public       Active   62m
8
kube-system       Active   62m
9
mongodb           Active   41s

1
kubectl -n mongodb get sa
2
3
default                                     1         55s
4
mongodb-enterprise-operator-multi-cluster   1         52s

1
CLUSTER_ARRAY=($MDB_1 $MDB_2 $MDB_3)
2
for CLUSTER in "${CLUSTER_ARRAY[@]}"; do     
3
  kubectl label --context=$CLUSTER namespace mongodb istio-injection=enabled
4
done

1
kubectx $MASTER

1
helm upgrade --install mongodb-enterprise-operator-multi-cluster mongodb/enterprise-operator \
2
  --namespace mongodb \
3
  --set namespace=mongodb \
4
  --version="${HELM_CHART_VERSION}" \
5
  --set operator.name=mongodb-enterprise-operator-multi-cluster \
6
  --set "multiCluster.clusters={${CLUSTERS}}" \
7
  --set operator.createOperatorServiceAccount=false \
8
  --set multiCluster.performFailover=false

1
kubectl -n mongodb get pods

1
NAME                                                   READY STATUS    RESTARTS   AGE
2
mongodb-enterprise-operator-multi-cluster-688d48dfc6    1/1  Running 0  8s

1
kubectl -n mongodb get crd | grep multi

1
mongodbmulti.mongodb.com                         

1
POD=$(kubectl -n mongodb get po|grep operator|awk '{ print $1 }')
2
kubectl -n mongodb logs -f po/$POD

1
for CLUSTER in "${CLUSTER_ARRAY[@]}"; do
2
  helm template --show-only templates/database-roles.yaml mongodb/enterprise-operator --namespace "mongodb" | kubectl apply -f - --context=${CLUSTER} --namespace mongodb; 
3
done

1
kubectx $MASTER
2
URL=http://$(kubectl -n "${NAMESPACE}" get svc ops-manager-svc-ext -o jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}')
3
echo $URL

1
kubectl apply -f - <<EOF
2
apiVersion: v1
3
kind: Secret
4
metadata:
5
  name: multi-organization-secret
6
  namespace: mongodb
7
stringData:
8
  publicKey: <PUBLIC KEY>
9
  privateKey: <PRIVATE_KEY>
10
EOF

1
kubectl apply -f - <<EOF
2
apiVersion: v1
3
kind: ConfigMap
4
metadata:
5
  name: multi-project
6
  namespace: mongodb
7
data:
8
  baseUrl: "${URL}"
9
  orgId: <YOUR_ORG_ID>
10
EOF

1
MDB_VERSION=6.0.2-ent
2
kubectl apply -f - <<EOF
3
apiVersion: mongodb.com/v1
4
kind: MongoDBMulti
5
metadata:
6
   name: multi-replica-set
7
   namespace: mongodb
8
spec:
9
   version: "${MDB_VERSION}" 
10
   type: ReplicaSet
11
   persistent: true 
12
   duplicateServiceObjects: true 
13
   credentials: multi-organization-secret 
14
   opsManager:
15
     configMapRef:
16
       name: multi-project 
17
   clusterSpecList:
18
     clusterSpecs:
19
     - clusterName: ${MDB_1} 
20
       members: 1
21
     - clusterName: ${MDB_2}
22
       members: 1
23
     - clusterName: ${MDB_3}    
24
       members: 1
25
EOF

1
POD=$(kubectl -n mongodb get po|grep operator|awk '{ print $1 }')

1
kubectl -n mongodb logs -f po/$POD|grep IP_ADDRESS_NOT_ON_ACCESS_LIST

1
kubectl -n mongodb get mdbm

1
NAME            PHASE     AGE
2
multi-cluster   Running   4m25s

1
for CLUSTER in "${CLUSTER_ARRAY[@]}"; do
2
  kubectl -n mongodb --context=${CLUSTER}  get pods
3
done

1
gke_lustrous-spirit-371620_us-central1-a_mdb-cluster-2
2
NAME                READY   STATUS    RESTARTS      AGE
3
multi-cluster-0-0   2/2     Running   1 (13m ago)   13m
4
5
gke_lustrous-spirit-371620_us-east1-b_mdb-cluster-3
6
NAME                READY   STATUS    RESTARTS      AGE
7
multi-cluster-1-0   2/2     Running   1 (12m ago)   12m
8
9
gke_lustrous-spirit-371620_us-west2-a_mdb-cluster-1
10
NAME                READY   STATUS    RESTARTS      AGE
11
multi-cluster-2-0   2/2     Running   1 (12m ago)   12m

1
for CLUSTER in "${CLUSTER_ARRAY[@]}"; do
2
  kubectl -n mongodb --context=${CLUSTER}  get sts
3
done

1
gke_lustrous-spirit-371620_us-central1-a_mdb-cluster-2
2
NAME              READY   AGE
3
multi-cluster-0   1/1     16m
4
5
gke_lustrous-spirit-371620_us-east1-b_mdb-cluster-3
6
NAME              READY   AGE
7
multi-cluster-1   1/1     15m
8
9
gke_lustrous-spirit-371620_us-west2-a_mdb-cluster-1
10
NAME              READY   AGE
11
multi-cluster-2   1/1     15m

1
POD=$(kubectl -n mongodb get po|grep operator-multi|awk '{ print $1 }') 
2
kubectl -n mongodb logs -f po/$POD

1
kubectl logs $POD -n mongodb

1
kubectl exec -it $POD -n mongodb -- /bin/bash