Docker Swarm 下搭建 MongoDB 分片+副本+选举集群

一、环境准备

三台服务器，建立 Docker Swarm 集群，一个 Manager，两个 Worker。

• docker 版本：17-09
• mongo 版本：3.6

二、MongoDB 集群架构设计

高清图地址: https://www.processon.com/view/link/5a3c7386e4b0bf89b8530376

三、搭建集群

1、【Manager】创建集群网络

docker network create -d overlay --attachable mongo

–attachable 允许其他容器加入此网络

2、创建 9 个 Data 服务，3 个 Config 服务，1 个 Global 模式的 Mongos 服务

2.1、【所有机器】创建相关文件夹

mkdir /root/mongo/config /root/mongo/shard1 /root/mongo/shard2 /root/mongo/shard3

2.2、【Manager】创建 stack.yml

version: '3.3'
services:
  mongors1n1:
    # docker 中国的镜像加速地址
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard1 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard1:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        # 指定在服务器 manager 上启动
        constraints:
          - node.hostname==manager
  mongors2n1:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard2 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard2:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==manager
  mongors3n1:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard3 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard3:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==manager
  mongors1n2:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard1 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard1:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker1
  mongors2n2:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard2 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard2:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker1
  mongors3n2:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard3 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard3:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker1
  mongors1n3:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard1 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard1:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker2
  mongors2n3:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard2 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard2:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker2
  mongors3n3:
    image: registry.docker-cn.com/library/mongo
    command: mongod --shardsvr --replSet shard3 --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/shard3:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker2
  cfg1:
    image: registry.docker-cn.com/library/mongo
    command: mongod --configsvr --replSet cfgrs --smallfiles --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/config:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==manager
  cfg2:
    image: registry.docker-cn.com/library/mongo
    command: mongod --configsvr --replSet cfgrs --smallfiles --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/config:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker1
  cfg3:
    image: registry.docker-cn.com/library/mongo
    command: mongod --configsvr --replSet cfgrs --smallfiles --dbpath /data/db --port 27017
    networks:
      - mongo
    volumes:
      - /etc/localtime:/etc/localtime
      - /root/mongo/config:/data/db
    deploy:
      restart_policy:
        condition: on-failure
      replicas: 1
      placement:
        constraints:
          - node.hostname==worker2
  mongos:
    image: registry.docker-cn.com/library/mongo
    # mongo 3.6 版默认绑定IP为 127.0.0.1，此处绑定 0.0.0.0 是允许其他容器或主机可以访问
    command: mongos --configdb cfgrs/cfg1:27017,cfg2:27017,cfg3:27017 --bind_ip 0.0.0.0 --port 27017
    networks:
      - mongo
    # 映射宿主机的 27017 端口
    ports:
      - 27017:27017
    volumes:
      - /etc/localtime:/etc/localtime
    depends_on:
      - cfg1
      - cfg2
      - cfg3
    deploy:
      restart_policy:
        condition: on-failure
      # 在集群内的每一台服务器上都启动一个容器
      mode: global
networks:
  mongo:
    external: true

2.3、启动服务，在 Manager 上执行

docker stack deploy -c stack.yml mongo

2.4、【Manager】查看服务的启动情况

docker service ls

正常情况下，会出现如下结果：

[docker@manager ~]# docker service ls
ID                  NAME                MODE                REPLICAS            IMAGE                                         PORTS
z1l5zlghlfbi        mongo_cfg1          replicated          1/1                 registry.docker-cn.com/library/mongo:latest
lg9vbods29th        mongo_cfg2          replicated          1/1                 registry.docker-cn.com/library/mongo:latest
i6d6zwxsq0ss        mongo_cfg3          replicated          1/1                 registry.docker-cn.com/library/mongo:latest
o0lfdavd8kpj        mongo_mongors1n1    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
n85yeyod7mlu        mongo_mongors1n2    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
cwurdqng9tdk        mongo_mongors1n3    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
vu6al5kys28u        mongo_mongors2n1    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
xrjiep0vrf0w        mongo_mongors2n2    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
qqzifwcejjyk        mongo_mongors2n3    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
tddgw8hygv1b        mongo_mongors3n1    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
qrb6fjty03mw        mongo_mongors3n2    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
m8ikdzjssmhn        mongo_mongors3n3    replicated          1/1                 registry.docker-cn.com/library/mongo:latest
mnnlm49b7kyb        mongo_mongos        global              3/3                 registry.docker-cn.com/library/mongo:latest   *:27017->27017/tcp

3、初始化集群

3.1 【Manager】初始化 Mongo 配置集群

docker exec -it $(docker ps | grep "cfg1" | awk '{ print $1 }') bash -c "echo 'rs.initiate({_id: \"cfgrs\",configsvr: true, members: [{ _id : 0, host : \"cfg1\" },{ _id : 1, host : \"cfg2\" }, { _id : 2, host : \"cfg3\" }]})' | mongo"

3.2 【Manager】初始化三个 Mongo 数据集群

docker exec -it $(docker ps | grep "mongors1n1" | awk '{ print $1 }') bash -c "echo 'rs.initiate({_id : \"shard1\", members: [{ _id : 0, host : \"mongors1n1\" },{ _id : 1, host : \"mongors1n2\" },{ _id : 2, host : \"mongors1n3\", arbiterOnly: true }]})' | mongo"

docker exec -it $(docker ps | grep "mongors2n1" | awk '{ print $1 }') bash -c "echo 'rs.initiate({_id : \"shard2\", members: [{ _id : 0, host : \"mongors2n1\" },{ _id : 1, host : \"mongors2n2\" },{ _id : 2, host : \"mongors2n3\", arbiterOnly: true }]})' | mongo"

docker exec -it $(docker ps | grep "mongors3n1" | awk '{ print $1 }') bash -c "echo 'rs.initiate({_id : \"shard3\", members: [{ _id : 0, host : \"mongors3n1\" },{ _id : 1, host : \"mongors3n2\" },{ _id : 2, host : \"mongors3n3\", arbiterOnly: true }]})' | mongo"

3.3 【Manager】将三个数据集群当做分片加入 mongos

docker exec -it $(docker ps | grep "mongos" | awk '{ print $1 }') bash -c "echo 'sh.addShard(\"shard1/mongors1n1:27017,mongors1n2:27017,mongors1n3:27017\")' | mongo "

docker exec -it $(docker ps | grep "mongos" | awk '{ print $1 }') bash -c "echo 'sh.addShard(\"shard2/mongors2n1:27017,mongors2n3:27017,mongors2n3:27017\")' | mongo "

docker exec -it $(docker ps | grep "mongos" | awk '{ print $1 }') bash -c "echo 'sh.addShard(\"shard3/mongors3n1:27017,mongors3n2:27017,mongors3n3:27017\")' | mongo "

4、连接集群

4.1 内部：在 mongo 网络下的容器，通过 mongos:27017 连接

4.2 外部：通过 IP:27017 连接，IP 可以为三台服务的中的一个的 IP