一、环境
1$ cat /etc/redhat-release 2CentOS Linux release 7.0.1406 (Core) 3$ uname -a 4Linux zhaopin-2-201 3.10.0-123.el7.x86_64 #1 SMP Mon Jun 30 12:09:22 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux 5$ mongo --port 30000 6MongoDB shell version: 3.0.6 7connecting to: 127.0.0.1:30000/test 8mongos> sh.status(); 9--- Sharding Status --- 10 sharding version: { 11 "_id" : 1, 12 "minCompatibleVersion" : 5, 13 "currentVersion" : 6, 14 "clusterId" : ObjectId("561728b4030ea038bcb57fa0") 15} 16 shards: 17 { "_id" : "sh0", "host" : "sh0/172.30.2.201:27010" } 18 { "_id" : "sh1", "host" : "sh1/172.30.2.201:27011" } 19 balancer: 20 Currently enabled: yes 21 Currently running: no 22 Failed balancer rounds in last 5 attempts: 0 23 Migration Results for the last 24 hours: 24 No recent migrations 25 databases: 26 { "_id" : "admin", "partitioned" : false, "primary" : "config" } 27 { "_id" : "mydb", "partitioned" : true, "primary" : "sh0" }
二、准备
插入测试数据
1$ mongo --port 30000 2MongoDB shell version: 3.0.6 3connecting to: 127.0.0.1:30000/test 4mongos> use mydb 5switched to db mydb 6mongos> sh.shardCollection("mydb.test",{"Uid":1}); 7{ "collectionsharded" : "mydb.test", "ok" : 1 } 8mongos> for (var i = 1; i <= 100000; i++) db.test.save({"Uid":i,"Name":"zhanjindong","Age":13,"Date":new Date()}); 9WriteResult({ "nInserted" : 1 }) 10mongos> use config; 11switched to db config 12mongos> db.chunks.find().limit(3); 13{ "_id" : "mydb.test-Uid_MinKey", "lastmod" : Timestamp(10, 1), "lastmodEpoch" : ObjectId("56178dad030ea038bcb592c2"), "ns" : "mydb.test", "min" : { "Uid" : { "$minKey" : 1 } }, "max" : { "Uid" : 2 }, "shard" : "sh0" } 14{ "_id" : "mydb.test-Uid_2.0", "lastmod" : Timestamp(1, 2), "lastmodEpoch" : ObjectId("56178dad030ea038bcb592c2"), "ns" : "mydb.test", "min" : { "Uid" : 2 }, "max" : { "Uid" : 10 }, "shard" : "sh0" } 15{ "_id" : "mydb.test-Uid_10.0", "lastmod" : Timestamp(11, 1), "lastmodEpoch" : ObjectId("56178dad030ea038bcb592c2"), "ns" : "mydb.test", "min" : { "Uid" : 10 }, "max" : { "Uid" : 4691 }, "shard" : "sh1" }
三、Shard Server节点故障
1.分片中副本集的节点故障
参考MongoDB副本集故障测试和解决方案，Query Routers会自动更新副本集的配置，只需要在副本集里面解决故障
2.分片故障
分片故障是指分片中的节点全部不可用了（如果分片中配置了副本集的话，基本是不会出现这种情况的）
1）将sh0停机
1$ ps aux | grep mongo 2wenhang+ 1493 0.0 0.0 112640 976 pts/0 S+ 10:42 0:00 grep --color=auto mongo 3root 41492 0.5 0.1 1045696 118336 ? Sl Oct09 8:12 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 4root 41509 0.5 0.1 1028196 108720 ? Sl Oct09 7:30 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 5root 41855 0.6 0.0 566248 55208 ? Sl Oct09 9:14 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 6root 41870 0.5 0.0 565220 56180 ? Sl Oct09 8:29 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 7root 42170 0.5 0.0 565216 56872 ? Sl Oct09 8:25 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 8root 42233 0.3 0.0 260180 13200 ? Sl Oct09 5:40 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 9root 42286 0.3 0.0 259148 11564 ? Sl Oct09 4:31 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 10$ ps aux | grep mongo | grep -v grep 11root 41492 0.5 0.1 1045696 118336 ? Sl Oct09 8:12 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 12root 41509 0.5 0.1 1028196 108720 ? Sl Oct09 7:31 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 13root 41855 0.6 0.0 566248 54988 ? Sl Oct09 9:14 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 14root 41870 0.5 0.1 565220 72392 ? Sl Oct09 8:29 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 15root 42170 0.5 0.0 565216 56776 ? Sl Oct09 8:25 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 16root 42233 0.3 0.0 260180 13200 ? Sl Oct09 5:40 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 17root 42286 0.3 0.0 259148 11564 ? Sl Oct09 4:31 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 18$ sudo kill 41492 19$ ps aux | grep mongo | grep -v grep 20root 41509 0.5 0.1 1028196 108736 ? Sl Oct09 7:31 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 21root 41855 0.6 0.1 566248 87832 ? Sl Oct09 9:14 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 22root 41870 0.5 0.1 565220 89116 ? Sl Oct09 8:29 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 23root 42170 0.5 0.0 565216 56852 ? Sl Oct09 8:26 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 24root 42233 0.3 0.0 260180 13200 ? Sl Oct09 5:40 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 25root 42286 0.3 0.0 259148 11564 ? Sl Oct09 4:31 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf
2）查看集群状态
1$ mongo --port 30000 2MongoDB shell version: 3.0.6 3connecting to: 127.0.0.1:30000/test 4mongos> sh.status(); 5--- Sharding Status --- 6 sharding version: { 7 "_id" : 1, 8 "minCompatibleVersion" : 5, 9 "currentVersion" : 6, 10 "clusterId" : ObjectId("561728b4030ea038bcb57fa0") 11} 12 shards: 13 { "_id" : "sh0", "host" : "sh0/172.30.2.201:27010" } 14 { "_id" : "sh1", "host" : "sh1/172.30.2.201:27011" } 15 balancer: 16 Currently enabled: yes 17 Currently running: no 18 Failed balancer rounds in last 5 attempts: 5 19 Last reported error: socket exception [CONNECT_ERROR] for sh0/172.30.2.201:27010 20 Time of Reported error: Sat Oct 10 2015 10:44:56 GMT+0800 (CST) 21 Migration Results for the last 24 hours: 22 19 : Success 23 2 : Failed with error 'migration already in progress', from sh0 to sh1 24 databases: 25 { "_id" : "admin", "partitioned" : false, "primary" : "config" } 26 { "_id" : "mydb", "partitioned" : true, "primary" : "sh0" } 27 mydb.test 28 shard key: { "Uid" : 1 } 29 chunks: 30 sh0 11 31 sh1 10 32 too many chunks to print, use verbose if you want to force print
3）测试可用性
1mongos> db.test.find({Uid:12}); 2{ "_id" : ObjectId("56178e473af16d338338d3bf"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:07.205Z") } 3mongos> db.test.find({Uid:8}); 4Error: error: { 5 "$err" : "socket exception [CONNECT_ERROR] for sh0/172.30.2.201:27010", 6 "code" : 11002, 7 "shard" : "sh0" 8} 9mongos> db.test.insert({"Uid":12,"Name":"zhanjindong","Age":13,"Date":new Date()}); 10WriteResult({ "nInserted" : 1 }) 11mongos> db.test.insert({"Uid":8,"Name":"zhanjindong","Age":13,"Date":new Date()}); 12WriteResult({ 13 "nInserted" : 0, 14 "writeError" : { 15 "code" : 7, 16 "errmsg" : "could not contact primary for replica set sh0" 17 } 18}) 19mongos> 20bye
设计到sh0的操作都会报错，其他分片的可以正常执行
重新启动sh0
1$ sudo /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 2about to fork child process, waiting until server is ready for connections. 3forked process: 2534 4child process started successfully, parent exiting 5$ ps aux | grep mongo | grep -v grep 6root 2534 1.1 0.0 971800 64060 ? Sl 10:53 0:00 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 7root 41509 0.5 0.1 1028196 108804 ? Sl Oct09 7:33 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 8root 41855 0.6 0.0 566248 55572 ? Sl Oct09 9:17 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 9root 41870 0.5 0.0 565220 56480 ? Sl Oct09 8:33 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 10root 42170 0.5 0.0 565216 57240 ? Sl Oct09 8:29 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 11root 42233 0.3 0.0 260180 13388 ? Sl Oct09 5:42 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 12root 42286 0.3 0.0 259148 11572 ? Sl Oct09 4:32 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 13$ mongo --port 30000 14MongoDB shell version: 3.0.6 15connecting to: 127.0.0.1:30000/test 16mongos> use mydb; 17switched to db mydb 18mongos> db.test.find({Uid:8}); 19{ "_id" : ObjectId("56178e453af16d338338d3bb"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:05.539Z") } 20mongos> db.test.find({Uid:12}); 21{ "_id" : ObjectId("56178e473af16d338338d3bf"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:07.205Z") } 22{ "_id" : ObjectId("56187d6743c2e9badb0ca53b"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:52:23.497Z") } 23mongos> db.test.insert({"Uid":8,"Name":"zhanjindong","Age":13,"Date":new Date()}); 24WriteResult({ "nInserted" : 1 })
两个分片都可读写
四、Config Server故障
1.单节点故障
1）停掉cf0
1$ ps axu | grep mongo | grep -v grep 2root 2534 0.4 0.1 982052 67608 ? Sl 10:53 0:03 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 3root 41509 0.5 0.1 1028196 108808 ? Sl Oct09 7:37 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 4root 41855 0.6 0.0 566248 55444 ? Sl Oct09 9:23 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 5root 41870 0.5 0.0 565220 56456 ? Sl Oct09 8:38 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 6root 42170 0.5 0.0 565216 56820 ? Sl Oct09 8:34 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 7root 42233 0.3 0.0 261204 13408 ? Sl Oct09 5:45 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 8root 42286 0.3 0.0 259148 11596 ? Sl Oct09 4:35 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 9[wenhang.pan@zhaopin-2-201 ~]$ sudo kill 41855 10[wenhang.pan@zhaopin-2-201 ~]$ ps axu | grep mongo | grep -v grep 11root 2534 0.4 0.1 982052 67636 ? Sl 10:53 0:03 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 12root 41509 0.5 0.1 1028196 108808 ? Sl Oct09 7:37 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 13root 41870 0.5 0.0 565220 56324 ? Sl Oct09 8:38 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 14root 42170 0.5 0.0 565216 56600 ? Sl Oct09 8:34 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 15root 42233 0.3 0.0 261204 13408 ? Sl Oct09 5:45 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 16root 42286 0.3 0.0 259148 11596 ? Sl Oct09 4:35 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf
2）查看状态
1$ mongo --port 30000 2MongoDB shell version: 3.0.6 3connecting to: 127.0.0.1:30000/test 4mongos> sh.status(); 5--- Sharding Status --- 6 sharding version: { 7 "_id" : 1, 8 "minCompatibleVersion" : 5, 9 "currentVersion" : 6, 10 "clusterId" : ObjectId("561728b4030ea038bcb57fa0") 11} 12 shards: 13 { "_id" : "sh0", "host" : "sh0/172.30.2.201:27010" } 14 { "_id" : "sh1", "host" : "sh1/172.30.2.201:27011" } 15 balancer: 16 Currently enabled: yes 17 Currently running: yes 18 Balancer lock taken at Sat Oct 10 2015 11:08:14 GMT+0800 (CST) by zhaopin-2-201:30000:1444358323:1804289383:Balancer:1681692777 19 Failed balancer rounds in last 5 attempts: 0 20 Migration Results for the last 24 hours: 21 19 : Success 22 2 : Failed with error 'migration already in progress', from sh0 to sh1 23 databases: 24 { "_id" : "admin", "partitioned" : false, "primary" : "config" } 25 { "_id" : "mydb", "partitioned" : true, "primary" : "sh0" } 26 mydb.test 27 shard key: { "Uid" : 1 } 28 chunks: 29 sh0 11 30 sh1 10 31 too many chunks to print, use verbose if you want to force print
3）测试可用性
1mongos> use mydb; 2switched to db mydb 3mongos> db.test.find({Uid:12}); 4{ "_id" : ObjectId("56178e473af16d338338d3bf"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:07.205Z") } 5{ "_id" : ObjectId("56187d6743c2e9badb0ca53b"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:52:23.497Z") } 6mongos> db.test.find({Uid:8}); 7{ "_id" : ObjectId("56178e453af16d338338d3bb"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:05.539Z") } 8{ "_id" : ObjectId("56187e117ec37f9bb7244496"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:55:13.529Z") } 9mongos> db.test.insert({"Uid":8,"Name":"zhanjindong","Age":13,"Date":new Date()}); 10WriteResult({ "nInserted" : 1 }) 11mongos> db.test.insert({"Uid":12,"Name":"zhanjindong","Age":13,"Date":new Date()}); 12WriteResult({ "nInserted" : 1 }) 13mongos> 14bye
集群可正常使用
2.双节点故障
1）停掉cf1
1$ ps axu | grep mongo | grep -v grep 2root 2534 0.4 0.1 982052 68128 ? Sl 10:53 0:05 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 3root 41509 0.5 0.1 1028196 108892 ? Sl Oct09 7:39 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 4root 41870 0.5 0.1 565220 92328 ? Sl Oct09 8:40 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 5root 42170 0.5 0.1 565216 90392 ? Sl Oct09 8:36 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 6root 42233 0.3 0.0 261204 13508 ? Sl Oct09 5:46 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 7root 42286 0.3 0.0 260172 11716 ? Sl Oct09 4:37 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 8$ sudo kill 41870 9$ ps axu | grep mongo | grep -v grep 10root 2534 0.4 0.1 982052 68128 ? Sl 10:53 0:05 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 11root 41509 0.5 0.1 1028196 108892 ? Sl Oct09 7:39 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 12root 42170 0.5 0.1 565216 90408 ? Sl Oct09 8:36 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 13root 42233 0.3 0.0 261204 13508 ? Sl Oct09 5:46 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 14root 42286 0.3 0.0 260172 11716 ? Sl Oct09 4:37 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf
2）查看集群状态
1$ mongo --port 30000 2MongoDB shell version: 3.0.6 3connecting to: 127.0.0.1:30000/test 4mongos> sh.status(); 5--- Sharding Status --- 6 sharding version: { 7 "_id" : 1, 8 "minCompatibleVersion" : 5, 9 "currentVersion" : 6, 10 "clusterId" : ObjectId("561728b4030ea038bcb57fa0") 11} 12 shards: 13 { "_id" : "sh0", "host" : "sh0/172.30.2.201:27010" } 14 { "_id" : "sh1", "host" : "sh1/172.30.2.201:27011" } 15 balancer: 16 Currently enabled: yes 17 Currently running: yes 18 Balancer lock taken at Sat Oct 10 2015 11:08:14 GMT+0800 (CST) by zhaopin-2-201:30000:1444358323:1804289383:Balancer:1681692777 19 Failed balancer rounds in last 5 attempts: 0 20 Migration Results for the last 24 hours: 21 19 : Success 22 2 : Failed with error 'migration already in progress', from sh0 to sh1 23 databases: 24 { "_id" : "admin", "partitioned" : false, "primary" : "config" } 25 { "_id" : "mydb", "partitioned" : true, "primary" : "sh0" } 26 mydb.test 27 shard key: { "Uid" : 1 } 28 chunks: 29 sh0 11 30 sh1 10 31 too many chunks to print, use verbose if you want to force print
3）测试可用性
1mongos> use mydb; 2switched to db mydb 3mongos> db.test.find({Uid:8}); 4{ "_id" : ObjectId("56178e453af16d338338d3bb"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:05.539Z") } 5{ "_id" : ObjectId("56187e117ec37f9bb7244496"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:55:13.529Z") } 6{ "_id" : ObjectId("5618824d7bd66d65a90fb01c"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:13:17.564Z") } 7mongos> db.test.find({Uid:12}); 8{ "_id" : ObjectId("56178e473af16d338338d3bf"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:07.205Z") } 9{ "_id" : ObjectId("56187d6743c2e9badb0ca53b"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:52:23.497Z") } 10{ "_id" : ObjectId("561882527bd66d65a90fb01d"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:13:22.594Z") } 11mongos> db.test.insert({"Uid":12,"Name":"zhanjindong","Age":13,"Date":new Date()}); 12WriteResult({ "nInserted" : 1 }) 13mongos> db.test.insert({"Uid":8,"Name":"zhanjindong","Age":13,"Date":new Date()}); 14WriteResult({ "nInserted" : 1 }) 15mongos> 16bye
集群可以正常使用
4）重新启动cf0和cf1
1$ ps axu | grep mongo | grep -v grep 2root 2534 0.4 0.1 982052 68228 ? Sl 10:53 0:05 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 3root 41509 0.5 0.1 1028196 107976 ? Sl Oct09 7:39 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 4root 42170 0.5 0.1 565216 93140 ? Sl Oct09 8:37 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 5root 42233 0.3 0.0 261204 13540 ? Sl Oct09 5:47 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 6root 42286 0.3 0.0 260172 11752 ? Sl Oct09 4:37 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 7$ sudo /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 8about to fork child process, waiting until server is ready for connections. 9forked process: 7309 10child process started successfully, parent exiting 11$ sudo /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 12about to fork child process, waiting until server is ready for connections. 13forked process: 7355 14child process started successfully, parent exiting 15$ ps axu | grep mongo | grep -v grep 16root 2534 0.4 0.1 982052 68228 ? Sl 10:53 0:06 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 17root 7309 0.6 0.0 545696 37504 ? Sl 11:17 0:00 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 18root 7355 0.5 0.0 542616 33748 ? Sl 11:17 0:00 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 19root 41509 0.5 0.1 1028196 108188 ? Sl Oct09 7:40 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 20root 42170 0.5 0.0 565216 57992 ? Sl Oct09 8:37 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 21root 42233 0.3 0.0 261204 13540 ? Sl Oct09 5:47 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 22root 42286 0.3 0.0 260172 11768 ? Sl Oct09 4:38 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf
5）解决方案
参考：
http://docs.mongodb.org/manual/tutorial/replace-config-server/
http://docs.mongodb.org/manual/tutorial/migrate-config-servers-with-different-hostnames/
五、Query Routers故障
1）将ms0停机
1$ ps axu | grep mongo | grep -v grep 2root 2534 0.4 0.1 982052 68228 ? Sl 10:53 0:06 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 3root 7309 0.6 0.0 545696 37504 ? Sl 11:17 0:00 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 4root 7355 0.5 0.0 542616 33748 ? Sl 11:17 0:00 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 5root 41509 0.5 0.1 1028196 108188 ? Sl Oct09 7:40 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 6root 42170 0.5 0.0 565216 57992 ? Sl Oct09 8:37 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 7root 42233 0.3 0.0 261204 13540 ? Sl Oct09 5:47 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms0/mongos.conf 8root 42286 0.3 0.0 260172 11768 ? Sl Oct09 4:38 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf 9$ sudo kill 42233 10$ ps axu | grep mongo | grep -v grep 11root 2534 0.4 0.1 982052 68408 ? Sl 10:53 0:06 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh0/mongodb.conf 12root 7309 0.6 0.0 547752 39096 ? Sl 11:17 0:01 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf0/config.conf 13root 7355 0.6 0.0 544672 34256 ? Sl 11:17 0:01 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf1/config.conf 14root 41509 0.5 0.1 1028196 108164 ? Sl Oct09 7:40 /opt/mongodb/bin/mongod --config /data/mongodb/conf/sh1/mongodb.conf 15root 42170 0.5 0.0 565216 57796 ? Sl Oct09 8:38 /opt/mongodb/bin/mongod --config /data/mongodb/conf/cf2/config.conf 16root 42286 0.3 0.0 260172 11768 ? Sl Oct09 4:38 /opt/mongodb/bin/mongos --config /data/mongodb/conf/ms1/mongos.conf
2）查看集群状态
1$ mongo --port 30000 2MongoDB shell version: 3.0.6 3connecting to: 127.0.0.1:30000/test 42015-10-10T11:21:04.681+0800 W NETWORK Failed to connect to 127.0.0.1:30000, reason: errno:111 Connection refused 52015-10-10T11:21:04.683+0800 E QUERY Error: couldn't connect to server 127.0.0.1:30000 (127.0.0.1), connection attempt failed 6 at connect (src/mongo/shell/mongo.js:181:14) 7 at (connect):1:6 at src/mongo/shell/mongo.js:181 8exception: connect failed 9$ mongo --port 30001 10MongoDB shell version: 3.0.6 11connecting to: 127.0.0.1:30001/test 12mongos> sh.status(); 13--- Sharding Status --- 14 sharding version: { 15 "_id" : 1, 16 "minCompatibleVersion" : 5, 17 "currentVersion" : 6, 18 "clusterId" : ObjectId("561728b4030ea038bcb57fa0") 19} 20 shards: 21 { "_id" : "sh0", "host" : "sh0/172.30.2.201:27010" } 22 { "_id" : "sh1", "host" : "sh1/172.30.2.201:27011" } 23 balancer: 24 Currently enabled: yes 25 Currently running: no 26 Failed balancer rounds in last 5 attempts: 0 27 Migration Results for the last 24 hours: 28 19 : Success 29 2 : Failed with error 'migration already in progress', from sh0 to sh1 30 databases: 31 { "_id" : "admin", "partitioned" : false, "primary" : "config" } 32 { "_id" : "mydb", "partitioned" : true, "primary" : "sh0" } 33 mydb.test 34 shard key: { "Uid" : 1 } 35 chunks: 36 sh0 11 37 sh1 10 38 too many chunks to print, use verbose if you want to force print 39mongos> 40bye
可见30000端口的mongos已经宕机，不能提供服务了，但30001端口的mongos可以使用
3）测试可用性
1mongos> use mydb; 2switched to db mydb 3mongos> db.test.find({Uid:12}); 4{ "_id" : ObjectId("56178e473af16d338338d3bf"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:07.205Z") } 5{ "_id" : ObjectId("56187d6743c2e9badb0ca53b"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:52:23.497Z") } 6{ "_id" : ObjectId("561882527bd66d65a90fb01d"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:13:22.594Z") } 7{ "_id" : ObjectId("561882f9ff80495f50bec62f"), "Uid" : 12, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:16:09.930Z") } 8mongos> db.test.find({Uid:8}); 9{ "_id" : ObjectId("56178e453af16d338338d3bb"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-09T09:52:05.539Z") } 10{ "_id" : ObjectId("56187e117ec37f9bb7244496"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T02:55:13.529Z") } 11{ "_id" : ObjectId("5618824d7bd66d65a90fb01c"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:13:17.564Z") } 12{ "_id" : ObjectId("561882fdff80495f50bec630"), "Uid" : 8, "Name" : "zhanjindong", "Age" : 13, "Date" : ISODate("2015-10-10T03:16:13.788Z") } 13mongos> db.test.insert({"Uid":8,"Name":"zhanjindong","Age":13,"Date":new Date()}); 14WriteResult({ "nInserted" : 1 }) 15mongos> db.test.insert({"Uid":12,"Name":"zhanjindong","Age":13,"Date":new Date()}); 16WriteResult({ "nInserted" : 1 })
可见集群可以正常使用
4）解决方案
可以在代码中配置连接多台mongo实现failover，java代码如下：
1List<ServerAddress> mongoHostList = new ArrayList<ServerAddress>(); 2mongoHostList.add(new ServerAddress("ip",port)); 3mongoHostList.add(new ServerAddress("ip",port)); 4mongoHostList.add(new ServerAddress("ip",port)); 5MongoClient mg = new MongoClient(mongoHostList);
六、结论
1.Shard Server故障
1）分片节点故障，可以按照MongoDB副本集故障测试和解决方案中解决，仍可以实现可用性
2）整个分片故障，如果可以重新启动的话，仍然可以使用，否则只能从备份中恢复，必然会丢数据
2.Config Server故障
只要有可用的配置服务器，整个集群都可以正常使用
3.Query Routers故障
只要有可用的mongos服务器，整个集群就可以正常使用，要客户端也实现failover，需要将代码写成连接多台mongo的方式