Zookeeper核心原理-源码分析选举的过程
https://github.com/apache/zookeeper
1 public static void main(String[] args) { 2 QuorumPeerMain main = new QuorumPeerMain(); 3 try { 4 main.initializeAndRun(args); 5 ... 6 } 7 .... 8 }
判断是集群模式还是单机模式
1 protected void initializeAndRun(String[] args) 2 throws ConfigException, IOException, AdminServerException 3 { 4 QuorumPeerConfig config = new QuorumPeerConfig(); 5 if (args.length == 1) { 6 config.parse(args[0]); 7 } 8 9 // Start and schedule the the purge task 10 DatadirCleanupManager purgeMgr = new DatadirCleanupManager(config 11 .getDataDir(), config.getDataLogDir(), config 12 .getSnapRetainCount(), config.getPurgeInterval()); 13 purgeMgr.start(); 14 //判断是单机环境还是集群环境 15 if (args.length == 1 && config.isDistributed()) { 16 runFromConfig(config); 17 } else { 18 LOG.warn("Either no config or no quorum defined in config, running " 19 + " in standalone mode"); 20 // there is only server in the quorum -- run as standalone 21 ZooKeeperServerMain.main(args); 22 } 23 }
1 public void runFromConfig(QuorumPeerConfig config) 2 throws IOException, AdminServerException 3 { 4 try { 5 ManagedUtil.registerLog4jMBeans(); 6 } catch (JMException e) { 7 LOG.warn("Unable to register log4j JMX control", e); 8 } 9 10 LOG.info("Starting quorum peer"); 11 MetricsProvider metricsProvider; 12 try { 13 metricsProvider = MetricsProviderBootstrap 14 .startMetricsProvider(config.getMetricsProviderClassName(), 15 config.getMetricsProviderConfiguration()); 16 } catch (MetricsProviderLifeCycleException error) { 17 throw new IOException("Cannot boot MetricsProvider " + config.getMetricsProviderClassName(), 18 error); 19 } 20 try { 21 22 ServerCnxnFactory cnxnFactory = null; 23 ServerCnxnFactory secureCnxnFactory = null; 24 //为客户端提供读写的Server,也就是2181这个端口的访问功能 25 if (config.getClientPortAddress() != null) { 26 cnxnFactory = ServerCnxnFactory.createFactory(); 27 cnxnFactory.configure(config.getClientPortAddress(), 28 config.getMaxClientCnxns(), 29 false); 30 } 31 32 if (config.getSecureClientPortAddress() != null) { 33 secureCnxnFactory = ServerCnxnFactory.createFactory(); 34 secureCnxnFactory.configure(config.getSecureClientPortAddress(), 35 config.getMaxClientCnxns(), 36 true); 37 } 38 //zk的逻辑主线程，负责选举、投票 39 quorumPeer = getQuorumPeer(); 40 quorumPeer.setRootMetricsContext(metricsProvider.getRootContext()); 41 quorumPeer.setTxnFactory(new FileTxnSnapLog( 42 config.getDataLogDir(), 43 config.getDataDir())); 44 quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled()); 45 quorumPeer.enableLocalSessionsUpgrading( 46 config.isLocalSessionsUpgradingEnabled()); 47 //quorumPeer.setQuorumPeers(config.getAllMembers()); 48 quorumPeer.setElectionType(config.getElectionAlg()); 49 //设置myid 50 quorumPeer.setMyid(config.getServerId()); 51 //设置单位时间 52 quorumPeer.setTickTime(config.getTickTime()); 53 quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout()); 54 quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout()); 55 //设置初始化同步时间 56 quorumPeer.setInitLimit(config.getInitLimit()); 57 //设置 58 quorumPeer.setSyncLimit(config.getSyncLimit()); 59 quorumPeer.setObserverMasterPort(config.getObserverMasterPort()); 60 quorumPeer.setConfigFileName(config.getConfigFilename()); 61 quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory())); 62 quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false); 63 if (config.getLastSeenQuorumVerifier()!=null) { 64 quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false); 65 } 66 quorumPeer.initConfigInZKDatabase(); 67 quorumPeer.setCnxnFactory(cnxnFactory); 68 quorumPeer.setSecureCnxnFactory(secureCnxnFactory); 69 quorumPeer.setSslQuorum(config.isSslQuorum()); 70 quorumPeer.setUsePortUnification(config.shouldUsePortUnification()); 71 quorumPeer.setLearnerType(config.getPeerType()); 72 quorumPeer.setSyncEnabled(config.getSyncEnabled()); 73 quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs()); 74 if (config.sslQuorumReloadCertFiles) { 75 quorumPeer.getX509Util().enableCertFileReloading(); 76 } 77 78 // sets quorum sasl authentication configurations 79 quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl); 80 if(quorumPeer.isQuorumSaslAuthEnabled()){ 81 quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl); 82 quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl); 83 quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal); 84 quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext); 85 quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext); 86 } 87 quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize); 88 quorumPeer.initialize(); 89 //启动主线程，QuorumPeer重写了Thread.start方法 90 quorumPeer.start(); 91 quorumPeer.join(); 92 } catch (InterruptedException e) { 93 // warn, but generally this is ok 94 LOG.warn("Quorum Peer interrupted", e); 95 } finally { 96 if (metricsProvider != null) { 97 try { 98 metricsProvider.stop(); 99 } catch (Throwable error) { 100 LOG.warn("Error while stopping metrics", error); 101 } 102 } 103 } 104 }
1 @Override 2 public synchronized void start() { 3 if (!getView().containsKey(myid)) { 4 throw new RuntimeException("My id " + myid + " not in the peer list"); 5 } 6 //恢复db 7 loadDataBase(); 8 startServerCnxnFactory(); 9 try { 10 adminServer.start(); 11 } catch (AdminServerException e) { 12 LOG.warn("Problem starting AdminServer", e); 13 System.out.println(e); 14 } 15 //选举初始化 16 startLeaderElection(); 17 super.start(); //调用一个线程 18 19 }
主要是从本地文件中恢复数据，以及获取最新的zxid
private void loadDataBase() { try { zkDb.loadDataBase(); //从本地文件恢复DB // load the epochs //从最新的zxid恢复epoch变量、zxid 64位，前32位是epoch的值，后32位是zxid long lastProcessedZxid = zkDb.getDataTree().lastProcessedZxid; long epochOfZxid = ZxidUtils.getEpochFromZxid(lastProcessedZxid); try { //从文件中读取当前的epoch currentEpoch = readLongFromFile(CURRENT_EPOCH_FILENAME); } catch(FileNotFoundException e) { // pick a reasonable epoch number // this should only happen once when moving to a // new code version currentEpoch = epochOfZxid; LOG.info(CURRENT_EPOCH_FILENAME + " not found! Creating with a reasonable default of {}. This should only happen when you are upgrading your installation", currentEpoch); writeLongToFile(CURRENT_EPOCH_FILENAME, currentEpoch); } if (epochOfZxid > currentEpoch) { throw new IOException("The current epoch, " + ZxidUtils.zxidToString(currentEpoch) + ", is older than the last zxid, " + lastProcessedZxid); } try { acceptedEpoch = readLongFromFile(ACCEPTED_EPOCH_FILENAME); } catch(FileNotFoundException e) { // pick a reasonable epoch number // this should only happen once when moving to a // new code version acceptedEpoch = epochOfZxid; LOG.info(ACCEPTED_EPOCH_FILENAME + " not found! Creating with a reasonable default of {}. This should only happen when you are upgrading your installation", acceptedEpoch); writeLongToFile(ACCEPTED_EPOCH_FILENAME, acceptedEpoch); } if (acceptedEpoch < currentEpoch) { throw new IOException("The accepted epoch, " + ZxidUtils.zxidToString(acceptedEpoch) + " is less than the current epoch, " + ZxidUtils.zxidToString(currentEpoch)); } } catch(IOException ie) { LOG.error("Unable to load database on disk", ie); throw new RuntimeException("Unable to run quorum server ", ie); } }
1 synchronized public void startLeaderElection() { 2 try { 3 //如果当前节点状态是Looking,则投票给自己 4 if (getPeerState() == ServerState.LOOKING) { 5 //投票对象 6 currentVote = new Vote(myid, getLastLoggedZxid(), getCurrentEpoch()); 7 } 8 } catch(IOException e) { 9 RuntimeException re = new RuntimeException(e.getMessage()); 10 re.setStackTrace(e.getStackTrace()); 11 throw re; 12 } 13 //根据配置获取选举算法 14 this.electionAlg = createElectionAlgorithm(electionType); 15 }
可以通过zoo.cfg里面配置，默认是fast选举
1 protected Election createElectionAlgorithm(int electionAlgorithm){ 2 Election le=null; 3 4 //TODO: use a factory rather than a switch 5 switch (electionAlgorithm) { 6 case 1: 7 le = new AuthFastLeaderElection(this); 8 break; 9 case 2: 10 le = new AuthFastLeaderElection(this, true); 11 break; 12 case 3://Leader选举IO 负责类 13 QuorumCnxManager qcm = createCnxnManager(); 14 QuorumCnxManager oldQcm = qcmRef.getAndSet(qcm); 15 if (oldQcm != null) { 16 LOG.warn("Clobbering already-set QuorumCnxManager (restarting leader election?)"); 17 oldQcm.halt(); 18 } 19 QuorumCnxManager.Listener listener = qcm.listener; 20 if(listener != null){ 21 listener.start(); //启动已绑定的端口的选举线程，等待集群中的其他线程。 22 //基于TCP的选举算法 23 FastLeaderElection fle = new FastLeaderElection(this, qcm); 24 fle.start(); 25 le = fle; 26 } else { 27 LOG.error("Null listener when initializing cnx manager"); 28 } 29 break; 30 default: 31 assert false; 32 } 33 return le; 34 }
1 public FastLeaderElection(QuorumPeer self, QuorumCnxManager manager){ 2 this.stop = false; 3 this.manager = manager; 4 starter(self, manager); 5 }
1 private void starter(QuorumPeer self, QuorumCnxManager manager) { 2 this.self = self; 3 proposedLeader = -1; 4 proposedZxid = -1; 5 //业务层发送队列，业务对象ToSend 6 sendqueue = new LinkedBlockingQueue<ToSend>(); 7 //业务层接收队列，业务对象Notification 8 recvqueue = new LinkedBlockingQueue<Notification>(); 9 this.messenger = new Messenger(manager); 10 }
1 public void start() { 2 this.messenger.start(); 3 }
1 void start(){ 2 this.wsThread.start(); //启动业务层发送线程，将消息发送给IO负责类，QuorumCnxManager 3 this.wrThread.start(); //启动业务层接收线程，从IO负责类QuorumCnxManager接收消息 4 }
1 Messenger(QuorumCnxManager manager) { 2 3 this.ws = new WorkerSender(manager); 4 5 this.wsThread = new Thread(this.ws, 6 "WorkerSender[myid=" + self.getId() + "]"); 7 this.wsThread.setDaemon(true); 8 9 this.wr = new WorkerReceiver(manager); 10 11 this.wrThread = new Thread(this.wr, 12 "WorkerReceiver[myid=" + self.getId() + "]"); 13 this.wrThread.setDaemon(true); 14 }
1 private void starter(QuorumPeer self, QuorumCnxManager manager) { 2 this.self = self; 3 proposedLeader = -1; 4 proposedZxid = -1; 5 //业务层发送队列，业务对象ToSend 6 sendqueue = new LinkedBlockingQueue<ToSend>(); 7 //业务层接收队列，业务对象Notification 8 recvqueue = new LinkedBlockingQueue<Notification>(); 9 this.messenger = new Messenger(manager); 10 }
FastLeaderElection 初始化完成以后，调用super.start()，最终运行QuorumPeer的 run方法
1 @Override 2 public synchronized void start() { 3 if (!getView().containsKey(myid)) { 4 throw new RuntimeException("My id " + myid + " not in the peer list"); 5 } 6 //恢复db 7 loadDataBase(); 8 startServerCnxnFactory(); 9 try { 10 adminServer.start(); 11 } catch (AdminServerException e) { 12 LOG.warn("Problem starting AdminServer", e); 13 System.out.println(e); 14 } 15 //选举初始化 16 startLeaderElection(); 17 super.start(); //调用一个线程 18 19 }
1 @Override 2 public void run() { 3 updateThreadName(); 4 5 LOG.debug("Starting quorum peer"); 6 try { //此处通过JMX来监控一些属性 7 jmxQuorumBean = new QuorumBean(this); 8 MBeanRegistry.getInstance().register(jmxQuorumBean, null); 9 for(QuorumServer s: getView().values()){ 10 ZKMBeanInfo p; 11 if (getId() == s.id) { 12 p = jmxLocalPeerBean = new LocalPeerBean(this); 13 try { 14 MBeanRegistry.getInstance().register(p, jmxQuorumBean); 15 } catch (Exception e) { 16 LOG.warn("Failed to register with JMX", e); 17 jmxLocalPeerBean = null; 18 } 19 } else { 20 RemotePeerBean rBean = new RemotePeerBean(this, s); 21 try { 22 MBeanRegistry.getInstance().register(rBean, jmxQuorumBean); 23 jmxRemotePeerBean.put(s.id, rBean); 24 } catch (Exception e) { 25 LOG.warn("Failed to register with JMX", e); 26 } 27 } 28 } 29 } catch (Exception e) { 30 LOG.warn("Failed to register with JMX", e); 31 jmxQuorumBean = null; 32 } 33 34 try { 35 /* 36 * Main loop 37 */ 38 while (running) { 39 switch (getPeerState()) { //判断当前节点的状态 40 case LOOKING: //如果是looking,则进入选举流程 41 LOG.info("LOOKING"); 42 ServerMetrics.LOOKING_COUNT.add(1); 43 44 if (Boolean.getBoolean("readonlymode.enabled")) { 45 LOG.info("Attempting to start ReadOnlyZooKeeperServer"); 46 47 // Create read-only server but don't start it immediately 48 final ReadOnlyZooKeeperServer roZk = 49 new ReadOnlyZooKeeperServer(logFactory, this, this.zkDb); 50 51 // Instead of starting roZk immediately, wait some grace 52 // period before we decide we're partitioned. 53 // 54 // Thread is used here because otherwise it would require 55 // changes in each of election strategy classes which is 56 // unnecessary code coupling. 57 Thread roZkMgr = new Thread() { 58 public void run() { 59 try { 60 // lower-bound grace period to 2 secs 61 sleep(Math.max(2000, tickTime)); 62 if (ServerState.LOOKING.equals(getPeerState())) { 63 roZk.startup(); 64 } 65 } catch (InterruptedException e) { 66 LOG.info("Interrupted while attempting to start ReadOnlyZooKeeperServer, not started"); 67 } catch (Exception e) { 68 LOG.error("FAILED to start ReadOnlyZooKeeperServer", e); 69 } 70 } 71 }; 72 try { 73 roZkMgr.start(); 74 reconfigFlagClear(); 75 if (shuttingDownLE) { 76 shuttingDownLE = false; 77 startLeaderElection(); 78 } 79 //此处通过策略模式来决定当前哪个选举算法来进行领导选举 80 setCurrentVote(makeLEStrategy().lookForLeader()); 81 } catch (Exception e) { 82 LOG.warn("Unexpected exception", e); 83 setPeerState(ServerState.LOOKING); 84 } finally { 85 // If the thread is in the the grace period, interrupt 86 // to come out of waiting. 87 roZkMgr.interrupt(); 88 roZk.shutdown(); 89 } 90 } else { 91 try { 92 reconfigFlagClear(); 93 if (shuttingDownLE) { 94 shuttingDownLE = false; 95 startLeaderElection(); 96 } 97 setCurrentVote(makeLEStrategy().lookForLeader()); 98 } catch (Exception e) { 99 LOG.warn("Unexpected exception", e); 100 setPeerState(ServerState.LOOKING); 101 } 102 } 103 break; 104 case OBSERVING: 105 try { 106 LOG.info("OBSERVING"); 107 setObserver(makeObserver(logFactory)); 108 observer.observeLeader(); 109 } catch (Exception e) { 110 LOG.warn("Unexpected exception",e ); 111 } finally { 112 observer.shutdown(); 113 setObserver(null); 114 updateServerState(); 115 116 // Add delay jitter before we switch to LOOKING 117 // state to reduce the load of ObserverMaster 118 if (isRunning()) { 119 Observer.waitForReconnectDelay(); 120 } 121 } 122 break; 123 case FOLLOWING: 124 try { 125 LOG.info("FOLLOWING"); 126 setFollower(makeFollower(logFactory)); 127 follower.followLeader(); 128 } catch (Exception e) { 129 LOG.warn("Unexpected exception",e); 130 } finally { 131 follower.shutdown(); 132 setFollower(null); 133 updateServerState(); 134 } 135 break; 136 case LEADING: 137 LOG.info("LEADING"); 138 try { 139 setLeader(makeLeader(logFactory)); 140 leader.lead(); 141 setLeader(null); 142 } catch (Exception e) { 143 LOG.warn("Unexpected exception",e); 144 } finally { 145 if (leader != null) { 146 leader.shutdown("Forcing shutdown"); 147 setLeader(null); 148 } 149 updateServerState(); 150 } 151 break; 152 } 153 start_fle = Time.currentElapsedTime(); 154 } 155 } finally { 156 LOG.warn("QuorumPeer main thread exited"); 157 MBeanRegistry instance = MBeanRegistry.getInstance(); 158 instance.unregister(jmxQuorumBean); 159 instance.unregister(jmxLocalPeerBean); 160 161 for (RemotePeerBean remotePeerBean : jmxRemotePeerBean.values()) { 162 instance.unregister(remotePeerBean); 163 } 164 165 jmxQuorumBean = null; 166 jmxLocalPeerBean = null; 167 jmxRemotePeerBean = null; 168 } 169 }
1 /** 2 * Starts a new round of leader election. Whenever our QuorumPeer 3 * changes its state to LOOKING, this method is invoked, and it 4 * sends notifications to all other peers. 5 * 真正的投票逻辑 6 */ 7 public Vote lookForLeader() throws InterruptedException { 8 try { 9 self.jmxLeaderElectionBean = new LeaderElectionBean(); 10 MBeanRegistry.getInstance().register( 11 self.jmxLeaderElectionBean, self.jmxLocalPeerBean); 12 } catch (Exception e) { 13 LOG.warn("Failed to register with JMX", e); 14 self.jmxLeaderElectionBean = null; 15 } 16 if (self.start_fle == 0) { 17 self.start_fle = Time.currentElapsedTime(); 18 } 19 try { 20 //收到的投票 21 Map<Long, Vote> recvset = new HashMap<Long, Vote>(); 22 //存储选举的结果 23 Map<Long, Vote> outofelection = new HashMap<Long, Vote>(); 24 25 int notTimeout = finalizeWait; 26 27 synchronized(this){ 28 logicalclock.incrementAndGet(); //增加逻辑时钟 29 updateProposal(getInitId(), getInitLastLoggedZxid(), getPeerEpoch()); 30 } 31 32 LOG.info("New election. My id = " + self.getId() + 33 ", proposed zxid=0x" + Long.toHexString(proposedZxid)); 34 sendNotifications(); //发送投票，包括发送给自己 35 36 SyncedLearnerTracker voteSet; 37 38 /* 39 * Loop in which we exchange notifications until we find a leader 40 */ 41 42 while ((self.getPeerState() == ServerState.LOOKING) && 43 (!stop)){ //主循环，直到选举出Leader 44 /* 45 * Remove next notification from queue, times out after 2 times 46 * the termination time 47 */ 48 //从IO线程里拿到投票消息，自己的投票也在这里处理 49 //LinkedBlockedQueue() 接收 50 Notification n = recvqueue.poll(notTimeout, 51 TimeUnit.MILLISECONDS); 52 53 /* 54 * Sends more notifications if haven't received enough. 55 * Otherwise processes new notification. 56 */ 57 if(n == null){ 58 //如果空闲情况，消息发完了，继续发送，一直到选出leader为止 59 if(manager.haveDelivered()){ 60 sendNotifications(); 61 } else { 62 //消息还没投递出去，可能是其他server 还没启动，尝试再连接 63 manager.connectAll(); 64 } 65 66 /* 67 * Exponential backoff 68 */ 69 //延长超时时间 70 int tmpTimeOut = notTimeout*2; 71 notTimeout = (tmpTimeOut < maxNotificationInterval? 72 tmpTimeOut : maxNotificationInterval); 73 LOG.info("Notification time out: " + notTimeout); 74 } 75 else if (validVoter(n.sid) && validVoter(n.leader)) { 76 /* 77 * Only proceed if the vote comes from a replica in the current or next 78 * voting view for a replica in the current or next voting view. 79 */ 80 switch (n.state) { //判断收到消息的节点状态 81 case LOOKING: 82 if (getInitLastLoggedZxid() == -1) { 83 LOG.debug("Ignoring notification as our zxid is -1"); 84 break; 85 } 86 if (n.zxid == -1) { 87 LOG.debug("Ignoring notification from member with -1 zxid" + n.sid); 88 break; 89 } 90 //判断接收到的节点epoch 大于logicalclock,则表示当前是新一轮的选举 91 // If notification > current, replace and send messages out 92 if (n.electionEpoch > logicalclock.get()) { 93 logicalclock.set(n.electionEpoch); //更新本地的logicalclock 94 recvset.clear(); //清空接收队列 95 //检查接收到的这个消息是否可以胜出，一次比较epoch,zxid,myid 96 if(totalOrderPredicate(n.leader, n.zxid, n.peerEpoch, 97 getInitId(), getInitLastLoggedZxid(), getPeerEpoch())) { 98 //胜出以后，把投票改为对方的投票 99 updateProposal(n.leader, n.zxid, n.peerEpoch); 100 } else { //否则票据不变 101 updateProposal(getInitId(), 102 getInitLastLoggedZxid(), 103 getPeerEpoch()); 104 } 105 //继续广播消息，让其他节点知道我现在的票据 106 sendNotifications(); 107 } else if (n.electionEpoch < logicalclock.get()) {//如果收到的消息epoch小于当前节点的epoch,则忽略这条消息 108 if(LOG.isDebugEnabled()){ 109 LOG.debug("Notification election epoch is smaller than logicalclock. n.electionEpoch = 0x" 110 + Long.toHexString(n.electionEpoch) 111 + ", logicalclock=0x" + Long.toHexString(logicalclock.get())); 112 } 113 break; 114 //如果是epoch相同的话，就是比较zxid,myid,如果胜出，则更新自己的票据,并且发出广播 115 } else if (totalOrderPredicate(n.leader, n.zxid, n.peerEpoch, 116 proposedLeader, proposedZxid, proposedEpoch)) { 117 updateProposal(n.leader, n.zxid, n.peerEpoch); 118 sendNotifications(); 119 } 120 121 if(LOG.isDebugEnabled()){ 122 LOG.debug("Adding vote: from=" + n.sid + 123 ", proposed leader=" + n.leader + 124 ", proposed zxid=0x" + Long.toHexString(n.zxid) + 125 ", proposed election epoch=0x" + Long.toHexString(n.electionEpoch)); 126 } 127 128 // don't care about the version if it's in LOOKING state 129 recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch)); 130 //判断选举是否结束，默认算法是超过半数server同意 131 voteSet = getVoteTracker( 132 recvset, new Vote(proposedLeader, proposedZxid, 133 logicalclock.get(), proposedEpoch)); 134 135 if (voteSet.hasAllQuorums()) { 136 137 // Verify if there is any change in the proposed leader 138 //一直等到新的notification的到达，直到超时 139 while((n = recvqueue.poll(finalizeWait, 140 TimeUnit.MILLISECONDS)) != null){ 141 if(totalOrderPredicate(n.leader, n.zxid, n.peerEpoch, 142 proposedLeader, proposedZxid, proposedEpoch)){ 143 recvqueue.put(n); 144 break; 145 } 146 } 147 148 /* 149 * This predicate is true once we don't read any new 150 * relevant message from the reception queue 151 */ 152 //判断是否为leader 153 if (n == null) { 154 //修改状态，LEADING 变为Follower 155 setPeerState(proposedLeader, voteSet); 156 //返回最终投票结果 157 Vote endVote = new Vote(proposedLeader, 158 proposedZxid, logicalclock.get(), 159 proposedEpoch); 160 leaveInstance(endVote); 161 return endVote; 162 } 163 } 164 break; 165 //如果收到的消息选票状态不是LOOKING,比如这台机器刚加入一个已经正在运行的ZK集群时 166 //observer 不参与选举 167 case OBSERVING: 168 LOG.debug("Notification from observer: " + n.sid); 169 break; 170 171 case FOLLOWING: 172 case LEADING: 173 /* 174 * Consider all notifications from the same epoch 175 * together. 176 */ 177 if(n.electionEpoch == logicalclock.get()){ //判断epoch是否相同， 178 //加入到本机的投票集合 179 recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch)); 180 //投票是否结束，如果结束，再确认Leader是否有效 181 //如果结束，修改自己的状态并返回投票结果 182 voteSet = getVoteTracker(recvset, new Vote(n.version, 183 n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); 184 185 if (voteSet.hasAllQuorums() && 186 checkLeader(outofelection, n.leader, n.electionEpoch)) { 187 setPeerState(n.leader, voteSet); 188 Vote endVote = new Vote(n.leader, 189 n.zxid, n.electionEpoch, n.peerEpoch); 190 leaveInstance(endVote); 191 return endVote; 192 } 193 } 194 195 /* 196 * Before joining an established ensemble, verify that 197 * a majority are following the same leader. 198 */ 199 outofelection.put(n.sid, new Vote(n.version, n.leader, 200 n.zxid, n.electionEpoch, n.peerEpoch, n.state)); 201 voteSet = getVoteTracker(outofelection, new Vote(n.version, 202 n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); 203 204 if (voteSet.hasAllQuorums() && 205 checkLeader(outofelection, n.leader, n.electionEpoch)) { 206 synchronized(this){ 207 logicalclock.set(n.electionEpoch); 208 setPeerState(n.leader, voteSet); 209 } 210 Vote endVote = new Vote(n.leader, n.zxid, 211 n.electionEpoch, n.peerEpoch); 212 leaveInstance(endVote); 213 return endVote; 214 } 215 break; 216 default: 217 LOG.warn("Notification state unrecoginized: " + n.state 218 + " (n.state), " + n.sid + " (n.sid)"); 219 break; 220 } 221 } else { 222 if (!validVoter(n.leader)) { 223 LOG.warn("Ignoring notification for non-cluster member sid {} from sid {}", n.leader, n.sid); 224 } 225 if (!validVoter(n.sid)) { 226 LOG.warn("Ignoring notification for sid {} from non-quorum member sid {}", n.leader, n.sid); 227 } 228 } 229 } 230 return null; 231 } finally { 232 try { 233 if(self.jmxLeaderElectionBean != null){ 234 MBeanRegistry.getInstance().unregister( 235 self.jmxLeaderElectionBean); 236 } 237 } catch (Exception e) { 238 LOG.warn("Failed to unregister with JMX", e); 239 } 240 self.jmxLeaderElectionBean = null; 241 LOG.debug("Number of connection processing threads: {}", 242 manager.getConnectionThreadCount()); 243 } 244 }
FastLeaderElection
实现了Election接口，实现了各服务器之间基于TCP协议进行选举
Notification
内部类，Notification 表示收到的选举投票信息。
ToSend
表示发送给其他服务器的选举投票信息
Messenger
WorkSender
实现Runnable 接口，是选票发送器。
从sendqueue中获取待发送的选票，并将 其传递到底层QuorumCnxManager中 。
WorkerReceiver
实现Runnable 接口，是选票接收器。
从QuorumCnxManager 中获取其他服务器发来的选举消息，并将其转换成一个选票，然后保存到 recvqueue中
浙公网安备 33010602011771号