Hadoop RPC Server端源码分析

　　上一篇说了关于RPC和Client相关的类的解析，下面进行server端的解析，如下文。

 

　　Server端要从RPCEngine中的getserver开始，选择对应的rpc引擎writable或者protocol，最终构造方法是在server类中的构造方法(ProtoBuffRpcEngine中和Rpc中都有Server类，他们之间的父子关系是 ProtoBuffRpcEngine.Server 继承 Rpc.Server 继承 Server，最终调用的是父类的构造方法)。

Server端与client端类似，其中也包含许多内部类，例如类似Client的call、connection，Server还有专门处理请求的一些类，Listener监听请求、Listener.Reader读取请求并给Handler处理，handler处理请求，Responder请求返回结果返回到客户端。

Server类图如下

 

INITIAL_RESP_BUF_SIZE 初始化最大的应答缓存字节数。

SERVER 当前Server

PROTOCOL_CACHE 协议缓存

CurCall 当前线程正在处理的Call

bindAddress Server端绑定的地址

port Server端绑定的端口

handlerCount 同时运行的handler的线程数

readThreads 同时运行readThread的线程数

readerPendingConnectionQueue 每个reader等待队列的长度

rpcRequestClass 客户端请求的类型

conf Configuration 配置对象

maxQueueSize 请求call队列最大值

maxRespSize 应答队列最大值

socketSendBufferSize socket的缓冲区大小

maxDataLength 数据最大长度，客户端与服务端发送数据采用的是变长格式，所以发送消息时，先发送消息的长度，然后发送消息的内容，Hadoop固定消息长度大小为4字节（dataLength大小，dataLength记录着客户端发送过来的数据的大小），maxDataLength指定的是，在4字节内允许的最大数据长度，一般是占满4字节。

tcpNoDelay 是否采用negal 算法，一般不采用。

Running 当前是否正常运行

callQueue 当前处理的call的队列

connectionManager connection的管理类下面会详细说明

listener 监听客户端的请求

responder 将处理结果反馈给客户端

handlers 处理请求的线程组

 

server构造方法

server构造方法依旧是初始化一些信息

protected Server(String bindAddress, int port,

      Class<? extends Writable> rpcRequestClass, int handlerCount,

      int numReaders, int queueSizePerHandler, Configuration conf,

      String serverName, SecretManager<? extends TokenIdentifier> secretManager,

      String portRangeConfig)

throws IOException {

//初始化参数信息，地址、端口、configuration对象、handler个数、reader个数等等

    this.bindAddress = bindAddress;

    this.conf = conf;

    this.portRangeConfig = portRangeConfig;

    this.port = port;

    this.rpcRequestClass = rpcRequestClass;

    this.handlerCount = handlerCount;

    this.socketSendBufferSize = 0;

    this.maxDataLength = conf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH,

        CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT);

    if (queueSizePerHandler != -1) {

      this.maxQueueSize = queueSizePerHandler;

    } else {

      this.maxQueueSize = handlerCount * conf.getInt(

          CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,

          CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT);     

    }

    this.maxRespSize = conf.getInt(

        CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY,

        CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT);

    if (numReaders != -1) {

      this.readThreads = numReaders;

    } else {

      this.readThreads = conf.getInt(

          CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY,

          CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_DEFAULT);

    }

    this.readerPendingConnectionQueue = conf.getInt(

        CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY,

        CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_DEFAULT);

 

    // Setup appropriate callqueue

final String prefix = getQueueClassPrefix();

//构造call队列CallQueueManager是一个泛型类，增加了灵活性，里面只是包含了一个BlockingQueue队列对象和两个原子引用对象AtomicReference，一个take一个put，能够更灵活的存取队列中的信息。

    this.callQueue = new CallQueueManager<Call>(getQueueClass(prefix, conf),

        maxQueueSize, prefix, conf);

 

    this.secretManager = (SecretManager<TokenIdentifier>) secretManager;

    this.authorize =

      conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,

                      false);

 

    // configure supported authentications

    this.enabledAuthMethods = getAuthMethods(secretManager, conf);

    this.negotiateResponse = buildNegotiateResponse(enabledAuthMethods);

   

// Start the listener here and let it bind to the port

//构建listener

    listener = new Listener();

this.port = listener.getAddress().getPort();  

//构建connection管理类实例

    connectionManager = new ConnectionManager();

    this.rpcMetrics = RpcMetrics.create(this, conf);

    this.rpcDetailedMetrics = RpcDetailedMetrics.create(this.port);

    this.tcpNoDelay = conf.getBoolean(

        CommonConfigurationKeysPublic.IPC_SERVER_TCPNODELAY_KEY,

        CommonConfigurationKeysPublic.IPC_SERVER_TCPNODELAY_DEFAULT);

 

    // 构建responder

    responder = new Responder();

   

    if (secretManager != null || UserGroupInformation.isSecurityEnabled()) {

      SaslRpcServer.init(conf);

      saslPropsResolver = SaslPropertiesResolver.getInstance(conf);

    }

   

    this.exceptionsHandler.addTerseExceptions(StandbyException.class);

  }

 

构建完server对象后需要手动启动start方法，这样才能让server端真正的运行起来。

 

public synchronized void start() {

    //开启发送线程

responder.start();

//开启监听线程

listener.start();

//开启处理线程

    handlers = new Handler[handlerCount];

   

    for (int i = 0; i < handlerCount; i++) {

      handlers[i] = new Handler(i);

      handlers[i].start();

    }

  }

 

call方法

Call方法是一个抽象方法。实现在RPC.Server（此类暂不讲述）中，实现比较简单如下

public Writable call(RPC.RpcKind rpcKind, String protocol,

        Writable rpcRequest, long receiveTime) throws Exception {

      return getRpcInvoker(rpcKind).call(this, protocol, rpcRequest,

          receiveTime);

}

public static RpcInvoker  getRpcInvoker(RPC.RpcKind rpcKind) {

           //找到对应的RPCInvoker实现类，RPCKindMap中存储着两个键值对，一个writable相关的另一个protocol相关。

    RpcKindMapValue val = rpcKindMap.get(rpcKind);

    return (val == null) ? null : val.rpcInvoker;

}

 

调用RPCInvoker对应实现类中的call方法。实现方法到上面rpc讲解中查看。最终返回请求处理的结果。即调用服务端协议接口实现类的对应方法。并将结果返回。

 

setupResponse

此方法是对请求应答的初始化。

private void setupResponse(ByteArrayOutputStream responseBuf,

                             Call call, RpcStatusProto status, RpcErrorCodeProto erCode,

                             Writable rv, String errorClass, String error)

  throws IOException {

    responseBuf.reset();

    //用于填充返回结果的输出流，填充到responseBuf中

    DataOutputStream out = new DataOutputStream(responseBuf);

    //构造应答消息的头信息

    RpcResponseHeaderProto.Builder headerBuilder = 

        RpcResponseHeaderProto.newBuilder();

     //客户端id，处理的是哪个客户端

    headerBuilder.setClientId(ByteString.copyFrom(call.clientId));

    //处理的是客户端的那个call

    headerBuilder.setCallId(call.callId);

    //重试次数

    headerBuilder.setRetryCount(call.retryCount);

    //状态，是否成功存在这里

    headerBuilder.setStatus(status);

    //版本

    headerBuilder.setServerIpcVersionNum(CURRENT_VERSION);

           //如果处理成功

    if (status == RpcStatusProto.SUCCESS) {

      //构建头信息并计算头信息长度

      RpcResponseHeaderProto header = headerBuilder.build();

      final int headerLen = header.getSerializedSize();

      int fullLength  = CodedOutputStream.computeRawVarint32Size(headerLen) +

          headerLen;

      try {

                     //protocol引擎处理方式

        if (rv instanceof ProtobufRpcEngine.RpcWrapper) {

          ProtobufRpcEngine.RpcWrapper resWrapper =

              (ProtobufRpcEngine.RpcWrapper) rv;

                      //将头信息和rv最终返回结果写到out中的responseBuf缓冲中

          fullLength += resWrapper.getLength();

          out.writeInt(fullLength);

          header.writeDelimitedTo(out);

          rv.write(out);

         //writable处理方式

        } else { // Have to serialize to buffer to get len

          final DataOutputBuffer buf = new DataOutputBuffer();

          rv.write(buf);

          byte[] data = buf.getData();

          fullLength += buf.getLength();

          out.writeInt(fullLength);

          header.writeDelimitedTo(out);

          out.write(data, 0, buf.getLength());

        }

      } catch (Throwable t) {

        LOG.warn("Error serializing call response for call " + call, t);

        // Call back to same function - this is OK since the

        // buffer is reset at the top, and since status is changed

        // to ERROR it won't infinite loop.

        setupResponse(responseBuf, call, RpcStatusProto.ERROR,

            RpcErrorCodeProto.ERROR_SERIALIZING_RESPONSE,

            null, t.getClass().getName(),

            StringUtils.stringifyException(t));

        return;

      }

    } else { //Rpc失败

      //设置应答失败信息

      headerBuilder.setExceptionClassName(errorClass);

      headerBuilder.setErrorMsg(error);

      headerBuilder.setErrorDetail(erCode);

      RpcResponseHeaderProto header = headerBuilder.build();

      int headerLen = header.getSerializedSize();

      final int fullLength  =

          CodedOutputStream.computeRawVarint32Size(headerLen) + headerLen;

      out.writeInt(fullLength);

      header.writeDelimitedTo(out);

    }

    if (call.connection.useWrap) {

      wrapWithSasl(responseBuf, call);

    }

//将序列化后的返回结果放到当前处理的call中，call是处理远程调用的，所以与调用相关的东西会存储到此处。

    call.setResponse(ByteBuffer.wrap(responseBuf.toByteArray()));

  }

 

Listener

Listener是采用javaNIO的方式处理请求的，listener中的selector监听accept事件，当有事件发生时，将请求发送到reader（reader是listener的内部类）线程组中让其中的某个线程处理。Reader在接收到请求后，监听read事件，如果客户端发送的内容可读，则将请求信息读入calls队列中，等待handler县城处理。

Listener类图如下

 

 

acceptChnnel NIO serverSocket的通道，向selector中注册监听事件，就可以在一个线程中监听多个事件。

Readers reader线程组，用读取客户端请求

Selector 事件选择器

currentReader 当前reader数量

address server端的网络地址

backlogLength 监听队列最大值（当服务器忙时，会将客户端来的请求放到队列中等待listener处理）超过最大值让不予处理

 

listener构造方法

构造server时，其中包含了listener的构造，那么看看listener里都创建了些什么。

public Listener() throws IOException {

             //创建server网络地址

      address = new InetSocketAddress(bindAddress, port);

      //创建NIO通道

      acceptChannel = ServerSocketChannel.open();

             //将通道设置成异步

      acceptChannel.configureBlocking(false);

 

      // 将socket绑定到address上

      bind(acceptChannel.socket(), address, backlogLength, conf, portRangeConfig);

      port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port

      // 创建selector;

      selector= Selector.open();

             //创建readers线程组

      readers = new Reader[readThreads];

      for (int i = 0; i < readThreads; i++) {

        Reader reader = new Reader(

            "Socket Reader #" + (i + 1) + " for port " + port);

        readers[i] = reader;

        reader.start();

      }

 

      // 注册accept监听事件

      acceptChannel.register(selector, SelectionKey.OP_ACCEPT);

      this.setName("IPC Server listener on " + port);

      //将线程设置为守护线程，不同于Linux中的守护进程

      this.setDaemon(true);

}

 

 

此方法很简单，就是构建socket和异步通道，构建reader线程组并开启每个线程，将通道注册到accept事件中。

Listener.run

下面看一下listener中的run（继承自thread）是如何运行的。

public void run() {

      LOG.info(Thread.currentThread().getName() + ": starting");

      SERVER.set(Server.this);

             //开启connectionManager对空闲的connection的处理，这是一个定时任务。

      connectionManager.startIdleScan();

             //确保server端是活跃的

      while (running) {

        SelectionKey key = null;

        try {

          getSelector().select();

                       //获取selector中的所有已准备好的事件

          Iterator<SelectionKey> iter = getSelector().selectedKeys().iterator();

          while (iter.hasNext()) {

            key = iter.next();

            iter.remove();

            try {

              if (key.isValid()) {

                                          //如果是accept则处理

                if (key.isAcceptable())

                  doAccept(key);

              }

            } catch (IOException e) {

            }

            key = null;

          }

        } catch (OutOfMemoryError e) {

          // we can run out of memory if we have too many threads

          // log the event and sleep for a minute and give

          // some thread(s) a chance to finish

          LOG.warn("Out of Memory in server select", e);

                       //如果出现问题则关闭当前连接

          closeCurrentConnection(key, e);

                       //直接调用manager中的关闭空闲连接。

          connectionManager.closeIdle(true);

          try { Thread.sleep(60000); } catch (Exception ie) {}

        } catch (Exception e) {

          closeCurrentConnection(key, e);

        }

      }

      LOG.info("Stopping " + Thread.currentThread().getName());

 

      synchronized (this) {

        try {

                       //server端关闭，则关闭通道和socket，关闭selector

          acceptChannel.close();

          selector.close();

        } catch (IOException e) { }

 

        selector= null;

        acceptChannel= null;

       

        //关闭所有连接，停止空闲连接扫描任务

        connectionManager.stopIdleScan();

        connectionManager.closeAll();

      }

}

 

此方法就是在server可用的情况下，不断的扫描selector中的事件，如果有accept事件则处理。如果server端关闭则关闭管道和selector，关闭所有连接。

 

doAccept

doAccept是专门处理accept事件的方法。

void doAccept(SelectionKey key) throws InterruptedException, IOException,  OutOfMemoryError {   

             //获取通道（NIO常规操作）并设置通道相关参数

      ServerSocketChannel server = (ServerSocketChannel) key.channel();

             //获取发送到此通道的连接

      SocketChannel channel;

      while ((channel = server.accept()) != null) {

 

        channel.configureBlocking(false);

        channel.socket().setTcpNoDelay(tcpNoDelay);

        channel.socket().setKeepAlive(true);

        //从reader中获取一个reader

        Reader reader = getReader();

                     //从manager中构建一个connection，connection可能是复用的。

        Connection c = connectionManager.register(channel);

        // If the connectionManager can't take it, close the connection.

        if (c == null) {

          if (channel.isOpen()) {

            IOUtils.cleanup(null, channel);

          }

          continue;

        }

                     //将connection放置到附件中，当出现问题是可直接关闭该connection

        key.attach(c);  // so closeCurrentConnection can get the object

                     //将connection添加到reader中处理reader事件。

        reader.addConnection(c);

      }

}

 

closeCurrentConnection

当连接或者内存等问题，则需要关闭当前连接

private void closeCurrentConnection(SelectionKey key, Throwable e) {

      if (key != null) {

                     //doAccept中将连接放到了attach中。

        Connection c = (Connection)key.attachment();

        if (c != null) {

                       //关闭连接，该方法最终调用connectionManager中的close方法，连接的管理全由connectionManager管理

          closeConnection(c);

          c = null;

        }

      }

}

 

doRead

该方法并非由listener自身调用，而是由reader调用（稍后分析）

void doRead(SelectionKey key) throws InterruptedException {

      int count = 0;

      ////doAccept中将连接放到了attach中

Connection c = (Connection)key.attachment();

      if (c == null) {

        return; 

      }

      //设置最近活跃时间，与connectionManager的闲置处理相关(稍后分析)

      c.setLastContact(Time.now());

     

      try {

                     //真正处理请求的地方。返回的是数据dataLength，如果是负数表示错误

        count = c.readAndProcess();

      } catch (InterruptedException ieo) {

        LOG.info(Thread.currentThread().getName() + ": readAndProcess caught InterruptedException", ieo);

        throw ieo;

      } catch (Exception e) {

        // a WrappedRpcServerException is an exception that has been sent

        // to the client, so the stacktrace is unnecessary; any other

        // exceptions are unexpected internal server errors and thus the

        // stacktrace should be logged

        LOG.info(Thread.currentThread().getName() + ": readAndProcess from client " +

            c.getHostAddress() + " threw exception [" + e + "]",

            (e instanceof WrappedRpcServerException) ? null : e);

        count = -1; //so that the (count < 0) block is executed

      }

      if (count < 0) {

                    //关闭连接

        closeConnection(c);

        c = null;

      }

      else {

//更新最近活跃时间，

        c.setLastContact(Time.now());

      }

}

 

 

此方法只是个中间过程，真正处理的地方在Connection的readAndProcess中。

 

Listener.Reader

Reader是专门处理reader事件的，每个reader都有各自的Connection队列（blockingQueue队列），        每个reader将处理不了的请求暂时放置到此队列中，队列有固定的长度，如果超出则不予处理。

Reader类图如下

 

 

pendingConnections 等待队列

readSelector read选择器

 

Reader构造方法

相对简单不做解析

Reader(String name) throws IOException {

        super(name);

           // readerPendingConnectionQueue server构造是已有此参数。

        this.pendingConnections =

            new LinkedBlockingQueue<Connection>(readerPendingConnectionQueue);

        this.readSelector = Selector.open();

}

 

Reader.run

Run方法将主要操作放到doRunLoop中。

public void run() {

        LOG.info("Starting " + Thread.currentThread().getName());

        try {

          doRunLoop();

        } finally {

          try {

                                //任务完成关闭selector

            readSelector.close();

          } catch (IOException ioe) {

            LOG.error("Error closing read selector in " + Thread.currentThread().getName(), ioe);

          }

        }

      }

 

doRunLoop

private synchronized void doRunLoop() {

        while (running) {

          SelectionKey key = null;

          try {

            //尽可能选择当前时间点队列的连接，避免肆无忌惮的接受那些饥饿的连接(并不表示不接受，而是在下次的 调用中在处理)

            int size = pendingConnections.size();

                                //迭代等待的连接，并将链接中的通道注册read事件。

            for (int i=size; i>0; i--) {

              Connection conn = pendingConnections.take();

              conn.channel.register(readSelector, SelectionKey.OP_READ, conn);

            }

            readSelector.select();

                                //迭代已经准备好的事件

            Iterator<SelectionKey> iter = readSelector.selectedKeys().iterator();

            while (iter.hasNext()) {

              key = iter.next();

              iter.remove();

              if (key.isValid()) {

                if (key.isReadable()) {

                                            //处理read时间，doread在Listener中。

                  doRead(key);

                }

              }

              key = null;

            }

          } catch (InterruptedException e) {

            if (running) {                      // unexpected -- log it

              LOG.info(Thread.currentThread().getName() + " unexpectedly interrupted", e);

            }

          } catch (IOException ex) {

            LOG.error("Error in Reader", ex);

          }

        }

      }

 

此方法只是监听read事件，然后将read处理交给doread。

Connection

Connection是解析（不是处理）请求的最终地方。请求最终的处理地方在handler中，handler会调用Server子类ProtoBuffRPCEngine中的Server内部类的call方法，这是请求最终的处理的地方，后面会解析。

Connection 类图如下

 

 

connectionHeaderRead header是否已经读入

connectionContextRead 上下文是否已经读入

channel 读取数据的通道

data 数据存放地

dataLengthBuffer 存放长度信息有可能存储的是魔数hrpc

RPCCount 当前处理的请求数，此值与ConnectionManager管理空闲Connection有关

lastContact 最近处理请求的时间，此值与ConnectionManager管理空闲Connection有关

dataLength 数据长度

socket socket连接

hostAddress client端的地址

remotePort client端的端口

addr 网络地址

connectionContext 上下文

protocolName 协议名

connectionHeaderBuf header信息存放地

serverClass 与服务端一致

authFailedCall 发生错误时返回的友好错误信息

 

Connection构造方法

public Connection(SocketChannel channel, long lastContact) {

      this.channel = channel;

      this.lastContact = lastContact;

      this.data = null;

      //4字节client.connection中开头解释过

      this.dataLengthBuffer = ByteBuffer.allocate(4);

      this.unwrappedData = null;

      this.unwrappedDataLengthBuffer = ByteBuffer.allocate(4);

      this.socket = channel.socket();

      this.addr = socket.getInetAddress();

      if (addr == null) {

        this.hostAddress = "*Unknown*";

      } else {

        this.hostAddress = addr.getHostAddress();

      }

      this.remotePort = socket.getPort();

      this.responseQueue = new LinkedList<Call>();

      if (socketSendBufferSize != 0) {

        try {

          socket.setSendBufferSize(socketSendBufferSize);

        } catch (IOException e) {

          LOG.warn("Connection: unable to set socket send buffer size to " +

                   socketSendBufferSize);

        }

      }

    }

 

构造方法只是初始化了一些socket、通道、返回队列等信息，就是对上面的属性进行一系列的初始化。

 

ReadAndProcess

下面看一下真正解析请求的地方到底做了些什么。此方法会在doread中被调用

public int readAndProcess()

        throws WrappedRpcServerException, IOException, InterruptedException {

      while (true) {

        /*至多读取一个RPC请求，如果header信息没有完整的读完，那么一直循环迭代直到读到一个RPC或者没有数据为止。

         */   

        int count = -1;

        if (dataLengthBuffer.remaining() > 0) {

          count = channelRead(channel, dataLengthBuffer);

                 //count<0表示没有信息可读，dataLengthBuffer.remaining>0表示发送过来的请求不完整。返回等待下一个新的请求或者当前请求的下一个数据包的到来。

          if (count < 0 || dataLengthBuffer.remaining() > 0)

            return count;

        }

        //处理ConnectionHeader

        if (!connectionHeaderRead) {

          //Every connection is expected to send the header.

          if (connectionHeaderBuf == null) {

                                //客户端一共发送7字节的头信息（具体信息看writeConnectionHeader），其中头4个字节是hrpc魔数，这里的connectionHeaderBuf存储后3个字节。魔数存放在dataLengthBuffer中。

            connectionHeaderBuf = ByteBuffer.allocate(3);

          }

                       //从通道中读取数据

          count = channelRead(channel, connectionHeaderBuf);

          if (count < 0 || connectionHeaderBuf.remaining() > 0) {

            return count;

          }

          int version = connectionHeaderBuf.get(0);

          // TODO we should add handler for service class later

          this.setServiceClass(connectionHeaderBuf.get(1));

          dataLengthBuffer.flip();

         

          // 如果是http get请求，我们将发送一条友好的错误提示信息。

          if (HTTP_GET_BYTES.equals(dataLengthBuffer)) {

            setupHttpRequestOnIpcPortResponse();

            return -1;

          }

          //如果ConnectionHeader不符合规定则返回错误-1.

          if (!RpcConstants.HEADER.equals(dataLengthBuffer)

              || version != CURRENT_VERSION) {

            //Warning is ok since this is not supposed to happen.

            LOG.warn("Incorrect header or version mismatch from " +

                     hostAddress + ":" + remotePort +

                     " got version " + version +

                     " expected version " + CURRENT_VERSION);

            setupBadVersionResponse(version);

            return -1;

          }

         

          // this may switch us into SIMPLE

          authProtocol = initializeAuthContext(connectionHeaderBuf.get(2));         

         

          //正确处理完ConnectionHeader，清空dataLengthBuffer数据，标记header已处理完

dataLengthBuffer.clear();

          connectionHeaderBuf = null;

          connectionHeaderRead = true;

          continue;

        }

        //如果处理的不是header，而是正常的请求则如下处理

        if (data == null) {

                       //将dataLengthBuffer置成可度状态

          dataLengthBuffer.flip();

                 //读取数据长度

          dataLength = dataLengthBuffer.getInt();

                       //检验数据是否超长

          checkDataLength(dataLength);

            //分配请求存储空间

          data = ByteBuffer.allocate(dataLength);

        }

        //读取数据到data中

        count = channelRead(channel, data);

        //如果数据读取完，客户端发送请求时可能要多个数据包才能将一个请求发送到服务端

        if (data.remaining() == 0) {

          dataLengthBuffer.clear();

          data.flip();

          boolean isHeaderRead = connectionContextRead;

                      //解析请求

          processOneRpc(data.array());

          data = null;

          if (!isHeaderRead) {

            continue;

          }

        }

        return count;

      }

}

 

 

此方法先判断是否有请求数据，没有或者数据不全则返回给上层（这里指的是dataLengthBuffer不是全部的请求），等待下次请求。如果dataLengthBuffer以完整读取，判断是否是Connectionheader，如果是则按照内部的约定判断是否正确。如果不是则读取请求数据部分，直到数据完整，开始解析请求信息。

 

 

processOneRpc

下面分析processOneRpc方法，终于到了最终解析请求的地方，其实此方法就是把请求反序列化，然后按照传过来的call对象在server端照搬的创建一个类似的对象。

private void processOneRpc(byte[] buf)

        throws IOException, WrappedRpcServerException, InterruptedException {

      int callId = -1;

      int retry = RpcConstants.INVALID_RETRY_COUNT;

      try {

                     //构建输入流用于读取buf中的数据。

        final DataInputStream dis =

            new DataInputStream(new ByteArrayInputStream(buf));

                     //读取请求头

        final RpcRequestHeaderProto header =

            decodeProtobufFromStream(RpcRequestHeaderProto.newBuilder(), dis);

                     //读取发送过来的callId和retry

        callId = header.getCallId();

        retry = header.getRetryCount();

        if (LOG.isDebugEnabled()) {

          LOG.debug(" got #" + callId);

        }

        checkRpcHeaders(header);

        //如果callId<0可能是ConnectionContext信息，或者是错误信息

        if (callId < 0) { // callIds typically used during connection setup

                       //错误处理

          processRpcOutOfBandRequest(header, dis);

        } else if (!connectionContextRead) {

          throw new WrappedRpcServerException(

              RpcErrorCodeProto.FATAL_INVALID_RPC_HEADER,

              "Connection context not established");

        } else {

                       //将请求转化为call的方法，下面将要分析

          processRpcRequest(header, dis);

        }

      } catch (WrappedRpcServerException wrse) { // inform client of error

        Throwable ioe = wrse.getCause();

        final Call call = new Call(callId, retry, null, this);

        setupResponse(authFailedResponse, call,

            RpcStatusProto.FATAL, wrse.getRpcErrorCodeProto(), null,

            ioe.getClass().getName(), ioe.getMessage());

        responder.doRespond(call);

        throw wrse;

      }

}

 

processRpcRequest

此处是将请求解析成call的最终的地方，并将解析出来的Call存储到callQueue（Server类属性成员保存着当前server正在处理的所有的call）中。

private void processRpcRequest(RpcRequestHeaderProto header,

        DataInputStream dis) throws WrappedRpcServerException,

        InterruptedException {

             //根据引擎类型反序列化结果

      Class<? extends Writable> rpcRequestClass =

          getRpcRequestWrapper(header.getRpcKind());

      if (rpcRequestClass == null) {

        LOG.warn("Unknown rpc kind "  + header.getRpcKind() +

            " from client " + getHostAddress());

        final String err = "Unknown rpc kind in rpc header"  +

            header.getRpcKind();

        throw new WrappedRpcServerException(

            RpcErrorCodeProto.FATAL_INVALID_RPC_HEADER, err);  

      }

             //请求的内容，包含要调用的方法和参数值

      Writable rpcRequest;

      try { //Read the rpc request

                     //读取请求内容到RPCRequest中

        rpcRequest = ReflectionUtils.newInstance(rpcRequestClass, conf);

        rpcRequest.readFields(dis);

      } catch (Throwable t) { // includes runtime exception from newInstance

        LOG.warn("Unable to read call parameters for client " +

                 getHostAddress() + "on connection protocol " +

            this.protocolName + " for rpcKind " + header.getRpcKind(),  t);

        String err = "IPC server unable to read call parameters: "+ t.getMessage();

        throw new WrappedRpcServerException(

            RpcErrorCodeProto.FATAL_DESERIALIZING_REQUEST, err);

      }

       

      Span traceSpan = null;

      if (header.hasTraceInfo()) {

        // If the incoming RPC included tracing info, always continue the trace

        TraceInfo parentSpan = new TraceInfo(header.getTraceInfo().getTraceId(),

                                             header.getTraceInfo().getParentId());

        traceSpan = Trace.startSpan(rpcRequest.toString(), parentSpan).detach();

      }

 

             //将请求构造成call对象，等待handler线程处理

      Call call = new Call(header.getCallId(), header.getRetryCount(),

          rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),

          header.getClientId().toByteArray(), traceSpan);

             //放入call队列等待handler的处理

      callQueue.put(call);              // queue the call; maybe blocked here

             //增加当前Connection处理rpc的数量（此值与ConnectionManager管理闲置Connection有关）

      incRpcCount();  // Increment the rpc count

}

 

isIdle

判断当前连接是否是空闲的状态，rpcCount=0空闲

private boolean isIdle() {

      return rpcCount.get() == 0;

}

 

Handler

Handler是真正处理请求的地方。类图如下

 

 

Handler构造方法

public Handler(int instanceNumber) {

      this.setDaemon(true);

      this.setName("IPC Server handler "+ instanceNumber + " on " + port);

}

 

相对简单，只是设置守护线程。

Handler.run

public void run() {

      LOG.debug(Thread.currentThread().getName() + ": starting");

      SERVER.set(Server.this);

             //设置应答消息的大小

      ByteArrayOutputStream buf =

        new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);

      while (running) {

        TraceScope traceScope = null;

        try {

                       //从call队列中获取一个call

          final Call call = callQueue.take(); // pop the queue; maybe blocked here

          if (LOG.isDebugEnabled()) {

            LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);

          }

                       //确保call对应的Connection中的客户端通道打开的

          if (!call.connection.channel.isOpen()) {

            LOG.info(Thread.currentThread().getName() + ": skipped " + call);

            continue;

          }

          String errorClass = null;

          String error = null;

          RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;

          RpcErrorCodeProto detailedErr = null;

                       //处理结果值

          Writable value = null;

                       //设置当前处理的call

          CurCall.set(call);

          if (call.traceSpan != null) {

            traceScope = Trace.continueSpan(call.traceSpan);

          }

 

          try {

            // Make the call as the user via Subject.doAs, thus associating

            // the call with the Subject

            if (call.connection.user == null) {

                                  //调用server的call方法处理请求，最终会走向writableRPCEnging或者ProtoBufRPCEngine中对应的实现类处理。此方法在server处有详细的讲解。

              value = call(call.rpcKind, call.connection.protocolName, call.rpcRequest,

                           call.timestamp);

            } else {

              value =

                call.connection.user.doAs

                  (new PrivilegedExceptionAction<Writable>() {

                     @Override

                     public Writable run() throws Exception {

                       // make the call

                       return call(call.rpcKind, call.connection.protocolName,

                                   call.rpcRequest, call.timestamp);

 

                     }

                   }

                  );

            }

          } catch (Throwable e) {

            if (e instanceof UndeclaredThrowableException) {

              e = e.getCause();

            }

            String logMsg = Thread.currentThread().getName() + ", call " + call;

            if (exceptionsHandler.isTerse(e.getClass())) {

              // Don't log the whole stack trace. Way too noisy!

              LOG.info(logMsg + ": " + e);

            } else if (e instanceof RuntimeException || e instanceof Error) {

              // These exception types indicate something is probably wrong

              // on the server side, as opposed to just a normal exceptional

              // result.

              LOG.warn(logMsg, e);

            } else {

              LOG.info(logMsg, e);

            }

            if (e instanceof RpcServerException) {

              RpcServerException rse = ((RpcServerException)e);

              returnStatus = rse.getRpcStatusProto();

              detailedErr = rse.getRpcErrorCodeProto();

            } else {

              returnStatus = RpcStatusProto.ERROR;

              detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;

            }

            errorClass = e.getClass().getName();

            error = StringUtils.stringifyException(e);

            // Remove redundant error class name from the beginning of the stack trace

            String exceptionHdr = errorClass + ": ";

            if (error.startsWith(exceptionHdr)) {

              error = error.substring(exceptionHdr.length());

            }

          }

                       //当前处理call完成，置空

          CurCall.set(null);

                       //同步防止当前连接发送多个应答是出现混乱

          synchronized (call.connection.responseQueue) {

            // setupResponse() needs to be sync'ed together with

            // responder.doResponse() since setupResponse may use

            // SASL to encrypt response data and SASL enforces

            // its own message ordering.

                                //初始化应答消息，为发送做准备，此方法在Server处有详细的讲解

            setupResponse(buf, call, returnStatus, detailedErr,

                value, errorClass, error);

            

            //防止应答缓冲大小超过最大值

            if (buf.size() > maxRespSize) {

              LOG.warn("Large response size " + buf.size() + " for call "

                  + call.toString());

              buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);

            }

                                //将消息发送到客户端

            responder.doRespond(call);

          }

        } catch (InterruptedException e) {

          if (running) {                          // unexpected -- log it

            LOG.info(Thread.currentThread().getName() + " unexpectedly interrupted", e);

            if (Trace.isTracing()) {

              traceScope.getSpan().addTimelineAnnotation("unexpectedly interrupted: " +

                  StringUtils.stringifyException(e));

            }

          }

        } catch (Exception e) {

          LOG.info(Thread.currentThread().getName() + " caught an exception", e);

          if (Trace.isTracing()) {

            traceScope.getSpan().addTimelineAnnotation("Exception: " +

                StringUtils.stringifyException(e));

          }

        } finally {

          if (traceScope != null) {

            traceScope.close();

          }

          IOUtils.cleanup(LOG, traceScope);

        }

      }

      LOG.debug(Thread.currentThread().getName() + ": exiting");

}

 

此方法只是取出call队列中的一个，并进行一系列的校验，然后交给Server的call处理（上面有解析），将返回值封装到responder中即setupResponse()（server上面有解析），然后调用responder的doRespond方法发送应答消息。

Responder

此类是请求处理的地方。

类图如下

 

 

 

writeSelector  write选择器

pending 等候的注册的Connection数。

doRespond

下面看一下上面handler的run方法调用的doRespond是如何把消息发送到客户端的。

void doRespond(Call call) throws IOException {

             ////同步防止当前连接发送多个应答是出现混乱

      synchronized (call.connection.responseQueue) {

        //将call添加到当前call所在连接的应答队列中。

        call.connection.responseQueue.addLast(call);

        //这里做了一步优化，当应答队列中的call数量是1个时，就在handler的线程中直接把应答消息发送到客户端，原因是当前连接不是很忙，如果>1则在responder的线程中处理，不要占用handler线程。

        if (call.connection.responseQueue.size() == 1) {

                       //直接调用该方法将数据发送到客户端

          processResponse(call.connection.responseQueue, true);

        }

      }

    }    

 

 

ProcessResponse

下面是responder很重要的一个方法，将消息发送给客户端。

private boolean processResponse(LinkedList<Call> responseQueue,

                                    boolean inHandler) throws IOException {

      boolean error = true;

      boolean done = false;       // there is more data for this channel.

      int numElements = 0;

      Call call = null;

      try {

                     ///同步防止当前连接发送多个应答是出现混乱

        synchronized (responseQueue) {

          //

          // If there are no items for this channel, then we are done

          //查看当前队列数量，没有则直接返回

          numElements = responseQueue.size();

          if (numElements == 0) {

            error = false;

            return true;              // no more data for this channel.

          }

          //

          // Extract the first call

          //去除第一个应答call

          call = responseQueue.removeFirst();

                       //获取客户端socket通道，写数据时用

          SocketChannel channel = call.connection.channel;

          if (LOG.isDebugEnabled()) {

            LOG.debug(Thread.currentThread().getName() + ": responding to " + call);

          }

          //

          // Send as much data as we can in the non-blocking fashion

          //将数据发送到客户端通过通道，call.rpcResponse是最终的返回结果，结果是在Rpc处ProtoBufRpcInvoker.Call中产生的。

          int numBytes = channelWrite(channel, call.rpcResponse);

                       //小于0（此处的小于0表示rpcResponse中没有数据）表示已经没有数据可写了或者channel已经关闭了，直接返回

          if (numBytes < 0) {

            return true;

          }

                       //如果RPCResponse中的数据已经全部发送完毕。

          if (!call.rpcResponse.hasRemaining()) {

            //清空RPCResponse

            call.rpcResponse = null;

                              //减少当前连接处理的请求数量

            call.connection.decRpcCount();

           //如果应答队列只有一个call，则表示当前连接没有需要在处理的请求了。

            if (numElements == 1) {    // last call fully processes.

                                  //表示当前连接已经完成了所有他应该处理的请求

              done = true;             // no more data for this channel.

            } else {

              done = false;            // more calls pending to be sent.

            }

            if (LOG.isDebugEnabled()) {

              LOG.debug(Thread.currentThread().getName() + ": responding to " + call

                  + " Wrote " + numBytes + " bytes.");

            }

          } else {

            //

            // 如果没有将应答消息全部写出去，则将call再次插入到队列头部，下次处理时将先处理当前的call让其返回完整的信息。

            //

            call.connection.responseQueue.addFirst(call);

            //如果是在handler线程中处理

            if (inHandler) {

              // set the serve time when the response has to be sent later

              call.timestamp = Time.now();

              //增加pending表示当前正有一个Connection正在处理中，其他Connection等待

              incPending();

              try {

                // 唤醒在run方法中阻塞住的selector，.

                writeSelector.wakeup();

                                          //将channel注册到write时间中，将call放到附件中，在run处理时可以从附件中直接取出待处理的call

                channel.register(writeSelector, SelectionKey.OP_WRITE, call);

              } catch (ClosedChannelException e) {

                //Its ok. channel might be closed else where.

                done = true;

              } finally {

                                          //处理完成减去pending，表示当前Connection已经完成部分处理（注意并不是全部）。

                decPending();

              }

            }

            if (LOG.isDebugEnabled()) {

              LOG.debug(Thread.currentThread().getName() + ": responding to " + call

                  + " Wrote partial " + numBytes + " bytes.");

            }

          }

          error = false;              // everything went off well

        }

      } finally {

        if (error && call != null) {

          LOG.warn(Thread.currentThread().getName()+", call " + call + ": output error");

          done = true;               // error. no more data for this channel.

          closeConnection(call.connection);

        }

      }

             //返回当前Connection是否已经处理完所有的请求（每个Connection都有各自的应答队列和请求队列）。

      return done;

}

 

Responder.run

//run方法是处理应答发送的主要地方，

public void run() {

      LOG.info(Thread.currentThread().getName() + ": starting");

      SERVER.set(Server.this);

      try {

                     //将应答处理移交给doRunLoop

        doRunLoop();

      } finally {

        LOG.info("Stopping " + Thread.currentThread().getName());

        try {

          writeSelector.close();

        } catch (IOException ioe) {

          LOG.error("Couldn't close write selector in " + Thread.currentThread().getName(), ioe);

        }

      }

    }

 

doRunLoop

    

private void doRunLoop() {

      long lastPurgeTime = 0;   // last check for old calls.

             //确保server运行

      while (running) {

        try {

          waitPending();     // 如果有一个channel注册了当前selector则等待

                       //select阻塞等待最多15分钟(PURGE_INTERVAL)，超过15分钟将要开启清理工作

          writeSelector.select(PURGE_INTERVAL);

          Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();

          while (iter.hasNext()) {

            SelectionKey key = iter.next();

            iter.remove();

            try {

              if (key.isValid() && key.isWritable()) {

                  //开始异步写应答消息

doAsyncWrite(key);

              }

            } catch (IOException e) {

              LOG.info(Thread.currentThread().getName() + ": doAsyncWrite threw exception " + e);

            }

          }

          long now = Time.now();

                       //如果当前时间离方法开始一直持续不到15分钟则继续循环等待待处理的call

          if (now < lastPurgeTime + PURGE_INTERVAL) {

            continue;

          }

                      

          lastPurgeTime = now;

          //

          // If there were some calls that have not been sent out for a

          // long time, discard them.

          //

          if(LOG.isDebugEnabled()) {

            LOG.debug("Checking for old call responses.");

          }

          ArrayList<Call> calls;

          ////开始清理工作

          // get the list of channels from list of keys.

          synchronized (writeSelector.keys()) {

                                //获取注册到selector上的所有待处理所有的call

            calls = new ArrayList<Call>(writeSelector.keys().size());

            iter = writeSelector.keys().iterator();

            while (iter.hasNext()) {

              SelectionKey key = iter.next();

              Call call = (Call)key.attachment();

              if (call != null && key.channel() == call.connection.channel) {

                calls.add(call);

              }

            }

          }

          //开始清理

          for(Call call : calls) {

            doPurge(call, now);

          }

        } catch (OutOfMemoryError e) {

          //

          // we can run out of memory if we have too many threads

          // log the event and sleep for a minute and give

          // some thread(s) a chance to finish

          //

          LOG.warn("Out of Memory in server select", e);

          try { Thread.sleep(60000); } catch (Exception ie) {}

        } catch (Exception e) {

          LOG.warn("Exception in Responder", e);

        }

      }

    }

 

doAsyncWrite

开始处理写工作，将消息写到客户端，最终调用的仍是processResponse方法

    

private void doAsyncWrite(SelectionKey key) throws IOException {

      Call call = (Call)key.attachment();

      if (call == null) {

        return;

      }

             //确保当前处理的call和Connection是同一个通道的

      if (key.channel() != call.connection.channel) {

        throw new IOException("doAsyncWrite: bad channel");

      }

 

      synchronized(call.connection.responseQueue) {

                     //写工作具体处理的地方

        if (processResponse(call.connection.responseQueue, false)) {

          try {

                                //如果返回true表示当前Connection所有的应答消息都已处理完毕，那么注册到selector上的事件就要清空了。

            key.interestOps(0);

          } catch (CancelledKeyException e) {

            /* The Listener/reader might have closed the socket.

             * We don't explicitly cancel the key, so not sure if this will

             * ever fire.

             * This warning could be removed.

             */

            LOG.warn("Exception while changing ops : " + e);

          }

        }

      }

    }

 

 

    //

    // 删除那些已经长时间等待到responseQueue队列待处理的call。15分钟可能代表这连接出现了异常无法正常处理。

    //

    

private void doPurge(Call call, long now) {

      LinkedList<Call> responseQueue = call.connection.responseQueue;

      synchronized (responseQueue) {

        Iterator<Call> iter = responseQueue.listIterator(0);

        while (iter.hasNext()) {

          call = iter.next();

          if (now > call.timestamp + PURGE_INTERVAL) {

            //关闭连接，长时间等待未处理可能是连接出现了问题。

closeConnection(call.connection);

            break;

          }

        }

      }

    }