Chap13-ChatServer

ChatServer

这一节我们使用Asio编写ChatServer.

我们从main入手

#include "global/ConfigManager.h"
#include "server/AsioPool.h"
#include "server/Server.h"

#include <boost/asio.hpp>
#include <boost/asio/signal_set.hpp>
#include <grpc++/grpc++.h>
#include <spdlog/spdlog.h>
#include <thread>

int main()
{
    spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%^%l%$] [%s:%#] %v");
    spdlog::set_level(spdlog::level::debug);

    {
        auto& cfg = ConfigManager::GetInstance();
        auto pool = AsioPool::GetInstance();
        boost::asio::io_context ioc;
        boost::asio::signal_set signals(ioc, SIGINT, SIGTERM);
        signals.async_wait([&ioc, pool](const boost::system::error_code& /*error*/, int /*signal_number*/) {
            pool->Stop();
            ioc.stop();
        });

        auto port = cfg["ChatServer"]["port"];
        std::make_shared<Server>(ioc, std::stoi(port))->Start();
        ioc.run();
    }
    return 0;
}

很简单，就是启动了一个Server.以及一个AsioPool

我们先来看AsioPool

// .h
#ifndef ASIOPOOL_H
#define ASIOPOOL_H

#include "../global/Singleton.h"
#include <boost/asio.hpp>
#include <boost/asio/executor_work_guard.hpp>
#include <thread>
#include <vector>

class AsioPool : public Singleton<AsioPool> {
    friend Singleton<AsioPool>;

public:
    ~AsioPool();
    AsioPool(const AsioPool&) = delete;
    AsioPool& operator=(const AsioPool&) = delete;

    boost::asio::io_context& GetIOService();
    void Stop();

private:
    AsioPool(std::size_t size = std::thread::hardware_concurrency());

private:
    std::vector<boost::asio::io_context> _services;
    std::vector<std::thread> _threads;
    std::vector<std::unique_ptr<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>>> _works;
    std::size_t _next_services;
};

#endif

// .cpp
#include "AsioPool.h"
#include <iostream>
#include <memory>
#include <spdlog/spdlog.h>

AsioPool::AsioPool(std::size_t size)
    : _services(size)
    , _works()
    , _next_services(0)
{
    _works.reserve(size);
    for (std::size_t i = 0; i < size; ++i) {
        _works.emplace_back(std::make_unique<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>>(boost::asio::make_work_guard(_services[i])));
    }
    _threads.reserve(size);
    for (std::size_t i = 0; i < size; i++) {
        _threads.emplace_back([this, i] {
            _services[i].run();
        });
    }
}

boost::asio::io_context& AsioPool::GetIOService()
{
    auto& service = _services[_next_services];
    _next_services = (_next_services + 1) % _services.size();
    return service;
}

void AsioPool::Stop()
{
    for (auto& work : _works) {
        work->get_executor().context().stop();
        work.reset();
    }
    for (auto& thread : _threads) {
        thread.join();
    }
}

AsioPool::~AsioPool()
{
    Stop();
    SPDLOG_INFO("AsioPool Destruct");
}

我们知道，asio的socket创建的时候，需要给他一个io_context上下文服务才能进行通信。当socket有信息发送的时候，对应的io_context底层应有一个缓冲区将这些发送请求一一发送给对端。所以我们创建了多个线程，多个io_context,每个线程跑着一个io_context，做到并行。

那么main中也创建了一个io_context，这个io_context不用于通信，有两个用途：

优雅退出：我们注册了信号，接收信号并退出程序
用于接受别的连接：即使是接受别人发来的连接请求也需要io_context处理，我们用这个单独处理连接请求。连接请求处理后，Server内部会获取一个io_context分配给socket通信。

再来看Server

// h
#ifndef GATEWAYSERVER_H
#define GATEWAYSERVER_H

#include "../global/const.h"
#include "../session/Session.h"
#include <memory>
#include <mutex>
#include <unordered_map>

class Server : public std::enable_shared_from_this<Server> {
public:
    Server(net::io_context& ioc, uint16_t port);
    void Start();
    void ClearSession(const std::string& session_id);
    bool CheckValid(const std::string& session_id);

private:
    net::ip::tcp::acceptor _acceptor;
    net::io_context& _ioc;
    uint16_t _port;
    std::unordered_map<std::string, std::shared_ptr<Session>> _sessions;
    std::mutex _mutex;
};

#endif
// cpp
#include "Server.h"
#include "../session/Session.h"
#include "AsioPool.h"
#include <spdlog/spdlog.h>

Server::Server(net::io_context& ioc, uint16_t port)
    : _ioc(ioc)
    , _acceptor(ioc, net::ip::tcp::endpoint(net::ip::tcp::v4(), port))
    , _port(port)
{
    SPDLOG_INFO("Server Start Success,Listen on port:{}", _port);
}

void Server::Start()
{
    auto& io_context = AsioPool::GetInstance()->GetIOService();
    std::shared_ptr<Session> conn = std::make_shared<Session>(io_context, this);
    _acceptor.async_accept(conn->GetSocket(), [this, conn, self = shared_from_this()](const boost::system::error_code& ec) {
        try {
            if (ec) {
                self->Start();
                return;
            }
            conn->Start();
            SPDLOG_INFO("New connection from {},session:{}", conn->GetSocket().remote_endpoint().address().to_string(),conn->GetSessionId());

            std::unique_lock<std::mutex> lock(_mutex);
            _sessions.insert(std::make_pair(conn->GetSessionId(), conn));
            lock.unlock();

            self->Start();
        } catch (std::exception& e) {
            SPDLOG_ERROR("Exception: {}", e.what());
        }
    });
}

void Server::ClearSession(const std::string& session_id)
{
    std::lock_guard<std::mutex> lock(_mutex);
    _sessions.erase(session_id);
}

bool Server::CheckValid(const std::string& session_id)
{
    auto it = _sessions.find(session_id);
    if (it == _sessions.end()) {
        return false;
    }
    return true;
}

Start中就是AsioPool::GetInstance()->GetIOService();获取一个连接，创建一个Session用于通信。

接下来看Session

// h
#ifndef SESSION_H
#define SESSION_H

#include "../global/const.h"
#include "MsgNode.h"
#include <memory>
#include <queue>

class Server;
class LogicSystem;

class Session : public std::enable_shared_from_this<Session> {
    friend class LogicSystem;

public:
    Session(boost::asio::io_context& ioc, Server* server);
    ~Session();
    std::string GetSessionId() noexcept;
    void Start();
    void Close();
    void Send(const char* msg, int max_length, uint16_t msg_id);
    void Send(std::string msg, uint16_t msg_id);
    void HandleWrite(boost::system::error_code ec, std::shared_ptr<Session> self);
    net::ip::tcp::socket& GetSocket() noexcept;
    std::queue<std::shared_ptr<SendNode>>& GetSendQueue() noexcept;

private:
    void AsyncHead(std::size_t len);
    void AsyncBody(std::size_t len);

private:
    std::shared_ptr<RecvNode> _recv_msg_node;
    std::shared_ptr<MsgNode> _recv_head_node;
    char _data[MAX_LENGTH];

    std::queue<std::shared_ptr<SendNode>> _send_queue;
    std::mutex _send_lock;

    net::ip::tcp::socket _socket;
    std::string _session_id;
    int _uid;

    Server* _server;
    bool _stop;
    bool _head_parse;
};

class LogicNode {
    friend class LogicSystem;

public:
    LogicNode(std::shared_ptr<Session>, std::shared_ptr<RecvNode>);

private:
    std::shared_ptr<Session> _session;
    std::shared_ptr<RecvNode> _recv_node;
};

#endif


// cpp
#include "../session/Session.h"
#include "../server/LogicSystem.h"
#include "../server/Server.h"
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>

#include "Session.h"
#include <spdlog/spdlog.h>

Session::Session(boost::asio::io_context& ioc, Server* server)
    : _socket(ioc)
    , _server(server)
    , _stop(false)
    , _head_parse(false)
    , _uid(0)
{
    _session_id = boost::uuids::to_string(boost::uuids::random_generator()());
    _recv_head_node = std::make_shared<MsgNode>(HEAD_TOTAL_LEN);
}

Session::~Session()
{
    Close();
}

void Session::Start()
{
    AsyncHead(HEAD_TOTAL_LEN);
}

void Session::Close()
{
    if (_stop) {
        return;
    }
    _stop = true;
    {
        std::lock_guard<std::mutex> lock(_send_lock);
        while (!_send_queue.empty()) {
            _send_queue.pop();
        }
    }

    // 安全关闭socket
    boost::system::error_code ec;
    _socket.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
    if (ec) {
        SPDLOG_WARN("Socket shutdown error: {}", ec.message());
    }

    _socket.close(ec);
    if (ec) {
        SPDLOG_WARN("Socket close error: {}", ec.message());
    }
    SPDLOG_INFO("Session {} disconnected!", _session_id);
}

void Session::Send(const char* msg, int max_length, uint16_t msg_id)
{
    std::lock_guard<std::mutex> lock(_send_lock);
    int size = _send_queue.size();
    if (size > MAX_SEND_SIZE) {
        SPDLOG_WARN("Session:{} Send Queue Fulled,This Failed", _session_id);
        return;
    }
    _send_queue.push(std::make_shared<SendNode>(msg, max_length, msg_id));
    if (size > 0) {
        return;
    }
    auto msgnode = _send_queue.front();
    boost::asio::async_write(_socket, boost::asio::buffer(msgnode->_data, msgnode->_total_len), std::bind(&Session::HandleWrite, this, std::placeholders::_1, shared_from_this()));
}

void Session::Send(std::string msg, uint16_t msg_id)
{
    Send(msg.data(), msg.size(), msg_id);
}

void Session::HandleWrite(boost::system::error_code ec, std::shared_ptr<Session> self)
{
    try {
        if (ec) {
            SPDLOG_WARN("Handle Write Filed,Errir is {}", ec.what());
            Close();
            _server->ClearSession(_session_id);
        }
        std::lock_guard<std::mutex> lock(_send_lock);
        _send_queue.pop();
        if (!_send_queue.empty()) {
            auto msgnode = _send_queue.front();
            boost::asio::async_write(_socket, boost::asio::buffer(msgnode->_data, msgnode->_total_len),
                std::bind(&Session::HandleWrite, this, std::placeholders::_1, shared_from_this()));
        }

    } catch (std::exception& e) {
        SPDLOG_WARN("Exception code:{}", e.what());
        Close();
        _server->ClearSession(_session_id);
    }
}

net::ip::tcp::socket& Session::GetSocket() noexcept
{
    return _socket;
}

std::queue<std::shared_ptr<SendNode>>& Session::GetSendQueue() noexcept
{
    return _send_queue;
}

void Session::AsyncHead(std::size_t len)
{
    boost::asio::async_read(_socket, boost::asio::buffer(_recv_head_node->_data, len), [self = shared_from_this(), this](boost::system::error_code error, size_t bytes_transferred) {
        if (error) {
            if (error == boost::asio::error::eof) {
                SPDLOG_WARN("Connection closed by perr");
            } else {
                SPDLOG_WARN("Error Reading header:{},bytes transferred:{}", error.message(), bytes_transferred);
            }
            _server->ClearSession(_session_id);
            Close();
            return;
        }
        if (bytes_transferred < HEAD_TOTAL_LEN) {
            SPDLOG_WARN("Read Length not matched");
            _server->ClearSession(_session_id);
            Close();
            return;
        }
        if (!_server->CheckValid(_session_id)) {
            SPDLOG_WARN("Invalid SessionId");
            _server->ClearSession(_session_id);
            Close();
            return;
        }

        uint16_t msg_id = 0;
        memcpy(&msg_id, _recv_head_node->_data, HEAD_ID_LEN);
        msg_id = boost::asio::detail::socket_ops::network_to_host_short(msg_id);
        uint16_t data_len = 0;
        memcpy(&data_len, _recv_head_node->_data + HEAD_ID_LEN, HEAD_DATA_LEN);
        data_len = boost::asio::detail::socket_ops::network_to_host_short(data_len);

        _recv_msg_node = std::make_shared<RecvNode>(data_len, msg_id);
        AsyncBody(data_len);
    });
}

void Session::AsyncBody(std::size_t len)
{
    boost::asio::async_read(_socket, boost::asio::buffer(_data, len), [self = shared_from_this(), this](boost::system::error_code error, size_t bytes_transferred) {
        if (error) {
            if (error == boost::asio::error::eof) {
                SPDLOG_WARN("Connection closed by perr");
            } else {
                SPDLOG_WARN("Error Reading Body:{},bytes transferred:{}", error.message(), bytes_transferred);
            }
            _server->ClearSession(_session_id);
            Close();
            return;
        }
        if (bytes_transferred < HEAD_TOTAL_LEN) {
            SPDLOG_WARN("Read Length not matched");
            _server->ClearSession(_session_id);
            Close();
            return;
        }
        if (!_server->CheckValid(_session_id)) {
            SPDLOG_WARN("Invalid SessionId");
            _server->ClearSession(_session_id);
            Close();
            return;
        }

        memcpy(_recv_msg_node->_data, _data, bytes_transferred);
        _recv_msg_node->_cur_len += bytes_transferred;

        auto logic_node = std::make_shared<LogicNode>(shared_from_this(), _recv_msg_node);
        LogicSystem::GetInstance()->PostMsgToQueue(logic_node);

        /* 继续接受head->body->head->body... */
        AsyncHead(HEAD_TOTAL_LEN);
    });
}

std::string Session::GetSessionId() noexcept
{
    return _session_id;
}

LogicNode::LogicNode(std::shared_ptr<Session> session, std::shared_ptr<RecvNode> recv_node)
    : _session(session)
    , _recv_node(recv_node)
{
}

关于读写，后面再说。先看这里的send函数。

_send_queue.push(std::make_shared<SendNode>(msg, max_length, msg_id));

我们将其放入了一个队列之中，为什么呢？因为多个数据，我们需要保证异步发送数据的有序性。比如hello world如果没有顺序保证，可能就会发送成为world hello.

仔细观察，在AsyncBody最后有一个步骤，我们将构造一个逻辑节点，发送给LogicSystem的队列。

LogicSystem::GetInstance()->PostMsgToQueue(logic_node);

我们为了解耦，这个Session负责接受原始数据，真正的逻辑处理交给逻辑类进行处理。逻辑类处理完之后，再次发送个这个Session的队列进行字节流的发送。

那么AsyncHead和AsyncBody操作呢，简单来说，为了防止粘包（读取部分或读取过多），我们使用async_read可以读取指定字节的数据。这样就可以配合一定的规则，解决粘包。我们的方法是，首先接受头部，一共4个字节，前两个字节是发送请求的id,后两个字节是接下来要接受的数据长度len.接下来再次调用async_read读取指定的len长度的字节就可以了。AsyncHead就是读取前四个字节的头部信息，AsyncBody就是读取数据内容的。

为了配合读取头部，和读取实际数据我们创建了MsgNode结构方便处理:

// h
#ifndef MSGNODE_H
#define MSGNODE_H

#include "../global/const.h"

#include <boost/asio.hpp>
#include <string>

class LogicSystem;
class MsgNode {
    friend class Session;
    friend class LogicSystem;

public:
    MsgNode(uint16_t max_len);
    virtual ~MsgNode();
    void Clear();

protected:
    uint16_t _cur_len;
    uint16_t _total_len;
    char* _data;
};

class RecvNode : public MsgNode {
    friend class Session;
    friend class LogicSystem;

public:
    RecvNode(uint16_t max_len, uint16_t msg_id);

private:
    uint16_t _msg_id;
};

class SendNode : public MsgNode {
    friend class Session;
    friend class LogicSystem;

public:
    SendNode(const char* msg, uint16_t max_len, uint16_t msg_id);

private:
    uint16_t _msg_id;
};

#endif

// cpp
#include "MsgNode.h"

MsgNode::MsgNode(uint16_t max_len)
    : _total_len(max_len)
    , _cur_len(0)
    , _data(new char[_total_len + 1])
{
    _data[_total_len] = '\0';
}

MsgNode::~MsgNode()
{
    delete[] _data;
}

void MsgNode::Clear()
{
    ::memset(_data, 0, _total_len);
    _cur_len = 0;
}

RecvNode::RecvNode(uint16_t max_len, uint16_t msg_id)
    : MsgNode(max_len)
    , _msg_id(msg_id)
{
}

SendNode::SendNode(const char* msg, uint16_t max_len, uint16_t msg_id)
    : MsgNode(max_len)
    , _msg_id(msg_id)
{
    uint16_t msg_id_host = boost::asio::detail::socket_ops::host_to_network_long(msg_id);
    memcpy(_data, &msg_id_host, HEAD_ID_LEN);

    uint16_t max_len_host = boost::asio::detail::socket_ops::host_to_network_long(max_len);
    memcpy(_data + HEAD_ID_LEN, &max_len_host, HEAD_DATA_LEN);
    memcpy(_data + HEAD_TOTAL_LEN, msg, max_len);
}

好的我们再来看LogicSystem类的实现

// h
#include "../global/Singleton.h"
#include "../global/const.h"
#include "../session/Session.h"
#include <condition_variable>
#include <functional>
#include <mutex>
#include <nlohmann/json.hpp>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>

typedef std::function<void(std::shared_ptr<Session>, uint16_t msg_id, const std::string& msg)> FuncBack;

class LogicSystem : public Singleton<LogicSystem> {
    friend class Singleton<LogicSystem>;

public:
    void PostMsgToQueue(std::shared_ptr<LogicNode> msg);
    void RegisterCallBacks();
    void DealMsg();

public:
    LogicSystem(std::size_t size = std::thread::hardware_concurrency());
    ~LogicSystem();

private:
    std::queue<std::shared_ptr<LogicNode>> _queue;
    std::mutex _mutex;
    std::condition_variable _cv;
    std::vector<std::thread> _work_threads;
    std::size_t _size;
    bool _stop;
    std::unordered_map<MsgId, FuncBack> _function_callbacks;
};

// cpp
#include "LogicSystem.h"
#include <spdlog/spdlog.h>
void LogicSystem::PostMsgToQueue(std::shared_ptr<LogicNode> msg)
{
    std::unique_lock<std::mutex> lock(_mutex);
    _queue.push(msg);
    _cv.notify_one();
}

void LogicSystem::RegisterCallBacks()
{
    // 登陆请求
    _function_callbacks[MsgId::ID_CHAT_LOGIN] = [this](std::shared_ptr<Session> session, uint16_t msg_id, const std::string& msg) {
        json j(msg);
        auto uid = j["uid"].get<int>();
        auto token = j["token"].get<std::string>();
        SPDLOG_INFO("User {} Login with token {}", uid, token);

        std::string return_str = j.dump(4);
        session->Send(return_str, msg_id);
    };
}

void LogicSystem::DealMsg()
{
    while (true) {
        std::unique_lock<std::mutex> lock(_mutex);
        _cv.wait(lock, [this]() {
            return _stop || !_queue.empty();
        });
        if (_stop && _queue.empty()) {
            break;
        }

        if (!_queue.empty()) {
            std::queue<std::shared_ptr<LogicNode>> local_queue;
            local_queue.swap(_queue);
            lock.unlock();
            while (!local_queue.empty()) {
                auto msg = local_queue.front();
                local_queue.pop();

                auto it = _function_callbacks.find(static_cast<MsgId>(msg->_recv_node->_msg_id));
                if (it != _function_callbacks.end()) {
                    it->second(msg->_session, msg->_recv_node->_msg_id, std::string(msg->_recv_node->_data, msg->_recv_node->_total_len));
                }
            }
        }
    }
}

LogicSystem::LogicSystem(std::size_t size)
    : _stop(false)
    , _size(size)
{
    RegisterCallBacks();
    _work_threads.reserve(size);
    for (std::size_t i = 0; i < size; ++i) {
        _work_threads.emplace_back(&LogicSystem::DealMsg, this);
    }
}

LogicSystem::~LogicSystem()
{
    _stop = true;
    _cv.notify_all();
    for (auto& p : _work_threads) {
        p.join();
    }

    SPDLOG_INFO("LogicSystem Stopped");
}

我们在逻辑类中创建了多个线程，每个线程都在等待获取队列中的锁，得到锁之后，去除一个任务，然后释放锁，前去处理任务。我们创建了一个存放处理类型和对应回调函数的容器，线程只需要根据这个类型id,就能找到回调进行处理。

处理完成之后，还要Send回原来的Session，进行字节流的发送。