一个简单的传输文件服务程序

Server

fileserver.py

from socketserver import ThreadingTCPServer
from socketserver import BaseRequestHandler
import log
import json
import optparse
import os
import struct
import time


#消息格式
# 长度len: 4字节, 表示数据的长度, 不包含消息头
# 消息类型msg: 4字节, 如: 0x1000 表示退出程序
# 消息子类型itemType: 4字节
# 数据: n 字节, 不定长
FMT = "<III{}s"

MSG_HEAD_SIZE = 12

MSG_EXIT = 0x1000
MSG_FILE = 0x1001
MSG_DIR = 0x1002

ITEM_FILE_BEG = 0x01
ITEM_FILE_MID = 0x02
ITEM_FILE_END = 0x03

ITEM_DIR_MKDIR = 0x01


g_cfg = {}

class FileStreamHandler(BaseRequestHandler):
    def __init__(self, request, client_address, server):
        log.d("handler created")
        self.buffer = b""
        self.filename = ""
        self.myfile = None
        self.myfilesize = 0
        self.start = time.perf_counter()
        super().__init__(request, client_address, server)

    def setup(self):
        log.d("connected, addr:%s", self.client_address)

    def finish(self):
        log.d("disconnected, addr:%s", self.client_address)
        self._delLastIncompleteFile()
        self.request.close()

    def handle(self):
        while True:
            byte_data = self.request.recv(g_cfg["rbufsize"])
            # if byte_data is None:
            #     assert False, "unexpect"
            if byte_data != b"":
                # log.d("recv datalen:%s", len(byte_data))
                self._parseData(byte_data)
            else:
                break

    def _parseData(self, data):
        self.buffer += data
        while True:
            # log.d(self.buffer)
            if len(self.buffer) < MSG_HEAD_SIZE:
                return

            i_data_len, *_ = struct.unpack("<I", self.buffer[:4])
            # log.d("datalen:", i_data_len)
            if len(self.buffer) < i_data_len + MSG_HEAD_SIZE:
                # log.d("len({}) too short".format(i_data_len))
                return

            str_fmt = FMT.format(i_data_len)
            frame_data = self.buffer[:i_data_len + MSG_HEAD_SIZE]
            self.buffer = self.buffer[i_data_len + MSG_HEAD_SIZE:]
            i_data_len, i_msg, i_item_type, byte_data, *_ = struct.unpack(str_fmt, frame_data)
            self._dispatchData(i_msg, i_item_type, byte_data)

    def _dispatchData(self, msg, itemtype, bytedata):
        # log.d("msg:{}, itemtype:{}, data:{}".format(msg, itemtype, bytedata))
        if msg == MSG_EXIT:
            self.server.shutdown()
        elif msg == MSG_FILE:
            self._handleFileMsg(msg, itemtype, bytedata)
        elif msg == MSG_DIR:
            self._handlDirMsg(msg, itemtype, bytedata)
        else:
            log.d("unknown msg:%d, itemtype:%d", msg, itemtype)

    def _handleFileMsg(self, msg, itemtype, bytedata: bytes):
        if itemtype == ITEM_FILE_MID:
            if self.myfile:
                self.myfilesize += len(bytedata)
                self.myfile.write(bytedata)
                self.myfile.flush()
            else:
                pass  # log.d("file not opened")
        elif itemtype == ITEM_FILE_BEG:
            self._delLastIncompleteFile()
            self.start = time.perf_counter()
            self.myfilesize = 0
            self.filename = os.path.join(g_cfg["path"], bytedata.decode("utf-8"))
            self.myfile = open(self.filename, mode="wb")
            if not self.myfile:
                log.d("open file<{}> failed".format(self.filename.replace("/", "\\")))
            else:
                log.d("open file<{}> success".format(self.filename.replace("/", "\\")))
        elif itemtype == ITEM_FILE_END:
            self._closeFile()
            elapsed = time.perf_counter() - self.start
            if elapsed <= 0:
                elapsed = 1
            speed = self.myfilesize / elapsed # Byte/s
            if speed > 1024 * 1024:
                str_speed = "{:0.2f}MB/s".format(speed / 1024 / 1024)
            elif speed > 1024:
                str_speed = "{:0.2f}KB/s".format(speed / 1024)
            else:
                str_speed = "{:0.2f}B/s".format(speed)
            log.d("recv {} finished, used {:0.3}s, speed:{}".format(self.filename.replace("/", "\\"), elapsed, str_speed))
            self.filename = ""
        else:
            log.d("unknown file msg, itemtype:%d", itemtype)

    def _handlDirMsg(self, msg, itemtype, bytedata: bytes):
        if itemtype == ITEM_DIR_MKDIR:
            dirname = bytedata.decode("utf-8")
            dirname = os.path.join(g_cfg["path"], dirname)
            if not os.path.exists(dirname):
                os.mkdir(dirname)
        else:
            log.d("unknown dir msg, itemtype:%d", itemtype)

    def _closeFile(self):
        if self.myfile:
            self.myfile.close()
            self.myfile = None

    def _delLastIncompleteFile(self):
        if self.myfile:
            self._closeFile()
            if os.path.exists(self.filename):
                log.d("rm file<%s>", self.filename)
                os.remove(self.filename)


def main():
    parser = optparse.OptionParser()
    parser.add_option("-p", "--port", action="store", type="int", dest="port", default=None)
    options, args = parser.parse_args()

    with open("server_envconfig.json", mode="r", encoding="utf-8") as fp:
        g_cfg.update(json.load(fp))
        g_cfg.setdefault("port", 22)
        g_cfg.setdefault("path", "")
        g_cfg.setdefault("rbufsize", 10485760)

    if options.port:
        g_cfg["port"] = options.port

    log.d("str_path:%s", g_cfg["path"])
    log.d("port:%s", g_cfg["port"])

    if not os.path.exists(g_cfg["path"]):
        os.mkdir(g_cfg["path"])

    server = ThreadingTCPServer(("", g_cfg["port"]), FileStreamHandler)
    server.serve_forever()


if __name__ == '__main__':
    main()

 

server_envconfig.json

{
    "path": "d:/recvfiles",
    "rbufsize": 10485760,
    "port": 5500
}

 

Client 

fileclient.py

import log
import json
import optparse
import os
import socket
import struct


#消息格式
# 长度len: 4字节, 表示数据的长度, 不包含消息头
# 消息类型msg: 4字节, 如: 0x1000 表示退出程序
# 消息子类型itemType: 4字节
# 数据: n 字节, 不定长
FMT = "<III{}s"

MSG_HEAD_SIZE = 12

MSG_EXIT = 0x1000
MSG_FILE = 0x1001
MSG_DIR = 0x1002

ITEM_FILE_BEG = 0x01
ITEM_FILE_MID = 0x02
ITEM_FILE_END = 0x03

ITEM_DIR_MKDIR = 0x01

g_cfg = {}

def sendTo(s, msg, itemtype, bytedata = b""):
    i_data_len = len(bytedata)
    str_fmt = FMT.format(i_data_len)
    msg = struct.pack(str_fmt, i_data_len, msg, itemtype, bytedata)
    s.sendall(msg)


def sendFile(s, filename, absfilename = ""):
    if not absfilename:
        absfilename = filename

    try:
        with open(absfilename, mode="rb") as fp:
            log.d("send {} begin".format(filename))
            byte_filename = filename.encode(encoding="utf-8")
            sendTo(s, MSG_FILE, ITEM_FILE_BEG, byte_filename)
            while True:
                data = fp.read(g_cfg["wbufsize"])
                if data:
                    sendTo(s, MSG_FILE, ITEM_FILE_MID, data)
                    # log.d("send %d bytes", len(data))
                else:
                    break
            sendTo(s, MSG_FILE, ITEM_FILE_END)
            log.d("send {} finished".format(filename))
    except FileNotFoundError as e:
        log.d("open file<%s> failed", filename)


def sendMkdirMsg(s, dirname):
    byte_dirname = dirname.encode(encoding="utf-8")
    sendTo(s, MSG_DIR, ITEM_DIR_MKDIR, byte_dirname)


def sendDir(s, dirname):
    str_root_path = os.path.abspath(dirname)

    str_root_path = str_root_path.replace("\\", "/")
    if not str_root_path.endswith("/"):
        str_root_path += "/"

    for root, dirs, files in os.walk(str_root_path):
        root = root.replace("\\", "/")

        rel_path = root[len(str_root_path):]
        if rel_path:
            sendMkdirMsg(s, rel_path)

        for file in files:
            filename = os.path.join(rel_path, file).replace("\\", "/")
            absfilename = os.path.join(root, file).replace("\\", "/")
            sendFile(s, filename, absfilename)


def sendExitMsg(s):
    sendTo(s, MSG_EXIT, 0)


def startTransfer(addr, options):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect(addr)

    if options.filename:
        sendFile(s, options.filename)
    if options.dirname:
        sendDir(s, options.dirname)

    s.close()
    log.d("finished")


def main():
    parser = optparse.OptionParser()
    parser.add_option("-f", "--file", action="store", type="string", dest="filename")
    parser.add_option("-d", "--dir", action="store", type="string", dest="dirname")
    parser.add_option("-o", "--host", action="store", type="string", dest="host")
    parser.add_option("-p", "--port", action="store", type="int", dest="port")
    options, args = parser.parse_args()

    if not options.filename and not options.dirname:
        parser.print_help()
        exit(0)

    with open("client_envconfig.json", mode="r", encoding="utf-8") as fp:
        g_cfg.update(json.load(fp))

    if options.host:
        g_cfg["host"] = options.host
    if options.port:
        g_cfg["port"] = options.port

    startTransfer((g_cfg["host"], g_cfg["port"]), options)


if __name__ == '__main__':
    main()
 

client_envconfig.json

{
    "host": "127.0.0.1",
    "wbufsize": 5242880,
    "port": 5500
}

 

log.py

import logging
import sys

str_def_fmt = "%(asctime)s.%(msecs)03d %(thread)d %(levelname)s " \
              "%(module)s %(filename)s:%(lineno)d %(funcName)s: %(message)s"
logging.basicConfig(level=logging.DEBUG
                    , format=str_def_fmt
                    , datefmt="%Y:%m:%d %H:%M:%S"
                    , stream=sys.stdout)
logger = logging.getLogger("log")


def setAsDebugLevel():
    logger.setLevel(logging.DEBUG)

def setAsInfoLevel():
    logger.setLevel(logging.INFO)

def setAsWarnLevel():
    logger.setLevel(logging.WARN)

def setAsErrorLevel():
    logger.setLevel(logging.ERROR)

def setAsFatalLevel():
    logger.setLevel(logging.FATAL)

d = logger.debug
i = logger.info
w = logger.warning
e = logger.error
f = logger.fatal


def main():
    d("test debug msg")
    i("test info msg")
    w("test warn msg")
    e("test err msg")
    f("test fatal msg")


if __name__ == "__main__":
    main()