FTP 文件传输服务
昨晚心血来潮，尝试用python写了一个ftp文件传输服务，可以接收指令，从远程ftp服务器同步指定目录数据，最后没用上，开源出来。
https://github.com/jadepeng/ftp_transfer_service.git
运行原理
- 'task_server' 是一个web服务器，可以接收传入任务，接收到任务后，将task写入mysql
- 启动任务后，'task_server'会扫描ftp文件列表，写入redis队列
transfer_client是传输执行程序，可以多点部署，该程序会读取redis队列，进行文件下载
使用
配置
修改 .env 文件, 配置mysql和redis地址
REDIS_SERVER=""
REDIS_PORT=6380
REDIS_PASSWORD=""
MYSQL_HOST=""
MYSQL_PORT=3306
MYSQL_PASSWORD=""
MYSQL_USER=""
MYSQL_DB=""
启动服务
server 端
python3 task_server.py
传输端，可以部署多个
python3 transfer_client.py
接收任务
POST /task/
{
"taskId": "9",
"serverPath": "/weblog",
"storagePath": "/data",
"host": "ftpServer",
"port": 21,
"user": "user",
"password": "password"
}
启动传输
GET /task/{taskId}/start
查看进度
GET /task/{taskId}/progress
实现简介
第一次用fastapi来写web服务，这里记录下有意思的地方。
配置
可以通过配置类实现app的配置参数，pydantic还可以加载env文件更新配置
setting.py
from pydantic import BaseSettings
class APISettings(BaseSettings):
mysql_host: str = "127.0.0.1"
mysql_port: int = 3306
mysql_password: str
mysql_user: str
mysql_db: str
redis_server: str = "127.0.0.1"
redis_port: int = 6380
redis_password: str
max_wait_time_count: int = 10
class Config:
env_file = ".env"
env_file_encoding = 'utf-8'
redis 队列
通过list实现队列，rpush，blpop
import redis
class RedisQueue(object):
def __init__(self, name, namespace='queue', **redis_kwargs):
self.__db= redis.Redis(**redis_kwargs)
self.key = '%s:%s' %(namespace, name)
def qsize(self):
return self.__db.llen(self.key) # 返回队列里面list内元素的数量
def put(self, item):
self.__db.rpush(self.key, item) # 添加新元素到队列最右方
def get_wait(self, timeout=None):
item = self.__db.blpop(self.key, timeout=timeout)
return item
def get_nowait(self):
item = self.__db.lpop(self.key)
return item
redis BloomFilter
BloomFilter 可以用来去重
import mmh3
import redis
class BloomFilter(object):
def __init__(self, bf_key, bit_size=2000000, hash_count=4, start_seed=41, **redis_kwargs):
self.bit_size = bit_size
self.hash_count = hash_count
self.start_seed = start_seed
self.client = redis.Redis(**redis_kwargs)
self.bf_key = bf_key
def add(self, data):
bit_points = self._get_hash_points(data)
for index in bit_points:
self.client.setbit(self.bf_key, index, 1)
def madd(self, m_data):
if isinstance(m_data, list):
for data in m_data:
self.add(data)
else:
self.add(m_data)
def exists(self, data):
bit_points = self._get_hash_points(data)
result = [
self.client.getbit(self.bf_key, index) for index in bit_points
]
return all(result)
def mexists(self, m_data):
result = {}
if isinstance(m_data, list):
for data in m_data:
result[data] = self.exists(data)
else:
result[m_data] = self.exists[m_data]
return result
def _get_hash_points(self, data):
return [
mmh3.hash(data, index) % self.bit_size
for index in range(self.start_seed, self.start_seed +
self.hash_count)
]
python的orm框架sqlalchemy
sqlalchemy 需要先定义ORM类
class TransferTask(Base):
__tablename__ = 'transfer_task'
taskId = Column(String(255), primary_key=True, index=True)
serverPath = Column(String(255), nullable=False)
storagePath = Column(String(255), nullable=False)
host = Column(String(255), nullable=False)
port = Column(Integer, nullable=False)
user = Column(String(255), nullable=False)
password = Column(String(255), nullable=False)
time = Column(DateTime, nullable=False, default=datetime.now)
class TransferFailedFile(Base):
__tablename__ = 'transfer_failed_file'
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
taskId = Column(String(255), index=True)
filePath = Column(String(1024), nullable=False)
time = Column(DateTime, nullable=False, default=datetime.now)
class TransferProgress(Base):
__tablename__ = 'transfer_task_progress'
taskId = Column(String(255), primary_key=True, index=True)
total = Column(Integer, nullable=False)
status = Column(Integer, nullable=False)
finished = Column(Integer, nullable=False)
failed = Column(Integer, nullable=False)
time = Column(DateTime, nullable=False, default=datetime.now)
if __name__ == '__main__':
settings = APISettings()
db = Database(settings.mysql_host, settings.mysql_port, settings.mysql_user, settings.mysql_password,
settings.mysql_db)
Base.metadata.create_all(db.engine)
使用了sqlalchemy CRUD就比较方便了, 可以通过query，filter来查询和过滤
def get_or_create_progress(self, task: TransferTask):
db = self.database.get_session()
dbitem = db.query(TransferProgress).filter(TransferProgress.taskId == task.taskId).first()
if not dbitem:
dbitem = TransferProgress()
dbitem.taskId = task.taskId
dbitem.total = 0
dbitem.status = TaskStatus.SCANNING.value
dbitem.finished = 0
dbitem.failed = 0
db.add(dbitem)
db.commit()
return dbitem
这里需要注意的是，session需要close，不然session过多会报错，可以封装一个get_session，利用yield来自动释放
见database.py
def get_db(self):
db = self.SessionLocal()
try:
yield db
finally:
db.close()
def get_session(self):
return next(self.get_db())
python ftp操作
python有个ftplib，可以用来操作ftp，这里简单封装一个client类, 实现listfiles和下载文件
import ftplib
import os
from datetime import datetime
import ntpath
class FtpClient:
def __init__(self, host: str, port: int, user: str, password: str):
self.host = host
self.port = port
self.user = user
self.password = password
self.connect()
def connect(self):
self.ftp = ftplib.FTP()
self.ftp.connect(host=self.host, port=self.port)
self.ftp.login(self.user, self.password)
self.ftp.encoding = "utf-8"
def list_files(self, dir):
self.ftp.cwd(dir)
for file_data in self.ftp.mlsd():
file_name, meta = file_data
file_type = meta.get("type")
if file_type == "file":
try:
self.ftp.voidcmd("TYPE I")
file_size = self.ftp.size(file_name)
yield f"{dir}/{file_name}", file_size
except Exception as e:
print(e)
else:
yield from self.list_files(dir + "/" + file_name)
def download_file(self, file_name:str, local_file_name:str):
try:
self.ftp.retrbinary('RETR %s' % file_name, open(local_file_name, 'wb').write)
except ftplib.error_perm:
print('ERROR: cannot read file "%s"' % file_name)
os.unlink(local_file_name)
下载程序
作为redis mq的消费者，要考虑的是下载失败了如何处理，异常退出如何处理？进度如何更新？
针对异常退出，这里用一个简单的方案，获取mq消息后，先将item写入到本地文件，这样如果client程序异常退出，下次进来还能继续
针对下载失败，这里失败后先重新放入队列，retryCount+1，如果超过最大重试次数，则写到错误记录。
进度更新，则依靠update+1执行。
def transfer_task_item(ftp, local_path, queque, task, task_item):
try:
local_file_name = local_path + task_item['fileName']
print("transfer %s to %s" % (task_item['fileName'], local_file_name))
# 文件已存在
if os.path.exists(local_file_name):
# 比较大小
size = os.path.getsize(local_file_name)
if size == task_item["fileSize"]:
db_service.update_finished(task.taskId, 1)
return
dir = os.path.abspath(os.path.dirname(local_file_name))
os.makedirs(dir, exist_ok=True)
ftp.download_file(task_item['fileName'], local_file_name)
# 更新进度
db_service.update_finished(task.taskId, 1)
except Exception as e:
print(e)
if task_item['retryCount'] < 3:
task_item['retryCount'] = task_item['retryCount'] + 1
queque.put(json.dumps(task_item))
else:
print(task_item['fileName'] + " transfer failed with max_retry_count")
db_service.add_failed_file(task.taskId, task_item['fileName'])
finally:
remove_lock()
