aiomultiprocess+ aiohttp + pyinvoke + deploykit 的一个简单集成

实际上就是一个简单的集成，测试下是否方便

参考玩法

通过http 请求，发起基于aiomultiprocess pool 的pyinvoke task 调用，task 里边是基于deploykit 的ssh 任务出来

• app.py

from invoke.executor import Executor
from invoke import Collection
import importlib
from aiohttp import web
import asyncio
import weakref
from aiomultiprocess import Worker, Pool

app = web.Application()

# 此处使用了动态加载模块的玩法，同时通过编程动态调用pyinvoke 
async def do_task(module_name, task_name):
    module = importlib.import_module(module_name)
    ns = Collection.from_module(module)
    importlib.reload(module)
    executor = Executor(ns)
    weakref.finalize(executor, lambda: print(f"Executor for {module_name} finalized"))
    weakref.finalize(ns, lambda: print(f"Collection for {module_name} finalized"))
    weakref.finalize(module, lambda: print(f"module for {module_name} finalized"))
    result = executor.execute(task_name)
    return  [{'name':key.name,'value':value} for key,value in result.items()]

async def run_task(module_name, task_name):
    result = await Worker(
        target=do_task,
        args=(module_name, task_name),
        kwargs={}
    )
    return result


async def run_pool_task(pool: Pool, module_name, task_name):
    results = await pool.apply(do_task, (module_name, task_name))
    return results

async def handle(request):
    name  = request.query.get('name', "demo")
    task = asyncio.create_task(run_pool_task(app['pool'], name, "taskb"))
    task.add_done_callback(lambda f: print("result:", f.result()))   
    return web.json_response({'result': "Task submitted"})

app = web.Application()
app.add_routes([web.get('/', handle),
                web.get('/{name}', handle)])

async def init_pool(app):
    app['pool'] = Pool(processes=5,maxtasksperchild=2)
    await app['pool'].__aenter__()

async def cleanup_pool(app):
    await app['pool'].__aexit__(None, None, None)

app.on_startup.append(init_pool)
app.on_cleanup.append(cleanup_pool)

if __name__ == '__main__':
    web.run_app(app)

• demo.py

任务定义

from invoke import task, call
from deploykit import parse_hosts
import subprocess
@task
def build(c):
    print("Building!")
    
@task
def taska(c,name,age):
    results = []
    hosts = parse_hosts("root@me")
    runs = hosts.run("uptime", stdout=subprocess.PIPE)
    for r in runs:
        results.append({'host': r.host.host, 'uptime': r.result.stdout,'version': "v2"})
    return results

@task(pre=[call(taska,"taska",age=30)])
def taskb(c):
    print("Executing Task B")
    return "Executing Task B----"

说明

示例代码比较简单，核心就是几个东西的集成，方便学习的不同框架的特性

参考资料

https://github.com/numtide/deploykit/tree/main

https://docs.pyinvoke.org/en/stable/index.html

https://docs.aiohttp.org/en/stable/

https://aiomultiprocess.omnilib.dev/en/stable/