celery

pip3 install celery

# 默认只支持linux，若想win跑需装eventlet

pip3 install eventlet

处理任务比较多/处理任务耗时，可以先接受请求，交给broker去处理，并立即返回一个凭证。以后拿着凭证去获取任务的执行结果。

Celery is a simple, flexible, and reliable distributed system to process vast amounts of messages,
while providing operations with the tools required to maintain such a system.

celery 是一个基于python开发的简单、灵活、和可靠的消息分发系统，可以处理大量的消息

其中broker  /backend （依赖rabbitmq/redis）。

Celery version 4.0 runs on
        Python ❨2.7, 3.4, 3.5❩
        PyPy ❨5.4, 5.5❩
    This is the last version to support Python 2.7, and from the next version (Celery 5.x) Python 3.5 or newer is required.

    If you’re running an older version of Python, you need to be running an older version of Celery:

        Python 2.6: Celery series 3.1 or earlier.
        Python 2.5: Celery series 3.0 or earlier.
        Python 2.4 was Celery series 2.2 or earlier.

    Celery is a project with minimal funding, so we don’t support Microsoft Windows. Please don’t open any issues related to that platform.

 

 

先创建worker:
celery worker -A  s1  -l info  -P eventlet
-A 指定队列名
-l 显示信息
-P win专用

# s1.py
import time
from celery import Celery
app = Celey("tasks",borker="redis://:beta@140.143.227.206:8888/0",
　　　　　　　　　　　　　　　　backend="redis://:beta@140.143.227.206:8888/0")

# 注册app可以处理的任务,可以很多
@app.task
def x1(x,y):
    time.sleep(6)
    return x+y

celery worker -A  s1  -l info  -P eventlet

from s1 import x1

# 立即告知celery去执行x1任务，并传入两个参数，下边有定时任务
result = x1.delay(4,4)
# result就是拿到的凭证

 

from celery.result import AsyncResult
from s1 import app

async = AsyncResult(id="574c582c-ce2b-45b9-8b9d-4fe1df252971", app=app)

if async.successful():
    result = async.get()
    print(result)
    # result.forget() # 将结果删除
elif async.failed():
    print('执行失败')
elif async.status == 'PENDING':
    print('任务等待中被执行')
elif async.status == 'RETRY':
    print('任务异常后正在重试')
elif async.status == 'STARTED':
    print('任务已经开始被执行')

 

 

多任务结构：

pro_cel
    ├── celery_tasks　　 # celery相关文件夹
    │   ├── celery.py   # celery连接和配置相关文件，名称不允许改变
    │   └── tasks.py    # 所有任务函数
    ├── check_result.py # 检查结果
    └── send_task.py    # 触发任务

3# pro_cel/celery_tasks/celery.py
from celery import Celery

celery = Celery('xxxxxx',
                broker='redis://192.168.0.111:6379',
                backend='redis://192.168.0.111:6379',
                include=['celery_tasks.tasks'，]) 进行任务的分发

 

4#pro_cel/celery_tasks/tasks.py
import time 
from ./celery import celery

@celery .task
def xxxxx(*args,*kwargs):
    time.sleep(5)
    return "任务结果"

@celery .task
def hhhhh(*args,*kwargs):
    time.sleep(5)
    return "任务结果"

 

5#pro_cel/check_result.py

from celery.result import AsyncResult
from celery_tasks.celery import celery

async = AsyncResult(id="ed88fa52-11ea-4873-b883-b6e0f00f3ef3", app=celery)

if async.successful():
    result = async.get()
    print(result)
    # result.forget() # 将结果删除
elif async.failed():
    print('执行失败')
elif async.status == 'PENDING':
    print('任务等待中被执行')
elif async.status == 'RETRY':
    print('任务异常后正在重试')
elif async.status == 'STARTED':
    print('任务已经开始被执行')

 

 

定时任务

- 创建定时器   
    celery  beat -A  celery_tasks
- 创建worker
    celery worker -A celery_tasks -l info -P eventlet 

 

定时执行任务

执行一次apply_async　　# 默认使用utc时间，

import datetime
from clery_tasks.tasks import xxxxx

ctime = datetime.datetime.now()
utc_ctime = datetime.datetime.utcfromtimestamp(ctime.timestamp())
 
s10 = datetime.timedelta(seconds=10)
ctime_x = utc_ctime + s10
 
# 使用apply_async并设定时间
result = xxxxx.apply_async(args=[1, 3], eta=ctime_x)
print(result.id)

 

 

类似于 linux 中 contab  的定时任务

from celery import Celery
from celery.schedules import crontab

cel = Celery('tasks',
             broker='redis://:beta@140.143.227.206:8888/0',
             backend='redis://:beta@140.143.227.206:8888/0',
             include=['celery_tasks.tasks', 'celery_tasks.xxx']
             )

cel.conf.beat_schedule = {
    'add-every-10-seconds': {
        'task': 'celery_tasks.tasks.x2',
        'args': (98, 10),
        'schedule': 10.0,　　　　#  每10秒
    },
    'add-every-12-seconds': {
        'task': 'celery_tasks.tasks.x2',
        'schedule': crontab(minute=42, hour=8, day_of_month=11, month_of_year=4),  # 每年的4月11号8点42分
        'args': (26, 16)
    },
}

 

 

 

 

 禁用UTC

# 时区
celery.conf.timezone = 'Asia/Shanghai'
# 是否使用UTC
celery.conf.enable_utc = False

1