异步任务 | Butterfly 开发手册

async_job(异步任务)

场景

agent 接到请求需要保序执行一些任务，比如实例的部署有以下流程

none
allocated【创建容器】发往容器管理服务进行创建容器，容器启动后会部署 runtime 包，启动 agent
deployed【拉取 program】执行 agent bin 目录的 unit-deploy，进行拉取 program 包(redis/twemproxy/..)
installed【执行 hook】
ready
running

实例部署例子

代码

基于 concurrent.futures

handler 代码

传送门

https://python-parallel-programmning-cookbook.readthedocs.io/zh_CN/latest/chapter4/02_Using_the_concurrent.futures_Python_modules.html

Last updated 3 years ago

场景

agent 接到请求需要保序执行一些任务，比如实例的部署有以下流程

none
allocated【创建容器】发往容器管理服务进行创建容器，容器启动后会部署 runtime 包，启动 agent
deployed【拉取 program】执行 agent bin 目录的 unit-deploy，进行拉取 program 包(redis/twemproxy/..)
installed【执行 hook】
ready
running

实例部署例子

代码

基于 concurrent.futures

handler 代码

#!/usr/bin/python
# coding=utf8
"""
# Author: wangbin34(meetbill)
# Created Time : 2021-01-21 16:32:27

# File Name: __init__.py
# Description:

    async_job --发起异步任务，由单独线程进行处理

    1.0.1: async_job 1.0.1 版本，用于简易创建异步任务，异步任务进行执行本地 Shell/Python 脚本
"""

import os
import logging

from xlib.httpgateway import Request
from xlib import retstat
from xlib.middleware import funcattr
from xlib.util import concurrent
from xlib.util import shell_util


__info = "async_job"
__version = "1.0.1"

executor = concurrent.ThreadPoolExecutor(max_workers=10)


def run_cmd(reqid, cmd):
    """
    执行本地命令
    Args:
        job_id  : (Str) job_id
        job_name: (Str) job_name
        cmd     : (Str) "python/bash file_path args"
        errlog  : (object) errlog logger
    """
    # 设置 1 小时超时
    cmd_result = shell_util.run(cmd, timeout=3600, reqid=reqid)


def _check_cmd(cmd):
    """
    检查 cmd 命令合法性, 仅支持执行 Python/Shell 脚本
    Args:
        cmd: (Str)
            example: "python/bash file_path args"
    Returns:
        bool
    """
    if ";" in cmd or "&&" in cmd or "||" in cmd:
        return False

    cmd_list = cmd.split()
    if len(cmd_list) < 2:
        return False

    file_type = cmd_list[0]
    file_path = cmd_list[1]
    if file_type not in ["python", "bash"]:
        return False

    if not os.path.exists(file_path):
        return False

    return True


def executor_callback(worker):
    """
    记录 worker 执行异常
    """
    logging.info("called worker callback function")
    worker_exception = worker.exception()
    if worker_exception:
        logging.exception("Worker return exception: {}".format(worker_exception))


@funcattr.api
def add_job(req, cmd):
    """
    添加任务

    cmd: (str) bash xxx.sh/python xxx.py  本地需要有此脚本
    """
    isinstance(req, Request)

    if not _check_cmd(cmd):
        return retstat.ERR, {}, [(__info, __version)]

    task = executor.submit(run_cmd, reqid=req.reqid, cmd=cmd)
    task.add_done_callback(executor_callback)

    return retstat.OK, {}, [(__info, __version)]

------------------------------------------------------------------------------------------- client NOTICE: 2021-01-29 11:32:10.3003238 +0800 CST: [logid=3285043710 command=service-add-unit method=add_unit ret={"error_code":0,"error_msg":"add unit(s) success","data":[{"unit_id":143735}],"job_id":108447335}] ================================================================================================== -------------------------------------------------------------------------------------------- agent start [Info] [2021-01-29 11:32:37.659864719 +0800 CST] [@ main.go:49]init_env done, ready to serv [Debug] [2021-01-29 11:32:37.660011768 +0800 CST] [@ main.go:50]start -------------------------------------------------------------------------------------------- deploy(log_id:3290504241) ===========(1) got one request [Info] [2021-01-29 11:33:02.684401481 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619872] log_id:[3290504241]] {"type":"yiqiu","cmd":"unit-deploy","env":{"YIQIU_UNIT_DEPLOY_SOURCE":"ftp:\/\/{CHUNFENG_SERVER_ADDR}\/\/home\/work\/yiqiu\/data\/deploy\/25039622\/archer.tar.gz","YIQIU_UNIT_DEPLOY_PATH":"\/home\/work","YIQIU_UNIT_DEPLOY_FILE":"\/home\/work\/nfs_client\/mnt\/online\/deploy\/\/package\/25039622\/archer.tar.gz","YIQIU_UNIT_DEPLOY_SIGN":"twemproxy_2.0.0.22","YIQIU_UNIT_ID":143735,"YIQIU_UNIT_HOME":"\/home\/work","YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":null,"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":{"service":"deploy","method":"callback_deploy","on_succ":{"job_id":25039622,"unit_id":"143735","op_type":"install"},"on_fail":{"job_id":25039622,"unit_id":"143735","op_type":"install"}},"task_id":180619872}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:02.6848853 +0800 CST] [@ request.go:60]handle request: [3290504241 unit-deploy yiqiu map[YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_UNIT_DEPLOY_SOURCE:ftp://{CHUNFENG_SERVER_ADDR}//home/work/yiqiu/data/deploy/25039622/archer.tar.gz YIQIU_UNIT_DEPLOY_PATH:/home/work YIQIU_UNIT_DEPLOY_FILE:/home/work/nfs_client/mnt/online/deploy//package/25039622/archer.tar.gz YIQIU_UNIT_DEPLOY_SIGN:twemproxy_2.0.0.22 YIQIU_UNIT_ID:143735 YIQIU_UNIT_HOME:/home/work YIQIU_PROGRAM_HOME:/home/work/program] <nil> 2021-01-29 11:33:02.684733124 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:09.913963198 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619872 unit-deploy]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:09.928235571 +0800 CST] [@ request.go:231]callback request=[map[token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:map[service:deploy method:callback_deploy on_succ:map[op_type:install job_id:2.5039622e+07 unit_id:143735] on_fail:map[job_id:2.5039622e+07 unit_id:143735 op_type:install]] task_id:1.80619872e+08 exit_code:0 product_name:yiqiu]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:09.92849919 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:09.944892037 +0800 CST] [@ request.go:236]callback respond=[map[data:[] error_code:0 error_msg:ok]] -------------------------------------------------------------------------------------------- install(env-install) ===========(1) got one request [Info] [2021-01-29 11:33:10.209887052 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619874] log_id:[3290504241]] {"type":"hook","cmd":"env-install","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619874}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:10.210165897 +0800 CST] [@ request.go:60]handle request: [3290504241 env-install hook map[YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_PROGRAM_HOME:/home/work/program] [] 2021-01-29 11:33:10.210096291 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:15.988614877 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619874 env-install]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:16.002122191 +0800 CST] [@ request.go:231]callback request=[map[reaction:[] task_id:1.80619874e+08 exit_code:0 product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:16.002323468 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:16.023512325 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(install) ===========(1) got one request [Info] [2021-01-29 11:33:10.218333335 +0800 CST] [@ server.go:202]got one request [query, content]=[map[log_id:[3290504241] req_id:[108447335_180619875]] {"type":"hook","cmd":"install","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619875}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:16.023563767 +0800 CST] [@ request.go:60]handle request: [3290504241 install hook map[YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_PROGRAM_HOME:/home/work/program] [] 2021-01-29 11:33:10.218539592 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:17.689937634 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619875 install]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:17.703886628 +0800 CST] [@ request.go:231]callback request=[map[product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:[] task_id:1.80619875e+08 exit_code:0]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:17.70418994 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:17.712410256 +0800 CST] [@ request.go:236]callback respond=[map[error_msg:ok data:[] error_code:0]] -------------------------------------------------------------------------------------------- install(config-changed) ===========(1) got one request [Info] [2021-01-29 11:33:10.226758702 +0800 CST] [@ server.go:202]got one request [query, content]=[map[log_id:[3290504241] req_id:[108447335_180619876]] {"type":"hook","cmd":"config-changed","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619876}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:17.712470188 +0800 CST] [@ request.go:60]handle request: [3290504241 config-changed hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:10.226974664 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:17.950438015 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619876 config-changed]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:17.964158373 +0800 CST] [@ request.go:231]callback request=[map[task_id:1.80619876e+08 exit_code:0 product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:[]]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:17.964365734 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:17.992496533 +0800 CST] [@ request.go:236]callback respond=[map[data:[] error_code:0 error_msg:ok]] -------------------------------------------------------------------------------------------- install(start) ===========(1) got one request [Info] [2021-01-29 11:33:10.234793019 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619877] log_id:[3290504241]] {"type":"hook","cmd":"start","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":{"service":"service","method":"unit_stat_transform","on_succ":{"unit_id":"143735","unit_stat":"installed","extra":{"breakpoint":"","job_id":"108447335","ptask_id":180619870}},"on_fail":{"unit_id":"143735","unit_stat":"error"}},"task_id":180619877}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:17.992586859 +0800 CST] [@ request.go:60]handle request: [3290504241 start hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:10.234936446 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:18.203049951 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619877 start]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:18.216600473 +0800 CST] [@ request.go:231]callback request=[map[product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:map[service:service method:unit_stat_transform on_succ:map[unit_id:143735 unit_stat:installed extra:map[breakpoint: job_id:108447335 ptask_id:1.8061987e+08]] on_fail:map[unit_id:143735 unit_stat:error]] task_id:1.80619877e+08 exit_code:0]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:18.216832947 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:18.223870129 +0800 CST] [@ request.go:236]callback respond=[map[error_msg:ok data:[] error_code:0]] -------------------------------------------------------------------------------------------- install(whitelist-auth-changed) ===========(1) got one request [Info] [2021-01-29 11:33:19.327140385 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619894] log_id:[3290504241]] {"type":"hook","cmd":"whitelist-auth-changed","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619894}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:19.327483454 +0800 CST] [@ request.go:60]handle request: [3290504241 whitelist-auth-changed hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:19.327391452 +0800 CST] ===========(3-1) callback request [Debug] [2021-01-29 11:33:19.327716349 +0800 CST] [@ request.go:231]callback request=[map[token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:[] task_id:1.80619894e+08 exit_code:0 product_name:yiqiu]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:19.327818934 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:19.335113373 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(mcq-relation-unit-changed-pre) ===========(1) got one request [Info] [2021-01-29 11:33:19.353135176 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619895] log_id:[3290504241]] {"type":"hook","cmd":"mcq-relation-unit-changed-pre","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RELATION_ID":"14044","YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619895}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:19.353666236 +0800 CST] [@ request.go:60]handle request: [3290504241 mcq-relation-unit-changed-pre hook map[YIQIU_RELATION_ID:14044 YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735] [] 2021-01-29 11:33:19.353468918 +0800 CST] ===========(3-1) callback request [Debug] [2021-01-29 11:33:19.353958986 +0800 CST] [@ request.go:231]callback request=[map[product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:[] task_id:1.80619895e+08 exit_code:0]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:19.354126233 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:19.361893637 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(mcq-relation-unit-changed) // loop ===========(1) got one request [Info] [2021-01-29 11:33:19.377234264 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619896] log_id:[3290504241]] {"type":"hook","cmd":"mcq-relation-unit-changed","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RELATION_ID":"14044","YIQIU_REMOTE_UNIT_ID":"110794","YIQIU_REMOTE_SERVICE":"rdtest-redis-bac-wangbin34-1.yiqiu.tc","YIQIU_REMOTE_UNIT_DEAD":0,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619896}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:19.377474822 +0800 CST] [@ request.go:60]handle request: [3290504241 mcq-relation-unit-changed hook map[YIQIU_RELATION_ID:14044 YIQIU_REMOTE_UNIT_ID:110794 YIQIU_REMOTE_SERVICE:rdtest-redis-bac-wangbin34-1.yiqiu.tc YIQIU_REMOTE_UNIT_DEAD:0 YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735] [] 2021-01-29 11:33:19.377419935 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:20.231633955 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619896 mcq-relation-unit-changed]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:20.245127599 +0800 CST] [@ request.go:231]callback request=[map[product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:[] task_id:1.80619896e+08 exit_code:0]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:20.245289345 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:20.256787194 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(mcq-relation-unit-changed-post) ===========(1) got one request [Info] [2021-01-29 11:33:19.430633148 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619898] log_id:[3290504241]] {"type":"hook","cmd":"mcq-relation-unit-changed-post","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RELATION_ID":"14044","YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619898}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:21.165010998 +0800 CST] [@ request.go:60]handle request: [3290504241 mcq-relation-unit-changed-post hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RELATION_ID:14044 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:19.430811485 +0800 CST] ===========(2-2) hook exec [Debug] [2021-01-29 11:33:23.43008929 +0800 CST] [@ request.go:170]hook exec done %!(EXTRA []interface {}=[3290504241 108447335_180619898 mcq-relation-unit-changed-post]) ===========(3-1) callback request [Debug] [2021-01-29 11:33:23.444169657 +0800 CST] [@ request.go:231]callback request=[map[job_id:1.08447335e+08 unit_id:143735 reaction:[] task_id:1.80619898e+08 exit_code:0 product_name:yiqiu token:XzTokenExample]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:23.444326197 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:23.453230514 +0800 CST] [@ request.go:236]callback respond=[map[error_msg:ok data:[] error_code:0]] // other relation change -------------------------------------------------------------------------------------------- install(on-ready) ===========(1) got one request [Info] [2021-01-29 11:33:19.737230887 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619909] log_id:[3290504241]] {"type":"hook","cmd":"on-ready","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":{"service":"service","method":"unit_stat_transform","on_succ":{"unit_id":"143735","unit_stat":"ready","extra":{"breakpoint":"","job_id":"108447335","ptask_id":180619870}},"on_fail":{"unit_id":"143735","unit_stat":"error"}},"task_id":180619909}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:33.6621357 +0800 CST] [@ request.go:60]handle request: [3290504241 on-ready hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:19.737417217 +0800 CST] ===========(3-1) callback request [Debug] [2021-01-29 11:33:33.662343148 +0800 CST] [@ request.go:231]callback request=[map[product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735 reaction:map[method:unit_stat_transform on_succ:map[extra:map[breakpoint: job_id:108447335 ptask_id:1.8061987e+08] unit_id:143735 unit_stat:ready] on_fail:map[unit_id:143735 unit_stat:error] service:service] task_id:1.80619909e+08 exit_code:0]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:33.662564127 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:33.671444246 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(on-running) ===========(1) got one request [Info] [2021-01-29 11:33:33.954288874 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619911] log_id:[3290504241]] {"type":"hook","cmd":"on-running","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":{"service":"service","method":"unit_stat_transform","on_succ":{"unit_id":"143735","unit_stat":"running","extra":{"breakpoint":"","job_id":"108447335","ptask_id":180619870}},"on_fail":{"unit_id":"143735","unit_stat":"error"}},"task_id":180619911}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:33.954612748 +0800 CST] [@ request.go:60]handle request: [3290504241 on-running hook map[YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RUNTIME_HOME:/home/work/.yiqiu] [] 2021-01-29 11:33:33.954518189 +0800 CST] ===========(3-1) callback request [Debug] [2021-01-29 11:33:33.954791636 +0800 CST] [@ request.go:231]callback request=[map[reaction:map[service:service method:unit_stat_transform on_succ:map[unit_id:143735 unit_stat:running extra:map[breakpoint: job_id:108447335 ptask_id:1.8061987e+08]] on_fail:map[unit_id:143735 unit_stat:error]] task_id:1.80619911e+08 exit_code:0 product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08 unit_id:143735]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:33.954937603 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:33.963651324 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] -------------------------------------------------------------------------------------------- install(mcq-relation-joined) ===========(1) got one request [Info] [2021-01-29 11:33:34.322398961 +0800 CST] [@ server.go:202]got one request [query, content]=[map[req_id:[108447335_180619916] log_id:[3290504241]] {"type":"hook","cmd":"mcq-relation-joined","env":{"YIQIU_PROGRAM_HOME":"\/home\/work\/program","YIQIU_UNIT_HOME":"\/home\/work","YIQIU_UNIT_ID":143735,"YIQIU_RELATION_ID":"14044","YIQIU_RELATION_QUICK_MODE":"no","YIQIU_RUNTIME_HOME":"\/home\/work\/.yiqiu"},"param":[],"callback":{"url":"http:\/\/{CHUNFENG_SERVER_ADDR}:8080\/api\/1.0\/callback\/callback_from_agent?unit_id=143735&log_id=3290504241","data":{"product_name":"yiqiu","token":"XzTokenExample","job_id":108447335,"unit_id":143735,"reaction":[],"task_id":180619916}}}] ===========(2-1) handle request [Info] [2021-01-29 11:33:34.322669304 +0800 CST] [@ request.go:60]handle request: [3290504241 mcq-relation-joined hook map[YIQIU_RELATION_QUICK_MODE:no YIQIU_RUNTIME_HOME:/home/work/.yiqiu YIQIU_PROGRAM_HOME:/home/work/program YIQIU_UNIT_HOME:/home/work YIQIU_UNIT_ID:143735 YIQIU_RELATION_ID:14044] [] 2021-01-29 11:33:34.322598149 +0800 CST] ===========(3-1) callback request [Debug] [2021-01-29 11:33:34.322851051 +0800 CST] [@ request.go:231]callback request=[map[unit_id:143735 reaction:[] task_id:1.80619916e+08 exit_code:0 product_name:yiqiu token:XzTokenExample job_id:1.08447335e+08]] ===========(3-2) callback exe [Info] [2021-01-29 11:33:34.322985706 +0800 CST] [@ client.go:125]call yiqiu, url: [http://{CHUNFENG_SERVER_ADDR}:8080/api/1.0/callback/callback_from_agent?unit_id=143735&log_id=3290504241] ===========(3-3) callback response [Debug] [2021-01-29 11:33:34.329526203 +0800 CST] [@ request.go:236]callback respond=[map[error_code:0 error_msg:ok data:[]]] // orther relation

#!/usr/bin/python # coding=utf8 """ # Author: wangbin34(meetbill) # Created Time : 2021-01-21 16:32:27 # File Name: __init__.py # Description: async_job --发起异步任务，由单独线程进行处理 1.0.1: async_job 1.0.1 版本，用于简易创建异步任务，异步任务进行执行本地 Shell/Python 脚本 """ import os import logging from xlib.httpgateway import Request from xlib import retstat from xlib.middleware import funcattr from xlib.util import concurrent from xlib.util import shell_util __info = "async_job" __version = "1.0.1" executor = concurrent.ThreadPoolExecutor(max_workers=10) def run_cmd(reqid, cmd): """ 执行本地命令 Args: job_id : (Str) job_id job_name: (Str) job_name cmd : (Str) "python/bash file_path args" errlog : (object) errlog logger """ # 设置 1 小时超时 cmd_result = shell_util.run(cmd, timeout=3600, reqid=reqid) def _check_cmd(cmd): """ 检查 cmd 命令合法性, 仅支持执行 Python/Shell 脚本 Args: cmd: (Str) example: "python/bash file_path args" Returns: bool """ if ";" in cmd or "&&" in cmd or "||" in cmd: return False cmd_list = cmd.split() if len(cmd_list) < 2: return False file_type = cmd_list[0] file_path = cmd_list[1] if file_type not in ["python", "bash"]: return False if not os.path.exists(file_path): return False return True def executor_callback(worker): """ 记录 worker 执行异常 """ logging.info("called worker callback function") worker_exception = worker.exception() if worker_exception: logging.exception("Worker return exception: {}".format(worker_exception)) @funcattr.api def add_job(req, cmd): """ 添加任务 cmd: (str) bash xxx.sh/python xxx.py 本地需要有此脚本 """ isinstance(req, Request) if not _check_cmd(cmd): return retstat.ERR, {}, [(__info, __version)] task = executor.submit(run_cmd, reqid=req.reqid, cmd=cmd) task.add_done_callback(executor_callback) return retstat.OK, {}, [(__info, __version)]