Python代码片段

1 记录函数执行耗时

def timeToRunWithParams(level, param_idx_lst):
    def inner(func):
        def wrapper(*args, **kwargs):
            s_date = datetime.datetime.now()
            result = func(*args, **kwargs)
            duration = datetime.datetime.now() - s_date
            cost_ms = round(duration.total_seconds() * 1000, 2)
            if "DEBUG" == level.upper():
                sub_params = [args[idx] for idx in param_idx_lst]
                log.info('func={} with sub_params={} cost={} milliseconds'.format(func.__name__, sub_params, cost_ms))
            else:
                log.info('func={} cost={} milliseconds'.format(func.__name__, cost_ms))
            return result
        return wrapper
    return inner

@timeToRunWithParams(level="info", param_idx_lst=[])
def job():
    # to do something
    pass

友好显示版本：

def calculate_time_spent(started_at):
    duration = datetime.datetime.now() - started_at
    total_seconds = duration.total_seconds()
    hours = int(total_seconds // 3600)
    remaining_seconds = total_seconds % 3600
    minutes = int(remaining_seconds // 60)
    seconds = int(remaining_seconds % 60)
    milliseconds = int((total_seconds - int(total_seconds)) * 1000)
    result_parts = []
    if hours:
        result_parts.append(f"{hours}Hour")
        if hours > 1:
            result_parts[-1] += "s"
    if minutes:
        result_parts.append(f"{minutes}Minute")
        if minutes > 1:
            result_parts[-1] += "s"
    if seconds:
        result_parts.append(f"{seconds}Second")
        if seconds > 1:
            result_parts[-1] += "s"
    if milliseconds:
        result_parts.append(f"{milliseconds}millisecond")
        if milliseconds > 1:
            result_parts[-1] += "s"
    if not result_parts:
        return "0 Seconds"
    return ",".join(result_parts)


started_at = datetime.datetime.now()
calculate_time_spent(started_at)
# Time spent in total: 2Minutes,44Seconds,317milliseconds

 

2 HTTP请求失败自动重试

def make_request(url, timeout=30, max_retries=3):
    retry_delay = 1  # 初始重试间隔时间为1秒
    for i in range(max_retries + 1):
        try:
            response = req.get(url, timeout=timeout)
            response.raise_for_status()  # 检查请求是否成功
            return response  # 返回成功的响应
        except req.exceptions.RequestException as e:
            if i == max_retries:
                raise  # 如果已达到最大重试次数，则将异常抛出
            else:
                log.error("failed to request: {} \n retrying {} times in {} seconds...".format(url, i+1, retry_delay))
                time.sleep(retry_delay)
                retry_delay *= 2  # 增加重试间隔时间

 

3 输出日志

def get_logger(path='./', filename='access.log', level='debug'):
    log_level = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'critical': logging.CRITICAL
    }
    level = 'debug' if level not in log_level else level
    if not os.path.exists(path):
        os.makedirs(path)
    log_file = os.path.join(path, filename)
    formatter = logging.Formatter("%(asctime)s.%(msecs)03d [line:%(lineno)d] %(levelname)s %(message)s",
                                    datefmt="%Y-%m-%d %H:%M:%S")
    # handler = logging.FileHandler(log_file, mode='a')
    # 创建一个时间轮转日志处理器
    handler = TimedRotatingFileHandler(
        log_file,
        when="midnight",
        interval=1,
        backupCount=32
    )
    handler.setFormatter(formatter)
    # logger = logging.getLogger(__name__)
    logger = logging.getLogger(filename)
    logger.setLevel(log_level[level])
    logger.addHandler(handler)
    return logger

 

4 自定义异常

class CustomException(Exception):
    def __init__(self, message):
        self.message = message
        super().__init__(self.message)
    def __str__(self):
        return "Custom Exception: {}".format(self.message)

raise CustomException("desc of panic")

　　

5 任务执行失败自动重试

import time
import datetime
import traceback

# 自定义重试次数和backoff时间
backoff = [
    [1, 1],
    [2, 2],
    [3, 4],
    [4, 8],
    [5, 16],
]

def date2ts(date_str, layout="%Y-%m-%d %H:%M:%S"):
    date_struct=time.strptime(date_str, layout)
    return int(time.mktime(date_struct))

def ts2date(ts, layout="%Y-%m-%d %H:%M:%S"):
    ts = int(ts)
    return datetime.datetime.fromtimestamp(ts).strftime(layout)

def generate_date_pairs(year, month):
    # 生成指定月份的所有日期
    start_date = datetime.datetime(year, month, 1)
    if month == 12:  # 如果是12月，下一个月是下一年的1月
        end_date = datetime.datetime(year + 1, 1, 1)
    else:
        end_date = datetime.datetime(year, month + 1, 1)
    
    date_range = pd.date_range(start_date, end_date, closed='left')
    date_pairs = []
    for i in range(len(date_range) - 1):
        date_pairs.append((date_range[i].strftime("%Y-%m-%d %H:%M:%S"), date_range[i + 1].strftime("%Y-%m-%d %H:%M:%S")))
    # 补上最后一天
    last_day = end_date - datetime.timedelta(days=1)
    date_pairs.append((last_day.strftime("%Y-%m-%d %H:%M:%S"), end_date.strftime("%Y-%m-%d %H:%M:%S")))
    return date_pairs

if __name__ == "__main__":
    pairs = generate_date_pairs(2024, 7)
    for pair in pairs:
        print(pair)
        day = ts2date(date2ts(pair[0]), layout="%Y%m%d")
        for x in backoff:
            try:
                do_something(*pair)
                break
            except Exception as e:
                print(traceback.format_exc())
                print(f"failed {x[0]} times, sleep {x[1]}")
                time.sleep(x[1])
        else:
            print(f"{day} failed!!!")
            continue
        print(f"{day} OK\n")

 

6 Dataframe常用操作-生成Excel文件

df = pd.DataFrame(series)

# 将column重新排列
column_order = [
    "app",
    "mode",
    "ppid",
    "pname",
    "pid",
    "name",
    "comm",
    "protocol",
    "bytes_received",
    "bytes_sent",
]
df_reordered = df[column_order]

# 按照多个column进行组合排序
df_sorted = df_reordered.sort_values(by=['name', 'pname', "bytes_sent"], ascending=[True, True, False])

# 按照指定column的value进行拆分
df_grouped = df_sorted.groupby("mode")
# 拆分的结果输出xlsx文件，每个子dataframe占用一个sheet
output_file = 'xxx.xlsx'
with pd.ExcelWriter(output_file, engine='xlsxwriter') as writer:
    for category, sub_df in df_grouped:
        sheet_name = "mode={}".format(str(category))
        sub_df.to_excel(writer, sheet_name=sheet_name, index=False)

 

7 Dataframe常用操作-读取Excel文件

xls = pd.ExcelFile("xxx.xlsx", engine='openpyxl')
name2df = {}
for sheet_name in xls.sheet_names:
    name2df[sheet_name] = xls.parse(sheet_name)

 

8 Dataframe常用操作-列名转换（翻译），并按照指定顺序排列

code2name = {
    "BaseName": "药物名称（中文）",
    "BaseName_EN": "药物名称（英文）",
    "ATCI": "ATCI",
    "ATCII": "ATCII",
    "ATCIII": "ATCIII",
    "ATCIV": "ATCIV",
    "Cases": "报告数（PS）",
    "AENUM": "不良事件例数（PS）",
    "path": "下载链接",
    "SEQ1": "SEQ1（网站链接相关）",
}
df2 = df.rename(columns=code2name)

order = [
    "药物名称（中文）",
    "药物名称（英文）",
    "ATCI",
    "ATCII",
    "ATCIII",
    "ATCIV",
    "报告数（PS）",
    "不良事件例数（PS）",
    "下载链接",
    "SEQ1（网站链接相关）",
]
df2 = df2[order]