tapd封装爬取逻辑

import requests
import json
from requests import sessions
import math
import schedule
import pandas as pd
import datetime
import time
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


class Tapd(object):

    def __init__(self):
        self.header = {
            "sec-ch-ua-platform": "macOS",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36"
        }

        data = {
            "data[Login][ref]": "xxxxxxxxxx",
            "data[Login][encrypt_key]": "xxx",
            "data[Login][encrypt_iv]": "xxxx",
            "data[Login][site]": "TAPD",
            "data[Login][via]": "encrypt_password",
            "data[Login][email]": "xxxxx",
            "data[Login][password]": "xxxxx",
            "data[Login][login]": "login",
            "dsc_token": "xxxxx"
        }
        # session request
        self.seq = sessions.Session()
        self.seq.request(url='https://www.tapd.cn/cloud_logins/login',
                         data=data, method="POST", headers=self.header
                         , verify=False
                         )

    def get_role_list(self):
        url_role = 'https://www.tapd.cn/api/basic/roles/get_role_group?workspace_id=69120889'

        rep_data = self.seq.get(url=url_role, headers=self.header, verify=False)
        # print(json.dumps(rep_data.json(),indent=3,ensure_ascii=False))

    def get_all_bugs(self):
        """
        查询全部缺陷清单基于：payload.json,和test_validate.json
        开发状态："新","接受/处理", "重新打开","转需求"
        测试状态： "已解决","待验收","已拒绝"
        :return:
        """

        # filter payload query params
        with open("./configs/payload.json", mode='r', encoding="utf-8") as f:
            payload_dev = json.load(f)
        with open("./configs/test_validate.json", mode='r', encoding="utf-8") as f:
            payload_verified = json.load(f)
        response_dev = self.seq.post(url="https://www.tapd.cn/api/entity/bugs/bugs_list",
                                     json=payload_dev, headers=self.header, verify=False)
        response_data_dev = response_dev.json()  # dict
        totals = response_data_dev.get("data", None).get("total_count", 0)
        response_data_test = self.seq.post(url="https://www.tapd.cn/api/entity/bugs/bugs_list", json=payload_verified,
                                           headers=self.header, verify=False).json()
        test_num = response_data_test.get("data", None).get("total_count", 0)
        page_size = 50
        dev_data, test_data = [], []
        if int(totals) > 0:
            pages_dev = math.ceil(int(totals) / page_size)
            dev_data = bug_query(pages_dev, payload_dev, response_data=response_data_dev)
        if int(test_num) > 0:
            pages_test = math.ceil(int(totals) / page_size)
            test_data = bug_query(pages_test, payload_verified, response_data=response_data_test)
        return [dev_data, totals, test_num, test_data]


def bug_query(pages, payload, response_data):
    data = []
    for current_page in range(1, pages + 1):
        payload["page"] = current_page
        bugs_list = response_data.get("data", '').get("bugs_list")
        for bug in bugs_list:
            bug_item_dict = bug.get("Bug", None)
            one_bug = {
                'bug_id': bug_item_dict.get('short_id'),
                'customer_type': bug_item_dict.get('custom_field_two', ''),
                'bug_type': bug_item_dict.get('bugtype', ''),
                'priority': bug_item_dict.get('priority', ''),
                'title': bug_item_dict.get("title", None),
                'status': bug_item_dict.get("status_alias", None),
                # 'severity':bug_item_dict.get('severity',''),
                'reporter': bug_item_dict.get("reporter", None),
                "create_time": bug_item_dict.get("created", None),
                'expected_fix_time': bug_item_dict.get("custom_field_one", ''),
                "deal_with": bug_item_dict.get("current_owner", None),
                "developer": bug_item_dict.get("de", ''),
                "tester": bug_item_dict.get("te", ''),
                "bug_link": "https://www.tapd.cn/69120889/bugtrace/bugs/view?bug_id=%s" % (
                    bug_item_dict.get("id"))
            }
            data.append(one_bug)
        return data


def group_bug_count(bug_data):
    """
    :param bug_data: [dev_data, totals, to_verified_num,test_data]
    :return:
    """
    dev_data, totals, to_verified_num = bug_data[0], bug_data[1], bug_data[2]
    if dev_data:
        with open("./configs/map_org.json", 'r') as f:
            members_map = json.load(f)
        Product_Counter = 0
        UI_Design_Counter = 0
        Test_Counter = 0
        Crawler_Counter = 0
        FrontDev_Counter = 0
        Backend_Counter = 0
        Bigdata_Counter = 0
        Reverse_Counter = 0
        AI_Counter = 0
        Null_Dealwith = 0
        for k, v in members_map.items():
            for bug_item in dev_data:
                dev = bug_item.get('developer', '')
                deal_with = bug_item.get('deal_with', '')
                tester = bug_item.get('tester', '')
                if deal_with in v and k == '产品组':
                    Product_Counter += 1
                if deal_with in v and k == '设计组':
                    UI_Design_Counter += 1
                if deal_with in v and k == '测试组':
                    Test_Counter += 1
                if deal_with in v and k == '采集组':
                    Crawler_Counter += 1
                if deal_with in v and k == '前端组':
                    FrontDev_Counter += 1
                if deal_with in v and k == '后端组':
                    Backend_Counter += 1
                if deal_with in v and k == '数据组':
                    Bigdata_Counter += 1
                if deal_with in v and k == '逆向组':
                    Reverse_Counter += 1
                if deal_with in v and k == '算法组':
                    AI_Counter += 1
                if not deal_with:
                    Null_Dealwith += 0

        data_res = {
            '总计未解决数': int(totals),
            '总未验证数': int(to_verified_num),
            '处理人空': Null_Dealwith,
            "产品组": Product_Counter,
            "设计组": UI_Design_Counter,
            '前端组': FrontDev_Counter,
            '后端组': Backend_Counter,
            '数据组': Bigdata_Counter,
            '算法组': AI_Counter,
            '逆向组': Reverse_Counter,
            '采集组': Crawler_Counter,
            '测试组': Test_Counter
        }
        sort_data = dict(sorted(data_res.items(),
                                key=lambda x: x[1], reverse=True
                                ))
        print(json.dumps(sort_data,
                         indent=4, ensure_ascii=False))
        return sort_data
    else:
        return {}



def group_bug_count_2(bug_data):
    """
    根据bug_data中的数据，统计不同组的bug数量和其他相关信息。

    :param bug_data: 一个包含dev_data, totals, to_verified_num的列表
    :return: 一个字典，包含不同组的bug数量和其他统计信息
    """
    dev_data, totals, to_verified_num = bug_data

    # 加载成员映射文件
    with open("./configs/map_org.json", 'r') as f:
        members_map = json.load(f)

        # 初始化计数器
    counters = {
        '产品组': 0,
        '设计组': 0,
        '测试组': 0,
        '采集组': 0,
        '前端组': 0,
        '后端组': 0,
        '数据组': 0,
        '逆向组': 0,
        '算法组': 0,
        '处理人空': 0
    }

    # 遍历dev_data中的bug项
    for bug_item in dev_data:
        dev = bug_item.get('developer', '')
        deal_with = bug_item.get('deal_with', '')
        tester = bug_item.get('tester', '')

        # 根据deal_with的值更新计数器
        for group_name, group_members in members_map.items():
            if deal_with in group_members:
                counters[group_name] += 1
                break  # 找到匹配的组后，跳出循环

        # 更新处理人空的计数器
        if not deal_with:
            counters['处理人空'] += 1

            # 添加总计数
    counters['总计未解决数'] = int(totals)
    counters['总未验证数'] = int(to_verified_num)

    return counters


def get_bug_data():
    """
    :return: [dev_data, totals, to_verified_num,test_data]
    """
    tapd = Tapd()
    data = tapd.get_all_bugs()
    return data or []


def generate_markdown_content(bug_data):
    """
    :param bug_data: [dev_data, totals, to_verified_num,test_data]
    :return:
    """
    # 自己会判断在内部 dev_data情况
    test_to_verify_num=bug_data[2]
    summary_deal_with = group_bug_count(bug_data)
    if summary_deal_with:
        total_un_link = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=978d9baf6689c9cf5944cb921f93e1ea'
        total_un_close = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=faf9690164ba552c0a88c63d3e481c8f'
        prd_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=2bee75ce21c0f54c9b2f8b831d09d1c9'
        test_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=fff2b8cb31f94a3f36e3cbbd4032e7c6'
        backend_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=6de148c97af43bdc29777eb00ec434bd'
        bigdata_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=0f2a9c2a083230f0e7eb211f9d896167'
        ai_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=3c41b7ea67b490f94fb2b2c93294ddaa'
        crawler_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=07d47d036e9ffb82544c80775813ac46'
        front_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=1f3a40b482841556072a30ad83034838'
        null_deal_link = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=0f44a2d7d55e7d2076693cd9de053284'
        ui_design_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=1f2a37509a410f5205bf75dd6c7da4b7'
        reverse_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=8322eb88e5e15396cd9bdeb9510d6036'
        deployer_un = 'https://www.tapd.cn/tapd_fe/69120889/bug/list?confId=1169120889001015948&page=1&queryToken=9ad18d000e022f9e5b2d1c190d185bed'

        markdown_body = {
            "msgtype": "markdown",
            "markdown": {
                "content": f"总未解决:  [{summary_deal_with.get('总计未解决数', 0)}]({total_un_link})\n"
                           f"总未验证:  [{summary_deal_with.get('总未验证数', 0)}]({total_un_close})\n"
                           f"产品组:   [{summary_deal_with.get('产品组', 0)}]({prd_un}) <font color=\"warning\">@xxx</font>\n"
                           f"测试组:   [{test_to_verify_num or summary_deal_with.get('测试组', 0)}]({test_un}) <font color=\"warning\">@xxx</font>\n"
                           f"后端组:   [{summary_deal_with.get('后端组', 0)}]({backend_un}) <font color=\"warning\">@xxx</font>\n"
                           f"数据组:   [{summary_deal_with.get('数据组', 0)}]({bigdata_un}) <font color=\"warning\">@xxxx</font>\n"
                           f"算法组:   [{summary_deal_with.get('算法组', 0)}]({ai_un}) <font color=\"warning\">@xxx</font>\n"
                           f"采集组:   [{summary_deal_with.get('采集组', 0)}]({crawler_un}) <font color=\"warning\">@xxx</font>\n"
                           f"前端组:   [{summary_deal_with.get('前端组', 0)}]({front_un}) <font color=\"warning\">@xxx</font>\n"
                           f"设计组:   [{summary_deal_with.get('设计组', 0)}]({ui_design_un}) <font color=\"warning\">@xxxx</font>\n"
                           f"逆向组:   [{summary_deal_with.get('逆向组', 0)}]({reverse_un}) <font color=\"warning\">@xxxx</font>\n"
                           f"处理人为空: [{summary_deal_with.get('处理人空', 0)}]({null_deal_link})"
            },
            "mentioned_list": [
                "@all"
            ],
            "mentioned_mobile_list": [
                "@all"
            ]
        }
        print(markdown_body)

        # url_online_group = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=a6d77220-23b1-428c-a1be-d6fb410f4f85'
        url_online_group_test = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=757afa3c-d39e-46c7-8d1f-60d6f2ddca55'
        ret = requests.post(url=url_online_group,
                            json=markdown_body, verify=False)

    else:
        pass


def generate_people_markdown(bug_data):
    """
    :param bug_data: [dev_data, totals, to_verified_num,test_data]
    :return:
    """
    dev_data, dev_total, test_total, test_data = bug_data[0], bug_data[1], bug_data[2], bug_data[3]
    data_mix = dev_data + test_data
    if data_mix:
        np_data = []
        if data_mix:
            for x in data_mix:
                deal_with = x.get('deal_with', '空')
                status = x.get('status', '空')
                bug_type = x.get('bug_type', '')
                create_time = x.get('create_time', '')
                unit_data = [deal_with, status, bug_type, create_time, 1]
                np_data.append(unit_data)
        # df = pd.DataFrame(np_data, columns=['处理人', '状态', '缺陷类型', '创建时间', "数量"])
        status_counts = {}
        for entry in np_data:
            name = entry[0]
            status = entry[1]
            if name in status_counts:
                if status in status_counts[name]:
                    status_counts[name][status] += 1
                else:
                    status_counts[name][status] = 1
            else:
                status_counts[name] = {status: 1}
        content = ""
        for name, statuses in status_counts.items():
            # print(f"{name}:")
            msg = f"\n<font color=\"warning\">@{name}</font>\n"
            for status, count in statuses.items():
                #  print(f"{status}:{count}")
                msg += f">{status}:{count}\n"
            content += msg
        print(content)

        markdown_message = {
            "msgtype": "markdown",
            "markdown": {
                "content": content
            },
            "mentioned_list": [
                "@all"
            ],
            "mentioned_mobile_list": [
                "@all"
            ]
        }

        ret = requests.post(
            url=url_rd_team_group,
            json=markdown_message, verify=False)


def notify_every_group():
    """
    通知各个小组线上缺陷
    :param data
    :return:
    """
    data = get_bug_data()

    try:
        generate_markdown_content(bug_data=data)
    except Exception as e:
        print(e)


def notify_status_count():
    """
    按人通知状态数量
    :return:
    """
    data = get_bug_data()
    try:
        generate_people_markdown(data)
    except Exception as f:
        print(f)


def create_schedule():
    """
    调度器：开始定时任务；每周二、周五 10：00
    :return:
    sample :
        # schedule.every(15).seconds.do(notify_every_group,data)
        # schedule.every(10).seconds.do(notify_status_count,data)
    """
    # data = get_bug_data()
    # todo tuesday
    schedule.every().tuesday.at("10:00").do(notify_status_count)
    schedule.every().tuesday.at("10:00").do(notify_every_group)
    # todo friday
    schedule.every().friday.at("10:00").do(notify_status_count)
    schedule.every().friday.at("10:00").do(notify_every_group)
    while True:
        schedule.run_pending()


if __name__ == '__main__':
    env = "prod"
    if env == "test":
        debug="xxxx"
        url_online_group = debug
        url_rd_team_group = debug
        schedule.every(60).seconds.do(notify_every_group)
        schedule.every(60).seconds.do(notify_status_count)
        while True:
            schedule.run_pending()
    elif env == "prod":
        url_online_group = 'xxxx'
        url_rd_team_group = "xxxxx"
        create_schedule()