TS爬虫爬取博客园博客列表

function onProcessData(context: IDataFlowScriptNodeContext): DbTableInfo | any[][] {

    const Jsoup = Java.type("org.jsoup.Jsoup");

    const url = "https://www.cnblogs.com/";

    let data = [];

    for (let i = 1; i <= 5; i++) {

        let res: string = post({

            url: "https://www.cnblogs.com/AggSite/AggSitePostList",

            data: {

                "CategoryType": "SiteHome",

                "ParentCategoryId": 0,

                "CategoryId": 808,

                "PageIndex": i.toFixed(),

                "TotalPostCount": 4000,

                "ItemListActionName": "AggSitePostList"

            },

            headers: {

                "Content-Type": "application/json"

            }

        }).responseText;

        let document = Jsoup.parse(res);

        let elements = document.getElementsByClass("post-item");

        for (let element of elements) {

            let row = [];

            let titleElement = element.getElementsByClass("post-item-title");

            let summaryElement = element.getElementsByClass("post-item-summary");

            let authorElement = element.select(".post-item-foot > .post-item-author");

            //let authorElement = element.select("section > footer > a.:nth-child(1)")

            let createDateElement = element.select("section > footer > span.post-meta-item > span");

            let commentCountElement = element.select("section > footer > a:nth-child(4) > span");

            let viewCountElement = element.select("section > footer > a:nth-child(5) > span");

            // 博客 ID

            let id = element.attr("data-post-id");

            // 博客标题

            let title = titleElement.text();

            // 内容简介

            let summary = summaryElement.get(0).ownText();

            let authorUrl = authorElement.select("a").get(0).attr("href");

            // 作者 ID

            let authorId = authorUrl.substring(url.length, authorUrl.length() - 1);

            // 作者网名

            let authorName = authorElement.select("span").get(0).text();

            // 创建时间

            let createDate = createDateElement.text();

            // 点赞数

            let diggCount = element.getElementById("digg_count_" + id).text();

            // 评论数

            let commentCount = commentCountElement.text();

            // 浏览量

            let viewCount = viewCountElement.text();

            print("-------------------------------------------")

            row.push(id);

            row.push(title);

            row.push(summary);

            row.push(authorUrl);

            row.push(authorId);

            row.push(authorName);

            row.push(createDate);

            row.push(diggCount);

            row.push(commentCount);

            row.push(viewCount);

            print("row======" + row);

            data.push(row);

        }

    }

    return data;

}