Node 获取其他服务器的数据的方法

//爬取一个页面

const http = require('http');

const fs = require('fs');

 

const _url="www.baidu.com";

const paths="";

const header={

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',

    'Accept-Encoding': 'gzip, deflate, br',

    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',

    'Cache-Control': 'max-age=0',

    'Connection': 'keep-alive',

    'Cookie': 'BIDUPSID=0DECECDA9A3FCA655964F7045DDAF9EF; PSTM=1604032144; BDUSS=tSaEp2UEdxRFM4U2tFeDZrdEhibEN6bmFpckdUVUV4Z29jSWNoMmo5ODNqdTVmSVFBQUFBJCQAAAAAAAAAAAEAAAA3gkUEbHVveWltaW5nbW0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADcBx183AcdfRm; BDUSS_BFESS=tSaEp2UEdxRFM4U2tFeDZrdEhibEN6bmFpckdUVUV4Z29jSWNoMmo5ODNqdTVmSVFBQUFBJCQAAAAAAAAAAAEAAAA3gkUEbHVveWltaW5nbW0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADcBx183AcdfRm; BAIDUID=4F6E443F8C4558717B3554BB3976CB63:FG=1; __yjs_duid=1_ecafc4e03c8b5f79ef8be19d4d6539d11619406343758; BD_UPN=12314753; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; MCITY=-%3A; BDSFRCVID_BFESS=AsKOJeC62Gh6sa5ed8PyhMY3mc5TikjTH6aouSbtJzc6T1-0OkhmEG0PVU8g0KubwqBlogKK3gOTH4PF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tJKeoKt5tC83fP36qR6BKnL_hxJt5C62aKDs2tbnBhcqEIL4K6jE-ft7bq6Z-hc3BG5U5K5x0JOZMUbSj4QoDjb-QNuLKjJKfR-Ool6KKl5nhMJ33j7JDMP0-R3z3bvy523iob3vQpPMVhQ3DRoWXPIqbN7P-p5Z5mAqKl0MLPbtbb0xb6_0DTO0DN_ft58sb5vfstbHaJ7KqnnpK4__-P6M0H7lWMT-0bFHLC3j-R8-etc_y65sXqFiLgcZ-xvGJHn7_JjOLpn0jtQy3-7oDnKv2GKehUQxtNR40DnjtpkMH43syl5obUPUDUJ9LUkJLgcdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtKDaMCPRjjD35n-WqlO05468KCo2BnD8Kb7VbnRTDMnkbfJBDR6vbxIfbDTlQM_K2p6ZSp840xTd54t7yajhJ-7vyI5-LbIh0-5OEf7VDp3pQT8r3lAOK5OibCrHBCONab3vOIOTXpO1jh8zBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksDMDtqj_efnCeVI_QKbRofJ-k-4QEbbQH-UnLq-ottgOZ0l8KtJT4VItmM4ofbq3-M-ckJ-rwyJFqBnrmWIQHDPQh0noxjx4FLHOR-x7DLC64KKJxHRLWeIJojKc6DCuBhUJiB5JMBan7_UJIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8xj5KbD5cBeUvK2tvtK6RjsJOOaCvNqpvOy4oTj6D1Maje56Q-5nbTXRnzyCOVHI52MtR83MvB-fntQ-jJWjQ2_f-hHCJcjnnzQft20hKAeMtjBbQab56GWn7jWhk5ep72y5OmQlRX5q79atTMfNTJ-qcH0KQpsIJM5-DWbT8EjH62btt_JR4e_D5P; kleck=3a4c71cc2af9fd2d64152c4617a81e5e; H_PS_PSSID=34130_34100_33967_31253_33848_34113_33607_34107_34134_34094_26350; delPer=0; BD_CK_SAM=1; PSINO=7; BAIDUID_BFESS=4F6E443F8C4558717B3554BB3976CB63:FG=1; ab_sr=1.0.1_YTg4MmQ5OGExZTg1OTNkOTE3M2I3NGE4NzY5YzMzOGYxNGZiYzVhNTM5YTUwNjg4MDMzNmQ3YzFlMWNiODc5NTI1NjRhYjk4MThkYzQ4OGYyMDQwMTExMWYzNjQ1Njk1OTUzNWJhYjZhMzdkNjI0MGJlOGZjY2Y2MWYwYjc1Nzg4MTY2NDVlNTZmNmE4Y2I0Y2EzYzM5Mjg1MjFmODM4ZA==; BD_HOME=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_645EC=c31aneegfTpnKBVM3nRptKOEo1jdb2rpeEncVmdPCMMWYXCMJUIJuATEKc%2F6E5W4ScoA; WWW_ST=1625022231988; sugstore=0; BA_HECTOR=ak0k0g8l81ag8h84uj1gdnnop0r',

    'Host': 'www.baidu.com',

    'Referer': 'https://www.baidu.com/s?wd=getaddrinfo%20ENOTFOUND%20https%3A%2F%2Fwww.baidu.com%2F&rsv_spt=1&rsv_iqid=0xc3a6ccea00167817&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&rqlang=cn&tn=baiduhome_pg&rsv_enter=0&rsv_dl=tb&oq=UnhandledPromiseRejectionWarning%253A%2520Unhandled%2520promise%2520rejection.%2520This%2520error%2520or&rsv_btype=t&inputT=1433&rsv_t=4cf117kWrmdcOxYFc04CykZTZhwlnEEejCWaMpdEdlhYJIkGHfDXLLna2J2%2Bjju1k0EV&rsv_sug3=4&rsv_n=2&rsv_pq=92dd821800056d64&rsv_sug4=1505',

    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',

    'sec-ch-ua-mobile': '?0',

    'Sec-Fetch-Dest': 'document',

    'Sec-Fetch-Mode': 'navigate',

    'Sec-Fetch-Site': 'same-origin',

    'Sec-Fetch-User': '?1',

    'Upgrade-Insecure-Requests': 1,

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'

}

 

let data = requestApi('get',_url,80,{},paths,{});

data.then(res=>{

    console.log('返回内容',res);

    fs.writeFile('html/baidu2.txt', res, (err) => {

        if (err) throw err;

        console.log(_url+'页面爬取成功!');

    });

})

 

function requestApi(method, hostname, port, headers, path, params){

 

    return new Promise((resolve, reject) => {

        const content = JSON.stringify(params),

            options = {

                hostname: hostname,

                port: port,

                path: path,

                method: method,

                headers: headers

            };

        let result = '';

 

        let req = http.request(options, function (res) {

            res.setEncoding("utf8");

 

            //res.setEncoding("binary");

            

 

            res.on('data', function (chunk) {

                result += chunk;//JSON.parse(chunk.toString());

            });

            res.on('agentRemove', function (chunk) {

                console.log('agentRemove', chunk);

            });

            res.on('timeout', function (chunk) {

                console.log('timeout', chunk);

            });

            res.on('free', function (chunk) {

                console.log('free', chunk);

            });

            res.on('error', function (e) {

                result = e.message;

            });

            res.on('drain', function (chunk) {

                console.log('drain', chunk);

            });

            res.on('end', function (chunk) {

                if (result=='') {

                    result = res.statusCode;

                }

                resolve(result);

            });

            res.on('close', function (chunk) {

                //console.log('close', chunk);

            });

        });

 

        req.on('error', function (e) {

            reject(e.message);

        });

        req.write(content);

        req.end();

    }).catch(err=>{

        console.error(err);

    });

}

 

 

 

//爬取一个图片

const http = require('https');

const fs = require('fs');

/**

 * 下载图片

 * @param {*请求图片地址} url 

 * @param {*保存图片名称} name 

 */

let updataImg = (url, name) => {

    //先访问图片

    http.get(url, (res) => {

        //用来存储图片二进制编码

        let imgData = '';

        //设置图片编码格式

        res.setEncoding("binary");

        //检测请求的数据

        res.on('data', (chunk) => {

            imgData += chunk;

           

        })

        //请求完成执行的回调

        res.on('end', () => {

            console.log('下载内容',imgData)

            // 通过文件流操作保存图片

            fs.writeFile(`./image/${name}.jpg`, imgData, 'binary', (error) => {

                if (error) {

                    console.log('下载失败');

                } else {

                    console.log('下载成功！')

                }

            })

        })

    })

}

const url='https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fpic.jj20.com%2Fup%2Fallimg%2Fmx12%2F0F320213159%2F200F3213159-1.jpg&refer=http%3A%2F%2Fpic.jj20.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1627550175&t=e0e1a863fe9abf219d2159f3695237a3';

updataImg(url,'3');