LaunchOptions options = new LaunchOptions
{
Headless = true,
ExecutablePath = @".local-chromium\Win64-901912\chrome-win\chrome.exe",
Args = new[] {
string.Format("--proxy-server={0}",ip),
"--start-maximized",//最大窗口
"--disable-infobars",//--隐藏自动化标题
"--no-sandbox",
"–single-process",
"–disable-gpu",
"--disable-setuid-sandbox",
"--ignore-certificate-errors",
"--app=https://www.baidu.com/"
},
IgnoreHTTPSErrors = true
};
using (var browser = await Puppeteer.LaunchAsync(options))
{
try
{
string userAgent = UAList[new Random().Next(0, UAList.Count())];
var newPage = await browser.NewPageAsync();
newPage.DefaultTimeout = 15000;
await newPage.SetUserAgentAsync(userAgent);
ViewPortOptions vOptions = new ViewPortOptions
{
Width = 1920,
Height = 1080
};
await newPage.SetViewportAsync(vOptions);
//隐藏webdriver特征
await newPage.EvaluateExpressionOnNewDocumentAsync("delete navigator.__proto__.webdriver;");
foreach (var href in link)
{
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
await newPage.GoToAsync(href, WaitUntilNavigation.DOMContentLoaded);
var jsSelectAllAnchors = @"Array.from(document.querySelectorAll('.p-card-href')).map(a => a.href);";
var urls = await newPage.EvaluateExpressionAsync<string[]>(jsSelectAllAnchors);
if (urls != null && urls.Where(a => a.Contains("b2bsearch/jump?url")).Count() > 0)
{
string[] filter_urls = urls.Where(a => a.Contains("b2bsearch/jump?url")).ToArray();
string productUrl = filter_urls[new Random().Next(0, filter_urls.Count())];
await newPage.GoToAsync(productUrl, WaitUntilNavigation.DOMContentLoaded);
await ScrollPage(newPage, 500, 800, 3, 200, 800);
}
sw.Stop();
Console.WriteLine("耗时:" + sw.ElapsedMilliseconds + "ms");
}
catch (Exception ex)
{
Console.WriteLine("当前打开页面链接异常:" + ex.Message);
if (ex.Message.Contains("ERR_TUNNEL_CONNECTION_FAILED") || ex.Message.Contains("ERR_PROXY_CONNECTION_FAILED"))
{
await browser.CloseAsync();//关闭浏览器
//重新运行任务
var d = Task.Run(() => TaskRun(link.Skip(link.IndexOf(href)).ToList()));
d.Wait();
return;
}
}
}
if (newPage != null && !newPage.IsClosed)
{
await newPage.CloseAsync();
}
if (browser != null && !browser.IsClosed)
{
await browser.CloseAsync();//关闭浏览器
}
Console.WriteLine("全部任务已完成....");
}
catch (Exception ex)
{
Console.WriteLine("global:" + ex.Message);
if (browser != null && !browser.IsClosed)
{
await browser.CloseAsync();
var d = Task.Run(() => TaskRun(link));//递归执行
d.Wait();
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + ";" + ex.InnerException);
}
}
/// <summary>
/// 模拟滚动页面
/// </summary>
/// <param name="page">当前页面</param>
/// <param name="times">次数</param>
/// <returns></returns>
async static Task ScrollPage(Page page, int minPix, int maxPix, int times, int minSleep = 1000, int maxSleep = 2000)
{
Func<Task> scroll = null;
int i = 0;
scroll = new Func<Task>(async () =>
{
int top = new Random().Next(minPix, maxPix);
await page.EvaluateExpressionAsync("window.scrollBy({top:" + top + ",behavior:'smooth'})");
Thread.Sleep(new Random().Next(minSleep, maxSleep));
i++;
if (i < times)
{
await scroll();
}
});
await scroll();
//再向上随机滚动
await page.EvaluateExpressionAsync("window.scrollBy({top:" + (0 - new Random().Next(500, 1500)) + ",behavior:'smooth'})");
}