利用github给国外文件下载加速
前言
作为一名程序员，经常需要下载一些编程相关的环境，而国内的网络环境大家都知道，有的文件用浏览器是下载不动的，于是我有了利用github下载文件的想法。
我的demo项目地址：https://github.com/bobowire/wireboy.remote.download （为保护github这片净土，已设置为私有）
参考资料：
- NodeJS使用node-fetch下载文件并显示下载进度示例：https://www.jianshu.com/p/4b58711cb72a
- nodejs——发送邮件（带附件）：https://www.cnblogs.com/yourstars/p/6728931.html
觉得好玩的，大家可以点个赞~
下载加速原理
使用github的Action远程执行文件下载（下载qt环境速度可以达到3mb/s），然后将下载的文件进行分片，每片15mb，分片后的文件以邮件附件的方式发送到国内邮箱，我们通过下载邮箱中的附件，将分片的附件合并成完整文件，从而实现不FQ、不用下载器也能下载国外文件的目的。
简单点说
- github远程下载
- 文件分片
- 通过邮箱发到国内
- 对附件进行合并
使用方法
- 新建github项目
- 创建js文件download.js文件，内容请查阅后文
- 创建workflows/RunDownload.yml文件，内容请查阅后文
- 修改download.js中的fileURL 变量值，此为文件url地址
- 在项目github->settings->Secrets中，点击右上方“new responsitory secret”按钮，添加"EMAILPASS","SENDEMAIL","TOEMAIL"变量（授权码、发送邮箱、目标邮箱）
- 以上全部完成后，我们每次修改download.js文件的fileURL地址，github都会自动进行一次下载。原理请自行百度“github action”。
注意：
- 授权码（EMAILPASS）是指“邮箱第三方登录授权码”，如何获取授权码，以QQ邮箱为例，请点击：http://jingyan.baidu.com/article/fedf0737af2b4035ac8977ea.html
github Action文件（RunDownload.yml）
name: Github wireboy.remote.download
on:
push:
branches:
- main
schedule:
- cron: '* * * * *'
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout codes
uses: actions/checkout@v2
- name: Use Node.js
uses: actions/setup-node@v1
with:
node-version: '12.x'
- name: Run
run: npm install
- run: node download.js
env:
EMAILPASS: ${{ secrets.EMAILPASS }}
SENDEMAIL: ${{ secrets.SENDEMAIL }}
TOEMAIL: ${{ secrets.TOEMAIL }}
源代码（download.js）
const fetch = require("node-fetch");
const fs = require("fs");
const path = require("path");
const progressStream = require('progress-stream');
const nodemailer = require('nodemailer');
//下载 的文件 地址 （https://nodejs.org/dist/v12.18.3/node-v12.18.3-x64.msi）
//vscode地址：https://az764295.vo.msecnd.net/stable/ea3859d4ba2f3e577a159bc91e3074c5d85c0523/VSCodeUserSetup-x64-1.52.1.exe
let fileURL = 'https://az764295.vo.msecnd.net/stable/ea3859d4ba2f3e577a159bc91e3074c5d85c0523/VSCodeUserSetup-x64-1.52.1.exe';
//分割后文件集合
let attachments = [];
//每个邮件附件最大数量
let perEmailAttachmentMaxCount = 1;
//每个附件最大大小
let attachmentMaxSize = 1024 * 1024 * 45;
//下载保存的文件路径
let fileSavePath = path.join(__dirname, path.basename(fileURL));
//缓存文件路径
let tmpFileSavePath = fileSavePath + ".tmp";
//创建写入流
const fileStream = fs.createWriteStream(tmpFileSavePath).on('error', function (e) {
console.error('error==>', e)
}).on('ready', function () {
console.log("开始下载:", fileURL);
}).on('finish', function () {
//下载完成后重命名文件
fs.renameSync(tmpFileSavePath, fileSavePath);
console.log('文件下载完成:', fileSavePath);
const readstream = fs.createReadStream(fileSavePath);
let i = 0;
console.time('readtime');
let patchIndex = 0;
readstream.on('readable', () => {
{
let chunk = readstream.read(attachmentMaxSize);
while (null !== chunk) {
patchIndex = patchIndex + 1;
// console.log('read times:'+patchIndex)
// console.log(fileSavePath+'.email_'+patchIndex);
let emailFilePath = fileSavePath+'.email_'+patchIndex;
let emailFile = fs.createWriteStream(emailFilePath);
emailFile.write(chunk);
emailFile.end();
attachments.push({
filename: patchIndex+'_'+path.basename(fileURL),
path: emailFilePath,
});
chunk = readstream.read(attachmentMaxSize);
}
}
});
readstream.on('close', () => {
console.timeEnd('readtime');
if(attachments.length > 1)
{
attachments.push({
filename: '文件分割合并器(SplitMergeFile).exe',
path: path.join(__dirname, 'SplitMergeFile.exe')
});
}
let sendIndex = 1;
let sendFiles = [];
let total = attachments.length / perEmailAttachmentMaxCount;
var i = 0;
for(i = 0; i < attachments.length; i++)
{
sendFiles.push(attachments[i]);
if(sendFiles.length >= perEmailAttachmentMaxCount)
{
sendEmail(sendFiles,sendIndex,total);
sendFiles = [];
sendIndex += 1;
}
}
if(sendFiles.length > 0)
{
sendEmail(sendFiles,sendIndex,total);
sendFiles = [];
}
});
});
var sendEmail = function(sendFiles,patchIndex,total){
let msg = createEmailMessage(path.basename(fileURL) + '_Part' + patchIndex + '/' + total, sendFiles);
console.log('Send Mail Part_' + patchIndex + '/' + total + ' ' + path.basename(fileURL));
var i;
for(i = 0; i < msg.attachments.length; i++)
{
console.log(msg.attachments[i].path);
}
var transporter = createTransporter();
transporter.sendMail(msg, (error, info) => {
if (error) {
console.log('Error occurred');
console.log(error.message);
return;
}
console.log(path.basename(fileURL) + '_Part' + patchIndex + ' sent successfully! ');
// console.log('Server responded with "%s"', info.response);
transporter.close();
});
};
//请求文件
fetch(fileURL, {
method: 'GET',
headers: { 'Content-Type': 'application/octet-stream' },
// timeout: 100,
}).then(res => {
//获取请求头中的文件大小数据
let fsize = res.headers.get("content-length");
//创建进度
let str = progressStream({
length: fsize,
time: 100 /* ms */
});
// 下载进度
str.on('progress', function (progressData) {
//不换行输出
let percentage = Math.round(progressData.percentage) + '%';
console.log(percentage);
// process.stdout.write('\033[2J'+);
// console.log(progress);
/*
{
percentage: 9.05,
transferred: 949624,
length: 10485760,
remaining: 9536136,
eta: 42,
runtime: 3,
delta: 295396,
speed: 949624
}
*/
});
res.body.pipe(str).pipe(fileStream);
}).catch(e => {
//自定义异常处理
console.log(e);
});
var createTransporter = function(){
return nodemailer.createTransport({
service: 'smtp.163.com',
host: "smtp.163.com",
secureConnection: true,
port:465,
auth: {
user: process.env.SENDEMAIL,//发送者邮箱
pass: process.env.EMAILPASS //邮箱第三方登录授权码
},
debug: true
},{
from: process.env.SENDEMAIL,//发送者邮箱
headers: {
'X-Laziness-level': 1000
}
});
}
console.log('SMTP Configured');
var createEmailMessage = function(subject,sendFiles){
var message = {
// Comma separated lsit of recipients 收件人用逗号间隔
to: process.env.TOEMAIL,
// Subject of the message 信息主题
subject: subject,
// plaintext body
// text: '请查阅附件',
// Html body
html: '<p>下载文件成功!</p><p>下载地址：' + fileURL + '</p><p>感谢github提供的下载渠道，详见博客《利用github给国外文件下载加速》 - https://www.cnblogs.com/zhuxiaoxiao/p/14280136.html</p>',
// Apple Watch specific HTML body 苹果手表指定HTML格式
// watchHtml: '<b>Hello</b> to myself',
// An array of attachments 附件
attachments: sendFiles
// [
// String attachment
// {
// filename: 'notes.txt',
// content: 'Some notes about this e-mail',
// contentType: 'text/plain' // optional,would be detected from the filename 可选的，会检测文件名
// },
// // Binary Buffer attchment
// {
// filename: 'image.png',
// content: Buffer.from('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' +
// '//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' +
// 'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'),
// cid: '00001' // should be as unique as possible 尽可能唯一
// },
// File Stream attachment
// {
// filename: filename,
// path: filepath,
// // cid: '00002' // should be as unique as possible 尽可能唯一
// }
// ]
};
return message;
};
效果图
收件箱
github的action运行日志
下载VisualCode运行图
