文件分片 浏览器文件大小限制 自定义请求头 在一个资源的加载进度停止之后被触发 arrayBuffer 异步 二进制数据缓存区
js 整数限制 浏览器文件大小限制
https://w3c.github.io/FileAPI/#dom-blob-arraybuffer
https://developer.mozilla.org/en-US/docs/Web/API/Blob
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>upload</title>
</head>
<body>
<input type="file" name="file" id="file">
<button id="upload" onClick="upload()">upload</button>
<script>
let bytesPerPiece = 1024 * 1024 * 2; // 每块Part最小100KB（最后一块可以比100KB小），最大5GB。
const UploadId = "D91419EF432C4F208DC09EDF869EE224";
const apiUrl = 'http://test:8082/api/cloud-resource-oss/upload-part/';
// https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch
// https://developer.mozilla.org/zh-CN/docs/Web/API/File/Using_files_from_web_applications
// https://stackoverflow.com/questions/42506376/custom-headers-are-not-added-to-request-object
async function postData(url = '', data = {}) {
// Default options are marked with *
const response = await fetch(url, {
method: 'POST', // *GET, POST, PUT, DELETE, etc.
// mode: 'cors', // no-cors, *cors, same-origin
mode: 'same-origin', // no-cors, *cors, same-origin
cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached
// credentials: 'omit', // include, *same-origin, omit
credentials: 'include', // include, *same-origin, omit
headers: {
'Content-Type': 'application/json',
// 'Content-Type': 'application/x-www-form-urlencoded',
'X-ACCESS-TOKEN': '1',
},
redirect: 'follow', // manual, *follow, error
referrer: 'no-referrer', // no-referrer, *client
body: JSON.stringify(data) // body data type must match "Content-Type" header
});
// return await response.json(); // parses JSON response into native JavaScript objects
return await response;
}
function transformArrayBufferToBase64(buffer) {
var binary = '';
var bytes = new Uint8Array(buffer);
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
//发送请求
async function upload() {
let ele = document.getElementById("file")
const blob = ele.files[0];
let start = 0;
let end;
let index = 0;
const filesize = blob.size;
const filename = blob.name;
//计算文件切片总数
const totalPieces = Math.ceil(filesize / bytesPerPiece);
console.log("filesize, bytesPerPiece, filesize / bytesPerPiece", filesize, bytesPerPiece, filesize / bytesPerPiece)
while (index < totalPieces) {
end = start + bytesPerPiece;
if (end > filesize) {
end = filesize;
}
console.log("index", index)
// The default value is 0.
// The default value is size.
let chunk = blob.slice(start, end);//切割文件
let sliceIndex = blob.name + index;
console.log('chunk', chunk)
let data = {};
data["region_id"] = "cn-beijing";
data["BucketName"] = "aa34";
data["ObjectName"] = "32.6M.zip";
data["PartNumber"] = index + 1;
data["PartSize"] = bytesPerPiece;
data["TotalSize"] = filesize;
data["UploadId"] = UploadId;
let reader = new FileReader();
// 处理loadend事件。该事件在读取操作结束时（要么成功，要么失败）触发。
reader.onloadend = function (evt) {
data["PartBytes"] = transformArrayBufferToBase64(evt.target.result)
try {
const ret = postData(apiUrl, data);
} catch (error) {
console.error(error);
}
}
reader.readAsArrayBuffer(chunk);
start = end;
index++;
}
}
</script>
</body>
</html>
文件校验
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>upload</title>
</head>
<body>
<input type="file" name="file" id="file">
<button id="upload" onClick="upload()">upload</button>
<script>
let bytesPerPiece = 1024 * 1024 * 1; // 每块Part最小100KB（最后一块可以比100KB小），最大5GB。
const UploadId = "8509183A4A5C4DFF8EEC19DF78B5E8EC";
const apiUrl = 'http://test:8082/api/cloud-resource-oss/upload-part/';
// https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch
// https://developer.mozilla.org/zh-CN/docs/Web/API/File/Using_files_from_web_applications
// https://stackoverflow.com/questions/42506376/custom-headers-are-not-added-to-request-object
async function postData(url = '', data = {}) {
// Default options are marked with *
const response = await fetch(url, {
method: 'POST', // *GET, POST, PUT, DELETE, etc.
// mode: 'cors', // no-cors, *cors, same-origin
mode: 'same-origin', // no-cors, *cors, same-origin
cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached
// credentials: 'omit', // include, *same-origin, omit
credentials: 'include', // include, *same-origin, omit
headers: {
'Content-Type': 'application/json',
// 'Content-Type': 'application/x-www-form-urlencoded',
'X-ACCESS-TOKEN': 'token9999999',
'X-Appid': 'aliid1',
'X-Proxy-Url': 'http://120.79.10.103:8004',
},
redirect: 'follow', // manual, *follow, error
referrer: 'no-referrer', // no-referrer, *client
body: JSON.stringify(data) // body data type must match "Content-Type" header
});
// return await response.json(); // parses JSON response into native JavaScript objects
return await response;
}
function transformArrayBufferToBase64(buffer) {
let binary = '';
const bytes = new Uint8Array(buffer);
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
let data = {};
data["region_id"] = "cn-beijing";
data["BucketName"] = "aaaaaaaaaaa434343434";
data["ObjectName"] = "32.6M.zip";
var ele;
var blob;
async function handlerOrder(jobList) {
// https://developers.google.com/web/fundamentals/primers/async-functions
/*
async function logInOrder(urls) {
for (const url of urls) {
const response = await fetch(url);
console.log(await response.text());
}
}
代码简洁得多，但我的第二次获取要等到第一次获取读取完毕才能开始，以此类推
*/
for (let i in jobList) {
console.log("i", i)
const job = jobList[i]
// blob, start, end, index,
const start = job["start"]
const end = job["end"]
const index = job["index"]
const partSize = job["PartSize"]
const response = await worker(start, end, index, partSize);
console.log("job,response", job, await response)
}
}
async function worker(start, end, index, partSize) {
// The default value is 0.
// The default value is size.
let chunk = blob.slice(start, end);//切割文件
data["PartNumber"] = index + 1;
data["PartSize"] = partSize;
let reader = new FileReader();
reader.onloadstart = function (evt) {
}
// 处理loadend事件。该事件在读取操作结束时（要么成功，要么失败）触发。
reader.onloadend = function (evt) {
data["PartBytes"] = transformArrayBufferToBase64(evt.target.result)
try {
return postData(apiUrl, data);
} catch (error) {
console.error(error);
}
}
reader.readAsArrayBuffer(chunk);
}
async function upload() {
ele = document.getElementById("file")
blob = ele.files[0];
filesize = blob.size;
filename = blob.name;
let start = 0;
let end;
let index = 0;
//计算文件切片总数
const totalPieces = Math.ceil(filesize / bytesPerPiece);
console.log("filesize, bytesPerPiece, filesize / bytesPerPiece", filesize, bytesPerPiece, filesize / bytesPerPiece)
data["TotalSize"] = filesize;
data["UploadId"] = UploadId;
let partSize = bytesPerPiece;
let passIndex = []
let jobList = []
// blob, start, end, index, bytesPerPiece
// 建立任务池
while (index < totalPieces) {
end = start + bytesPerPiece;
if (end > filesize) {
end = filesize;
partSize = filesize - start;
}
console.log("index", index)
data["PartNumber"] = index + 1;
data["PartSize"] = partSize;
let d = {}
d["start"] = start
d["end"] = end
d["index"] = index
d["PartSize"] = partSize
jobList.push(d)
index++;
start = end;
}
// 处理任务
// 文件校验
await handlerOrder(jobList)
}
</script>
</body>
</html>
TODO 零拷贝
https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/TypedArray
一个TypedArray 对象描述一个底层的二进制数据缓存区的一个类似数组(array-like)视图。事实上，没有名为 TypedArray的全局对象，也没有一个名为的 TypedArray构造函数。
当传入length参数时，一个内部数组缓冲区会被创建在内存中。该缓存区的大小是传入的length乘以数组中每个元素的字节数（BYTES_PER_ELEMENT），每个元素的值都为0。(译者注：每个元素的字节数是由具体的构造函数决定的，比如Int16Array的每个元素的字节数为2，Int32Array的每个元素的字节数为4)
FileReader 对象允许Web应用程序异步读取存储在用户计算机上的文件（或原始数据缓冲区）的内容，使用 File 或 Blob 对象指定要读取的文件或数据。
其中File对象可以是来自用户在一个<input>元素上选择文件后返回的FileList对象,也可以来自拖放操作生成的 DataTransfer对象,还可以是来自在一个HTMLCanvasElement上执行mozGetAsFile()方法后返回结果。
let message = 'abcdABCD';let enc = new TextEncoder();let tr=enc.encode(message);
undefined
tr
Uint8Array(8) [97, 98, 99, 100, 65, 66, 67, 68]
tr[0]
97
字符串转ArrayBuffer Uint8Array
浙公网安备 33010602011771号