node 实现大体积文件上传

demo下载地址: http://chaoliumeihai.com/demo/node_update.rar

 

首先是后台服务创建 采用的是express框架 + node

const express = require('express'); //1
const url  = require('url');
// 引入body-parser模块
const bodyParser = require('body-parser');

const updatefile = require('./src/updateFile.js')

var app = express();

// 支持跨域 //req, res, next
app.all('*', function ( res , req ,next) {
    req.header('Access-Control-Allow-Origin', '*');
    req.header('Access-Control-Allow-Headers', 'Content-Type,Authorization');
    req.header('Access-Control-Allow-Methods', '*');
    // res.header('Content-Type', 'application/json;charset=utf-8');
    // res.header('Content-Type', 'text/html;charset=utf-8');
    next();
});

// 支持静态访问路径
app.use(express.static(__dirname.split("/src")[0]));

// 配置body-parser模块
app.use(bodyParser.urlencoded({limit: '500mb', extended: false }));
app.use(bodyParser.json({limit: '500mb'}));

app.post("/updateFile", function(res,req){
    // 获取前段传入的参数
    let param = res.body;
    updatefile.addFile({
        name: param.name, 
        id: param.id, 
        base64: param.base64, 
        length: param.length, 
        index: param.index
    }).then((data)=>{
        req.json({is: true , id: data.id});
    })
})

app.get("/mergeFile", function(res,req){
    // 获取前段传入的参数
    let requset_url = res.url;
    let param = url.parse(requset_url,true).query;
    updatefile.merge(param.id).then((data)=>{
        req.json({data: data});
    });
})

var server = app.listen(3333, function(data){

    let host = server.address().address;
    host === "::" ? host = "127.0.0.1" : host;
    const port = server.address().port;

    console.log(server.address())

    console.log("---------------------------")
    console.log("成功开启服务器：")
    console.log(host + ":" + port);
    console.log("测试上传demo地址：")
    console.log(host + ":" + port+ "/demo/update.html");
    console.log("---------------------------")

})

 

这里的updateFile是自己写的小插件

var fs = require("fs")

// 存储结果配置
let fileData = [];

// 常用配置
let _config = {
    // 上传文件临时存储位置
    path: './src/updateFile/temporary',
    // 上传文件最终保存地址
    paths: './src/updateFile/document'
}

// 创建一个上传的实例
let updateFileProcess = function(length, name, savepath, path){
    
    // 随机一个id数字
    this.id = "updateFile_" + new Date()*1 + parseInt( Math.random()*100000 );
    
    // 名称
    this.name = name;

    // 编码
    this.code = '';
    
    // 文件段数
    this.fileCount = 0;

    // 临时文件存储路径
    this.savepath = savepath;

    // 实际文件保存路径
    this.path = path;
    
    // 文件的总大小字节
    this.fileLength = length;

    // 临时文件个数
    this.fileCount = 0;
    
    // 接受数据
    this.setFileBase64 = setFileBase64;
    
    // 创建临时文件
    this.createTemporaryBase64 = createTemporaryBase64;

    // 删除临时文件
    this.deleteFolder = deleteFolder;

    // 合并数据
    this.mergeFile = mergeFile;
    
}


let deleteFolder = function() {
    var path = this.savepath+"/"+this.id;
    let files = [];
    if( fs.existsSync(path) ) {
        files = fs.readdirSync(path);
        files.forEach(function(file,index){
            let curPath = path + "/" + file;
            if(fs.statSync(curPath).isDirectory()) {
                deleteFolder(curPath);
            } else {
                fs.unlinkSync(curPath);
            }
        });
        fs.rmdirSync(path);
    }
}

// 合并数据
let mergeFile = function(){

    return new Promise((a)=>{
        // 获取临时存储路径
        var paths = this.savepath;
        // 获取实际保存路径
        var path = this.path;
        // 获取临时存储文件个数
        var index = this.fileCount;
        // console.log("fileCount:" + index);
        // 开始遍历文件数据
        // var base64 = "";
        // 文件id；
        var id = this.id;
        // for( var i = 0; i < index ; i++ ){
        //     console.log("获取文件开始："+paths+"/"+id+"/"+i+".das");
        //     base64 += fs.readFileSync(paths+"/"+id+"/"+i+".das","utf8");
        // }
        // console.log("获取文件字节："+base64.length);
        var date = new Date();
        var timeString = date.getFullYear().toString()+"_"+(date.getMonth()+1).toString()+"_"+date.getDate().toString();
        fs.mkdir(path+"/"+timeString,{recursive: true}, ()=>{
            // 获取所有的base64合集之后 进行保存文件步骤
            // 将base64保存为字节进行存储
            // var dataBuffer = new Buffer.from(base64, 'base64');
            // console.log("合并文件开始："+path+"/"+id+"/"+this.name);
            // console.log("合并文件字节："+dataBuffer.byteLength());
            // fs.writeFile(path+"/"+timeString+"/"+this.name,"",function(){
                let p = path+"/"+timeString+"/"+this.id+"."+this.name.split(".")[1];
                let dhh = fs.createWriteStream(p);

                let i = 0;
                // recursive function
                let main = ()=> {
                    if ( i >= index) {
                        dhh.end();
                        // console.log("合并完成")
                        this.deleteFolder();
                        // console.log("删除掉缓存文件")
                        // console.log("路径:"+"/"+timeString+"/"+id+"."+this.name.split(".")[1])
                        a({ path: "/"+timeString+"/"+id+"."+this.name.split(".")[1] });
                        return;
                    }
                    currentfile = paths+"/"+id+"/"+i+".das";
                    // console.log(currentfile);
                    i++;
                    stream = fs.createReadStream(currentfile);
                    stream.pipe(dhh, {end: false});
                    stream.on("end", function() {
                        // console.log(currentfile + ' appended' + '-' + index);
                        main();        
                    });
                }
                main();
            // });
        })

    })

}

// 接受数据 
let setFileBase64 = function(param){
    return new Promise((a,b)=>{
        // 获取这个文件的段数
        var fileCountIndex = param.index;
        // 获取文件的段的base64文件
        var fileBase64 = param.base64;
        // 创建临时文件
        this.createTemporaryBase64(fileBase64,fileCountIndex).then(()=>{
            a({
                id: this.id
            });
        });
    })
}

let createTemporaryBase64 = function(base64,index){
    return new Promise((a)=>{
        // 获取临时下载文件保存路径
        var path = this.savepath;
        // 文件id；
        var id = this.id;
        // console.log("path:"+path+"/"+id)
        fs.mkdir(path+"/"+id,{recursive: true}, ()=>{
            // console.log("创建成功")
            // console.log(base64)
            // var data = fs.readFileSync('./data1/1.files', 'utf8');
            // console.log("读取成功");
            // 将base64保存为字节进行存储
            var dataBuffer = new Buffer.from(base64, 'base64');
            // 将base64存储进入临时文件中 并创建
            fs.writeFile(path+"/"+id+"/"+index+".das",dataBuffer,()=>{
                // console.log(path+"/"+id+"/"+index+".das");
                // console.log("*****************");
                this.fileCount ++;
                a();
            })
        });
    })
}

// 开始一个新的对象
let addFile = function(param){

    return new Promise((a,b)=>{
        // 默认缓存路径
        var path = _config.paths;
        // 完成保存路径
        var savepath = _config.path;
        // 名称
        var name = param.name;
        // 文件字节大小
        var length = param.length;
        // 文件下标
        var index = param.index;
        // 文件id
        var id = param.id;
        // 文件base64文件
        var base64 = param.base64;
        // 根据id查询是否已经创建了对象
        let updatefs = fileData.find((e)=>{ return e.id === id });
        if( !updatefs ){
            // 如果实例化对象不存在
            updatefs = new updateFileProcess(length, name, savepath, path);
            // 讲对象储存在集合中
            fileData.push(updatefs)
        }
        // 如果updatefs实例对象是存在的则继续添加创建
        updatefs.setFileBase64({
            base64,
            index,
        }).then((res)=>{
            // 成功完成后
            a(res);
        })
    })
}

let merge = function(id){
    return new Promise ((a,b)=>{
        // 从和集中根据id查询出实例
        let updatefs = fileData.find((e)=>{ return e.id === id });
        // 触发实例的合并功能
        updatefs.mergeFile().then((e)=>{
            // console.log(JSON.stringify(e));
            // 回调
            a({isSuccess: true, path: e.path, id: id });
        });
    })
}

module.exports = { updateFileProcess , addFile , merge }

 

然后就是前段调用这个接口服务的代码

<!DOCTYPE html>
<html>

    <head>
        <meta charset="UTF-8">
        <!-- import CSS -->
        <link rel="stylesheet" href="./js/index.css">
        <!-- import Vue before Element -->
        <script src="./js/vue.js"></script>
        <!-- import JavaScript -->
        <script src="./js/index.js"></script>
        <!-- import axios -->
        <script src="./js/axios.min.js"></script>
        <script src="./js/updateFile.js" rel='stylesheet'></script>
    </head>

    <style>
        .updateBox {
            height: 150px;
            padding: 15px;
            line-height: 150px;
            text-align: center;
        }
        .ralte {
            height: 20px;
            background: #ffff00;
            width: 30%;
            border-radius: 15px;
            text-align: center;
            transition: width 1s;
        }
        .info {
            margin-top: 5px;
            color: #000000;
        }
    </style>

    <body>
        <div id="app">
            <el-card class="box-card">
                <div slot="header" class="clearfix">
                    <span>上传测试</span>
                </div>
                <div class="text">
                    <div class='updateBox' @click='updateClick' @drop='dropFile' ondragover='event.preventDefault()'>点击上传(或者直接拖進來)</div>
                    <div class='ralte' v-show='rate > 0' style='width: 0%;' :style='"width:"+rate*100+"%"'>{{rate * 100}}%</div>
                    <div v-show='path' class='info' >{{"当前文件保存路径为：/src/updateFile/" + path}}</div>
                </div>
            </el-card>

        </div>
    </body>
    <script>

        document.ondragover = function (e) {
            e.preventDefault();  //只有在ondragover中阻止默认行为才能触发 ondrop 而不是   ondragleave
        };
        document.ondrop = function (e) {
            e.preventDefault();  //阻止 document.ondrop的默认行为  *** 在新窗口中打开拖进的图片
        };
        /*拖拽的源对象----- 客户端的一张图片 */
        /*拖拽目标对象-----div#container  若图片释放在此元素上方，则需要在其中显示*/
        // container.ondragover = function (e) {
        //     e.preventDefault();
        // };

        new Vue({
            el: '#app',
            data: function () {
                return {
                    visible: false,
                    tableData: [],
                    inputFile: null,
                    rate: 0,
                    path: ""
                }
            },
            mounted (){
                // this.getTable();
            },
            methods: {
                updateClick (){
                    this.rate = 0;
                    this.path = "";
                    var ue = new update();
                    ue.oncomplete = (res)=>{
                        this.rate = res.rate;
                        if( res.isload === true ){
                            this.path = res.path;
                        }
                    }
                    ue.selectFlie().then(()=>{
                        ue.init();
                    });
                },

                dropFile (res){
                    this.rate = 0;
                    this.path = "";
                    var ue = new update();
                    ue.oncomplete = (res)=>{
                        this.rate = res.rate;
                        if( res.isload === true ){
                            this.path = res.path;
                        }
                    }
                    ue.setFiles(res.dataTransfer.files[0]).then(()=>{
                        ue.init();
                    });
                    
                },

                selectFile (){
                    
                },

                inputFileChange (){

                },
            }
        })
    </script>

</html>

当然 前段还得使用配套后台写的引入插件

// import { updateVideo , mergeVideo } from '@/api/updateFile'
let update = window.update = function(name,base64,type){

    this.name = name;

    this.type = type;

    this.id = "";

    this.base64 = base64;

    // 每次上传字节长度
    this.countLength = 5000000;

    // 分段处理数据结果
    this.baseArray = [];

    // 处理文件进行分段处理
    this.handleFile = handleFile;

    // 加载文件
    this.updateFile = updateFile;

    // 合并文件
    this.mergeFile = mergeFile;

    // 每次成功的回调函数
    this.oncomplete = null;

    // 选择文件
    this.selectFlie = selectFlie;
    this.setFiles = setFiles;

    // 初始化事件
    this.init = init;

    // 保存的服务器路径
    this.path = '';

    // 初始化处理
    // this.init();

    return this;

}

const selectFlie = function(){
    var update = this;
    return new Promise((suc)=>{
        var file = document.createElement("input");
        file.type = 'file';
        file.onchange = function(){
            update.setFiles(this.files[0]).then(suc)   
        }
        file.click();
    })
}

const setFiles = function(files){
    var update = this;
    return new Promise((suc)=>{
        var rs = new FileReader();
        rs.onload = function(res){
            update.base64 = res.currentTarget.result.split(",")[1];
            suc(update.base64, update); 
        }
        update.name = files.name;
        update.type = files.type || files.name.split(".").pop();
        rs.readAsDataURL(files);
    })
}

const init = function(){
    // 对数据进行分段
    this.handleFile()
    // 开始对数据进行上传
    this.updateFile();
}

// 将文件进行分段处理
const handleFile = function(){
    var countLength = this.countLength;
    var base64 = this.base64;
    var min = 0;
    var max = countLength;
    var b = "";
    for( ;; ){
        b = base64.substring(min,max);
        if( b == "" ){ 
            break;
        }
        this.baseArray.push( b );
        min = max;
        max = min + countLength;
    }
    console.log("视频文件数据分段处理结束");
    console.log("分段数量为：" + this.baseArray.length);
}

const updateFile = function(index = 0){
    var name = this.name;
    var base64 = this.baseArray[index];
    if( !base64 ){ 
        this.mergeFile().then((res)=>{
            this.oncomplete && this.oncomplete({
                index, name, id, res: null, isload: true,
                rate: (index)/this.baseArray.length,
                path: res.data.path
            });
            console.log("合成成功");  
        })
        return console.log("请求结束"); 
    }
    var length = this.base64.length;
    var id = this.id;
    updateVideo({
        name: name,
        id: id,
        base64: base64,
        length: length,
        index: index
    }).then((res)=>{
        var rate = (index+1)/this.baseArray.length;
        this.oncomplete && this.oncomplete({
            index, name, id, res, 
            isload: false, rate: rate
        });
        this.id = res.data.id;
        setTimeout(()=>{
            this.updateFile(++index);
        },10)
    })
}

const updateVideo = function(param){
    return axios({
        url: "/updateFile",
        method: "post",
        data: param || {
            name: "123456",
            id: "1",
            base64: "53as54da5s4d65a4sdasd",
            length: 2000,
            index: 1
        }
    })
}

const mergeFile = function(){
    return new Promise((a)=>{
        var id = this.id;
        if( !id ) return;
        mergeVideo({
            id: id,
            type: this.type,
            name: this.name
        }).then((res)=>{
            a(res.data);
        })
    })
}

const mergeVideo = function(param){
    return axios({
        url: "/mergeFile",
        method: "get",
        params: param
    })
}

// export { update }

具体的可以下载我上传的实体demo文件运行看看  
demo下载地址: http://chaoliumeihai.com/demo/node_update.rar