React：使用 Next.js 损坏的分块文件上传

Question

我正在使用 axios 和 Next.js 开发文件上传功能（任何类型的文件）。我将上传文件的块大小限制为 768kB（因为 next.js server dev 最多只允许 1MB ）对于图像和视频文件，它可以正常工作。上传的文件与原始文件的文件大小完全相同。但对于 Android 应用程序 (.apk)，它不会。上传的APK文件比原来的小。

前端代码： src/lib/api-helpers.js

const _=require('lodash');

const helpers={
  /**
   * Helper for uploading chunked file
   * @param {string} url 
   * @param {File} file 
   * @param {(progress:number)=>void} onUploadProgress 
   * @param {(attachment:import('models/Attachments').IAttachmentsSafe)=>void} onUploadFinish 
   * @returns void
   */
  chunkUpload(url,file,onUploadProgress,onUploadFinish)
  {
    const config=typeof url=='object'?_.extend({"chunkSize":786432},url):{
      "url":url,
      "chunkSize":786432
    };
    if(!file && config['file']) {
      file=config.file;
      delete config.file
    };
    if(!onUploadProgress && config['onUploadProgress'])
    {
      onUploadProgress=config.onUploadProgress;
      delete config.onUploadProgress;
    }
    if(!onUploadFinish && config['onUploadFinish'])
    {
      onUploadFinish=config.onUploadFinish;
      delete config.onUploadFinish;
    }

    if(!onUploadProgress)
    {
      console.error('No onUploadProgress');
      return;
    }

    if(!onUploadFinish)
    {
      console.error('No onUploadFinish');
      return;
    }
    
    var n,response;
    config.url=helpers.getFullUrl(config.url);
    const maxChunk=Math.ceil(file.size/config.chunkSize);
    const fileData={
      tempId:helpers.randHex(8),
      fileName:file.name,
      mime:file.type,
      fileSize:file.size,
      parts:maxChunk,
      index:0,
      content:""
    };

    if(config['additionalData']) _.extend(fileData,config.additionalData);

    const canceller={
      c:null,
      cancelled:false,
      cancel()
      {
        if(!this.c) return;

        this.c();
        this.cancelled=true;
      }
    }
    const CancelToken = new axios.CancelToken((c)=>{
      canceller.c=c;
    });

    const chunkRe=new RegExp('.{1,'+config.chunkSize+'}','g');

    (async function(){
      var base64File=await helpers.toBase64(file);
      base64File=base64File.replace(base64File.substr(0,base64File.search(',')+1),'');
      const base64Arr=base64File.match(chunkRe);

      for(const [i,item] of base64Arr.entries())
      {
        if(canceller.cancelled) break;
        fileData.index=i;
        fileData.content=item;
    
        try{
          response=await axios({
            url:config.url,
            method:"POST",
            data:fileData
          },{
            cancelToken:CancelToken
          });
        }catch(error){
          console.log(error);
        }finally{
          if(response.status==200 && response.data.success)
          {
            if(i==maxChunk-1) onUploadFinish(response.data.result);
            else if(typeof response.data.result.size=='number') onUploadProgress((i+1)*100/maxChunk);
          }
        }
      }
    })();

    return canceller;
  },
  /**
   * Get full path based on file system, only runs on server
   * @param {string} path 
   * @returns string
   */
  getFullPath(path)
  {
    if(path.indexOf(global.WORKSPACE_PATH)===0) return path;
    return pathJoin(global.WORKSPACE_PATH,path);  
  },
  /**
   * Get full file URL
   * @param {string} url 
   * @returns string
   */
  getFullUrl(url)
  {
    if(/^http/.test(url)||(/^\//.test(url))) return url;
  
    var baseUri;
  
    //if run at web browser
    if(typeof document=='object'?document && settings:false) baseUri=settings.baseUrl;
    //if run at server
    else if(process?process.env:false) baseUri=process.env.BASE_URI!=undefined?process.env.BASE_URI:'';

    return baseUri.replace(/[\/]+$/,'')+'/'+url.replace(/^[\/]+/,'');
  },
  randHex(size)
  {
    var maxlen = 16,
      min = Math.pow(16,Math.min(size,maxlen)-1),
      max = Math.pow(16,Math.min(size,maxlen)) - 1,
      n   = Math.floor( Math.random() * (max-min+1) ) + min,
      r   = n.toString(16);

    while ( r.length < size ) r = r + randHex( size - maxlen );

    return r;
  },
  toBase64(file)
  {
    return new Promise((resolve, reject) => {
      const reader = new FileReader();
      reader.onload = () => resolve(reader.result);
      reader.onerror = error => reject(error);
      reader.readAsDataURL(file);
    });
  }
};
module.exports=helpers;

src/models/database.js

const mongoose=require('mongoose');

var cached = global.mongo;
if (!cached) cached = global.mongo = { conn: null, promise: null };

const database={
  async function setUpDb() {
    if(cached.conn?cached.conn.client.isConnected():false) {
      return cached.conn;
    }
  
    if (!cached.promise) {
      cached.conn= await mongoose.createConnection(process.env.MONGODB_URI,{
        useNewUrlParser:true,
        useUnifiedTopology:true,
        dbName:process.env.MONGODB_DB
      });
    
      if(cached.conn.readyState!==1) throw new Error("connection not connected");
    }

    return cached.conn;
  },
  async function setupModel(modelName,modelDef,collectionName)
  {
    const conn=await database.setUpDb();

    if(conn.models[modelName]) return conn.models[modelName];

    return conn.model(modelName,modelDef,collectionName);
  }
}
module.exports=database;

src/models/Attachment.js

if(typeof window=='object'?window && document:false)
{
    module.exports={};
}else{
  const {setupModel}=require('../middlewares/database');
  const Mongoose=require('mongoose');
  const {getFullUrl}=require('../lib/api-helpers');

  const attachmentSchema=new Mongoose.Schema({
    name:{
        type:String,
        index:true,
        unique:true
    },
    fileName:String,
    filePath:String,
    fileSize:Number,
    fileUrl:String,
    mime:{
        type:String,
        index:true
    },
    attributes:Object,
    isSong:{
        type:Boolean,
        default:false,
        index:true
    },
    thumbnail:Object,
    posterPic:{
        type:Mongoose.Types.ObjectId,
        ref:'Attachments'
    }
  });

  //Covert to "safe" JS object, without filePath
  attachmentSchema.virtual('safe').get(async function()
  {
        const ret=JSON.parse(JSON.stringify(this.toObject()));
        delete ret.filePath;
        ret.fileUrlFull=getFullUrl(this.fileUrl);

        if(this.thumbnail) ret.thumbnail={
            fileName:this.thumbnail.fileName,
            fileSize:this.thumbnail.fileSize,
            fileUrlFull:getFullUrl(this.thumbnail.fileUrl),
            attributes:this.thumbnail.attributes
        };

        if(this.posterPic) {
            if(!this.populated('posterPic')) await this.populate('posterPic').execPopulate();
            ret.posterPic=await this.posterPic.safe;
        }
        
        return ret;
    }
  );

  async function asyncAttachments()
  {
    return await setupModel('Attachments',attachmentSchema,'attachments');
  };

  const AttachmentModule=module.exports=asyncAttachments;

  AttachmentModule.createFromFile=async function(fullPath,attrs)
  {
    if(!attrs) attrs={
        fileSize:0,
        mime:''
    };
    let comparePath=getFullPath('public'),karaokeChannel='left';

    const path=require('path');
    const fs=require('fs');
    const contentType=require('mime-types').contentType;

    if(fullPath.substr(0,comparePath.length)!=comparePath) fullPath=path.join(comparePath,fullPath);

    if(!fs.existsSync(fullPath)) throw ReferenceError('FILE_NOT_FOUND');

    var fileSize=0,mime='';

    if(attrs?!(fileSize=(attrs['fileSize']?attrs.fileSize:'')):true)
    {
        const stats=fs.statSync(fullPath);
        fileSize=stats.size;
    }

    if(attrs?!(mime=attrs['mime']?attrs.mime:''):true) mime=contentType(path.basename(fullPath));

    if(attrs['fileSize']!=undefined) delete attrs.fileSize;
    if(attrs['mime']!=undefined) delete attrs.mime;

    const doc={
        name:path.basename(fullPath),
        fileName:path.basename(fullPath),
        filePath:fullPath.substr(getFullPath('').length+1).replace(/\/g,'/'),
        fileSize,
        fileUrl:"",
        mime,
        attributes:null
    };

    if(attrs['posterPic'])
    {
        doc.posterPic=attrs.posterPic;
        delete attrs.posterPic;
    }

    let names=path.basename(fullPath).split('.');
    let ext=names.pop();
    doc.name=names.join('.');
    //if Windows server
    doc.fileUrl=(/^public[\/]/i.test(doc.filePath)?doc.filePath.substr(7):doc.filePath).replace(/\/g,'/');

    if(/^image\//.test(attrs.mime))
    {
        const sharp=require('sharp')(fullPath);
        const metadata=await sharp.metadata();

        doc.attributes={
            width:metadata.width,
            height:metadata.height
        };
        const thumbnailSize=await (require('../lib/Settings')).getItem('thumbnailSize');

        if(doc.attributes.width>thumbnailSize.width||doc.attributes.height>thumbnailSize.height)
        {
            let scale=1;

            if(doc.attributes.width>doc.attributes.height) scale=thumbnailSize.width/doc.attributes.width;
            else scale=thumbnailSize.height/doc.attributes.height;

            const nw=doc.attributes.width*scale,nh=doc.attributes.height*scale;
            const thumbnailFile=fullPath.substr(0,fullPath.length-(ext.length+1))+`-${nw}x${nh}.${ext}`;

            if(await sharp.resize(nw,nh).toFile(thumbnailFile))
            {
                doc.thumbnail={
                    fileName:path.basename(thumbnailFile),
                    filePath:fullPath.substr(getFullPath('').length+1),
                    fileSize:0,
                    fileUrl:"",
                    attributes:{
                        width:nw,
                        height:nh
                    }
                };
                
                doc.thumbnail.fileUrl=doc.thumbnail.filePath.substr(0,7)=='public/'?doc.thumbnail.filePath.substr(7):doc.thumbnail.filePath;
            }
        }
    }

    return await (await asyncAttachments()).create(doc);
  }
}

src/pages/api/attachments.js

import nc from 'next-connect';
import {join as pathJoin,dirname} from 'path';
import { openSync, closeSync, appendFileSync,existsSync as fsExists,renameSync,chmodSync,rmdirSync } from 'fs';
import {getFullPath} from 'lib/api-helpers';
import {mkdir} from 'shelljs';
import {platform} from 'os';
import asyncAtt, {createFromFile} from 'models/Attachments';
import { ObjectId } from 'mongodb';

const handler=nc();
handler.post(async function(req,res){
  if(req.session.files==undefined) req.session.files={};

  let fd=0,sess;
  let sessFiles=req.session.get('files');
  const autoAttach=typeof req.body['autoAttach']=='boolean'?req.body.autoAttach:true;

  if(sessFiles?req.session.files[req.body.tempId]==undefined:true)
  {
      sess={...req.body};

      if(sess['autoAttach']!=undefined) delete sess.autoAttach;

      delete sess.tempId;
      delete sess.content;
      sess.mime=sess.mime.split(';')[0];
      sess.tempFile=getFullPath(pathJoin('.tmp','uploads',req.body.tempId,sess.fileName));

      if(!sessFiles) sessFiles={};

      sessFiles[req.body.tempId]=sess;
  }else{
      sessFiles[req.body.tempId].index=req.body.index;
      sess=sessFiles[req.body.tempId]; 
  }

  req.session.set('files',sessFiles);

  if(!fsExists(dirname(sess.tempFile))) mkdir('-p',dirname(sess.tempFile));

  const buffer=Buffer.from(req.body.content,'base64');
  let result={
      index:req.body.index,
      size:buffer.length
  };
        
  fd=openSync(sess.tempFile,'as');
  appendFileSync(fd,buffer);
  //ensure file is written correctly, wait for 500ms
  await (new Promise((resolve)=>{
      setTimeout(() => {
         resolve();
      }, 500);
  }));
  closeSync(fd);
        
  if(sess.index==sess.parts-1)
  {
    if(autoAttach)
    {
        const names=sess.fileName.split('.'),now=new Date();
        const ext=names.pop(),
        //baseDir is public/uploads/YYYY-MM/
        baseDir=getFullPath(pathJoin('public','uploads',now.getFullYear()+'-'+(now.getMonth()+1).toString().padStart(2,'0')));                
            
        if(!fsExists(baseDir)) {
            await mkdir('-p',baseDir);

            if(platform()!='win32') chmodSync(baseDir,0o775);
        }

        const newFile=pathJoin(baseDir,names.join('.')+'-'+req.body.tempId+'.'+ext);
        //move uploaded file from temp directory to destined uploads dir
        renameSync(sess.tempFile,newFile);
        //remove temporary directory
        rmdirSync(dirname(sess.tempFile));

        if(platform()!='win32') chmodSync(newFile,0o664);

        delete req.session.files[req.body.tempId];

        try{
           const attachment=await createFromFile(newFile,{
                 fileSize:sess.fileSize,
                 mime:sess.mime
           });

           if(attachment) result=await attachment.safe;
        }catch(err){
           res.json({
               success:false,
               result:err
           });
           return;
        }
    }else{
        result.tempFile=sess.tempFile;
    }
 }

 res.status(200).json({
    success:true,
    result
 });
});
export default handler;

原始MP4文件：

已上传的 MP4 文件：

原始APK文件：

上传的APK文件：

是不是少了什么？无论如何提前谢谢

Answer 1

我终于找到了自己的答案。 Slice 文件，将其转换为BASE64，然后上传。有了这个，它将发送一个精确的副本，一个字节一个字节。我将尝试制作一个用于分块文件上传的节点模块。

React：使用 Next.js 损坏的分块文件上传

React: Corrupted Chunked File Upload with Next.js

chunked

node.js

reactjs

axios

next.js