试图获取一个非常大的文件的哈希 (SHA-512)，在 javascript 中超过 2.5 G

Question

我正在尝试获取一个大文件的 SHA512。 2.5 G 甚至更大的文件。我的方法是创建一个数组缓冲区，由 crypto.subtle.digest API.

消化

问题是我总是有一个

Array buffer allocation failed

是不是我的块大小，数组缓冲区有限制。我不知道了？或者也许有更好的方法来获取哈希摘要而不是使用完整的数组缓冲区？

// received a file object 
    function CalculateHash(file)
    {
 var obj = { File : file}; 
    
     var chunkSize = 10485760;
      const chunksQuantity = Math.ceil(obj.File.size / chunkSize);
      const chunksQueue = new Array(chunksQuantity).fill().map((_, index) => index).reverse();
      
    
    
      
      var buffer = null;
      
      reader.onload =  async function (evt) {
        if (buffer == null) {
          buffer = evt.currentTarget.result;
        } else {
          var tmp = new Uint8Array(buffer.byteLength + evt.currentTarget.result.byteLength);
          tmp.set(new Uint8Array(buffer), 0);
          tmp.set(new Uint8Array(evt.currentTarget.result), buffer.byteLength);
          buffer = tmp;
        }    
        readNext();
      }
      
    
        var readNext = async function () {
          if (chunksQueue.length > 0) {
            const chunkId = chunksQueue.pop();
            const sentSize = chunkId * chunkSize;
            const chunk = obj.File.slice(sentSize, sentSize + chunkSize);
            reader.readAsArrayBuffer(chunk);
    
          } else {
            var x = await digestMessage(buffer);
            hash.SHA512 = x.toUpperCase();   
            buffer = null;
           
          }
      }
    
      readNext();
    }
    
    async function digestMessage(file) {  
      const hashBuffer = await crypto.subtle.digest('SHA-512', file);           // hash the message
      const hashArray = Array.from(new Uint8Array(hashBuffer));                     // convert buffer to byte array
      const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string
      return hashHex;
    }

Answer 1

基于@ArtjomB。回答，问题是progressiveHash。 BufferArray和浏览器的限制。

这是最终的工人代码。它将这两种方法与比 cryptoJS 库更快的本地摘要混合在一起。如果文件大于 1Gb，我们使用 CryptoJS 库，否则我们使用本机浏览器摘要。欢迎任何建议！

    var window = self;
        var document = {};
        self.importScripts("/Crypto.min.js");
        
        
        onmessage = async function (args) {
          var obj = args.data;  
      var reader = new FileReader();
      var hash = {};
      var chunkSize = 10485760;
      var largeFileTrigger = 1048576000;
      const chunksQuantity = Math.ceil(obj.File.size / chunkSize);
      const chunksQueue = new Array(chunksQuantity).fill().map((_, index) => index).reverse();
      var isLargeFile = obj.File.size > largeFileTrigger;
      var buffer = null;
      var progressiveArray = [];
      reader.onload = async function (evt) {
        if (isLargeFile) {
          progressiveArray.push(evt.currentTarget.result);
        } else {
          if (buffer == null) {
            buffer = evt.currentTarget.result;
          } else {
            var tmp = new Uint8Array(buffer.byteLength + evt.currentTarget.result.byteLength);
            tmp.set(new Uint8Array(buffer), 0);
            tmp.set(new Uint8Array(evt.currentTarget.result), buffer.byteLength);
            buffer = tmp;
          }
        }    
        readNext();
      }
    
        var readNext = async function () {
          if (chunksQueue.length > 0) {
            const chunkId = chunksQueue.pop();
            const sentSize = chunkId * chunkSize;
            const chunk = obj.File.slice(sentSize, sentSize + chunkSize);
            reader.readAsArrayBuffer(chunk);
    
          } else {
            var hexHash = null;
            if (isLargeFile) {
              var sha = CryptoJS.algo.SHA512.create();
              for (var i = 0; i < progressiveArray.length; i++) {
                sha.update(arrayBufferToWordArray(progressiveArray[i]));
              }
              hexHash = sha.finalize().toString();          
            } else {
              hexHash = await digestMessage(buffer);
            }     
           
            SHA512 = hexHash.toUpperCase();    
            buffer = null;
            progressiveArray = null;
            postMessage({ Hash: SHA512 });
          }
      }
    
      readNext();
    }
    
    async function digestMessage(file) {  
      const hashBuffer = await crypto.subtle.digest('SHA-512', file);           // hash the message
      const hashArray = Array.from(new Uint8Array(hashBuffer));                     // convert buffer to byte array
      const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string
      return hashHex;
    }
    
    function arrayBufferToWordArray(ab) {
      var i8a = new Uint8Array(ab);
      var a = [];
      for (var i = 0; i < i8a.length; i += 4) {
        a.push(i8a[i] << 24 | i8a[i + 1] << 16 | i8a[i + 2] << 8 | i8a[i + 3]);
      }
      return CryptoJS.lib.WordArray.create(a, i8a.length);
    }

试图获取一个非常大的文件的哈希 (SHA-512)，在 javascript 中超过 2.5 G

Trying to get the Hash (SHA-512) of a very Large file, more that 2.5 G in javascript

javascript

hash

cryptography

filereader

arraybuffer