在 Android 中合并多个 Azure 的云块 blob

Merging multiple Azure's Cloud block blobs in Android

我是 Azure 的 Blob 云的新手。我基本上想将视频文件从我的 android 应用程序上传到 Azure 云,但我不能,因为一旦大小达到 32MB,它就会停止并抛出 OutOfMemory 异常。所以我做了一些关于如何解决这个问题的研究,我想出了一个解决方案,将文件分解成字节,而不是将它作为多个 blob 上传。最后将它们编译成一个 blob。但我不知道该怎么做。我尝试使用 commitBlockList 但为此我无法获取每个 blob 的 ID。

try {
        // Setup the cloud storage account.
        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        int maxSize = 64 * Constants.MB;

        // Create a blob service client
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        blobClient.getDefaultRequestOptions().setSingleBlobPutThresholdInBytes(maxSize);
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new     BlobContainerPermissions();
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        container.uploadPermissions(containerPermissions);
        CloudBlockBlob finalFile = container.getBlockBlobReference("1.jpg");
        CloudBlob b = container.getBlockBlobReference("temp");
        String Lease = b.getSnapshotID();
        b.uploadFromFile(URL);
        List<BlockEntry> blockEntryIterator = new ArrayList<>();
        blockEntryIterator.add(new BlockEntry(Lease));
        finalFile.commitBlockList(blockEntryIterator);
    } catch (Throwable t) {

    }

~~~更新~~~

我试图将文件分成几部分,但现在出现此错误 "The specified blob or block content is invalid"。 public void splitTest(String URL) 抛出 IOException、URISyntaxException、InvalidKeyException、StorageException {

    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream( URL );
        File sourceFile = new File( URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 3 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android_" + getFileNameFromUrl(URL);
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        blob.deleteIfExists();

        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<BlockEntry>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {

            blockNumber++;
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "blockid: 000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "blockid: 00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid00" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "blockid0: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid0" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "blockid: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.DEFAULT).replace("\n","").toLowerCase();
            traceLine( "Base 64["+x+"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);

            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);

            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;

        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    }
}

~~~工作更新~~~

任何想重复使用此方法打开文件并逐字节读取的人都可以使用此方法。我能够上传 ~1GB 的文件,但之后它给我错误 500 并崩溃。如果有人有任何解决方案,请告诉我。

public boolean splitTest(String URL) throws IOException, URISyntaxException, InvalidKeyException, StorageException {
    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream(URL);
        File sourceFile = new File(URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 512 * 1024; // 256K
        //int blockSize = 1 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android/Android_" + getFileNameFromUrl(URL).replace("\n","").replace(" ","_").replace("-","").toLowerCase();
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        traceLine("Title of blob -> " + title);

        if(blob.exists())
            blob.deleteIfExists();

        blob.setStreamWriteSizeInBytes(blockSize);
        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "tempblobid0000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "tempblobid000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid000" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "tempblobid00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid00" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "tempblobid0" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0" + blockNumber;
            }
            else if(blockNumber>=10000 && blockNumber<=99999){
                traceLine( "tempblobid" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.NO_WRAP).replace("\n","").toLowerCase();
            traceLine( "Base 64["+ x +"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);
            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);
            notification2(a,pctDone);
            //a.update(pctDone);
            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;
            blockNumber++;
        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead + "\t total bytes -> " + fileSize + "\tBytes Left -> " + bytesLeft);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return true;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        traceLine("HTTP Status code -> " + storageException.getHttpStatusCode());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    }

}

这取决于您使用的 phone 类型。如果您 运行 内存不足 32MB,那么您的内存非常小 phone 或使用了很多其他进程。查看您的 phone 以及您有多少可用内存,就像 Gaurav 提到的和我的其他答案提到的那样,将阈值降低到该水平。把自己分块对你想做的事情没有真正的帮助。

您要查看的两个设置是 blob 本身的 singleBlobPutThresholdInBytes 和 setStreamWriteSizeInBytes。 singleBlobPutThresholdInBytes 影响我们何时开始分块与放置整个 blob,setStreamWriteSizeInBytes 影响分块发生时块的大小。放置阈值默认为 64MB,写入大小默认为 4MB。请注意,如果减小写入大小,您可能无法获得最大块 blob 大小,因为块 blob 被限制为 50k 块。尝试将 singleBlobPutThreshold 减少到你可以处理的内存量,直到 4MB - 如果它小于 4MB,你真的有麻烦并且需要减少流写入大小。