Azure SDK v12 与存储数据移动库的性能对比？

Question

我知道存储数据移动库在向 blob 存储上传文件和从 blob 存储下载文件时应该更快，但与 Azure SDK v12 相比，我没有看到它的性能优势。我使用 Azure SDK v12 的平均时间为 37.463 秒，使用存储数据移动库 (SDML) 的平均时间为 41.863 秒。

这里是使用 SDML 的代码：

namespace FunctionApp
{
    using Microsoft.AspNetCore.Mvc;
    using Microsoft.Azure.Storage;
    using Microsoft.Azure.Storage.Blob;
    using Microsoft.Azure.Storage.DataMovement;
    using Microsoft.Azure.WebJobs;
    using Microsoft.Azure.WebJobs.Extensions.Http;
    using Microsoft.Extensions.Logging;

    using System;
    using System.Diagnostics;
    using System.IO;
    using System.IO.Compression;
    using System.Net;
    using System.Net.Http;
    using System.Threading;
    using System.Threading.Tasks;
    using System.Web.Http;

    public static class Function1
    {
        [FunctionName("A")]
        public static async Task<IActionResult> HttpStart(
            [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testRoute")] HttpRequestMessage req,
            ILogger log)
        {
            Stopwatch timer = new Stopwatch();
            timer.Start();
            try
            {
                ServicePointManager.Expect100Continue = false;
                ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
                TransferManager.Configurations.ParallelOperations = 64;

                string fileToDownload = "<URI to zip file in blob storage containing two 300MB files";
                string connectionString = "<connection string to storage account>";
                string containerName = "<container to upload files to>";

                using MemoryStream test = new MemoryStream();
                CloudBlockBlob sourceBlob = new CloudBlockBlob(new Uri(fileToDownload));
                await TransferManager.DownloadAsync(sourceBlob, test);

                CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
                CloudBlobClient blobClient = account.CreateCloudBlobClient();
                CloudBlobContainer container = blobClient.GetContainerReference(containerName);

                using ZipArchive zipArchive = new ZipArchive(test);
                foreach (ZipArchiveEntry file in zipArchive.Entries)
                {
                    if (!string.IsNullOrEmpty(file.Name))
                    {
                        CloudBlockBlob destBlob = container.GetBlockBlobReference(file.FullName);
                        using Stream stream = file.Open();
                        await TransferManager.UploadAsync(stream, destBlob);
                    }
                }
            }
            catch (Exception exception)
            {
                return new InternalServerErrorResult();
            }

            timer.Stop();
            return new OkObjectResult(timer.ElapsedMilliseconds);
        }
    }
}

这是使用 Azure SDK v12 的代码：

namespace FunctionApp
{
    using Azure.Storage.Blobs;
    using Azure.Storage.Blobs.Specialized;

    using Microsoft.AspNetCore.Mvc;
    using Microsoft.Azure.WebJobs;
    using Microsoft.Azure.WebJobs.Extensions.Http;
    using Microsoft.Extensions.Logging;

    using System;
    using System.Diagnostics;
    using System.IO;
    using System.IO.Compression;
    using System.Net;
    using System.Net.Http;
    using System.Threading;
    using System.Threading.Tasks;
    using System.Web.Http;

    public static class Function1
    {
        [FunctionName("A")]
        public static async Task<IActionResult> HttpStart(
            [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testRoute")] HttpRequestMessage req,
            ILogger log)
        {
            Stopwatch timer = new Stopwatch();
            timer.Start();
            try
            {
                ServicePointManager.Expect100Continue = false;
                ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;

                string fileToDownload = "<URI to zip file in blob storage containing two 300MB files";
                string connectionString = "<connection string to storage account>";
                string containerName = "<container to upload files to>";

                using MemoryStream test = new MemoryStream();
                BlockBlobClient client = new BlockBlobClient(new Uri(fileToDownload));

                await client.DownloadToAsync(test);
                BlobContainerClient containerClient = new BlobContainerClient(connectionString, containerName);

                using ZipArchive zipArchive = new ZipArchive(test);
                foreach (ZipArchiveEntry file in zipArchive.Entries)
                {
                    if (!string.IsNullOrEmpty(file.Name))
                    {
                        BlockBlobClient blockBlobClient = containerClient.GetBlockBlobClient(file.FullName);
                        using Stream stream = file.Open();
                        await blockBlobClient.UploadAsync(stream);
                    }
                }

            }
            catch (Exception exception)
            {
                return new InternalServerErrorResult();
            }

            timer.Stop();
            return new OkObjectResult(timer.ElapsedMilliseconds) ;
        }
    }
}

Answer 1

对于数据移动库，您可以设置ParallelOperations和BlockSize，如下所示：

TransferManager.Configurations.ParallelOperations = 20;
TransferManager.Configurations.BlockSize = 20971520*2; //20M

我这边测试过，SDML更快