使用 AWS C++ SDK 分段上传 S3

Question

我正在尝试使用 AWS C++ SDK 中的分段上传功能将文件上传到 S3。我可以找到 JAVA、.NET、PHP、RUBY 和 Rest API 的示例，但没有找到关于如何在 C++ 中执行此操作的任何线索。你能给我一个实现相同目标的方向吗？

Answer 1

我一般建议使用 Transfer Manager。

但如果您出于某种原因不想这样做，那么您可以查看传输管理器的 source，了解如何直接使用 S3 API 进行分段上传。

Answer 2

传输管理器需要将文件存储在磁盘上并具有文件名。这不利于流式传输。这是我用来做原型的代码模板。

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/core/utils/HashingUtils.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/UploadPartRequest.h>

#include <sstream>
#include <iostream>
#include <string>

using namespace Aws::S3::Model;

int main(int argc, char *argv[]) {
    // new api
    Aws::SDKOptions options;
    Aws::InitAPI(options);

    // use default credential provider chains
    Aws::Client::ClientConfiguration clientConfiguration;
    clientConfiguration.region = "<your-region>";
    clientConfiguration.endpointOverride = "<endpoint-override>";
    Aws::S3::S3Client s3_client(clientConfiguration);

    std::string bucket = "<bucket>";
    std::string key = "<key>";

    // initiate the process
    Aws::S3::Model::CreateMultipartUploadRequest create_request;
    create_request.SetBucket(bucket.c_str());
    create_request.SetKey(key.c_str());
    create_request.SetContentType("text/plain");

    auto createMultipartUploadOutcome =
      s3_client.CreateMultipartUpload(create_request);

    std::string upload_id = createMultipartUploadOutcome.GetResult().GetUploadId();
    std::cout << "multiplarts upload id is:" << upload_id << std::endl;

    // start upload
    Aws::S3::Model::UploadPartRequest my_request;
    my_request.SetBucket(bucket.c_str());
    my_request.SetKey(key.c_str());
    my_request.SetPartNumber(1);
    my_request.SetUploadId(upload_id.c_str());

    Aws::StringStream ss;
    // just have a small chunk of data to verify everything works
    ss << "to upload";

    std::shared_ptr<Aws::StringStream> stream_ptr =
      Aws::MakeShared<Aws::StringStream>("WriteStream::Upload" /* log id */, ss.str());

    my_request.SetBody(stream_ptr);

    Aws::Utils::ByteBuffer part_md5(
        Aws::Utils::HashingUtils::CalculateMD5(*stream_ptr));
    my_request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5));

    auto start_pos = stream_ptr->tellg();
    stream_ptr->seekg(0LL, stream_ptr->end);
    my_request.SetContentLength(static_cast<long>(stream_ptr->tellg()));
    stream_ptr->seekg(start_pos);

    auto uploadPartOutcomeCallable1 = s3_client.UploadPartCallable(my_request);

    // finish upload
    Aws::S3::Model::CompleteMultipartUploadRequest completeMultipartUploadRequest;
    completeMultipartUploadRequest.SetBucket(bucket.c_str());
    completeMultipartUploadRequest.SetKey(key.c_str());
    completeMultipartUploadRequest.SetUploadId(upload_id.c_str());

    UploadPartOutcome uploadPartOutcome1 = uploadPartOutcomeCallable1.get();
    CompletedPart completedPart1;
    completedPart1.SetPartNumber(1);
    auto etag = uploadPartOutcome1.GetResult().GetETag();
    // if etag must not be empty
    assert(etag.empty());
    completedPart1.SetETag(etag);

    completeMultipartUploadRequest.SetBucket(bucket.c_str());
    completeMultipartUploadRequest.SetKey(key.c_str());
    completeMultipartUploadRequest.SetUploadId(upload_id.c_str());

    CompletedMultipartUpload completedMultipartUpload;
        completedMultipartUpload.AddParts(completedPart1);

    completeMultipartUploadRequest.WithMultipartUpload(completedMultipartUpload);

    auto completeMultipartUploadOutcome =
      s3_client.CompleteMultipartUpload(completeMultipartUploadRequest);

    if (!completeMultipartUploadOutcome.IsSuccess()) {
      auto error = completeMultipartUploadOutcome.GetError();
      std::stringstream ss;
      ss << error << error.GetExceptionName() << ": " << error.GetMessage() << std::endl;
      return -1;
    }
}

使用 AWS C++ SDK 分段上传 S3

Multipart Upload S3 using AWS C++ SDK

c++

multipart

amazon-s3

aws-sdk-cpp