如何使用 Lambda + Java 从 S3 收集文件并将其发送到外部 SFTP？

Question

我每天都会将一些文件放在 S3 存储桶中，我需要创建一个 lambda 函数来将此文件转发到外部 SFTP。 （使用 java）

我的问题是我不确定如何从我的 lambda 建立与 S3 的连接以收集文件（最终编辑它，例如重命名）然后转发到 SFTP。如果我们需要调用另一个 Lambda 函数，是否可以像我们一样调用它？ Example。或者我是否必须像在 AWS 环境之外一样进行连接？

如果您有一些建议或者一些与此接近的简单实施示例，那就太好了！

Answer 1

首先，您需要configure your S3 bucket将新的对象事件发送到您的 Lambda 函数。

在您的 Lambda 函数中，您将从事件对象中提取 S3 对象路径。然后您需要使用 AWS SDK for Java to download the file from S3 to the Lambda function's /tmp folder. Then have your function perform whatever edits you need to on the file in the /tmp folder. Finally use a SFTP library for Java 将文件从 /tmp 文件夹发送到 SFTP 服务器。

Answer 2

这是从 S3 获取文件并将其保存到 SFTP 的完整示例。

我添加了以下库以使这个 Java 模块工作：

• Lambda Core

• Google Gson

• AWS S3 SDK

• SSHJ

  import com.amazonaws.regions.Regions;
  import com.amazonaws.services.lambda.runtime.Context;
  import com.amazonaws.services.lambda.runtime.LambdaLogger;
  import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
  import com.amazonaws.services.s3.AmazonS3;
  import com.amazonaws.services.s3.AmazonS3ClientBuilder;
  import com.amazonaws.services.s3.model.S3Object;
  import com.google.gson.Gson;
  import com.google.gson.GsonBuilder;
  import net.schmizz.sshj.SSHClient;
  import net.schmizz.sshj.sftp.SFTPClient;
  import net.schmizz.sshj.xfer.FileSystemFile;
  
  import java.io.*;
  import java.nio.charset.StandardCharsets;
  import java.nio.file.Files;
  import java.nio.file.StandardCopyOption;
  
  public class S3ToSFTPTest implements RequestStreamHandler {
  
      private LambdaLogger logger;
  
      @Override
      public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
          logger = context.getLogger();
  
          AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();
  
          // s3 client
          if (s3Client == null) {
              logger.log("S3 Client is null - can't continue!");
              sendResponse(context, new S3ToSFTPTestResponse(false), output);
              return;
          }
  
          String bucketName = "==== S3 BUCKET NAME ====";
  
          // s3 bucket - make sure it exist
          if (!s3Client.doesBucketExistV2(bucketName)) {
              logger.log("S3 Bucket does not exists - can't continue!");
              sendResponse(context, new S3ToSFTPTestResponse(false), output);
              return;
          }
  
          String fileName = "==== S3 FILE NAME ====";
          File localFile = null;
  
          try {
              localFile = File.createTempFile(fileName, "");
  
              // get S3Object
              S3Object s3Object = s3Client.getObject(bucketName, fileName);
  
              // get stream from S3Object
              InputStream inputStream = s3Object.getObjectContent();
  
              // write S3Object stream into a temp file
              Files.copy(inputStream, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
          } catch (Exception e) {
              logger.log("Failed to get file from S3: " + e.toString());
          }
  
          if(localFile == null) {
              sendResponse(context, new S3ToSFTPTestResponse(false), output);
              return;
          }
  
  
          // now, you have the file stored locally
          // modify it as you need to
          // .....
  
  
  
          // finally, send the file to SFTP
          boolean fileSaved = saveFilesToSFTP(context, localFile);
          logger.log("fileSaved: " + fileSaved);
  
          sendResponse(context, new S3ToSFTPTestResponse(true), output);
      }
  
  
      public boolean saveFilesToSFTP(Context context, File... files) {
          // this is for test only - In real application, I would suggest that
          // do NOT store these information in the code.
          // You should use service like Secrets Manager or Parameter Store
          final String sftpHostname = "==== SFTP Hostname ====";
          final String sftpUsername = "==== SFTP Username ====";
          final String sftpPassword = "==== SFTP Password ====";
  
          String remoteFolderPath = "/root/S3Files/";
  
          try {
              SSHClient ssh = new SSHClient();
              ssh.addHostKeyVerifier((hostname1, port, key) -> true);
  
              ssh.connect(sftpHostname);
              logger.log("SSHClient Connected!");
  
              try {
                  ssh.authPassword(sftpUsername, sftpPassword);
                  logger.log("SSHClient Authenticated!");
  
                  try (SFTPClient sftp = ssh.newSFTPClient()) {
                      for(File file : files) {
                          sftp.put(new FileSystemFile(file), remoteFolderPath);
                      }
                  } catch (Exception e) {
                      logger.log("failed to get SFTPClient: " + e.toString());
                      return false;
                  }
              } finally {
                  ssh.disconnect();
              }
          } catch (Exception e) {
              logger.log("SFTP connection failed: " + e.toString());
              return false;
          }
  
          return true;
      }
  
  
      public void sendResponse(Context context, S3ToSFTPTestResponse response, OutputStream output) {
          Gson gson = new GsonBuilder().setPrettyPrinting().create();
          String responseStr = gson.toJson(response);
  
          logger.log("response:\n" + responseStr);
  
          try {
              OutputStreamWriter writer = new OutputStreamWriter(output, StandardCharsets.UTF_8);
              writer.write(responseStr);
              writer.close();
          } catch (Exception e) {
              logger.log("failed to send response: " + e.toString());
          }
      }
  
  
      public static class S3ToSFTPTestResponse implements Serializable {
          private boolean success;
  
          public S3ToSFTPTestResponse() {
          }
  
          public S3ToSFTPTestResponse(boolean success) {
              this.success = success;
          }
  
          public boolean isSuccess() {
              return success;
          }
      }
  }