无法将 s3 对象读取流通过管道传递给 PUT 请求

Question

我有一个简单的场景。我需要从 S3 读取一个对象并将其输出通过管道传输到 PUT 请求。这是我使用 request.

的代码

// client.js
let AWS = require('aws-sdk')
let request = require('request')

let bucket = 'my_bucket'
let filename = 'path/to/file.zip'

let host = 'localhost'
let port = 8080

let s3 = new AWS.S3({
  . . .
})

let readStream = s3.getObject({
  Bucket: bucket,
  Key: filename
}).createReadStream()

let formData = {
  applicationType: 'my_app_type',
  applicationName: 'my_app_name',
  upload: {
    value: readStream,
    options: {
      filename: 'my_file_name.zip',
      contentType: 'application/zip'
    }
  }
}

request.put({
  url: 'http://' + host + ':' + port + '/bootstrap',
  formData: formData
}, function (error, response, body) {
  if (error) throw error
  console.log(body)
})

而且，这是我的 server.js 代码。

// server.js
let http = require('http')
let Busboy = require('busboy')
let events = require('events')
let fs = require('fs')

let host = 'localhost'
let port = 8080

let compressedCodeLocation = './code.zip'

let handleRequest = function (request, response) {
  let eventEmitter = new events.EventEmitter()
  let inputStreamWriter = fs.createWriteStream(compressedCodeLocation)
  inputStreamWriter.on('finish', function () {
    eventEmitter.emit('input.stream.saved')
  })
  let busboy = new Busboy({
    headers: request.headers
  })

  busboy.on('file', function (field, file) {
    file.pipe(inputStreamWriter)
  })
  busboy.on('field', function (field, val) {
    console.log(field + ': ' + val)
  })
  eventEmitter.on('input.stream.saved', function () {
    let stats = fs.statSync(compressedCodeLocation)
    response.statusCode = 200
    response.end(JSON.stringify(stats))
  })

  request.pipe(busboy)
}

let server = http.createServer(handleRequest)
server.listen(port, host, function () {
  console.log('Server started on ' + host + ':' + port)
})

let handleShutdown = function () {
  server.close(function () {
    console.log('Server stopped on ' + host + ':' + port)
  })
}
process.on('SIGTERM', handleShutdown)
process.on('SIGINT', handleShutdown)

Server 收到以下 headers:

{ host: 'localhost:8080',
  'content-type': 'multipart/form-data; boundary=--------------------------870259812928253745629174',
  'content-length': '465',
  connection: 'close' }

我在 Server 端收到此错误：

File [upload] got 58 bytes
events.js:160
      throw er; // Unhandled 'error' event
      ^

Error: Unexpected end of multipart data
    at /pots/cnc/node_modules/dicer/lib/Dicer.js:62:28
    at _combinedTickCallback (internal/process/next_tick.js:67:7)
    at process._tickCallback (internal/process/next_tick.js:98:9)

并且，Client 收到以下错误：

Error: read ECONNRESET
    at exports._errnoException (util.js:1018:11)
    at TCP.onread (net.js:568:26)

有趣的是，如果我先将文件保存在本地，然后 createReadStream 保存该本地文件，它会起作用：

let formData = {
  ...
  upload: {
    value: fs.createReadStream(localPath + "/" + filename),
    options: {
      ...
    }
  }
};

Answer 1

我在使用请求时遇到了问题。但是工作得很好。试试这个：

const {S3} = require('aws-sdk'),
      got = require('got'),
      FormData = require('form-data'),
;
const form = new FormData(),
      readStream = s3.getObject({
          Bucket: bucket,
          Key: filename
      }).createReadStream()
;
form.append('applicationType', 'my_app_type')
form.append('applicationName', 'my_app_name')
form.append('upload', readStream,{
  filename: 'my_file_name.zip',
  contentType: 'application/zip'
})
got.put('http://' + host + ':' + port + '/bootstrap', {body: form})

Answer 2

解决方案 2：

我今天找到的另一个答案要简单得多。只需使用属性 knownLength 让请求模块提前知道文件的大小。

upload: {
  value: readStream,
  options: {
    filename: 'my_file_name.zip',
    contentType: 'application/zip'
    knownLength: 423424
  }
}

解决方案 1

This solution is using the request module itself, in case you do not want to introduce a new library to your code.

我在的帮助下解决了这个问题。请将所有投票重定向到他的答案。

我需要将 require 模块的形式重置为 form-data 模块的形式。正如 request documentation 所述：

For advanced cases, you can access the form-data object itself via r.form().

一旦在我的请求中设置了新表单（我认为这会重写多部分边界），我就会将表单数据通过管道传输到我的请求中。 See Reference.

let AWS = require('aws-sdk')
let FormData = require('form-data')
let request = require('request')

let bucket = 'ppi-uploads'
let filename = 'introduction.zip'

let host = 'localhost'
let port = 8080

let s3 = new AWS.S3({
  . . .
})

let readStream = s3.getObject({
  Bucket: bucket,
  Key: filename
}).createReadStream()

let form = new FormData()
form.append('applicationType', 'html')
form.append('applicationName', 'introduction')
form.append('upload', readStream, {
  filename: 'introduction.zip',
  contentType: 'application/zip'
})

let putRequest = request.put({
  url: 'http://' + host + ':' + port + '/bootstrap',
  headers: form.getHeaders()
}, function (error, response, body) {
  if (error) throw error
  console.log(body)
})

form.pipe(putRequest)

无法将 s3 对象读取流通过管道传递给 PUT 请求

Cannot pipe s3 object readstream to PUT request

request

node.js

node-request

busboy

解决方案 2：

解决方案 1