如何在 go 中以流的形式上传文件？
How can you upload files as a stream in go?
有很多关于在 go 中使用 http.Request 编辑文件 post 的教程，但几乎总是这样开始的：
file, err := os.Open(path)
if err != nil {
return nil, err
}
fileContents, err := ioutil.ReadAll(file)
也就是说，您将整个文件读入内存，然后将其转换为 Buffer 并将其传递到请求中，如下所示：
func send(client *http.Client, file *os.File, endpoint string) {
body := &bytes.Buffer{}
io.Copy(body, file)
req, _ := http.NewRequest("POST", endpoint, body)
resp, _ := client.Do(req)
}
如果您想 post 一个大文件并避免将其读入内存，而是将文件分块处理...您会怎么做？
事实证明，您实际上可以将 *File（或任何类似流的）对象直接传递到 NewRequest。
但是请注意，NewRequest（如此处所示：https://golang.org/src/net/http/request.go?s=21674:21746#L695）实际上不会设置 ContentLength 除非流是 显式 一个的：
- *bytes.Buffer
- *bytes.Reader
- *strings.Reader
由于 *File 不是其中之一，请求将在没有内容长度的情况下发送，除非您手动设置它，这可能会导致某些服务器丢弃传入请求的主体，从而导致服务器上 '' 的正文似乎已从 go 端正确发送。
如果需要设置Content-Length，可以手动完成。以下代码段是将文件和额外参数作为流上传的示例（代码基于Buffer-less Multipart POST in Golang）
//NOTE: for simplicity, error check is omitted
func uploadLargeFile(uri, filePath string, chunkSize int, params map[string]string) {
//open file and retrieve info
file, _ := os.Open(filePath)
fi, _ := file.Stat()
defer file.Close()
//buffer for storing multipart data
byteBuf := &bytes.Buffer{}
//part: parameters
mpWriter := multipart.NewWriter(byteBuf)
for key, value := range params {
_ = mpWriter.WriteField(key, value)
}
//part: file
mpWriter.CreateFormFile("file", fi.Name())
contentType := mpWriter.FormDataContentType()
nmulti := byteBuf.Len()
multi := make([]byte, nmulti)
_, _ = byteBuf.Read(multi)
//part: latest boundary
//when multipart closed, latest boundary is added
mpWriter.Close()
nboundary := byteBuf.Len()
lastBoundary := make([]byte, nboundary)
_, _ = byteBuf.Read(lastBoundary)
//calculate content length
totalSize := int64(nmulti) + fi.Size() + int64(nboundary)
log.Printf("Content length = %v byte(s)\n", totalSize)
//use pipe to pass request
rd, wr := io.Pipe()
defer rd.Close()
go func() {
defer wr.Close()
//write multipart
_, _ = wr.Write(multi)
//write file
buf := make([]byte, chunkSize)
for {
n, err := file.Read(buf)
if err != nil {
break
}
_, _ = wr.Write(buf[:n])
}
//write boundary
_, _ = wr.Write(lastBoundary)
}()
//construct request with rd
req, _ := http.NewRequest("POST", uri, rd)
req.Header.Set("Content-Type", contentType)
req.ContentLength = totalSize
//process request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
} else {
log.Println(resp.StatusCode)
log.Println(resp.Header)
body := &bytes.Buffer{}
_, _ = body.ReadFrom(resp.Body)
resp.Body.Close()
log.Println(body)
}
}
如果请求必须有一个 Content-Length header（大多数文件主机拒绝没有它的上传请求），并且您想将文件作为流上传（不加载全部到内存），标准库帮不了你，还得自己算。
这是一个最小的工作示例（没有错误检查），它使用 io.MultiReader 将 os.File 与其他字段连接起来，同时保持对请求大小的标记。
支持常规字段（字符串内容）和文件字段，计算总请求body大小。只需添加一个新的 case 分支即可轻松使用其他值类型扩展它。
import (
"crypto/rand"
"fmt"
"io"
"io/fs"
"mime"
"path/filepath"
"strings"
)
type multipartPayload struct {
headers map[string]string
body io.Reader
size int64
}
func randomBoundary() string {
var buf [8]byte
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err)
}
return fmt.Sprintf("%x", buf[:])
}
// Multipart request has the following structure:
// POST /upload HTTP/1.1
// Other-Headers: ...
// Content-Type: multipart/form-data; boundary=$boundary
// \r\n
// --$boundary\r\n request body starts here
// Content-Disposition: form-data; name="field1"\r\n
// Content-Type: text/plain; charset=utf-8\r\n
// Content-Length: 4\r\n
// \r\n
// $content\r\n
// --$boundary\r\n
// Content-Disposition: form-data; name="field2"\r\n
// ...
// --$boundary--\r\n
func prepareMultipartPayload(fields map[string]interface{}) (*multipartPayload, error) {
boundary := randomBoundary()
headers := make(map[string]string)
totalSize := 0
headers["Content-Type"] = fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
parts := make([]io.Reader, 0)
CRLF := "\r\n"
fieldBoundary := "--" + boundary + CRLF
for k, v := range fields {
parts = append(parts, strings.NewReader(fieldBoundary))
totalSize += len(fieldBoundary)
if v == nil {
continue
}
switch v.(type) {
case string:
header := fmt.Sprintf(`Content-Disposition: form-data; name="%s"`, k)
parts = append(
parts,
strings.NewReader(header+CRLF+CRLF),
strings.NewReader(v.(string)),
strings.NewReader(CRLF),
)
totalSize += len(header) + 2*len(CRLF) + len(v.(string)) + len(CRLF)
continue
case fs.File:
stat, _ := v.(fs.File).Stat()
contentType := mime.TypeByExtension(filepath.Ext(stat.Name()))
header := strings.Join([]string{
fmt.Sprintf(`Content-Disposition: form-data; name="%s"; filename="%s"`, k, stat.Name()),
fmt.Sprintf(`Content-Type: %s`, contentType),
fmt.Sprintf(`Content-Length: %d`, stat.Size()),
}, CRLF)
parts = append(
parts,
strings.NewReader(header+CRLF+CRLF),
v.(fs.File),
strings.NewReader(CRLF),
)
totalSize += len(header) + 2*len(CRLF) + int(stat.Size()) + len(CRLF)
continue
}
}
finishBoundary := "--" + boundary + "--" + CRLF
parts = append(parts, strings.NewReader(finishBoundary))
totalSize += len(finishBoundary)
headers["Content-Length"] = fmt.Sprintf("%d", totalSize)
return &multipartPayload{headers, io.MultiReader(parts...), int64(totalSize)}, nil
}
然后准备请求，设置内容长度并发送：
file, err := os.Open("/path/to/file.ext")
if err != nil {
return nil, err
}
defer file.Close()
up, err := prepareMultipartPayload(map[string]interface{}{
"a_string": "field",
"another_field": "yep",
"file": file, // you can have multiple file fields
})
r, _ := http.NewRequest("POST", "https://example.com/upload", up.body)
for k, v := range up.headers {
r.Header.Set(k, v)
}
r.ContentLength = up.size
c := http.Client{}
res, err := c.Do(r)
有很多关于在 go 中使用 http.Request 编辑文件 post 的教程，但几乎总是这样开始的：
file, err := os.Open(path)
if err != nil {
return nil, err
}
fileContents, err := ioutil.ReadAll(file)
也就是说，您将整个文件读入内存，然后将其转换为 Buffer 并将其传递到请求中，如下所示：
func send(client *http.Client, file *os.File, endpoint string) {
body := &bytes.Buffer{}
io.Copy(body, file)
req, _ := http.NewRequest("POST", endpoint, body)
resp, _ := client.Do(req)
}
如果您想 post 一个大文件并避免将其读入内存，而是将文件分块处理...您会怎么做？
事实证明，您实际上可以将 *File（或任何类似流的）对象直接传递到 NewRequest。
但是请注意，NewRequest（如此处所示：https://golang.org/src/net/http/request.go?s=21674:21746#L695）实际上不会设置 ContentLength 除非流是 显式 一个的：
- *bytes.Buffer
- *bytes.Reader
- *strings.Reader
由于 *File 不是其中之一，请求将在没有内容长度的情况下发送，除非您手动设置它，这可能会导致某些服务器丢弃传入请求的主体，从而导致服务器上 '' 的正文似乎已从 go 端正确发送。
如果需要设置Content-Length，可以手动完成。以下代码段是将文件和额外参数作为流上传的示例（代码基于Buffer-less Multipart POST in Golang）
//NOTE: for simplicity, error check is omitted
func uploadLargeFile(uri, filePath string, chunkSize int, params map[string]string) {
//open file and retrieve info
file, _ := os.Open(filePath)
fi, _ := file.Stat()
defer file.Close()
//buffer for storing multipart data
byteBuf := &bytes.Buffer{}
//part: parameters
mpWriter := multipart.NewWriter(byteBuf)
for key, value := range params {
_ = mpWriter.WriteField(key, value)
}
//part: file
mpWriter.CreateFormFile("file", fi.Name())
contentType := mpWriter.FormDataContentType()
nmulti := byteBuf.Len()
multi := make([]byte, nmulti)
_, _ = byteBuf.Read(multi)
//part: latest boundary
//when multipart closed, latest boundary is added
mpWriter.Close()
nboundary := byteBuf.Len()
lastBoundary := make([]byte, nboundary)
_, _ = byteBuf.Read(lastBoundary)
//calculate content length
totalSize := int64(nmulti) + fi.Size() + int64(nboundary)
log.Printf("Content length = %v byte(s)\n", totalSize)
//use pipe to pass request
rd, wr := io.Pipe()
defer rd.Close()
go func() {
defer wr.Close()
//write multipart
_, _ = wr.Write(multi)
//write file
buf := make([]byte, chunkSize)
for {
n, err := file.Read(buf)
if err != nil {
break
}
_, _ = wr.Write(buf[:n])
}
//write boundary
_, _ = wr.Write(lastBoundary)
}()
//construct request with rd
req, _ := http.NewRequest("POST", uri, rd)
req.Header.Set("Content-Type", contentType)
req.ContentLength = totalSize
//process request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
} else {
log.Println(resp.StatusCode)
log.Println(resp.Header)
body := &bytes.Buffer{}
_, _ = body.ReadFrom(resp.Body)
resp.Body.Close()
log.Println(body)
}
}
如果请求必须有一个 Content-Length header（大多数文件主机拒绝没有它的上传请求），并且您想将文件作为流上传（不加载全部到内存），标准库帮不了你，还得自己算。
这是一个最小的工作示例（没有错误检查），它使用 io.MultiReader 将 os.File 与其他字段连接起来，同时保持对请求大小的标记。
支持常规字段（字符串内容）和文件字段，计算总请求body大小。只需添加一个新的 case 分支即可轻松使用其他值类型扩展它。
import (
"crypto/rand"
"fmt"
"io"
"io/fs"
"mime"
"path/filepath"
"strings"
)
type multipartPayload struct {
headers map[string]string
body io.Reader
size int64
}
func randomBoundary() string {
var buf [8]byte
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err)
}
return fmt.Sprintf("%x", buf[:])
}
// Multipart request has the following structure:
// POST /upload HTTP/1.1
// Other-Headers: ...
// Content-Type: multipart/form-data; boundary=$boundary
// \r\n
// --$boundary\r\n request body starts here
// Content-Disposition: form-data; name="field1"\r\n
// Content-Type: text/plain; charset=utf-8\r\n
// Content-Length: 4\r\n
// \r\n
// $content\r\n
// --$boundary\r\n
// Content-Disposition: form-data; name="field2"\r\n
// ...
// --$boundary--\r\n
func prepareMultipartPayload(fields map[string]interface{}) (*multipartPayload, error) {
boundary := randomBoundary()
headers := make(map[string]string)
totalSize := 0
headers["Content-Type"] = fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
parts := make([]io.Reader, 0)
CRLF := "\r\n"
fieldBoundary := "--" + boundary + CRLF
for k, v := range fields {
parts = append(parts, strings.NewReader(fieldBoundary))
totalSize += len(fieldBoundary)
if v == nil {
continue
}
switch v.(type) {
case string:
header := fmt.Sprintf(`Content-Disposition: form-data; name="%s"`, k)
parts = append(
parts,
strings.NewReader(header+CRLF+CRLF),
strings.NewReader(v.(string)),
strings.NewReader(CRLF),
)
totalSize += len(header) + 2*len(CRLF) + len(v.(string)) + len(CRLF)
continue
case fs.File:
stat, _ := v.(fs.File).Stat()
contentType := mime.TypeByExtension(filepath.Ext(stat.Name()))
header := strings.Join([]string{
fmt.Sprintf(`Content-Disposition: form-data; name="%s"; filename="%s"`, k, stat.Name()),
fmt.Sprintf(`Content-Type: %s`, contentType),
fmt.Sprintf(`Content-Length: %d`, stat.Size()),
}, CRLF)
parts = append(
parts,
strings.NewReader(header+CRLF+CRLF),
v.(fs.File),
strings.NewReader(CRLF),
)
totalSize += len(header) + 2*len(CRLF) + int(stat.Size()) + len(CRLF)
continue
}
}
finishBoundary := "--" + boundary + "--" + CRLF
parts = append(parts, strings.NewReader(finishBoundary))
totalSize += len(finishBoundary)
headers["Content-Length"] = fmt.Sprintf("%d", totalSize)
return &multipartPayload{headers, io.MultiReader(parts...), int64(totalSize)}, nil
}
然后准备请求，设置内容长度并发送：
file, err := os.Open("/path/to/file.ext")
if err != nil {
return nil, err
}
defer file.Close()
up, err := prepareMultipartPayload(map[string]interface{}{
"a_string": "field",
"another_field": "yep",
"file": file, // you can have multiple file fields
})
r, _ := http.NewRequest("POST", "https://example.com/upload", up.body)
for k, v := range up.headers {
r.Header.Set(k, v)
}
r.ContentLength = up.size
c := http.Client{}
res, err := c.Do(r)