golang中處理多個檔案同時上傳以及檔案上傳速度最佳化

這是一個建立於 的文章，其中的資訊可能已經有所發展或是發生改變。

首先給大家推薦一本書Go in Practice，通過一個個超級小巧而又非常實戰的案例提升golang內功。
在TECHNIQUE 48 Incrementally saving a file中，向讀者講述了如何處理多個檔案同時提交的處理方法
以下代碼和文中略有不同：

func uploadBigFile(w http.ResponseWriter, r *http.Request) {    mr, err := r.MultipartReader()    if err != nil {        fmt.Sprintln(err)        fmt.Fprintln(w, err)        return    }    values := make(map[string][]string, 0)    maxValueBytes := int64(10 << 20)    for {        part, err := mr.NextPart()        if err == io.EOF {            break        }        name := part.FormName()        if name == "" {            continue        }        fileName := part.FileName()        var b bytes.Buffer        if fileName == "" {            n, err := io.CopyN(&b, part, maxValueBytes)            if err != nil && err != io.EOF {                fmt.Sprintln(err)                fmt.Fprintln(w, err)                return            }            maxValueBytes -= n            if maxValueBytes <= 0 {                msg := "multipart message too large"                fmt.Fprint(w, msg)                return            }            values[name] = append(values[name], b.String())        }        dst, err := os.Create("/tmp/upload/" + fileName)        defer dst.Close()        for {            buffer := make([]byte, 100000)            cBytes, err := part.Read(buffer)            if err == io.EOF {                break            }            dst.Write(buffer[0:cBytes])        }    }}

在進行benchmark測試的時候，發現上述方式在處理單個檔案上傳的時候要比普通的處理方法要快
普通的檔案上傳處理介面

func upload(w http.ResponseWriter, r *http.Request) {    file, head, err := r.FormFile("my_file")    if err != nil {        fmt.Sprintln(err)        fmt.Fprintln(w, err)        return    }    localFileDir := "/tmp/upload/"    err = os.MkdirAll(localFileDir, 0777)    if err != nil {        fmt.Sprintln(err)        fmt.Fprintln(w, err)        return    }    localFilePath := localFileDir + head.Filename    localFile, err := os.Create(localFilePath)    if err != nil {        fmt.Sprintln(err)        fmt.Fprintln(w, err)        return    }    defer localFile.Close()    io.Copy(localFile, file)    fmt.Fprintln(w, localFilePath)}

基準測試代碼

func BenchmarkUpload(b *testing.B) {    for i := 0; i < b.N; i++ {        path := "/home/kes/code_1.13.1-1497464373_amd64.deb"        file, err := os.Open(path)        if err != nil {            b.Error(err)        }        defer file.Close()        body := &bytes.Buffer{}        writer := multipart.NewWriter(body)        part, err := writer.CreateFormFile("my_file", filepath.Base(path))        if err != nil {            b.Error(err)        }        io.Copy(part, file)        writer.Close()        req := httptest.NewRequest("POST", "/upload", body)        req.Header.Set("Content-Type", writer.FormDataContentType())        res := httptest.NewRecorder()        upload(res, req)        if res.Code != http.StatusOK {            b.Error("not 200")        }    }    // t.Log(res.Body.String())    // t.Log(io.read)}func BenchmarkUploadBig(b *testing.B) {    for i := 0; i < b.N; i++ {        path := "/home/kes/code_1.13.1-1497464373_amd64.deb"        file, err := os.Open(path)        if err != nil {            b.Error(err)        }        defer file.Close()        body := &bytes.Buffer{}        writer := multipart.NewWriter(body)        part, err := writer.CreateFormFile("my_file", filepath.Base(path))        if err != nil {            b.Error(err)        }        io.Copy(part, file)        writer.Close()        req := httptest.NewRequest("POST", "/upload", body)        req.Header.Set("Content-Type", writer.FormDataContentType())        res := httptest.NewRecorder()        uploadBigFile(res, req)        if res.Code != http.StatusOK {            b.Error("not 200")        }    }    // t.Log(res.Body.String())    // t.Log(io.read)}

測試的檔案大小大概44M
測試結果

$ go test -bench="." -vBenchmarkUpload-4              1        1074491528 ns/opBenchmarkUploadBig-4           2         517001745 ns/opPASSok      _/home/kes/test    2.966s$ go test -bench="." -vBenchmarkUpload-4              1        1295672144 ns/opBenchmarkUploadBig-4           2         537212487 ns/opPASSok      _/home/kes/test    3.218s