Golang multipart file form request

Question

I'm writing an API client against Mapbox, uploading a batch of svg images to a custom map. The api they provide for this is documented with an example cUrl call that works fine:

curl -F images=@include/mapbox/sprites_dark/aubergine_selected.svg "https://api.mapbox.com/styles/v1/<my_company>/<my_style_id>/sprite?access_token=$MAPBOX_API_KEY" --trace-ascii /dev/stdout

When attemting to do the same from golang I quickly came across that the multiform library is very limited, and wrote some code to make the request as similar to the cUrl request mentioned above.

func createMultipartFormData(fileMap map[string]string) (bytes.Buffer, *multipart.Writer) {
    var b bytes.Buffer
    var err error
    w := multipart.NewWriter(&b)
    var fw io.Writer
    for fileName, filePath := range fileMap {

        h := make(textproto.MIMEHeader)
        h.Set("Content-Disposition",
            fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "images", fileName))
        h.Set("Content-Type", "image/svg+xml")

        if fw, err = w.CreatePart(h); err != nil {
            fmt.Printf("Error creating form File %v, %v", fileName, err)
            continue
        }

        fileContents, err := ioutil.ReadFile(filePath)
        fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))

        blockSize := 64
        remainder := len(fileContents) % blockSize
        iterations := (len(fileContents) - remainder) / blockSize

        newBytes := []byte{}
        for i := 0; i < iterations; i++ {
            start := i * blockSize
            end := i*blockSize + blockSize
            newBytes = append(newBytes, fileContents[start:end]...)
            newBytes = append(newBytes, []byte("\n")...)
        }

        if remainder > 0 {
            newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
            newBytes = append(newBytes, []byte("\n")...)
        }

        if err != nil {
            fmt.Printf("Error reading svg file: %v: %v", filePath, err)
            continue
        }

        _, err = fw.Write(newBytes)

        if err != nil {
            log.Debugf("Could not write file to multipart: %v, %v", fileName, err)
            continue
        }
    }

    w.Close()

    return b, w
}

Along with setting the headers in the actual request:

    bytes, formWriter := createMultipartFormData(filesMap)

    req, err := http.NewRequest("Post", fmt.Sprintf("https://api.mapbox.com/styles/v1/%v/%v/sprite?access_token=%v", "my_company", styleID, os.Getenv("MAPBOX_API_KEY")), &bytes)

    if err != nil {
        return err
    }

    req.Header.Set("User-Agent", "curl/7.64.1")
    req.Header.Set("Accept", "*/*")
    req.Header.Set("Content-Length", fmt.Sprintf("%v", len(bytes.Bytes())))
    req.Header.Set("Content-Type", formWriter.FormDataContentType())

    byts, _ := httputil.DumpRequest(req, true)
    fmt.Println(string(byts))

    res, err := http.DefaultClient.Do(req)

Even want as far to limit the line length and replicate the encoding used by cUrl but so far no luck. Does anyone with experience know why this works from cUrl but not golang?

Answer 1

Just to be curious, What is this for?

        fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))

        blockSize := 64
        remainder := len(fileContents) % blockSize
        iterations := (len(fileContents) - remainder) / blockSize

        newBytes := []byte{}
        for i := 0; i < iterations; i++ {
            start := i * blockSize
            end := i*blockSize + blockSize
            newBytes = append(newBytes, fileContents[start:end]...)
            newBytes = append(newBytes, []byte("\n")...)
        }

        if remainder > 0 {
            newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
            newBytes = append(newBytes, []byte("\n")...)
        }

        if err != nil {
            fmt.Printf("Error reading svg file: %v: %v", filePath, err)
            continue
        }

Reading a entire file into memory is rarely a good idea (ioutil.ReadFile).

As @muffin-top says, how about those three lines of code?

    for fileName, filePath := range fileMap {

        // h := ...

        fw, _ := w.CreatePart(h) // TODO: handle error

        f, _ := os.Open(filePath) // TODO: handle error

        io.Copy(fw, f) // TODO: handle error

        f.Close() // TODO: handle error
    }