net/http: slow io.Copy(http.ResponseWriter, *os.File) with Content-Length · golang/go#41513

2020-09-21T04:09:54.000Z

### What version of Go are you using (`go version`)? $ go version go version go1.15.2 linux/amd64 ### Does this issue reproduce with the latest release? Yes ### What operating system and processor architecture are you using (`go env`)? go env Output $ go env GO111MODULE="" GOARCH="amd64" GOBIN="" GOCACHE="/root/.cache/go-build" GOENV="/root/.config/go/env" GOEXE="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="linux" GOINSECURE="" GOMODCACHE="/go/pkg/mod" GONOPROXY="" GONOSUMDB="" GOOS="linux" GOPATH="/go" GOPRIVATE="" GOPROXY="https://proxy.golang.org,direct" GOROOT="/usr/local/go" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" GCCGO="gccgo" AR="ar" CC="gcc" CXX="g++" CGO_ENABLED="1" GOMOD="" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build447228345=/tmp/go-build -gno-record-gcc-switches" ### What did you do? package http_test import ( "io" "io/ioutil" "log" "net/http" "net/http/httptest" "os" "strconv" "testing" ) func runHandlerBenchmark(b *testing.B, handler http.Handler) { server := httptest.NewServer(handler) defer server.Close() client := server.Client() for i := 0; i < b.N; i++ { response, err := client.Get(server.URL) if err != nil { log.Fatal(err.Error()) } _, err = ioutil.ReadAll(response.Body) if err != nil { log.Fatal(err.Error()) } } } type HandlerFile struct { callback func(response http.ResponseWriter, file *os.File) } func (handler *HandlerFile) ServeHTTP(response http.ResponseWriter, request *http.Request) { file, err := os.Open("./main_test.go") if err != nil { log.Fatal(err.Error()) } defer file.Close() handler.callback(response, file) _, err = io.Copy(response, file) if err != nil { log.Fatal(err.Error()) } } func BenchmarkFileWithoutContentLength(b *testing.B) { doNothing := func(response http.ResponseWriter, file *os.File) {} handler := HandlerFile{ callback: doNothing, } runHandlerBenchmark(b, &handler) } func BenchmarkFileWithContentLength(b *testing.B) { addContentLength := func(response http.ResponseWriter, file *os.File) { stat, err := file.Stat() if err != nil { log.Fatal(err.Error()) } response.Header().Set("Content-Length", strconv.FormatInt(stat.Size(), 10)) } handler := HandlerFile{ callback: addContentLength, } runHandlerBenchmark(b, &handler) } ### What did you expect to see? Both benchmarks to take the same amount of time ### What did you see instead? goos: linux goarch: amd64 BenchmarkFileWithoutContentLength-8 12226 95274 ns/op BenchmarkFileWithContentLength-8 20 52161420 ns/op PASS

(21 comments) (1 reaction) (0 assignees)Go (19,008 forks)batch import

NeedsInvestigationhelp wanted

Repository metrics

Stars: (133,883 stars)
PR merge metrics: (No merged PRs in 30d)

Description

What version of Go are you using (`go version`)?

Does this issue reproduce with the latest release?

Yes

What operating system and processor architecture are you using (`go env`)?

What did you do?

What did you expect to see?

Both benchmarks to take the same amount of time

What did you see instead?

Contributor guide

Research direction: Examine the net/http response writer's interaction with io.Copy when Content Length is set. Check for any unnecessary buffering or allocation that slows down the copy.
Tech stack: go
Domain: backendnetworking
Issue type: Bug
Difficulty: 2
Estimated time: 1-3 hours
Activity status: Active
Clarity: Clear
Prerequisites: GoHTTP
Newbie friendliness: 40