Reproducing EOF/write-pipe errors on Go HTTP client side, when server closes an idle connection

Makefile

default: run

gencert:
	go run /home/aryszka/go/src/crypto/tls/generate_cert.go --host localhost

cert.pem: gencert
key.pem: gencert

init: cert.pem key.pem

fmt:
	gofmt -w -s reof.go

run: fmt init
	go run reof.go

reof.go

/*
This code almost consistently reproduces an issue with http.Transport (same for http.Client)
when after making a successful request and keeping the connection in the pool, the server
closes the connection and the client makes a new request using the same connection. In these
cases, the client sometimes receives an error from http.Transport.RoundTrip() or
http.Client.Do(). The issue doesn't always happen, but relatively often. The error can be
either EOF, 'http: server closed idle connection' or 'write: broken pipe'.

Required conditions seem to be:
- the request must have content (headers???)
- the request must happen "immediately" after the server closed the connection that the client
tries to use from the pool (even if we wait for the connection closed state on the server side,
before making the next request)
*/

package main

import (
	"crypto/tls"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strconv"
	"time"
)

const (
	retries   = 243
	useTLS    = false
	useClient = false

	// set the request size to 0 for get requests without payload
	//
	// when the request size is 0, the issue cannot be reproduced
	requestSize  = 1 << 15
	responseSize = 1 << 18

	// set the delay of closing the idle connection on the server and making
	// the next request.
	//
	// when the delay after close is not 0, the issue cannot be reproduced
	delayBeforeClose = 3 * time.Millisecond
	delayAfterClose  = 0

	// this makes the issue happen rarelier, but doesn't eliminate it
	waitConnectionCloseComplete = false

	certFile = "cert.pem"
	keyFile  = "key.pem"

	serverAddress = "localhost:4499"
)

var (
	requestFeed  = rand.NewSource(0)
	responseFeed = rand.NewSource(1)
	closedToken  = struct{}{}
)

type server struct {
	url              string
	server           *http.Server
	lastConn         net.Conn
	lastState        http.ConnState
	connectionClosed chan struct{}
}

type client struct {
	server    *server
	transport http.RoundTripper
	client    *http.Client
}

func response(w http.ResponseWriter, r *http.Request) {
	log.Println("request received", r.Method)

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Println("failed to read request", err)
	}

	if len(b) != requestSize {
		log.Println("failed to read request", len(b), requestSize)
	}

	log.Println("sending response")

	_, err = io.Copy(w, io.LimitReader(rand.New(responseFeed), responseSize))
	if err != nil {
		log.Println("failed to write response", err)
	}
}

func startServer() *server {
	var u string
	if useTLS {
		u = "https://" + serverAddress
	} else {
		u = "http://" + serverAddress
	}

	s := &server{
		url:              u,
		connectionClosed: make(chan struct{}),
	}
	s.server = &http.Server{
		Addr:           serverAddress,
		Handler:        http.HandlerFunc(response),
		ReadTimeout:    12 * time.Second,
		WriteTimeout:   12 * time.Second,
		MaxHeaderBytes: 1 << 20,
		ConnState:      s.connection,
	}

	go func() {
		var err error
		if useTLS {
			err = s.server.ListenAndServeTLS(certFile, keyFile)
		} else {
			err = s.server.ListenAndServe()
		}

		if err != nil {
			log.Fatalln("listener failed", err)
		}
	}()

	// feeling lazy: wait for the server being ready
	time.Sleep(12 * time.Millisecond)
	log.Println("serving")

	return s
}

func (s *server) connection(c net.Conn, cs http.ConnState) {
	if c != s.lastConn {
		log.Println("new connection received", cs)
		s.lastConn = c
	} else {
		log.Println("connection state changed", cs)
	}

	s.lastState = cs
	if waitConnectionCloseComplete && cs == http.StateClosed {
		s.connectionClosed <- closedToken
	}
}

func (s *server) closeLastConnection() {
	if s.lastConn != nil {
		log.Println("closing last connection", s.lastState)
		err := s.lastConn.Close()
		if err != nil {
			log.Println("failed to close connection", err)
		}

		if waitConnectionCloseComplete {
			<-s.connectionClosed
		}
	}
}

func createClient(s *server) *client {
	c := &client{
		server: s,
		transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	if useClient {
		c.client = &http.Client{Transport: c.transport}
	}

	return c
}

func (c *client) makeRequest(requestSize int) {
	var (
		req *http.Request
		rsp *http.Response
		err error
	)

	if requestSize == 0 {
		req, err = http.NewRequest("GET", c.server.url, nil)
		if err != nil {
			if err == io.EOF {
				log.Fatalln("failed to make request", err)
				return
			}

			log.Println("failed to make request", err)
			return
		}
	} else {
		req, err = http.NewRequest(
			"POST",
			c.server.url,
			io.LimitReader(rand.New(requestFeed), int64(requestSize)),
		)
		if err != nil {
			if err == io.EOF {
				log.Fatalln("failed to make request", err)
				return
			}

			log.Println("failed to make request", err)
			return
		}

		req.Header.Set("Content-Length", strconv.Itoa(requestSize))
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	req.Header.Set("X-Test", "test")

	log.Println("sending request", c.server.url, req.Method)
	if useClient {
		rsp, err = c.client.Do(req)
	} else {
		rsp, err = c.transport.RoundTrip(req)
	}

	if err != nil {
		if err == io.EOF {
			log.Fatalln("failed to make request", err)
			return
		}

		log.Println("failed to make request", err)
		return
	}

	defer rsp.Body.Close()

	log.Println("response received", rsp.StatusCode)
	if rsp.StatusCode != http.StatusOK {
		log.Println("invalid status code", rsp.StatusCode, http.StatusOK)
	}

	b, err := ioutil.ReadAll(rsp.Body)
	if err != nil {
		log.Println("failed to read response body", err)
	}

	log.Println("response consumed")

	if len(b) != responseSize {
		log.Println("failed to read response", len(b), responseSize)
	}
}

func main() {
	s := startServer()
	c := createClient(s)

	for i := 0; i < retries; i++ {
		c.makeRequest(requestSize)
		time.Sleep(delayBeforeClose)
		s.closeLastConnection()
		time.Sleep(delayAfterClose)
	}
}

a Makefile that generates certificates: https://gist.github.com/aryszka/0d1a657ae6f7ac660a17cab79424173c