Skip to content

Instantly share code, notes, and snippets.

@decoomanj

decoomanj/httpclient.go

Forked from seantalts/httpclient.go
Created October 11, 2017 10:33
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save decoomanj/c85e5f6540936b7c6b32ecc59a0c3e95 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save decoomanj/c85e5f6540936b7c6b32ecc59a0c3e95 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @seantalts seantalts revised this gist Apr 25, 2014. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -3,7 +3,7 @@ package httptimeout
    import (
    "net/http"
    "time"
    "fmt""
    "fmt"
    )

    type TimeoutTransport struct {
  2. @seantalts seantalts revised this gist Apr 25, 2014. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -37,6 +37,7 @@ func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error)

    select {
    case <-timeout:// A round trip timeout has occurred.
    t.Transport.CancelRequest(req)
    return nil, netTimeoutError{
    error: fmt.Errorf("timed out after %s", t.RoundTripTimeout),
    }
  3. @seantalts seantalts revised this gist Apr 24, 2014. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    package json_client
    package httptimeout

    import (
    "io"
  4. @seantalts seantalts revised this gist Apr 24, 2014. 2 changed files with 55 additions and 33 deletions.
    52 changes: 29 additions & 23 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,40 +1,46 @@
    package httptimeout

    import (
    "net"
    "net/http"
    "time"
    "fmt""
    )

    type TimeoutConn struct {
    net.Conn
    timeout time.Duration
    type TimeoutTransport struct {
    http.Transport
    RoundTripTimeout time.Duration
    }

    func NewTimeoutConn(conn net.Conn, timeout time.Duration) *TimeoutConn {
    return &TimeoutConn{
    Conn: conn,
    timeout: timeout,
    }
    type respAndErr struct {
    resp *http.Response
    err error
    }

    func (c *TimeoutConn) Read(b []byte) (n int, err error) {
    c.SetReadDeadline(time.Now().Add(c.timeout))
    return c.Conn.Read(b)
    type netTimeoutError struct {
    error
    }

    func (c *TimeoutConn) Write(b []byte) (n int, err error) {
    c.SetWriteDeadline(time.Now().Add(c.timeout))
    return c.Conn.Write(b)
    }
    func (ne netTimeoutError) Timeout() bool { return true }

    type dialer func(netw, addr string) (net.Conn, error)
    // If you don't set RoundTrip on TimeoutTransport, this will always timeout at 0
    func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    timeout := time.After(t.RoundTripTimeout)
    resp := make(chan respAndErr, 1)

    go func() {
    r, e := t.Transport.RoundTrip(req)
    resp <- respAndErr{
    resp: r,
    err: e,
    }
    }()

    func NewTimeoutDial(readWriteTimeout, dialTimeout time.Duration) dialer {
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, dialTimeout)
    if err != nil {
    return nil, err
    select {
    case <-timeout:// A round trip timeout has occurred.
    return nil, netTimeoutError{
    error: fmt.Errorf("timed out after %s", t.RoundTripTimeout),
    }
    return NewTimeoutConn(conn, readWriteTimeout), nil
    case r := <-resp: // Success!
    return r.resp, r.err
    }
    }
    36 changes: 26 additions & 10 deletions httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    package httptimeout
    package json_client

    import (
    "io"
    @@ -22,31 +22,47 @@ func TestHttpTimeout(t *testing.T) {
    })
    ts := httptest.NewServer(http.DefaultServeMux)
    defer ts.Close()

    numDials := 0

    client := &http.Client{
    Transport: &http.Transport{
    Dial: func(netw, addr string) (net.Conn, error) {
    t.Logf("dial to %s://%s", netw, addr)
    numDials++
    return NewTimeoutDial(time.Millisecond*200, time.Millisecond*200)(netw,
    addr)
    Transport: &TimeoutTransport{
    Transport: http.Transport{
    Dial: func(netw, addr string) (net.Conn, error) {
    t.Logf("dial to %s://%s", netw, addr)
    numDials++ // For testing only.
    return net.Dial(netw, addr) // Regular ass dial.
    },
    },
    RoundTripTimeout: time.Millisecond * 200,
    },
    }

    addr := ts.URL

    SendTestRequest(t, client, "1st", addr, "normal")
    if numDials != 1 {
    t.Fatalf("Should only have 1 dial at this point.")
    }
    SendTestRequest(t, client, "2st", addr, "normal")
    if numDials != 1 {
    t.Fatalf("Should only have 1 dial at this point.")
    }
    SendTestRequest(t, client, "3st", addr, "timeout")
    if numDials != 1 {
    t.Fatalf("Should only have 1 dial at this point.")
    }
    SendTestRequest(t, client, "4st", addr, "normal")
    if numDials != 2 {
    t.Fatalf("Should have our 2nd dial.")
    }

    time.Sleep(time.Millisecond * 300)
    time.Sleep(time.Millisecond * 700)

    SendTestRequest(t, client, "5st", addr, "normal")
    if numDials
    if numDials != 2 {
    t.Fatalf("Should still only have 2 dials.")
    }
    }

    func SendTestRequest(t *testing.T, client *http.Client, id, addr, path string) {
  5. @seantalts seantalts revised this gist Apr 24, 2014. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    package json_client
    package httptimeout

    import (
    "net"
    2 changes: 1 addition & 1 deletion httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    package json_client
    package httptimeout

    import (
    "io"
  6. @seantalts seantalts revised this gist Apr 24, 2014. 3 changed files with 86 additions and 124 deletions.
    65 changes: 28 additions & 37 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,49 +1,40 @@
    package httpclient
    package json_client

    import (
    "net"
    "net/http"
    "time"
    "net"
    "time"
    )

    type Config struct {
    ConnectTimeout time.Duration
    ReadWriteTimeout time.Duration
    type TimeoutConn struct {
    net.Conn
    timeout time.Duration
    }

    func TimeoutDialer(config *Config) func(net, addr string) (c net.Conn, err error) {
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout)
    if err != nil {
    return nil, err
    }
    conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout))
    return conn, nil
    }
    func NewTimeoutConn(conn net.Conn, timeout time.Duration) *TimeoutConn {
    return &TimeoutConn{
    Conn: conn,
    timeout: timeout,
    }
    }

    func NewTimeoutClient(args ...interface{}) *http.Client {
    // Default configuration
    config := &Config{
    ConnectTimeout: 1 * time.Second,
    ReadWriteTimeout: 1 * time.Second,
    }
    func (c *TimeoutConn) Read(b []byte) (n int, err error) {
    c.SetReadDeadline(time.Now().Add(c.timeout))
    return c.Conn.Read(b)
    }

    // merge the default with user input if there is one
    if len(args) == 1 {
    timeout := args[0].(time.Duration)
    config.ConnectTimeout = timeout
    config.ReadWriteTimeout = timeout
    }
    func (c *TimeoutConn) Write(b []byte) (n int, err error) {
    c.SetWriteDeadline(time.Now().Add(c.timeout))
    return c.Conn.Write(b)
    }

    if len(args) == 2 {
    config.ConnectTimeout = args[0].(time.Duration)
    config.ReadWriteTimeout = args[1].(time.Duration)
    }
    type dialer func(netw, addr string) (net.Conn, error)

    return &http.Client{
    Transport: &http.Transport{
    Dial: TimeoutDialer(config),
    },
    }
    func NewTimeoutDial(readWriteTimeout, dialTimeout time.Duration) dialer {
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, dialTimeout)
    if err != nil {
    return nil, err
    }
    return NewTimeoutConn(conn, readWriteTimeout), nil
    }
    }
    120 changes: 58 additions & 62 deletions httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,84 +1,80 @@
    package httpclient
    package json_client

    import (
    "io"
    "io/ioutil"
    "net"
    "net/http"
    "sync"
    "net/http/httptest"
    "testing"
    "time"
    )

    var starter sync.Once
    var addr net.Addr

    func testHandler(w http.ResponseWriter, req *http.Request) {
    time.Sleep(500 * time.Millisecond)
    io.WriteString(w, "hello, world!\n")
    }

    func testDelayedHandler(w http.ResponseWriter, req *http.Request) {
    time.Sleep(2100 * time.Millisecond)
    io.WriteString(w, "hello, world ... in a bit\n")
    }

    func setupMockServer(t *testing.T) {
    http.HandleFunc("/test", testHandler)
    http.HandleFunc("/test-delayed", testDelayedHandler)
    ln, err := net.Listen("tcp", ":0")
    if err != nil {
    t.Fatalf("failed to listen - %s", err.Error())
    func TestHttpTimeout(t *testing.T) {
    http.HandleFunc("/normal", func(w http.ResponseWriter, req *http.Request) {
    // Empirically, timeouts less than these seem to be flaky
    time.Sleep(100 * time.Millisecond)
    io.WriteString(w, "ok")
    })
    http.HandleFunc("/timeout", func(w http.ResponseWriter, req *http.Request) {
    time.Sleep(250 * time.Millisecond)
    io.WriteString(w, "ok")
    })
    ts := httptest.NewServer(http.DefaultServeMux)
    defer ts.Close()

    numDials := 0

    client := &http.Client{
    Transport: &http.Transport{
    Dial: func(netw, addr string) (net.Conn, error) {
    t.Logf("dial to %s://%s", netw, addr)
    numDials++
    return NewTimeoutDial(time.Millisecond*200, time.Millisecond*200)(netw,
    addr)
    },
    },
    }
    go func() {
    err = http.Serve(ln, nil)
    if err != nil {
    t.Fatalf("failed to start HTTP server - %s", err.Error())
    }
    }()
    addr = ln.Addr()
    }

    func TestDefaultConfig(t *testing.T) {
    starter.Do(func() { setupMockServer(t) })
    addr := ts.URL

    httpClient := NewTimeoutClient()
    req, _ := http.NewRequest("GET", "http://"+addr.String()+"/test-delayed", nil)
    SendTestRequest(t, client, "1st", addr, "normal")
    SendTestRequest(t, client, "2st", addr, "normal")
    SendTestRequest(t, client, "3st", addr, "timeout")
    SendTestRequest(t, client, "4st", addr, "normal")

    httpClient = NewTimeoutClient()

    _, err := httpClient.Do(req)
    if err == nil {
    t.Fatalf("request should have timed out")
    }
    time.Sleep(time.Millisecond * 300)

    SendTestRequest(t, client, "5st", addr, "normal")
    if numDials
    }

    func TestHttpClient(t *testing.T) {
    starter.Do(func() { setupMockServer(t) })
    func SendTestRequest(t *testing.T, client *http.Client, id, addr, path string) {
    req, err := http.NewRequest("GET", addr+"/"+path, nil)

    httpClient := NewTimeoutClient()

    req, _ := http.NewRequest("GET", "http://"+addr.String()+"/test", nil)

    resp, err := httpClient.Do(req)
    if err != nil {
    t.Fatalf("1st request failed - %s", err.Error())
    t.Fatalf("new request failed - %s", err)
    }
    defer resp.Body.Close()

    connectTimeout := (250 * time.Millisecond)
    readWriteTimeout := (50 * time.Millisecond)

    httpClient = NewTimeoutClient(connectTimeout, readWriteTimeout)

    resp, err = httpClient.Do(req)
    if err == nil {
    t.Fatalf("2nd request should have timed out")
    }

    resp, err = httpClient.Do(req)
    if resp != nil {
    t.Fatalf("3nd request should not have timed out")
    req.Header.Add("Connection", "keep-alive")

    switch path {
    case "normal":
    if resp, err := client.Do(req); err != nil {
    t.Fatalf("%s request failed - %s", id, err)
    } else {
    result, err2 := ioutil.ReadAll(resp.Body)
    if err2 != nil {
    t.Fatalf("%s response read failed - %s", id, err2)
    }
    resp.Body.Close()
    t.Logf("%s request - %s", id, result)
    }
    case "timeout":
    if _, err := client.Do(req); err == nil {
    t.Fatalf("%s request not timeout", id)
    } else {
    t.Logf("%s request - %s", id, err)
    }
    }

    }
    25 changes: 0 additions & 25 deletions httpclient_usage.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,25 +0,0 @@
    /*
    This wrapper takes care of both the connection timeout and the readwrite timeout.
    WARNING: You must instantiate this every time you want to use it, otherwise it is
    likely that the timeout is reached before you actually make the call.
    One argument sets the connect timeout and the readwrite timeout to the same value.
    Other wise, 2 arguments are 1) connect and 2) readwrite
    It returns an *http.Client
    */

    package main

    import(
    "httpclient"
    "time"
    )

    func main() {
    httpClient := httpclient.NewWithTimeout(500*time.Millisecond, 1*time.Second)
    resp, err := httpClient.Get("http://google.com")
    if err != nil {
    fmt.Println("Rats! Google is down.")
    }
    }
  7. @dmichael dmichael revised this gist Jun 5, 2013. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -16,10 +16,10 @@ func TimeoutDialer(config *Config) func(net, addr string) (c net.Conn, err error
    conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout)
    if err != nil {
    return nil, err
    }
    }
    conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout))
    return conn, nil
    }
    }
    }

    func NewTimeoutClient(args ...interface{}) *http.Client {
  8. @dmichael dmichael revised this gist Jun 5, 2013. 1 changed file with 31 additions and 31 deletions.
    62 changes: 31 additions & 31 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,49 +1,49 @@
    package httpclient

    import (
    "net"
    "net/http"
    "time"
    "net"
    "net/http"
    "time"
    )

    type Config struct {
    ConnectTimeout time.Duration
    ReadWriteTimeout time.Duration
    ConnectTimeout time.Duration
    ReadWriteTimeout time.Duration
    }

    func TimeoutDialer(config *Config) func(net, addr string) (c net.Conn, err error) {
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout)
    if err != nil {
    return nil, err
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout)
    if err != nil {
    return nil, err
    }
    conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout))
    return conn, nil
    conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout))
    return conn, nil
    }
    }

    func NewTimeoutClient(args ...interface{}) *http.Client {
    // Default configuration
    config := &Config{
    ConnectTimeout: 1 * time.Second,
    ReadWriteTimeout: 1 * time.Second,
    }
    // Default configuration
    config := &Config{
    ConnectTimeout: 1 * time.Second,
    ReadWriteTimeout: 1 * time.Second,
    }

    // merge the default with user input if there is one
    if len(args) == 1 {
    timeout := args[0].(time.Duration)
    config.ConnectTimeout = timeout
    config.ReadWriteTimeout = timeout
    }
    // merge the default with user input if there is one
    if len(args) == 1 {
    timeout := args[0].(time.Duration)
    config.ConnectTimeout = timeout
    config.ReadWriteTimeout = timeout
    }

    if len(args) == 2 {
    config.ConnectTimeout = args[0].(time.Duration)
    config.ReadWriteTimeout = args[1].(time.Duration)
    }
    if len(args) == 2 {
    config.ConnectTimeout = args[0].(time.Duration)
    config.ReadWriteTimeout = args[1].(time.Duration)
    }

    return &http.Client{
    Transport: &http.Transport{
    Dial: TimeoutDialer(config),
    },
    }
    return &http.Client{
    Transport: &http.Transport{
    Dial: TimeoutDialer(config),
    },
    }
    }
  9. @dmichael dmichael revised this gist Jun 5, 2013. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    package httpclient

    import (
    "net"
    "net"
    "net/http"
    "time"
    )
    2 changes: 1 addition & 1 deletion httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,7 +1,7 @@
    package httpclient

    import (
    "io"
    "io"
    "net"
    "net/http"
    "sync"
  10. @dmichael dmichael revised this gist Jun 5, 2013. 1 changed file with 7 additions and 2 deletions.
    9 changes: 7 additions & 2 deletions httpclient_usage.go
    View file
    Original file line number Diff line number Diff line change
    @@ -2,17 +2,22 @@
    This wrapper takes care of both the connection timeout and the readwrite timeout.
    WARNING: You must instantiate this every time you want to use it, otherwise it is
    likely that the timeout is reached before you actually make the call.
    One argument sets the connect timeout and the readwrite timeout to the same value.
    Other wise, 2 arguments are 1) connect and 2) readwrite
    It returns an *http.Client
    */

    package main

    import(
    "httpclient"
    "time
    "time"
    )

    func main() {
    httpClient := httpclient.NewWithTimeout(500*time.Millisecond)
    httpClient := httpclient.NewWithTimeout(500*time.Millisecond, 1*time.Second)
    resp, err := httpClient.Get("http://google.com")
    if err != nil {
    fmt.Println("Rats! Google is down.")
  11. @dmichael dmichael revised this gist Jun 5, 2013. 1 changed file with 20 additions and 0 deletions.
    20 changes: 20 additions & 0 deletions httpclient_usage.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,20 @@
    /*
    This wrapper takes care of both the connection timeout and the readwrite timeout.
    WARNING: You must instantiate this every time you want to use it, otherwise it is
    likely that the timeout is reached before you actually make the call.
    */

    package main

    import(
    "httpclient"
    "time
    )

    func main() {
    httpClient := httpclient.NewWithTimeout(500*time.Millisecond)
    resp, err := httpClient.Get("http://google.com")
    if err != nil {
    fmt.Println("Rats! Google is down.")
    }
    }
  12. @dmichael dmichael created this gist Jun 5, 2013.
    49 changes: 49 additions & 0 deletions httpclient.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,49 @@
    package httpclient

    import (
    "net"
    "net/http"
    "time"
    )

    type Config struct {
    ConnectTimeout time.Duration
    ReadWriteTimeout time.Duration
    }

    func TimeoutDialer(config *Config) func(net, addr string) (c net.Conn, err error) {
    return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout)
    if err != nil {
    return nil, err
    }
    conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout))
    return conn, nil
    }
    }

    func NewTimeoutClient(args ...interface{}) *http.Client {
    // Default configuration
    config := &Config{
    ConnectTimeout: 1 * time.Second,
    ReadWriteTimeout: 1 * time.Second,
    }

    // merge the default with user input if there is one
    if len(args) == 1 {
    timeout := args[0].(time.Duration)
    config.ConnectTimeout = timeout
    config.ReadWriteTimeout = timeout
    }

    if len(args) == 2 {
    config.ConnectTimeout = args[0].(time.Duration)
    config.ReadWriteTimeout = args[1].(time.Duration)
    }

    return &http.Client{
    Transport: &http.Transport{
    Dial: TimeoutDialer(config),
    },
    }
    }
    84 changes: 84 additions & 0 deletions httpclient_test.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,84 @@
    package httpclient

    import (
    "io"
    "net"
    "net/http"
    "sync"
    "testing"
    "time"
    )

    var starter sync.Once
    var addr net.Addr

    func testHandler(w http.ResponseWriter, req *http.Request) {
    time.Sleep(500 * time.Millisecond)
    io.WriteString(w, "hello, world!\n")
    }

    func testDelayedHandler(w http.ResponseWriter, req *http.Request) {
    time.Sleep(2100 * time.Millisecond)
    io.WriteString(w, "hello, world ... in a bit\n")
    }

    func setupMockServer(t *testing.T) {
    http.HandleFunc("/test", testHandler)
    http.HandleFunc("/test-delayed", testDelayedHandler)
    ln, err := net.Listen("tcp", ":0")
    if err != nil {
    t.Fatalf("failed to listen - %s", err.Error())
    }
    go func() {
    err = http.Serve(ln, nil)
    if err != nil {
    t.Fatalf("failed to start HTTP server - %s", err.Error())
    }
    }()
    addr = ln.Addr()
    }

    func TestDefaultConfig(t *testing.T) {
    starter.Do(func() { setupMockServer(t) })

    httpClient := NewTimeoutClient()
    req, _ := http.NewRequest("GET", "http://"+addr.String()+"/test-delayed", nil)

    httpClient = NewTimeoutClient()

    _, err := httpClient.Do(req)
    if err == nil {
    t.Fatalf("request should have timed out")
    }

    }

    func TestHttpClient(t *testing.T) {
    starter.Do(func() { setupMockServer(t) })

    httpClient := NewTimeoutClient()

    req, _ := http.NewRequest("GET", "http://"+addr.String()+"/test", nil)

    resp, err := httpClient.Do(req)
    if err != nil {
    t.Fatalf("1st request failed - %s", err.Error())
    }
    defer resp.Body.Close()

    connectTimeout := (250 * time.Millisecond)
    readWriteTimeout := (50 * time.Millisecond)

    httpClient = NewTimeoutClient(connectTimeout, readWriteTimeout)

    resp, err = httpClient.Do(req)
    if err == nil {
    t.Fatalf("2nd request should have timed out")
    }

    resp, err = httpClient.Do(req)
    if resp != nil {
    t.Fatalf("3nd request should not have timed out")
    }

    }