Skip to content

Instantly share code, notes, and snippets.

@erikdubbelboer

erikdubbelboer/readme.md

Last active January 31, 2021 22:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save erikdubbelboer/fb1844198849d2122bd0f53cb6767f4b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save erikdubbelboer/fb1844198849d2122bd0f53cb6767f4b to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. erikdubbelboer revised this gist Jan 31, 2021. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1 @@
    See: https://github.com/valyala/fasthttp/issues/931#issuecomment-770458418
  2. erikdubbelboer created this gist Jan 31, 2021.
    168 changes: 168 additions & 0 deletions sendfile-fasthttp.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,168 @@
    package main

    import (
    "fmt"
    "io"
    "io/ioutil"
    "net/http"
    "os"
    "sync"
    "sync/atomic"
    "time"

    "github.com/valyala/fasthttp"
    )

    var (
    useBigFile bool
    fileSize int

    client = &http.Client{
    Transport: &http.Transport{
    MaxIdleConnsPerHost: 1024,
    },
    }
    )

    type bigFileReader struct {
    f *os.File
    }

    func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) {
    if rf, ok := w.(io.ReaderFrom); ok {
    return rf.ReadFrom(r.f)
    }
    panic("never reached")
    }

    func (r *bigFileReader) Read(p []byte) (int, error) {
    return r.f.Read(p)
    }

    func (r *bigFileReader) Close() error {
    return r.f.Close()
    }

    type smallFileReader struct {
    f *os.File
    }

    func (r *smallFileReader) Read(p []byte) (int, error) {
    return r.f.Read(p)
    }

    func (r *smallFileReader) Close() error {
    return r.f.Close()
    }

    func serve(ctx *fasthttp.RequestCtx) {
    f, err := os.Open(fmt.Sprintf("out%d.file", fileSize))
    if err != nil {
    panic(err)
    }

    var r io.Reader
    if useBigFile {
    r = &bigFileReader{f}
    } else {
    r = &smallFileReader{f}
    }

    ctx.SetBodyStream(r, fileSize)
    }

    func bench() float64 {
    seconds := 10
    end := time.Now().Add(time.Second * time.Duration(seconds))

    var requests int64
    var wg sync.WaitGroup

    worker := func() {
    for time.Now().Before(end) {
    resp, err := client.Get("http://localhost:8081/")
    if err != nil {
    panic(err)
    }

    if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
    panic(err)
    }

    if err := resp.Body.Close(); err != nil {
    panic(err)
    }

    atomic.AddInt64(&requests, 1)
    }
    wg.Done()
    }

    for i := 0; i < 32; i++ {
    wg.Add(1)
    go worker()
    }

    wg.Wait()

    return float64(requests) / float64(seconds)
    }

    func add(r []int, from, to, inc int) []int {
    for i := from; i <= to; i += inc {
    r = append(r, i)
    }
    return r
    }

    func size(b int) string {
    const unit = 1000
    div, exp := int64(unit), 0
    for n := b / unit; n >= unit; n /= unit {
    div *= unit
    exp++
    }
    return fmt.Sprintf("%.1f%cb", float64(b)/float64(div), "kmg"[exp])
    }

    func main() {
    go func() {
    if err := fasthttp.ListenAndServe("127.0.0.1:8081", serve); err != nil {
    panic(err)
    }
    }()

    time.Sleep(time.Second) // Give the server some time to start.

    fmt.Printf("%10s: %10s vs %10s %10s\n", "size", "sendfile", "r/w", "sendfile faster")

    sizes := make([]int, 0)

    sizes = add(sizes, 1_000, 10_000, 1_000) // 1kb - 10kb, 1kb increments
    sizes = add(sizes, 20_000, 60_000, 10_000) // 20kb - 60kb, 10kb increments
    sizes = add(sizes, 65_994, 65_995, 1)
    sizes = add(sizes, 70_000, 100_000, 10_000) // 70kb - 100kb, 10kb increments
    sizes = add(sizes, 200_000, 1_200_000, 100_000) // 200kb - 1.2mb, 100kb increments
    sizes = add(sizes, 1_500_000, 3_000_000, 500_000) // 1.5mb - 3mb, 500kb increments
    sizes = add(sizes, 100_000_000, 1_000_000_000, 100_000_000) // 100mb - 1gb, 100mb increments

    for _, fileSize = range sizes {
    if err := ioutil.WriteFile(fmt.Sprintf("out%d.file", fileSize), make([]byte, fileSize), 0666); err != nil {
    panic(err)
    }

    time.Sleep(time.Second)

    useBigFile = false
    smallRate := bench()

    time.Sleep(time.Second)

    useBigFile = true
    bigRate := bench()

    fmt.Printf("%10s: %10.2f vs %10.2f diff %10.2f%%\n", size(fileSize), bigRate, smallRate, 100-smallRate/bigRate*100)

    time.Sleep(time.Second)
    }
    }