Skip to content

Instantly share code, notes, and snippets.

@z-sector

z-sector/pget.go

Forked from montanaflynn/pget.go
Created September 28, 2023 09:15
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save z-sector/2a6abebc58993ad047960c17bc403fe1 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save z-sector/2a6abebc58993ad047960c17bc403fe1 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 40 additions and 47 deletions.
    87 changes: 40 additions & 47 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -24,12 +24,12 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    semaphoreChan := make(chan struct{}, concurrencyLimit)

    // this channel will not block and collect the http request results
    resultChan := make(chan *result)
    resultsChan := make(chan *result)

    // make sure we close these channels when we're done with them
    defer func() {
    close(semaphoreChan)
    close(resultChan)
    close(resultsChan)
    }()

    // keen an index and loop through every url we will send a request to
    @@ -49,9 +49,8 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    res, err := http.Get(url)
    result := &result{i, *res, err}

    // now send the result struct through the resultChan so we
    // can get the results without being able to return them
    resultChan <- result
    // now we can send the result struct through the resultsChan
    resultsChan <- result

    // once we're done it's we read from the semaphoreChan which
    // has the effect of removing one from the limit and allowing
    @@ -64,11 +63,10 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    // make a slice to hold the results we're expecting
    var results []result

    // start listening for any results over the resultChan
    // start listening for any results over the resultsChan
    // once we get a result append it to the result slice
    for {

    // once we get a result append it to the result slice
    result := <-resultChan
    result := <-resultsChan
    results = append(results, *result)

    // if we've reached the expected amount of urls then stop
    @@ -82,50 +80,45 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    return results[i].index < results[j].index
    })

    // now we're done we return the results
    return results
    }

    // send a bunch of requests and time how long they take
    func benchmarkBoundedParallelRequests(urls []string, concurrency int) string {
    boundedParallelTimeStart := time.Now()
    results := boundedParallelGet(urls, concurrency)
    seconds := time.Since(boundedParallelTimeStart).Seconds()
    tmplate := "%d bounded parallel requests: %d/%d in %v"
    return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds)
    }

    func main() {
    // we'll use the init function to set up the benchmark
    // by making a slice of 100 URLs to send requets to
    var urls []string

    // let's make a slice of URLs to send requets to
    var urls []string
    func init() {
    for i := 0; i < 100; i++ {
    urls = append(urls, "http://httpbin.org/get")
    }
    }

    // the main function sets up an anonymous benchmark func
    // that will time how long it takes to get all the URLs
    // at the specified concurrency level
    //
    // and you should see something like the following printed
    // depending on how fast your computer and internet is
    //
    // 5 bounded parallel requests: 100/100 in 5.533223255
    // 10 bounded parallel requests: 100/100 in 2.5115351219
    // 25 bounded parallel requests: 100/100 in 1.189462884
    // 50 bounded parallel requests: 100/100 in 1.17430002
    // 75 bounded parallel requests: 100/100 in 1.001383863
    // 100 bounded parallel requests: 100/100 in 1.3769354
    func main() {
    benchmark := func(urls []string, concurrency int) string {
    startTime := time.Now()
    results := boundedParallelGet(urls, concurrency)
    seconds := time.Since(startTime).Seconds()
    tmplate := "%d bounded parallel requests: %d/%d in %v"
    return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds)
    }

    // and now let's compare different concurrency limits
    fmt.Println(benchmarkBoundedParallelRequests(urls, 5))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 10))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 25))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 50))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 75))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 100))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 125))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 150))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 175))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 200))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 300))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 325))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 350))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 375))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 300))

    // and you should see something like the following printed
    // depending on how fast your computer and internet is

    // 5 bounded parallel requests: 100/100 in 5.533223255
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997
    // 25 bounded parallel requests: 100/100 in 1.189462884
    // 50 bounded parallel requests: 100/100 in 1.17430002
    // 75 bounded parallel requests: 100/100 in 1.001383863
    // 100 bounded parallel requests: 100/100 in 1.3769354
    fmt.Println(benchmark(urls, 10))
    fmt.Println(benchmark(urls, 25))
    fmt.Println(benchmark(urls, 50))
    fmt.Println(benchmark(urls, 75))
    fmt.Println(benchmark(urls, 100))
    }
  2. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 19 additions and 4 deletions.
    23 changes: 19 additions & 4 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -8,7 +8,7 @@ import (
    )

    // a struct to hold the result from each request including an index
    // which can be used for sorting the results after they come in
    // which will be used for sorting the results after they come in
    type result struct {
    index int
    res http.Response
    @@ -26,6 +26,12 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    // this channel will not block and collect the http request results
    resultChan := make(chan *result)

    // make sure we close these channels when we're done with them
    defer func() {
    close(semaphoreChan)
    close(resultChan)
    }()

    // keen an index and loop through every url we will send a request to
    for i, url := range urls {

    @@ -61,7 +67,7 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    // start listening for any results over the resultChan
    for {

    // once we've got a result append it to the result slice
    // once we get a result append it to the result slice
    result := <-resultChan
    results = append(results, *result)

    @@ -103,10 +109,19 @@ func main() {
    fmt.Println(benchmarkBoundedParallelRequests(urls, 50))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 75))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 100))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 125))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 150))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 175))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 200))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 300))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 325))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 350))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 375))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 300))

    // and you should see something like the following printed
    // depending on how fast your computer and internet is
    //
    // depending on how fast your computer and internet is

    // 5 bounded parallel requests: 100/100 in 5.533223255
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997
    // 25 bounded parallel requests: 100/100 in 1.189462884
  3. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -106,7 +106,7 @@ func main() {

    // and you should see something like the following printed
    // depending on how fast your computer and internet is

    //
    // 5 bounded parallel requests: 100/100 in 5.533223255
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997
    // 25 bounded parallel requests: 100/100 in 1.189462884
  4. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 11 additions and 11 deletions.
    22 changes: 11 additions & 11 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -96,21 +96,21 @@ func main() {
    urls = append(urls, "http://httpbin.org/get")
    }

    // and now let's compare different concurrency limits
    fmt.Println(benchmarkBoundedParallelRequests(urls, 5))
    // Output: 5 bounded parallel requests: 100/100 in 5.533223255

    fmt.Println(benchmarkBoundedParallelRequests(urls, 10))
    // Output: 10 bounded parallel requests: 100/100 in 2.5115351219999997

    fmt.Println(benchmarkBoundedParallelRequests(urls, 25))
    // Output: 25 bounded parallel requests: 100/100 in 1.189462884

    fmt.Println(benchmarkBoundedParallelRequests(urls, 50))
    // Output: 50 bounded parallel requests: 100/100 in 1.17430002

    fmt.Println(benchmarkBoundedParallelRequests(urls, 75))
    // Output: 75 bounded parallel requests: 100/100 in 1.001383863

    fmt.Println(benchmarkBoundedParallelRequests(urls, 100))
    // Output: 100 bounded parallel requests: 100/100 in 1.3769354

    // and you should see something like the following printed
    // depending on how fast your computer and internet is

    // 5 bounded parallel requests: 100/100 in 5.533223255
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997
    // 25 bounded parallel requests: 100/100 in 1.189462884
    // 50 bounded parallel requests: 100/100 in 1.17430002
    // 75 bounded parallel requests: 100/100 in 1.001383863
    // 100 bounded parallel requests: 100/100 in 1.3769354
    }
  5. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 6 additions and 6 deletions.
    12 changes: 6 additions & 6 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -96,21 +96,21 @@ func main() {
    urls = append(urls, "http://httpbin.org/get")
    }

    fmt.Println(benchmarkBoundedParallelRequests(5))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 5))
    // Output: 5 bounded parallel requests: 100/100 in 5.533223255

    fmt.Println(benchmarkBoundedParallelRequests(10))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 10))
    // Output: 10 bounded parallel requests: 100/100 in 2.5115351219999997

    fmt.Println(benchmarkBoundedParallelRequests(25))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 25))
    // Output: 25 bounded parallel requests: 100/100 in 1.189462884

    fmt.Println(benchmarkBoundedParallelRequests(50))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 50))
    // Output: 50 bounded parallel requests: 100/100 in 1.17430002

    fmt.Println(benchmarkBoundedParallelRequests(75))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 75))
    // Output: 75 bounded parallel requests: 100/100 in 1.001383863

    fmt.Println(benchmarkBoundedParallelRequests(100))
    fmt.Println(benchmarkBoundedParallelRequests(urls, 100))
    // Output: 100 bounded parallel requests: 100/100 in 1.3769354
    }
  6. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 15 additions and 17 deletions.
    32 changes: 15 additions & 17 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -7,16 +7,6 @@ import (
    "time"
    )

    // a slice for the urls we're going to fake
    var urls []string

    // make one hundred httpbin and put them in the slice
    func init() {
    for i := 0; i < 100; i++ {
    urls = append(urls, "http://httpbin.org/get")
    }
    }

    // a struct to hold the result from each request including an index
    // which can be used for sorting the results after they come in
    type result struct {
    @@ -89,7 +79,8 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    return results
    }

    func benchmarkBoundedParallelRequests(concurrency int) string {
    // send a bunch of requests and time how long they take
    func benchmarkBoundedParallelRequests(urls []string, concurrency int) string {
    boundedParallelTimeStart := time.Now()
    results := boundedParallelGet(urls, concurrency)
    seconds := time.Since(boundedParallelTimeStart).Seconds()
    @@ -98,21 +89,28 @@ func benchmarkBoundedParallelRequests(concurrency int) string {
    }

    func main() {

    // let's make a slice of URLs to send requets to
    var urls []string
    for i := 0; i < 100; i++ {
    urls = append(urls, "http://httpbin.org/get")
    }

    fmt.Println(benchmarkBoundedParallelRequests(5))
    // 5 bounded parallel requests: 100/100 in 5.533223255
    // Output: 5 bounded parallel requests: 100/100 in 5.533223255

    fmt.Println(benchmarkBoundedParallelRequests(10))
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997
    // Output: 10 bounded parallel requests: 100/100 in 2.5115351219999997

    fmt.Println(benchmarkBoundedParallelRequests(25))
    // 25 bounded parallel requests: 100/100 in 1.189462884
    // Output: 25 bounded parallel requests: 100/100 in 1.189462884

    fmt.Println(benchmarkBoundedParallelRequests(50))
    // 50 bounded parallel requests: 100/100 in 1.17430002
    // Output: 50 bounded parallel requests: 100/100 in 1.17430002

    fmt.Println(benchmarkBoundedParallelRequests(75))
    // 75 bounded parallel requests: 100/100 in 1.001383863
    // Output: 75 bounded parallel requests: 100/100 in 1.001383863

    fmt.Println(benchmarkBoundedParallelRequests(100))
    // 100 bounded parallel requests: 100/100 in 1.3769354
    // Output: 100 bounded parallel requests: 100/100 in 1.3769354
    }
  7. Montana Flynn revised this gist Dec 23, 2016. 1 changed file with 52 additions and 19 deletions.
    71 changes: 52 additions & 19 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -3,63 +3,96 @@ package main
    import (
    "fmt"
    "net/http"
    "sort"
    "time"
    )

    // a slice for the urls we're going to fake
    var urls []string

    // make one hundred httpbin and put them in the slice
    func init() {
    for i := 0; i < 100; i++ {
    urls = append(urls, "http://httpbin.org/get")
    }
    }

    // a struct to hold the result from each request including an index
    // which can be used for sorting the results after they come in
    type result struct {
    i int
    res http.Response
    err error
    index int
    res http.Response
    err error
    }

    type semaphore struct{}

    // boundedParallelGet sends requests in parallel but only up to a certain
    // limit, and furthermore it's only parallel up to the amount of CPUs but
    // is always concurrent up to the concurrency limit
    func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    semaphoreChan := make(chan semaphore, concurrencyLimit)
    resultChan := make(chan *result)

    client := http.Client{}
    // this buffered channel will block at the concurrency limit
    semaphoreChan := make(chan struct{}, concurrencyLimit)

    // this channel will not block and collect the http request results
    resultChan := make(chan *result)

    // keen an index and loop through every url we will send a request to
    for i, url := range urls {

    // start a go routine with the index and url in a closure
    go func(i int, url string) {
    semaphoreChan <- semaphore{}

    res, err := client.Get(url)
    defer res.Body.Close()
    // this sends an empty struct into the semaphoreChan which
    // is basically saying add one to the limit, but when the
    // limit has been reached block until there is room
    semaphoreChan <- struct{}{}

    // send the request and put the response in a result struct
    // along with the index so we can sort them later along with
    // any error that might have occoured
    res, err := http.Get(url)
    result := &result{i, *res, err}

    // now send the result struct through the resultChan so we
    // can get the results without being able to return them
    resultChan <- result


    // once we're done it's we read from the semaphoreChan which
    // has the effect of removing one from the limit and allowing
    // another goroutine to start
    <-semaphoreChan

    }(i, url)
    }

    // make a slice to hold the results we're expecting
    var results []result

    // start listening for any results over the resultChan
    for {
    select {
    case result := <-resultChan:
    results = append(results, *result)
    if len(results) == len(urls) {
    return results
    }

    // once we've got a result append it to the result slice
    result := <-resultChan
    results = append(results, *result)

    // if we've reached the expected amount of urls then stop
    if len(results) == len(urls) {
    break
    }
    }

    // let's sort these results real quick
    sort.Slice(results, func(i, j int) bool {
    return results[i].index < results[j].index
    })

    return results
    }

    func benchmarkBoundedParallelRequests(concurrency int) string {
    boundedParallelTimeStart := time.Now()
    results := boundedParallelGet(urls, concurrency)
    seconds := time.Now().Sub(boundedParallelTimeStart).Seconds()
    seconds := time.Since(boundedParallelTimeStart).Seconds()
    tmplate := "%d bounded parallel requests: %d/%d in %v"
    return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds)
    }
  8. Montana Flynn revised this gist Dec 21, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -37,8 +37,8 @@ func boundedParallelGet(urls []string, concurrencyLimit int) []result {

    result := &result{i, *res, err}
    resultChan <- result

    <-semaphoreChan

    }(i, url)
    }

  9. Montana Flynn revised this gist Dec 16, 2016. No changes.
  10. Montana Flynn created this gist Dec 16, 2016.
    85 changes: 85 additions & 0 deletions pget.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,85 @@
    package main

    import (
    "fmt"
    "net/http"
    "time"
    )

    var urls []string

    func init() {
    for i := 0; i < 100; i++ {
    urls = append(urls, "http://httpbin.org/get")
    }
    }

    type result struct {
    i int
    res http.Response
    err error
    }

    type semaphore struct{}

    func boundedParallelGet(urls []string, concurrencyLimit int) []result {
    semaphoreChan := make(chan semaphore, concurrencyLimit)
    resultChan := make(chan *result)

    client := http.Client{}

    for i, url := range urls {
    go func(i int, url string) {
    semaphoreChan <- semaphore{}

    res, err := client.Get(url)
    defer res.Body.Close()

    result := &result{i, *res, err}
    resultChan <- result
    <-semaphoreChan

    }(i, url)
    }

    var results []result
    for {
    select {
    case result := <-resultChan:
    results = append(results, *result)
    if len(results) == len(urls) {
    return results
    }
    }
    }

    return results
    }

    func benchmarkBoundedParallelRequests(concurrency int) string {
    boundedParallelTimeStart := time.Now()
    results := boundedParallelGet(urls, concurrency)
    seconds := time.Now().Sub(boundedParallelTimeStart).Seconds()
    tmplate := "%d bounded parallel requests: %d/%d in %v"
    return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds)
    }

    func main() {
    fmt.Println(benchmarkBoundedParallelRequests(5))
    // 5 bounded parallel requests: 100/100 in 5.533223255

    fmt.Println(benchmarkBoundedParallelRequests(10))
    // 10 bounded parallel requests: 100/100 in 2.5115351219999997

    fmt.Println(benchmarkBoundedParallelRequests(25))
    // 25 bounded parallel requests: 100/100 in 1.189462884

    fmt.Println(benchmarkBoundedParallelRequests(50))
    // 50 bounded parallel requests: 100/100 in 1.17430002

    fmt.Println(benchmarkBoundedParallelRequests(75))
    // 75 bounded parallel requests: 100/100 in 1.001383863

    fmt.Println(benchmarkBoundedParallelRequests(100))
    // 100 bounded parallel requests: 100/100 in 1.3769354
    }