-
-
Save z-sector/2a6abebc58993ad047960c17bc403fe1 to your computer and use it in GitHub Desktop.
Bounded Parallel Get Requests in Golang
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "fmt" | |
| "net/http" | |
| "time" | |
| ) | |
| var urls []string | |
| func init() { | |
| for i := 0; i < 100; i++ { | |
| urls = append(urls, "http://httpbin.org/get") | |
| } | |
| } | |
| type result struct { | |
| i int | |
| res http.Response | |
| err error | |
| } | |
| type semaphore struct{} | |
| func boundedParallelGet(urls []string, concurrencyLimit int) []result { | |
| semaphoreChan := make(chan semaphore, concurrencyLimit) | |
| resultChan := make(chan *result) | |
| client := http.Client{} | |
| for i, url := range urls { | |
| go func(i int, url string) { | |
| semaphoreChan <- semaphore{} | |
| res, err := client.Get(url) | |
| defer res.Body.Close() | |
| result := &result{i, *res, err} | |
| resultChan <- result | |
| <-semaphoreChan | |
| }(i, url) | |
| } | |
| var results []result | |
| for { | |
| select { | |
| case result := <-resultChan: | |
| results = append(results, *result) | |
| if len(results) == len(urls) { | |
| return results | |
| } | |
| } | |
| } | |
| return results | |
| } | |
| func benchmarkBoundedParallelRequests(concurrency int) string { | |
| boundedParallelTimeStart := time.Now() | |
| results := boundedParallelGet(urls, concurrency) | |
| seconds := time.Now().Sub(boundedParallelTimeStart).Seconds() | |
| tmplate := "%d bounded parallel requests: %d/%d in %v" | |
| return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds) | |
| } | |
| func main() { | |
| fmt.Println(benchmarkBoundedParallelRequests(5)) | |
| // 5 bounded parallel requests: 100/100 in 5.533223255 | |
| fmt.Println(benchmarkBoundedParallelRequests(10)) | |
| // 10 bounded parallel requests: 100/100 in 2.5115351219999997 | |
| fmt.Println(benchmarkBoundedParallelRequests(25)) | |
| // 25 bounded parallel requests: 100/100 in 1.189462884 | |
| fmt.Println(benchmarkBoundedParallelRequests(50)) | |
| // 50 bounded parallel requests: 100/100 in 1.17430002 | |
| fmt.Println(benchmarkBoundedParallelRequests(75)) | |
| // 75 bounded parallel requests: 100/100 in 1.001383863 | |
| fmt.Println(benchmarkBoundedParallelRequests(100)) | |
| // 100 bounded parallel requests: 100/100 in 1.3769354 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment