-
-
Save z-sector/2a6abebc58993ad047960c17bc403fe1 to your computer and use it in GitHub Desktop.
Bounded Parallel Get Requests in Golang
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "fmt" | |
| "net/http" | |
| "sort" | |
| "time" | |
| ) | |
| // a struct to hold the result from each request including an index | |
| // which will be used for sorting the results after they come in | |
| type result struct { | |
| index int | |
| res http.Response | |
| err error | |
| } | |
| // boundedParallelGet sends requests in parallel but only up to a certain | |
| // limit, and furthermore it's only parallel up to the amount of CPUs but | |
| // is always concurrent up to the concurrency limit | |
| func boundedParallelGet(urls []string, concurrencyLimit int) []result { | |
| // this buffered channel will block at the concurrency limit | |
| semaphoreChan := make(chan struct{}, concurrencyLimit) | |
| // this channel will not block and collect the http request results | |
| resultChan := make(chan *result) | |
| // make sure we close these channels when we're done with them | |
| defer func() { | |
| close(semaphoreChan) | |
| close(resultChan) | |
| }() | |
| // keen an index and loop through every url we will send a request to | |
| for i, url := range urls { | |
| // start a go routine with the index and url in a closure | |
| go func(i int, url string) { | |
| // this sends an empty struct into the semaphoreChan which | |
| // is basically saying add one to the limit, but when the | |
| // limit has been reached block until there is room | |
| semaphoreChan <- struct{}{} | |
| // send the request and put the response in a result struct | |
| // along with the index so we can sort them later along with | |
| // any error that might have occoured | |
| res, err := http.Get(url) | |
| result := &result{i, *res, err} | |
| // now send the result struct through the resultChan so we | |
| // can get the results without being able to return them | |
| resultChan <- result | |
| // once we're done it's we read from the semaphoreChan which | |
| // has the effect of removing one from the limit and allowing | |
| // another goroutine to start | |
| <-semaphoreChan | |
| }(i, url) | |
| } | |
| // make a slice to hold the results we're expecting | |
| var results []result | |
| // start listening for any results over the resultChan | |
| for { | |
| // once we get a result append it to the result slice | |
| result := <-resultChan | |
| results = append(results, *result) | |
| // if we've reached the expected amount of urls then stop | |
| if len(results) == len(urls) { | |
| break | |
| } | |
| } | |
| // let's sort these results real quick | |
| sort.Slice(results, func(i, j int) bool { | |
| return results[i].index < results[j].index | |
| }) | |
| return results | |
| } | |
| // send a bunch of requests and time how long they take | |
| func benchmarkBoundedParallelRequests(urls []string, concurrency int) string { | |
| boundedParallelTimeStart := time.Now() | |
| results := boundedParallelGet(urls, concurrency) | |
| seconds := time.Since(boundedParallelTimeStart).Seconds() | |
| tmplate := "%d bounded parallel requests: %d/%d in %v" | |
| return fmt.Sprintf(tmplate, concurrency, len(results), len(urls), seconds) | |
| } | |
| func main() { | |
| // let's make a slice of URLs to send requets to | |
| var urls []string | |
| for i := 0; i < 100; i++ { | |
| urls = append(urls, "http://httpbin.org/get") | |
| } | |
| // and now let's compare different concurrency limits | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 5)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 10)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 25)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 50)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 75)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 100)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 125)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 150)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 175)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 200)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 300)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 325)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 350)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 375)) | |
| fmt.Println(benchmarkBoundedParallelRequests(urls, 300)) | |
| // and you should see something like the following printed | |
| // depending on how fast your computer and internet is | |
| // 5 bounded parallel requests: 100/100 in 5.533223255 | |
| // 10 bounded parallel requests: 100/100 in 2.5115351219999997 | |
| // 25 bounded parallel requests: 100/100 in 1.189462884 | |
| // 50 bounded parallel requests: 100/100 in 1.17430002 | |
| // 75 bounded parallel requests: 100/100 in 1.001383863 | |
| // 100 bounded parallel requests: 100/100 in 1.3769354 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment