Make concurrent requests in Go

Our API is designed to allow you to have multiple concurrent scraping operations. That means you can speed up scraping for hundreds, thousands or even millions of pages per day, depending on your plan.

The more concurrent requests limit you have the more calls you can have active in parallel, and the faster you can scrape.

Making concurrent requests in GoLang is as easy as adding a “go” keyword before our scraping functions! The code below will make two concurrent requests to ScrapingBee’s pages, and save the content in an HTML file.

package main

import (


func save_page_to_html(target_url string, file_path string, wg *sync.WaitGroup) (interface{}, error) { // Using sync.Waitgroup to wait for goroutines to finish

  req, err := http.NewRequest("GET", SCRAPINGBEE_URL, nil)
  if err != nil {
    return nil, fmt.Errorf("Failed to build the request: %s", err)

  q := req.URL.Query()
  q.Add("api_key", API_KEY)
  q.Add("url", target_url)
  req.URL.RawQuery = q.Encode()

  client := &http.Client{}
  resp, err := client.Do(req)
  if err != nil {
    return nil, fmt.Errorf("Failed to request ScrapingBee: %s", err)
  defer resp.Body.Close()

  if resp.StatusCode != http.StatusOK {
    return nil, fmt.Errorf("Error request response with status code %d", resp.StatusCode)

  bodyBytes, err := io.ReadAll(resp.Body)

  file, err := os.Create(file_path)
  if err != nil {
      return nil, fmt.Errorf("Couldn't create the file ", err)

  l, err := file.Write(bodyBytes) // Write content to the file.
  if err != nil {
      return nil, fmt.Errorf("Couldn't write content to the file ", err)
  err = file.Close()
  if err != nil {
      return nil, fmt.Errorf("Couldn't close the file ", err)

  return l, nil

func main() {

    var wg sync.WaitGroup // New wait group
    wg.Add(2) // Using two goroutines

    go save_page_to_html("", "blog.html", &wg)
    go save_page_to_html("", "documentation.html", &wg)


Go back to tutorials