Getting started with ScrapingBee and Go

In this tutorial, we will see how you can use ScrapingBee’s API with GoLang, and use it to scrape web pages. As such, we will cover these topics:

• General structure of an API request
• Create your first API request.

Let’s get started!

1. General structure of an API request

The general structure of an API request made in Go will always look like this:

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "net/url"
)
func get_request() *http.Response {
    // Create client
    client := &http.Client{}

    my_url := url.QueryEscape("YOUR-URL") // Encoding the URL
    // Create request
    req, err := http.NewRequest("GET", "https://app.scrapingbee.com/api/v1/?api_key=YOUR-API-KEY&url="+my_url, nil) // Create the request the request

    parseFormErr := req.ParseForm()
    if parseFormErr != nil {
        fmt.Println(parseFormErr)
    }

    // Fetch Request
    resp, err := client.Do(req)

    if err != nil {
        fmt.Println("Failure : ", err)
    }

    return resp // Return the response
}

And you can do whatever you want with the response variable! For example:

func main() {
    request := get_request()
        // Read Response Body
    respBody, _ := ioutil.ReadAll(request.Body)

    // Display Results
    fmt.Println("response Status : ", request.Status)
    fmt.Println("response Headers : ", request.Header)
    fmt.Println("response Body : ", string(respBody))
}

2. Create your first API request:

Let’s create a tool that saves the HTML code of ScrapingBee’s blog:

func get_request(api_key string, user_url string) *http.Response {
    // Create client
    client := &http.Client{}

    my_url := url.QueryEscape(user_url) // Encoding the URL
    // Create request
    req, err := http.NewRequest("GET", "https://app.scrapingbee.com/api/v1/?api_key="+api_key+"&url="+my_url, nil) // Create the request the request

    parseFormErr := req.ParseForm()
    if parseFormErr != nil {
        fmt.Println(parseFormErr)
    }

    // Fetch Request
    resp, err := client.Do(req)

    if err != nil {
        fmt.Println("Failure : ", err)
    }
    return resp // Return the response
}

func main() {
    api_key := "YOUR-API-KEY"
    request := get_request(api_key, "https://scrapingbee.com/blog")
    // Read Response Body
    respBody, _ := ioutil.ReadAll(request.Body)

    file, err := os.Create("page.html")
    if err != nil {
        fmt.Println(err)
        return
    }
    l, err := file.WriteString(string(respBody)) // Write content to the file.
    if err != nil {
        fmt.Println(err)
        file.Close()
        return
    }
    fmt.Println("File has been saved successfully")
    err = file.Close()
    if err != nil {
        fmt.Println(err)
        return
    }
}