Documentation

How to use ScrapingBee's Google Search API

Overview

Here is the list of the different parameters you can use with ScrapingBee's Google Search API.

name [type](default) Description
api_key [string] required Your API key: Get your API key learn more
search [string] required The text you would put in the Google search bar. learn more
country_code [string] ("") Geolocation. learn more
nb_results [integer] (100) The number of results you wish to extract learn more
page [integer] (1) The page number you want to extract results from learn more
language [string] (en) Language the search results will be displayed in. learn more
add_html [boolean] (false) Add the full html of the page in the results learn more

Getting Started

Our Google Search API allows you to scrape search results pages in realtime.

To scrape the results, you only need two things:

Then, simply do this.

Keep in mind that each successful API call will cost you 25 api credits.


curl "https://app.scrapingbee.com/api/v1/?api_key=YOUR-API-KEY&search=YOUR-SEARCH"
         

#  Install the Python Requests library:
# `pip install requests`
import requests

def send_request():
    response = requests.get(
        url="https://app.scrapingbee.com/api/v1/store/google",
        params={
            "api_key": "YOUR-API-KEY",
            "search": "pizza new-york",
        },

    )
    print('Response HTTP Status Code: ', response.status_code)
    print('Response HTTP Response Body: ', response.content)
send_request()

// request Classic
const https = require('https')

const options = {
    hostname: 'app.scrapingbee.com',
    port: '443',
    path: '/api/v1/store/google?api_key=YOUR-API-KEY&search=pizza%20new%20york',
    method: 'GET',

}

const req = https.request(options, res => {
    console.log(`statusCode: ${ res.statusCode }`)
    res.on('data', d => {
        process.stdout.write(d)
    })
})

req.on('error', error => {
    console.error(error)
})

req.end()

import java.io.IOException;
import org.apache.http.client.fluent.*;

public class SendRequest
{
  public static void main(String[] args) {
    sendRequest();
  }

  private static void sendRequest() {

    // Classic (GET )

    try {

      // Create request
      Content content = Request.Get("https://app.scrapingbee.com/api/v1/store/google?api_key=YOUR-API-KEY&search=pizza%20new%20york")



      // Fetch request and return content
      .execute().returnContent();

      // Print content
      System.out.println(content);
    }
    catch (IOException e) { System.out.println(e); }
  }
}

require 'net/http'
require 'net/https'

# Classic (GET )
def send_request
    uri = URI('https://app.scrapingbee.com/api/v1/store/google:?api_key=YOUR-API-KEY&search=pizza%20new%20york')

    # Create client
    http = Net::HTTP.new(uri.host, uri.port)
    http.use_ssl = true
    http.verify_mode = OpenSSL::SSL::VERIFY_PEER

    # Create Request
    req =  Net::HTTP::Get.new(uri)

    # Fetch Request
    res = http.request(req)
    puts "Response HTTP Status Code: #{ res.code }"
    puts "Response HTTP Response Body: #{ res.body }"
rescue StandardError => e
    puts "HTTP Request failed (#{ e.message })"
end

send_request()

<?php

// get cURL resource
$ch = curl_init();

// set url
curl_setopt($ch, CURLOPT_URL, 'https://app.scrapingbee.com/api/v1/store/google?api_key=YOUR-API-KEY&search=pizza%20new%20york');

// set method
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

// return the transfer as a string
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);



// send the request and save response to $response
$response = curl_exec($ch);

// stop if fails
if (!$response) {
  die('Error: "' . curl_error($ch) . '" - Code: ' . curl_errno($ch));
}

echo 'HTTP Status Code: ' . curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
echo 'Response Body: ' . $response . PHP_EOL;

// close curl resource to free up system resources
curl_close($ch);

?>

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func sendClassic() {
	// Create client
	client := &http.Client{}

	// Create request
	req, err := http.NewRequest("GET", "https://app.scrapingbee.com/api/v1/store/google?api_key=YOUR-API-KEY&search=pizza%20new%20york", nil)


	parseFormErr := req.ParseForm()
	if parseFormErr != nil {
		fmt.Println(parseFormErr)
	}

	// Fetch Request
	resp, err := client.Do(req)

	if err != nil {
		fmt.Println("Failure : ", err)
	}

	// Read Response Body
	respBody, _ := ioutil.ReadAll(resp.Body)

	// Display Results
	fmt.Println("response Status : ", resp.Status)
	fmt.Println("response Headers : ", resp.Header)
	fmt.Println("response Body : ", string(respBody))
}

func main() {
    sendClassic()
}

The API will then respond with formatted JSON data:


{
    "statusCode": 200,
    "body": {
        "meta_data": {
            "url": "https://www.google.com/search?q=pizza%20new%20york&hl=en&gl=us&num=20&start=0",
            "number_of_results": 615000000,
            "location": "",
            "number_of_organic_results": 22,
            "number_of_ads": 0,
            "number_of_page": 7
        },
        "local_results": [{
            "title": "Joe's Pizza",
            "review": 4.5,
            "position": 0,
            "review_count": 5422
        },
        ...
        ],
        "organic_results": [{
            "url": "https://ny.eater.com/maps/nyc-best-iconic-pizza-pizzeria",
            "displayed_url": "ny.eater.com › maps › nyc-best-iconic-pizza-pizzeria",
            "description": "Jan 17, 2020 - Roberto Paciullo's Bronx trattoria serves wood-fired pizzas that have puffy brown crusts and floppy centers. Some pizza geeks think that Zero Otto ...",
            "position": 0,
            "title": "New York City's 27 Most Iconic Pizzerias - Eater NY"
        },
        ...
        {
            "url": "https://www.timeout.com/newyork/restaurants/best-new-york-pizza",
            "displayed_url": "www.timeout.com › restaurants › best-new-york-pizza",
            "description": "Aug 2, 2019 - There's nothing that says “I love NY” more than eating a classic slice of New York pizza in view of the Brooklyn Bridge. And who better to provide ...",
            "position": 17,
            "title": "29 Best Pizzas in NYC for Life-Changing Slices To Eat Today"
        }],
        "top_ads": [],
        "bottom_ads": [],
        "related_queries": [{
            "title": "best pizza new york",
            "url": "https://www.google.com/search?num=20&hl=en&gl=us&q=best+pizza+new+york&sa=X&ved=2ahUKEwiS_8TV6MrrAhVJ6uAKHdWjDEsQ1QIoAHoECBgQAQ"
        },
        ...
        {
            "title": "joe's pizza new york",
            "url": "https://www.google.com/search?num=20&hl=en&gl=us&q=joe%27s+pizza+new+york&sa=X&ved=2ahUKEwiS_8TV6MrrAhVJ6uAKHdWjDEsQ1QIoB3oECBgQCA"
        }],
        "questions": [{
            "text": "What is the best pizza in NY?",
            "position": 0
        },
        ...
        {
            "text": "How much is a slice of pizza in New York City?",
            "position": 3
        }]
    }
}

Every search that failed will be tried as many times as possible for 30 seconds.

So please be aware of this maximum timeout when writing your own code.

API Key: api_key [string] required

All requests are authenticated by using your private API key.

To get access to your API key, just create an account here and confirm your email.

The search parameter is the text you would put in the Google search bar. You can use anything that you would use in a regular Google search. (e.g., inurl:, site:, intitle:, etc.)

Never forget to correctly encode this parameter before calling the API, especially if you are using special characters such as + or :.

If you need help encoding your parameter you can find more information below:


sudo apt-get install gridsite-clients
urlencode "YOUR SEARCH"
         

import urllib.parse
encoded_url = urllib.parse.quote("YOUR SEARCH")

encoded_url = encodeURIComponent("YOUR SEARCH")

String encoded_url = URLEncoder.encode("YOUR SEARCH", "UTF-8");

require 'uri'
encoded_url = URI::encode("YOUR SEARCH")

<?php

$url_encoded = urlencode("YOUR SEARCH");

?>

package main

import (
	"net/url"
)

func main() {
	encoded_url := url.QueryEscape("YOUR SEARCH")
}

Country Code: country_code [string] (default=us)

country_code is the ISO 3166 country code, here are the currently supported country codes, do not hesitate to contact sale if you don't see the one you need.

Country code available:

country_code Country Name
br Brazil
ca Canada
fr France
de Germany
gr Greece
il Israel
it Italy
mx Mexico
nl Netherlands
ru Russia
es Spain
se Sweden
us UnitedStates
gb UnitedKingdom

Number of results: nb_results [integer] (default=100)

The number of results you want to get back from Google Search.

By default this number is 100, you can change it by using the nb_results parameter.

For example, if you only need the first 20 results, use nb_results=20

Warning: if you are looking to get ads informations, you will need to use nb_results=20 as Google do not show ads on page with more than 20 results.

Page number: page [integer] (default=1)

The page number you want to extract results from.

By default this number is 1, you can change it by using the page parameter.

If you want to get search information from the second page of results, juste use page=2.

Language: language [string] (default=en)

The language the search results will be displayed in.

By default the language is en (english), you can change it by using the language parameter.

The list of supported languages is available here.

Full HTML: add_html [boolean] (default=false)

Adding the full html of the page available in the results. Useful if you want to get some information we don't return in the JSON response.

By default this parameter's value is false, use add_html=true if you want to enable it.

Response data

Our API will return you a JSON object that will look like this

 
{
    "statusCode": 200, # Status code
    "body": {
        "meta_data": { # Search information
            "url": "https://www.google.com/search?q=pizza%20new%20york&hl=en&gl=us&num=20&start=0",
            "number_of_results": 615000000,
            "location": "",
            "number_of_organic_results": 22,
            "number_of_ads": 0,
            "number_of_page": 7
        },
        "local_results": [{ # Local results, in this example, restaurants
            "title": "Joe's Pizza",
            "review": 4.5,
            "position": 0,
            "review_count": 5422
        } ...
        ],
        "organic_results": [{ # Organic search results
            "url": "https://ny.eater.com/maps/nyc-best-iconic-pizza-pizzeria",
            "displayed_url": "ny.eater.com › maps › nyc-best-iconic-pizza-pizzeria",
            "description": "Jan 17, 2020 - Roberto Paciullo's Bronx trattoria serves wood-fired pizzas that have puffy brown crusts and floppy centers. Some pizza geeks think that Zero Otto ...",
            "position": 0,
            "title": "New York City's 27 Most Iconic Pizzerias - Eater NY"
        },
        ...
        {
            "url": "https://www.timeout.com/newyork/restaurants/best-new-york-pizza",
            "displayed_url": "www.timeout.com › restaurants › best-new-york-pizza",
            "description": "Aug 2, 2019 - There's nothing that says “I love NY” more than eating a classic slice of New York pizza in view of the Brooklyn Bridge. And who better to provide ...",
            "position": 17,
            "title": "29 Best Pizzas in NYC for Life-Changing Slices To Eat Today"
        }],
        "top_ads": [], # Ads at the top of the page
        "bottom_ads": [], # Ads at the bottoms of the page
        "related_queries": [{ # Related queries
            "title": "best pizza new york",
            "url": "https://www.google.com/search?num=20&hl=en&gl=us&q=best+pizza+new+york&sa=X&ved=2ahUKEwiS_8TV6MrrAhVJ6uAKHdWjDEsQ1QIoAHoECBgQAQ"
        },
        ...
        ],
        "questions": [{ # Related questions
            "text": "What is the best pizza in NY?",
            "position": 0
        },
        ...
        "full_html": # HTML of the page if full_html=true
        ]
    }
}