ScrapingBee vs Tavily

# pip install requests beautifulsoup4
import json
import os

import requests
from bs4 import BeautifulSoup

response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params={
        "url": "https://news.ycombinator.com/",
        "render_js": "false",
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=60,
)

response.raise_for_status()

soup = BeautifulSoup(response.text, "html.parser")

stories = [
    {
        "title": link.get_text(strip=True),
        "url": link.get("href"),
    }
    for link in soup.select(".titleline > a")[:10]
]

print(json.dumps(stories, indent=2))

const axios = require('axios');
const cheerio = require('cheerio');

axios.get('https://app.scrapingbee.com/api/v1/', {
    params: {
        url: 'https://news.ycombinator.com/',
        render_js: 'false'
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 60000
}).then(response => {
    const $ = cheerio.load(response.data);
    const stories = $('.titleline > a').slice(0, 10).map((i, el) => ({
        title: $(el).text().trim(),
        url: $(el).attr('href')
    })).get();
    console.log(JSON.stringify(stories, null, 2));
});

curl -G "https://app.scrapingbee.com/api/v1/" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "url=https://news.ycombinator.com/" \
  --data-urlencode "render_js=false"

[
  {
    "title": "Gpiozero Flow",
    "url": "https://bennuttall.com/blog/2026/07/gpiozero-flow/"
  },
  {
    "title": "Azulejo",
    "url": "https://en.wikipedia.org/wiki/Azulejo"
  },
  {
    "title": "AI's top startups are barely publishing their research",
    "url": "https://www.science.org/content/article/ai-s-top-startups-are-barely-publishing-their-research"
  },
  {
    "title": "Carolina Cloud pays SOFR on unused prepaid credits",
    "url": "https://docs.carolinacloud.io/organizations/prepaid-interest/"
  },
  {
    "title": "The coolest use for the Vision Pro",
    "url": "https://christianselig.com/2026/07/vision-pro-house/"
  }
]

# pip install requests beautifulsoup4
import json
import os

import requests
from bs4 import BeautifulSoup

response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params={
        "url": "https://weather.com/weather/today/",
        "render_js": "true",
        "premium_proxy": "true",
        "country_code": "us",
        "wait_for": '[data-testid="TemperatureValue"]',
        "wait": "2000",
        "block_ads": "true",
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=120,
)

response.raise_for_status()

soup = BeautifulSoup(response.text, "html.parser")

location_heading = soup.select_one("h1")

temperatures = [
    element.get_text(" ", strip=True)
    for element in soup.select('[data-testid="TemperatureValue"]')
]

temperature = next(
    (value for value in temperatures if any(char.isdigit() for char in value)),
    None,
)

result = {
    "location": (
        location_heading.get_text(" ", strip=True).removesuffix(" Weather")
        if location_heading
        else None
    ),
    "temperature": (
        temperature.replace(" °", "°")
        if temperature
        else None
    ),
}

print(json.dumps(result, indent=2, ensure_ascii=False))

const axios = require('axios');
const cheerio = require('cheerio');

axios.get('https://app.scrapingbee.com/api/v1/', {
    params: {
        url: 'https://weather.com/weather/today/',
        render_js: 'true',
        premium_proxy: 'true',
        country_code: 'us',
        wait_for: '[data-testid="TemperatureValue"]',
        wait: '2000',
        block_ads: 'true'
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 120000
}).then(response => {
    const $ = cheerio.load(response.data);
    const location = $('h1').first().text().trim().replace(/ Weather$/, '');
    const temperature = $('[data-testid="TemperatureValue"]')
        .map((i, el) => $(el).text().trim())
        .get()
        .find(value => /\d/.test(value));
    console.log(JSON.stringify({ location, temperature }, null, 2));
});

curl -G "https://app.scrapingbee.com/api/v1/" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "url=https://weather.com/weather/today/" \
  --data-urlencode "render_js=true" \
  --data-urlencode "premium_proxy=true" \
  --data-urlencode "country_code=us" \
  --data-urlencode 'wait_for=[data-testid="TemperatureValue"]' \
  --data-urlencode "wait=2000" \
  --data-urlencode "block_ads=true"

{
  "location": "Roswell",
  "temperature": "76°"
}

# pip install requests
import json
import os

import requests

ai_rules = {
    "page_type": {
        "description": "Classify the type of webpage",
        "type": "string",
        "enum": [
            "product page",
            "article",
            "category page",
            "homepage",
            "other",
        ],
    },
    "title": {
        "description": "The main title of the page",
        "type": "string",
    },
    "summary": {
        "description": (
            "A concise sentence explaining what this page is "
            "and what information it contains"
        ),
        "type": "string",
    },
    "price_gbp": {
        "description": (
            "The current product price in British pounds, "
            "as a number without the currency symbol"
        ),
        "type": "number",
    },
    "in_stock": {
        "description": "Whether the product is currently in stock",
        "type": "boolean",
    },
}

response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params={
        "url": (
            "https://books.toscrape.com/catalogue/"
            "a-light-in-the-attic_1000/index.html"
        ),
        "render_js": "false",
        "ai_selector": "article.product_page",
        "ai_extract_rules": json.dumps(ai_rules),
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=120,
)

response.raise_for_status()

print(
    json.dumps(
        response.json(),
        indent=2,
        ensure_ascii=False,
    )
)

const axios = require('axios');

const aiRules = {
    page_type: {
        description: 'Classify the type of webpage',
        type: 'string',
        enum: ['product page', 'article', 'category page', 'homepage', 'other']
    },
    title: {
        description: 'The main title of the page',
        type: 'string'
    },
    summary: {
        description: 'A concise sentence explaining what this page is ' +
                     'and what information it contains',
        type: 'string'
    },
    price_gbp: {
        description: 'The current product price in British pounds, ' +
                     'as a number without the currency symbol',
        type: 'number'
    },
    in_stock: {
        description: 'Whether the product is currently in stock',
        type: 'boolean'
    }
};

axios.get('https://app.scrapingbee.com/api/v1/', {
    params: {
        url: 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
        render_js: 'false',
        ai_selector: 'article.product_page',
        ai_extract_rules: JSON.stringify(aiRules)
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 120000
}).then(response => {
    console.log(JSON.stringify(response.data, null, 2));
});

curl -G "https://app.scrapingbee.com/api/v1/" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "url=https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html" \
  --data-urlencode "render_js=false" \
  --data-urlencode "ai_selector=article.product_page" \
  --data-urlencode 'ai_extract_rules={"page_type":{"description":"Classify the type of webpage","type":"string","enum":["product page","article","category page","homepage","other"]},"title":{"description":"The main title of the page","type":"string"},"summary":{"description":"A concise sentence explaining what this page is and what information it contains","type":"string"},"price_gbp":{"description":"The current product price in British pounds, as a number without the currency symbol","type":"number"},"in_stock":{"description":"Whether the product is currently in stock","type":"boolean"}}'

{
  "page_type": "product page",
  "title": "A Light in the Attic",
  "summary": "This page contains information about the book \"A Light in the Attic\" by Shel Silverstein, including its price, availability, and product description.",
  "price_gbp": 51.77,
  "in_stock": true
}

# pip install requests
import os

import requests


response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params={
        "url": "https://news.ycombinator.com/",

        # Let ScrapingBee choose a working configuration.
        "mode": "auto",

        # Do not let Auto Mode use a configuration
        # that costs more than 25 credits.
        "max_cost": 25,

        # Extract the requested information using AI.
        "ai_query": (
            "Return the first five stories displayed on the homepage. "
            "For each story, include its title, URL, score, author, "
            "and number of comments."
        ),
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=120,
)

response.raise_for_status()

print(
    "Auto Mode cost:",
    response.headers.get("Spb-auto-cost"),
    "credits",
)

print(
    "Total request cost:",
    response.headers.get("Spb-cost"),
    "credits",
)

print(response.text)

const axios = require('axios');

axios.get('https://app.scrapingbee.com/api/v1/', {
    params: {
        url: 'https://news.ycombinator.com/',

        // Let ScrapingBee choose a working configuration.
        mode: 'auto',

        // Cap what the Auto Mode configuration may cost.
        max_cost: 25,

        // Extract the requested information using AI.
        ai_query: 'Return the first five stories displayed on the homepage. ' +
                  'For each story, include its title, URL, score, author, ' +
                  'and number of comments.'
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 120000
}).then(response => {
    console.log('Auto Mode cost:', response.headers['spb-auto-cost'], 'credits');
    console.log('Total request cost:', response.headers['spb-cost'], 'credits');
    console.log(response.data);
});

curl -G -i "https://app.scrapingbee.com/api/v1/" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "url=https://news.ycombinator.com/" \
  --data-urlencode "mode=auto" \
  --data-urlencode "max_cost=25" \
  --data-urlencode "ai_query=Return the first five stories displayed on the homepage. For each story, include its title, URL, score, author, and number of comments."

Auto Mode cost: 6 credits
Total request cost: 6 credits
{"title": "Gpiozero Flow", "url": "https://bennuttall.com/blog/2026/07/gpiozero-flow/", "score": 53, "author": "benn_88", "comments": 14}
{"title": "GCC steering committee announces AI policy", "url": "https://lwn.net/Articles/1086041/", "score": 11, "author": "arto", "comments": 2}
{"title": "Carolina Cloud pays SOFR on unused prepaid credits", "url": "https://docs.carolinacloud.io/organizations/prepaid-interest/", "score": 26, "author": "bojangleslover", "comments": 11}
{"title": "AI's top startups are barely publishing their research", "url": "https://www.science.org/content/article/ai-s-top-startups-are-barely-publishing-their-research", "score": 517, "author": "YeGoblynQueenne", "comments": 266}
{"title": "The coolest use for the Vision Pro", "url": "https://christianselig.com/2026/07/vision-pro-house/", "score": 724, "author": "robbiet480", "comments": 278}

# pip install requests
import json
import os

import requests

response = requests.get(
    "https://app.scrapingbee.com/api/v1/fast_search",
    params={
        "search": "latest developments in web scraping",
        "country_code": "us",
        "language": "en",
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=60,
)

response.raise_for_status()

print(
    json.dumps(
        response.json(),
        indent=2,
        ensure_ascii=False,
    )
)

const axios = require('axios');

axios.get('https://app.scrapingbee.com/api/v1/fast_search', {
    params: {
        search: 'latest developments in web scraping',
        country_code: 'us',
        language: 'en'
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 60000
}).then(response => {
    console.log(JSON.stringify(response.data, null, 2));
});

curl -G "https://app.scrapingbee.com/api/v1/fast_search" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "search=latest developments in web scraping" \
  --data-urlencode "country_code=us" \
  --data-urlencode "language=en"

{
  "answer_box": {},
  "organic": [
    {
      "description": "",
      "extensions": [],
      "link": "https://www.browserless.io/blog/state-of-web-scraping-2026",
      "rank": 1,
      "title": "State of Web Scraping 2026: Trends, Challenges & What's Next"
    },
    {
      "description": "",
      "extensions": [],
      "link": "https://browser-use.com/posts/web-scraping-guide-2026",
      "rank": 2,
      "title": "The Ultimate Guide to Web Scraping (2026)"
    },
    {
      "description": "",
      "extensions": [],
      "link": "https://www.grepsr.com/blog/future-web-scraping-ai-automation/",
      "rank": 3,
      "title": "The Future of Web Scraping: AI & Automation"
    },
    {
      "description": "",
      "extensions": [],
      "link": "https://aimultiple.com/web-scraping-challenges",
      "rank": 5,
      "title": "The Most Common Web Scraping Challenges in 2026"
    }
  ],
  "people_also_ask": [],
  "related_searches": [
    {
      "query": "Latest developments in web scraping github"
    },
    {
      "query": "Bright Data"
    },
    {
      "query": "Tavily"
    }
  ],
  "status": "done",
  "top_stories": [],
  "url": ""
}

# pip install requests
import json
import os

import requests


js_scenario = {
    "instructions": [
        {"wait": 1500},
        {"scroll_y": 1000},
        {"wait_for_and_click": "#page-load-more"},
        {"wait": 1000},
        {"wait_for_and_click": "#page-load-more"},
        {"wait": 1000},
    ],
}

response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params={
        "url": "https://www.web-scraping.dev/reviews",
        "render_js": "true",

        # The page may not work correctly without its CSS.
        "block_resources": "false",

        "js_scenario": json.dumps(js_scenario),

        # Capture the final state after both clicks.
        "screenshot_full_page": "true",

        "window_width": 1440,
        "window_height": 900,
    },
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    timeout=120,
)

if not response.ok:
    print(response.text)
    response.raise_for_status()

output_path = "reviews-after-clicks.png"

with open(output_path, "wb") as file:
    file.write(response.content)

print(f"Screenshot saved to {output_path}")

const axios = require('axios');
const fs = require('fs');

const jsScenario = {
    instructions: [
        { wait: 1500 },
        { scroll_y: 1000 },
        { wait_for_and_click: '#page-load-more' },
        { wait: 1000 },
        { wait_for_and_click: '#page-load-more' },
        { wait: 1000 }
    ]
};

axios.get('https://app.scrapingbee.com/api/v1/', {
    params: {
        url: 'https://www.web-scraping.dev/reviews',
        render_js: 'true',

        // The page may not work correctly without its CSS.
        block_resources: 'false',

        js_scenario: JSON.stringify(jsScenario),

        // Capture the final state after both clicks.
        screenshot_full_page: 'true',

        window_width: 1440,
        window_height: 900
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    responseType: 'arraybuffer',
    timeout: 120000
}).then(response => {
    fs.writeFileSync('reviews-after-clicks.png', response.data);
    console.log('Screenshot saved to reviews-after-clicks.png');
});

curl -G "https://app.scrapingbee.com/api/v1/" \
  -H "Authorization: Bearer $SCRAPINGBEE_API_KEY" \
  --data-urlencode "url=https://www.web-scraping.dev/reviews" \
  --data-urlencode "render_js=true" \
  --data-urlencode "block_resources=false" \
  --data-urlencode 'js_scenario={"instructions":[{"wait":1500},{"scroll_y":1000},{"wait_for_and_click":"#page-load-more"},{"wait":1000},{"wait_for_and_click":"#page-load-more"},{"wait":1000}]}' \
  --data-urlencode "screenshot_full_page=true" \
  --data-urlencode "window_width=1440" \
  --data-urlencode "window_height=900" \
  --output reviews-after-clicks.png

Screenshot saved to reviews-after-clicks.png