Yandex Scraper: Real-Time Search Engine Data and Ranking Intelligence Collection

import requests
from bs4 import BeautifulSoup

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
    "Accept-Language": "en-US,en;q=0.9",
}

def scrape_yandex_serp(search_query):
    search_url = f"https://yandex.com/search/?text={search_query}"
    response = requests.get(search_url, headers=HEADERS)

    if response.status_code != 200:
        return None

    soup = BeautifulSoup(response.text, "lxml")
    results = []

    for item in soup.select("li.serp-item"):
        title_tag = item.select_one("h2 a")
        snippet_tag = item.select_one(".text-container")

        results.append({
            "Keyword": search_query,
            "Title": title_tag.get_text(strip=True) if title_tag else "N/A",
            "URL": title_tag.get("href") if title_tag else "N/A",
            "Snippet": snippet_tag.get_text(strip=True) if snippet_tag else "N/A"
        })

    return results

# Example keyword search
keyword = "best grocery delivery service"
serp_data = scrape_yandex_serp(keyword)

if serp_data:
    for row in serp_data[:5]:
        print(row)
else:
    print("Failed to fetch Yandex SERP data.")