Rome2Rio Scraper: Reliable Route and Fare Data Collection for Travel Analytics

import requests
from bs4 import BeautifulSoup

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
    "Accept-Language": "en-US,en;q=0.9",
}

def extract_rome2rio_route(url):
    try:
        response = requests.get(
            url,
            headers=HEADERS,
            timeout=15
        )
        response.raise_for_status()
    except requests.RequestException as error:
        print(f"Request failed: {error}")
        return None

    soup = BeautifulSoup(response.text, "lxml")

    def get_text(selector):
        element = soup.select_one(selector)
        return element.get_text(" ", strip=True) if element else "N/A"

    return {
        "Origin": get_text("[data-testid='origin']"),
        "Destination": get_text("[data-testid='destination']"),
        "Transport Options": get_text(
            "[data-testid='transport-mode']"
        ),
        "Journey Duration": get_text(
            "[data-testid='journey-duration']"
        ),
        "Estimated Fare": get_text(
            "[data-testid='journey-price']"
        ),
    }

# Example route URL; replace with an authorized route page.
rome2rio_route_url = "https://www.rome2rio.com/"

route_data = extract_rome2rio_route(rome2rio_route_url)
print(route_data)