Kayak Scraper: Dynamic Flight Price Monitoring and Hotel Insights

import requests
from bs4 import BeautifulSoup

DEFAULT_HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
    "Accept-Language": "en-US,en;q=0.8"
}

def scrape_kayak_flights(search_url):
    session = requests.Session()
    response = session.get(search_url, headers=DEFAULT_HEADERS, timeout=30)

    if response.status_code != 200:
        print("Failed to retrieve data")
        return []

    soup = BeautifulSoup(response.text, "html.parser")
    flights_data = []

    flight_cards = soup.find_all("div", {"class": "resultWrapper"})

    for card in flight_cards:
        airline = card.find("span", {"class": "codeshares-airline-names"})
        departure = card.find("span", {"class": "depart-time"})
        arrival = card.find("span", {"class": "arrival-time"})
        price = card.find("span", {"class": "price-text"})

        flights_data.append({
            "Airline": airline.get_text(strip=True) if airline else "N/A",
            "Departure Time": departure.get_text(strip=True) if departure else "N/A",
            "Arrival Time": arrival.get_text(strip=True) if arrival else "N/A",
            "Fare": price.get_text(strip=True) if price else "N/A"
        })

    return flights_data