import asyncio
import json
from crawlee import Request
from crawlee.http_crawler import HttpCrawler, HttpCrawlingContext
from crawlee.http_clients.curl_impersonate import CurlImpersonateHttpClient
from crawlee.http_clients import HttpxHttpClient

async def main() -> None:
    #http_client = CurlImpersonateHttpClient(
    #    persist_cookies_per_session=True,
    #)
    # Or use HttpxHttpClient:
    http_client = HttpxHttpClient(
        persist_cookies_per_session=True,
    )

    crawler = HttpCrawler(
        http_client=http_client,
        max_requests_per_crawl=20,
    )

    url = "https://www.viagogo.es/Entradas-Deportes/Futbol/Real-Madrid-C-F-Entradas/E-153769088"
    payload = {
        'ShowAllTickets': True,
        'HideDuplicateTicketsV2': False,
        'Quantity': 2,
        'IsInitialQuantityChange': False,
        'PageSize': 20,
        'CurrentPage': 2,
        'SortBy': 'NEWPRICE',
        'SortDirection': 0,
        'Sections': '',
        'Rows': '',
        'Seats': '',
        'SeatTypes': '',
        'TicketClasses': '',
        'ListingNotes': '',
        'PriceRange': '0,100',
        'InstantDelivery': False,
        'EstimatedFees': True,
        'BetterValueTickets': True,
        'PriceOption': '',
        'HasFlexiblePricing': False,
        'ExcludeSoldListings': False,
        'RemoveObstructedView': False,
        'NewListingsOnly': False,
        'PriceDropListingsOnly': False,
        'SelectBestListing': False,
        'ConciergeTickets': False,
        'Method': 'IndexSh'
    }
    payload_bytes = json.dumps(payload).encode()

    print(f"0. Start: {payload_bytes}")

    initial_req = Request.from_url(
        url=url,
        method="POST",
        payload=payload_bytes,
        use_extended_unique_key=True,
    )

    @crawler.router.default_handler
    async def request_handler(context: HttpCrawlingContext) -> None:
        # Handle the response here
        pass

    # Run the crawler
    await crawler.run([initial_req])

if __name__ == '__main__':
    asyncio.run(main())
    

    initial_req = Request.from_url(
        url=url,
        method="POST",
        payload=payload,  # payload as dictionary
        use_extended_unique_key=True,
    )