
    ik                    >	   U d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlZddlZddlZddlZddlZddlZ ej        d          Zej        ed<   d Z e            Z ed          Z G d	 d
ej                  Z G d dej                  Z G d dej                  Z  G d dej                  Z! G d dej        ee                   Z" G d dej                  Z# G d dej                  Z$ G d dej                  Z% G d dej                  Z& G d dej                  Z' G d dej                  Z( G d d ej                  Z) G d! d"ej                  Z* G d# d$ej                  Z+ G d% d&ej                  Z, G d' d(ej                  Z- G d) d*ej                  Z. G d+ d,ej                  Z/ G d- d.ej                  Z0 G d/ d0ej                  Z1 G d1 d2e&          Z2 G d3 d4e"e         ee                   Z3 G d5 d6ej                  Z4 G d7 d8ej                  Z5 G d9 d:ej                  Z6 G d; d<ej                  Z7 G d= d>ej                  Z8 G d? d@ej                  Z9 G dA dBej                  Z: G dC dDej                  Z; G dE dFej                  Z< G dG dHej                  Z= G dI dJej        ee                   Z> G dK dLej                  Z? G dM dNej                  Z@ G dO dPej                  ZA G dQ dRej                  ZB G dS dTej                  ZC G dU dVej                  ZD G dW dXej                  ZE G dY dZej                  ZF G d[ d\ej                  ZG G d] d^ej                  ZH G d_ d`ej                  ZI G da dbej                  ZJ G dc ddej                  ZK G de dfej                  ZL G dg dhej                  ZM G di djej                  ZN G dk dlej                  ZO G dm dNej                  Z@ G dn dHej                  Z= G do dp          ZP G dq dr          ZQ G ds dteP          ZR G du dveQ          ZSdS )wa"  
Firecrawl v1 API Client - Legacy Implementation

This module provides the legacy v1 implementation of the Firecrawl SDK.
It contains the complete `V1FirecrawlApp` class with all v1 API methods and types
for backward compatibility. This is used by the unified client to provide
version-specific access patterns like app.v1.scrape_url().

Classes:
    - V1FirecrawlApp: Legacy v1 client for interacting with the Firecrawl API.
    - AsyncV1FirecrawlApp: Async version of the v1 client.
    - CrawlWatcher: WebSocket-based crawl monitoring for v1.
    N)	AnyDictOptionalListUnionCallableLiteralTypeVarGeneric)datetime	firecrawlloggerc                     	 ddl m}  t          j                            t
                    } | t          j                            |d                                                    }t          j	        d|t          j
                  }|r'|                    d                                          S d S # t          $ r t          d           Y d S w xY w)Nr   )Pathz__init__.pyz"^__version__ = ['\"]([^'\"]*)['\"]   z&Failed to get version from __init__.py)pathlibr   ospathdirname__file__join	read_textresearchMgroupstrip	Exceptionprint)r   package_pathversion_fileversion_matchs       ]/home/agentuser/.hermes/hermes-agent/venv/lib/python3.11/site-packages/firecrawl/v1/client.pyget_versionr$      s    	W__X..lT"',,|]CCDDNNPPli E|UWUYZZm	 0$$Q''--//
/0 0	   4555TTs   B-B2 2CCTc                   H    e Zd ZU dZdZed         ed<   dZee	         ed<   dS )V1AgentOptionszConfiguration for the agent.FIRE-1modelNprompt)
__name__
__module____qualname____doc__r)   r	   __annotations__r*   r   str     r#   r'   r'   P   sA         &&'E78''' FHSM     r2   r'   c                   .    e Zd ZU dZdZed         ed<   dS )V1AgentOptionsExtract2Configuration for the agent in extract operations.r(   r)   Nr+   r,   r-   r.   r)   r	   r/   r1   r2   r#   r4   r4   U   -         <<'E78'''''r2   r4   c                   @    e Zd ZU dZee         ed<   ee         ed<   dS )V1ActionsResultz,Result of actions performed during scraping.screenshotspdfsN)r+   r,   r-   r.   r   r0   r/   r1   r2   r#   r9   r9   Y   s3         66c
s)OOOOOr2   r9   c                       e Zd ZU dZdZee         ed<   eed<   eed<   dZee	ee
f                  ed<    ej        dd          Zee
         ed	<   dS )
V1ChangeTrackingDataz.
    Data for the change tracking format.
    NpreviousScrapeAtchangeStatus
visibilitydiffjsonalias
json_field)r+   r,   r-   r.   r>   r   r0   r/   rA   r   r   pydanticFieldrE   r1   r2   r#   r=   r=   ^   s           '+hsm***OOO%)D(4S>
"))) .t6 B B BJBBBBBr2   r=   c                      e Zd ZU dZdZee         ed<   dZee         ed<   dZ	ee         ed<   dZ
ee         ed<   dZeee                  ed<   dZee         ed<    ej        dd	
          Zee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dS )V1FirecrawlDocumentz-Document retrieved or processed by Firecrawl.NurlmarkdownhtmlrawHtmllinksextractrB   rC   rE   
screenshotmetadataactionstitledescriptionchangeTracking)r+   r,   r-   r.   rJ   r   r0   r/   rK   rL   rM   rN   r   rO   r%   rF   rG   rE   rP   rQ   r   rR   r9   rS   rT   rU   r=   r1   r2   r#   rI   rI   h   s5        77C#"Hhsm"""D(3-!GXc]!!!!%E8DI%%%GXa[,hnT@@@J@@@ $J$$$"Hhsm""")-GXo&---E8C=!%K#%%%59NH1299999r2   rI   c                   T    e Zd ZU dZdZee         ed<   dZee	e                  ed<   dS )V1LocationConfigz$Location configuration for scraping.Ncountry	languages)
r+   r,   r-   r.   rX   r   r0   r/   rY   r   r1   r2   r#   rW   rW   x   sE         ..!GXc]!!!%)IxS	")))))r2   rW   c                       e Zd ZU dZeed<   dZeeeef                  ed<   dZ	eeeef                  ed<   dZ
eeed                           ed<   dS )V1WebhookConfigzConfiguration for webhooks.rJ   NheadersrQ   )	completedfailedpagestartedevents)r+   r,   r-   r.   r0   r/   r\   r   r   rQ   ra   r   r	   r1   r2   r#   r[   r[   }   s{         %%	HHH(,GXd38n%,,,)-HhtCH~&---PTFHT'"JKLMTTTTTr2   r[   c                       e Zd ZU dZdZeeed                           ed<    e	j
        dd          Zee         ed<   dZee         ed<   dZee         ed	<   dS )
V1ChangeTrackingOptionsz"Configuration for change tracking.N)zgit-diffrB   modesschemarC   schema_fieldr*   tag)r+   r,   r-   r.   rd   r   r   r	   r/   rF   rG   rf   r   r*   r0   rg   r1   r2   r#   rc   rc      s         ,,9=E8D!3456==="0(.X"F"F"FL(3-FFF FHSM   C#r2   rc   c                      e Zd ZU dZdZeeed                           ed<   dZ	ee
eef                  ed<   dZeee                  ed<   dZeee                  ed<   dZee         ed<   dZee         ed	<   d
Zee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZeed                  ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dS )V1ScrapeOptions#Parameters for scraping operations.N
rK   rL   rM   contentrN   rP   screenshot@fullPagerO   rB   rU   formatsr\   includeTagsexcludeTagsonlyMainContentwaitFor0u  timeoutlocationmobileskipTlsVerificationremoveBase64ImagesblockAdsbasicstealthenhancedautoproxychangeTrackingOptionsmaxAgestoreInCacheparsePDF)r+   r,   r-   r.   rn   r   r   r	   r/   r\   r   r0   ro   rp   rq   boolrr   intrt   ru   rW   rv   rw   rx   ry   r   r   rc   r   r   r   r1   r2   r#   ri   ri      s        -- eiGXd7  $_  `  a  b  i  i  i(,GXd38n%,,,'+K$s)$+++'+K$s)$+++&*OXd^***!GXc]!!!"GXc]"""+/Hh'(///!FHTN!!!*.$...)----#Hhtn###GKE8GBCDKKK?C8$;<CCC FHSM   #'L(4.'''#Hhtn#####r2   ri   c                   ^    e Zd ZU dZed         ed<   dZee         ed<   dZ	ee
         ed<   dS )V1WaitActionz'Wait action to perform during scraping.waittypeNmillisecondsselector)r+   r,   r-   r.   r	   r/   r   r   r   r   r0   r1   r2   r#   r   r      sP         11
&/"&L(3-&&&"Hhsm"""""r2   r   c                   ^    e Zd ZU dZed         ed<   dZee         ed<   dZ	ee
         ed<   dS )V1ScreenshotActionz-Screenshot action to perform during scraping.rP   r   NfullPagequality)r+   r,   r-   r.   r	   r/   r   r   r   r   r   r1   r2   r#   r   r      sQ         77
,
#Hhtn###!GXc]!!!!!r2   r   c                   4    e Zd ZU dZed         ed<   eed<   dS )V1ClickActionz(Click action to perform during scraping.clickr   r   Nr+   r,   r-   r.   r	   r/   r0   r1   r2   r#   r   r      s0         22
'
MMMMMr2   r   c                   4    e Zd ZU dZed         ed<   eed<   dS )V1WriteActionz(Write action to perform during scraping.writer   textNr   r1   r2   r#   r   r      s0         22
'

IIIIIr2   r   c                   4    e Zd ZU dZed         ed<   eed<   dS )V1PressActionz(Press action to perform during scraping.pressr   keyNr   r1   r2   r#   r   r      s0         22
'
	HHHHHr2   r   c                   Z    e Zd ZU dZed         ed<   ed         ed<   dZee         ed<   dS )V1ScrollActionz)Scroll action to perform during scraping.scrollr   )updown	directionNr   )	r+   r,   r-   r.   r	   r/   r   r   r0   r1   r2   r#   r   r      sN         33
(
|$$$$"Hhsm"""""r2   r   c                   *    e Zd ZU dZed         ed<   dS )V1ScrapeActionz)Scrape action to perform during scraping.scraper   N)r+   r,   r-   r.   r	   r/   r1   r2   r#   r   r      s)         33
(
r2   r   c                   4    e Zd ZU dZed         ed<   eed<   dS )V1ExecuteJavascriptActionz5Execute javascript action to perform during scraping.executeJavascriptr   scriptNr   r1   r2   r#   r   r      s1         ??
%
&&&&KKKKKr2   r   c                       e Zd ZU dZed         ed<   dZeed                  ed<   dZee	         ed<   dZ
ee         ed<   dS )	V1PDFActionz&PDF action to perform during scraping.pdfr   N)A0A1A2A3A4A5A6LetterLegalTabloidLedgerformat	landscapescale)r+   r,   r-   r.   r	   r/   r   r   r   r   r   floatr1   r2   r#   r   r      sk         00
%.rvFHWmnovvv $Ix~$$$!E8E?!!!!!r2   r   c                   .    e Zd ZU dZdZed         ed<   dS )V1ExtractAgentr5   r(   r)   Nr6   r1   r2   r#   r   r      r7   r2   r   c                       e Zd ZU dZdZee         ed<    ej	        dd          Z
ee         ed<   dZee         ed<   dZee         ed<   dS )	V1JsonConfigzConfiguration for extraction.Nr*   re   rC   rf   systemPromptagent)r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   r   r   r   r1   r2   r#   r   r      sz         '' FHSM   "0(.X"F"F"FL(3-FFF"&L(3-&&&&*E8N#*****r2   r   c                       e Zd ZU dZdZee         ed<   dZee         ed<   dZ	ee
eeeeeeeeeef	                           ed<   dZee         ed<   dZee         ed<   dS )V1ScrapeParamsrj   NrO   jsonOptionsrR   r   webhook)r+   r,   r-   r.   rO   r   r   r/   r   rR   r   r   r   r   r   r   r   r   r   r   r   r   r'   r   r[   r1   r2   r#   r   r      s         --&*GXl#****.K,'... }AGXd5/A=R_anp~  AO  Qj  lw  "w  x  y  z  A  A  A&*E8N#***)-GXo&-----r2   r   c                   V    e Zd ZU dZdZeed<   dZee	         ed<   dZ
ee	         ed<   dS )V1ScrapeResponsez"Response from scraping operations.TsuccessNwarningerror)r+   r,   r-   r.   r   r   r/   r   r   r0   r   r1   r2   r#   r   r      sP         ,,GT!GXc]!!!E8C=r2   r   c                       e Zd ZU dZdZee         ed<   dZee         ed<   dZ	e
ed<   dZee         ed<   dZeee                  ed<   dS )	V1BatchScrapeResponsez&Response from batch scrape operations.NidrJ   Tr   r   invalidURLs)r+   r,   r-   r.   r   r   r0   r/   rJ   r   r   r   r   r   r1   r2   r#   r   r      s}         00BC#GTE8C='+K$s)$+++++r2   r   c                       e Zd ZU dZdZeed<   ed         ed<   eed<   eed<   eed<   e	ed	<   d
Z
ee         ed<   ee         ed<   d
S )V1BatchScrapeStatusResponsez)Response from batch scrape status checks.Tr   scrapingr]   r^   	cancelledstatusr]   totalcreditsUsed	expiresAtNnextdatar+   r,   r-   r.   r   r   r/   r	   r   r   r   r   r0   r   rI   r1   r2   r#   r   r      s         33GTBCCCCNNNJJJD(3-
"
######r2   r   c                      e Zd ZU dZdZeee                  ed<   dZ	eee                  ed<   dZ
ee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed	<   dZee         ed
<   dZee         ed<   dZee         ed<   dZeeeef                  ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dS )V1CrawlParamsz#Parameters for crawling operations.NincludePathsexcludePathsmaxDepthmaxDiscoveryDepthlimitallowBackwardLinkscrawlEntireDomainallowExternalLinksignoreSitemapscrapeOptionsr   deduplicateSimilarURLsignoreQueryParametersregexOnFullURLdelaymaxConcurrencyallowSubdomains)r+   r,   r-   r.   r   r   r   r0   r/   r   r   r   r   r   r   r   r   r   r   r   ri   r   r   r[   r   r   r   r   r   r   r1   r2   r#   r   r      s        --(,L(49%,,,(,L(49%,,,"Hhsm"""'+x}+++E8C=)----(,x~,,,)----$(M8D>(((/3M8O,33359GXeC012999-1HTN111,08D>000%)NHTN)))E8C=$(NHSM(((&*OXd^*****r2   r   c                   p    e Zd ZU dZdZee         ed<   dZee         ed<   dZ	e
ed<   dZee         ed<   dS )V1CrawlResponsez"Response from crawling operations.Nr   rJ   Tr   r   )r+   r,   r-   r.   r   r   r0   r/   rJ   r   r   r   r1   r2   r#   r   r     sd         ,,BC#GTE8C=r2   r   c                       e Zd ZU dZdZeed<   ed         ed<   eed<   eed<   eed<   e	ed	<   d
Z
ee         ed<   ee         ed<   d
S )V1CrawlStatusResponsez"Response from crawl status checks.Tr   r   r   r]   r   r   r   Nr   r   r   r1   r2   r#   r   r     s         ,,GTBCCCCNNNJJJD(3-
"
######r2   r   c                   f    e Zd ZU dZeed<   dZee         ed<   eed<   dZ	ee         ed<   eed<   dS )V1CrawlErrorzA crawl error.r   N	timestamprJ   coder   )
r+   r,   r-   r.   r0   r/   r   r   r   r   r1   r2   r#   r   r   "  sZ         GGG$(Ix!(((	HHHD(3-JJJJJr2   r   c                   @    e Zd ZU dZee         ed<   ee         ed<   dS )V1CrawlErrorsResponsez2Response from crawl/batch scrape error monitoring.errorsrobotsBlockedN)r+   r,   r-   r.   r   r   r/   r0   r1   r2   r#   r   r   *  s9         <<9r2   r   c                       e Zd ZU dZdZee         ed<   dZee	         ed<   dZ
ee	         ed<   dZee	         ed<   dZee         ed<   dZee         ed	<   dZee	         ed
<   dZee         ed<   dS )V1MapParamsz"Parameters for mapping operations.Nr   r   includeSubdomainssitemapOnlyr   rs   rt   useIndexru   )r+   r,   r-   r.   r   r   r0   r/   r   r   r   r   r   r   rt   r   ru   rW   r1   r2   r#   r   r   /  s         ,, FHSM   $(M8D>((((,x~,,,"&K$&&&E8C="GXc]"""#Hhtn###+/Hh'(/////r2   r   c                   b    e Zd ZU dZdZeed<   dZee	e
                  ed<   dZee
         ed<   dS )V1MapResponsez!Response from mapping operations.Tr   NrN   r   )r+   r,   r-   r.   r   r   r/   rN   r   r   r0   r   r1   r2   r#   r   r   :  sU         ++GT!%E8DI%%%E8C=r2   r   c                      e Zd ZU dZdZee         ed<    ej	        dd          Z
ee         ed<   dZee         ed<   dZee         ed<   dZee         ed	<   dZee         ed
<   dZee         ed<   dZee         ed<   dZee         ed<   dS )V1ExtractParamsz0Parameters for extracting information from URLs.Nr*   re   rC   rf   r   r   enableWebSearchr   originshowSourcesr   )r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   r   r   r   r  r   r  r  r   ri   r1   r2   r#   r   r   @  s         :: FHSM   "0(.X"F"F"FL(3-FFF"&L(3-&&&)----&*OXd^***(,x~,,, FHSM   "&K$&&&/3M8O,33333r2   r   c                      e Zd ZU dZdZee         ed<   dZee	d                  ed<   dZ
ee         ed<   dZeed<   dZee         ed	<   dZee         ed
<   dZee         ed<   dZeeeef                  ed<   dZee         ed<   dS )V1ExtractResponsez!Response from extract operations.Nr   
processingr]   r^   r   r   Tr   r   r   r   sourcesr   )r+   r,   r-   r.   r   r   r0   r/   r   r	   r   r   r   r   r   r%   r   r   r  r   r   r   r   r1   r2   r#   r  r  L  s         ++BEIFHW@ABIII$(Ix!(((GTD(1+E8C=!GXc]!!!(,GXd38n%,,,!%K#%%%%%r2   r  c                      e Zd ZU eed<   dZee         ed<   dZee         ed<   dZ	ee         ed<   dZ
ee         ed<   d	Zee         ed
<   dZee         ed<   dZee         ed<   dZee         ed<   dZee         ed<   dS )V1SearchParamsquery   r   NtbsfilterenlangusrX   ru   apir  i`  rt   r   )r+   r,   r-   r0   r/   r   r   r   r  r  r  rX   ru   r  rt   r   ri   r1   r2   r#   r
  r
  X  s         JJJE8C=C# FHSM   D(3-!GXc]!!!"Hhsm"""!FHSM!!!"GXc]"""/3M8O,33333r2   r
  c                   l    e Zd ZU dZdZeed<   ee         ed<   dZ	e
e         ed<   dZe
e         ed<   dS )V1SearchResponsez Response from search operations.Tr   r   Nr   r   )r+   r,   r-   r.   r   r   r/   r   rI   r   r   r0   r   r1   r2   r#   r  r  d  sb         **GT
"
####!GXc]!!!E8C=r2   r  c                   h    e Zd ZU eed<   dZee         ed<   dZee         ed<   dZ	ee         ed<   dS )V1CreditUsageDataremaining_creditsNplan_creditsbilling_period_startbilling_period_end)
r+   r,   r-   r   r/   r  r   r  r0   r  r1   r2   r#   r  r  k  s\         "&L(3-&&&*.(3-...(,,,,,,r2   r  c                   $    e Zd ZU eed<   eed<   dS )V1CreditUsageResponser   r   N)r+   r,   r-   r   r/   r  r1   r2   r#   r  r  q  s'         MMM
r2   r  c                   h    e Zd ZU eed<   dZee         ed<   dZee         ed<   dZ	ee         ed<   dS )V1TokenUsageDataremaining_tokensNplan_tokensr  r  )
r+   r,   r-   r   r/   r   r   r  r0   r  r1   r2   r#   r  r  u  s\         !%K#%%%*.(3-...(,,,,,,r2   r  c                   $    e Zd ZU eed<   eed<   dS )V1TokenUsageResponser   r   N)r+   r,   r-   r   r/   r  r1   r2   r#   r"  r"  {  s'         MMM
r2   r"  c                   h    e Zd ZU dZee         ed<   dZee         ed<   dZee         ed<   e	ed<   dS )V1CreditUsageHistoricalPeriodN	startDateendDateapiKeyr   
r+   r,   r-   r%  r   r0   r/   r&  r'  r   r1   r2   r#   r$  r$    sZ         #Ix}###!GXc]!!! FHSM   r2   r$  c                   0    e Zd ZU eed<   ee         ed<   dS )V1CreditUsageHistoricalResponser   periodsN)r+   r,   r-   r   r/   r   r$  r1   r2   r#   r*  r*    s-         MMM/000000r2   r*  c                   h    e Zd ZU dZee         ed<   dZee         ed<   dZee         ed<   e	ed<   dS )V1TokenUsageHistoricalPeriodNr%  r&  r'  
tokensUsedr(  r1   r2   r#   r-  r-    sU         #Ix}###!GXc]!!! FHSM   OOOOOr2   r-  c                   0    e Zd ZU eed<   ee         ed<   dS )V1TokenUsageHistoricalResponser   r+  N)r+   r,   r-   r   r/   r   r-  r1   r2   r#   r0  r0    s-         MMM.//////r2   r0  c                   |    e Zd ZU dZdZee         ed<   dZee	         ed<   dZ
ee	         ed<   dZee	         ed	<   dS )
V1GenerateLLMsTextParamsz;
    Parameters for the LLMs.txt generation operation.
    
   maxUrlsFshowFullTextTcacheN._V1GenerateLLMsTextParams__experimental_stream)r+   r,   r-   r.   r4  r   r   r/   r5  r   r6  r7  r1   r2   r#   r2  r2    so            GXc]#(L(4.((( E8D>   ,08D>00000r2   r2  c                       e Zd ZU dZdZee         ed<   dZee         ed<   dZ	ee         ed<   dZ
ee         ed	<   dZee         ed
<   dZee         ed<   dS )V1DeepResearchParamsz5
    Parameters for the deep research operation.
       r   i  	timeLimit   r4  NanalysisPromptr   /_V1DeepResearchParams__experimental_streamSteps)r+   r,   r-   r.   r   r   r   r/   r;  r4  r=  r0   r   r>  r   r1   r2   r#   r9  r9    s            Hhsm"Ix}"""GXc]$(NHSM((("&L(3-&&&1555555r2   r9  c                   B    e Zd ZU dZeed<   eed<   dZee         ed<   dS )V1DeepResearchResponsez4
    Response from the deep research operation.
    r   r   Nr   )	r+   r,   r-   r.   r   r/   r0   r   r   r1   r2   r#   r@  r@    sD           MMMGGGE8C=r2   r@  c                       e Zd ZU dZeed<   dZeee	e
f                  ed<   e	ed<   dZee	         ed<   e	ed<   eed<   eed	<   eee	e
f                  ed
<   eee	e
f                  ed<   ee	         ed<   dS )V1DeepResearchStatusResponsez;
    Status response from the deep research operation.
    r   Nr   r   r   r   currentDepthr   
activitiesr  	summaries)r+   r,   r-   r.   r   r/   r   r   r   r0   r   r   r   r   r1   r2   r#   rB  rB    s           MMM%)D(4S>
")))KKKE8C=NNNMMMT#s(^$$$$$sCx.!!!!Cyr2   rB  c                   F    e Zd ZU dZdZeed<   eed<   dZe	e         ed<   dS )V1GenerateLLMsTextResponsez-Response from LLMs.txt generation operations.Tr   r   Nr   )
r+   r,   r-   r.   r   r   r/   r0   r   r   r1   r2   r#   rG  rG    sE         77GTGGGE8C=r2   rG  c                   4    e Zd ZU eed<   dZee         ed<   dS )$V1GenerateLLMsTextStatusResponseDatallmstxtNllmsfulltxt)r+   r,   r-   r0   r/   rK  r   r1   r2   r#   rI  rI    s/         LLL!%K#%%%%%r2   rI  c                   v    e Zd ZU dZdZeed<   dZee	         ed<   e
d         ed<   dZee         ed<   eed	<   dS )
 V1GenerateLLMsTextStatusResponsez4Status response from LLMs.txt generation operations.Tr   Nr   r  r   r   r   )r+   r,   r-   r.   r   r   r/   r   r   rI  r	   r   r0   r1   r2   r#   rM  rM    sk         >>GT;?D(7
8???78888E8C=NNNNNr2   rM  c                   x    e Zd ZU dZeed<   eeee	f                  ed<   dZ
ee         ed<   dZee         ed<   dS )r  z-
    Response from the search operation.
    r   r   Nr   r   )r+   r,   r-   r.   r   r/   r   r   r0   r   r   r   r   r1   r2   r#   r  r    sh           MMM
tCH~
!GXc]!!!E8C=r2   c                      e Zd ZU dZdZee         ed<    ej	        dd          Z
ee         ed<   dZee         ed<   dZee         ed	<   dZee         ed
<   dZee         ed<   dZee         ed<   dZeeeef                  ed<   dS )r   z/
    Parameters for the extract operation.
    Nr*   re   rC   rf   system_promptFallow_external_linksenable_web_searchr  show_sourcesr   )r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   rP  rQ  r   rR  r  rS  r   r   r1   r2   r#   r   r     s           !FHSM   "0(.X"F"F"FL(3-FFF#'M8C='''+0(4.000(-x~---&+OXd^+++#(L(4.(((&*E8DcN#*****r2   c            0          e Zd ZdZdZddee         dee         ddfdZdddddddddddddddddddddd	d
edeee	d                           dee
eef                  deee                  deee                  dee         dee         dee         dee         dee         dee         dee         dee         dee	d                  dee         dee         dee         deeeeeeeeeeeef	                           dee         dee         d ee         d!ee         dee         f.d"Zddddddddd#d$ed%ee         d&ee         d'ee         d(ee         d)ee         dee         dee         d*ee         defd+Zde fd,Z!de"fd-Z#dd/ede$fd0Z%dd/ede&fd1Z'ddddddddddddddddddd2dd3d
ed4eee                  d5eee                  d6ee         d7ee         d%ee         d8ee         d9ee         d:ee         d;ee         d*ee         d<eeee(f                  d=ee         d>ee         d?ee         d@ee         dAee         dBee         d!ee         dCee         dDee         de)f,dEZ*ddddddddddddddddddddFd
ed4eee                  d5eee                  d6ee         d7ee         d%ee         d8ee         d9ee         d:ee         d;ee         d*ee         d<eeee(f                  d=ee         d>ee         d?ee         d@ee         dAee         dBee         d!ee         dDee         de+f*dGZ,dHede)fdIZ-dHede.fdJZ/dHede
eef         fdKZ0ddddddddddddddddddddFd
ed4eee                  d5eee                  d6ee         d7ee         d%ee         d8ee         d9ee         d:ee         d;ee         d*ee         d<eeee(f                  d=ee         d>ee         d?ee         d@ee         dAee         dBee         d!ee         dDee         ddLf*dMZ1dddddddddNd
edOee         d;ee         dPee         dQee         d%ee         dee         dRee         dee         de2fdSZ3dddddddddddddddddd2ddddTdUee         deee	dV                           dee
eef                  deee                  deee                  dee         dee         dee         dee         dee         dee         dee         dee         dee	d                  dee         dee         deeeeeeeeeeeef	                           dWee4         dCee         dBee         d!ee         dDee         de5f.dXZ6dddddddddddddddddddddYdUee         deee	dV                           dee
eef                  deee                  deee                  dee         dee         dee         dee         dee         dee         dee         dee         dee	d                  dee         dee         deeeeeeeeeeeef	                           dWee4         dBee         dDee         d!ee         de7f,dZZ8ddddddddddddddddddddd[dUee         deee	dV                           dee
eef                  deee                  deee                  dee         dee         dee         dee         dee         dee         dee         dee         dee	d                  dee         dee         deeeeeeeeeeeef	                           dWee4         dBee         d!ee         dDee         ddLf,d\Z9dHede5fd]Z:dHede.fd^Z;	 ddddd.d.d.dd_dUeee                  d`ee         daee         dbee         d:ee         dcee         ddee         dWee
eef                  de<e         fdeZ=dfede<e         fdgZ>	 ddddd.d.d.dd_dUeee                  d`ee         daee         dbee         d:ee         dcee         ddee         dWee
eef                  de<e         fdhZ?dddddid
edjee         dkee         dlee         dmee         de@fdnZAdddddid
edjee         dkee         dlee         dmee         deBfdoZCdHede@fdpZD	 ddDee         de
eef         fdqZE	 	 dd
edte
eef         de
eef         duedveFdeGjH        fdwZI	 	 dd
ede
eef         duedveFdeGjH        f
dxZJ	 	 dd
ede
eef         duedveFdeGjH        f
dyZKdHede
eef         dCede)fdzZLd{eGjH        d|eddfd}ZMd~ed|edededef
dZNdddddddddd$ed6ee         dee         djee         dee         dbee         dee         deeOe
eef         gdf                  deeOe
eef         gdf                  dePfdZQdddddddd$ed6ee         dee         djee         dee         dbee         dee         de
eef         fdZRdHedePfdZSde
eef         deddfdZTd ZUddZVd ZWddZXd ZYd ZZd Z[ddZ\ddZ]dS )V1FirecrawlAppa  
    Legacy v1 Firecrawl client for backward compatibility.
    
    This class provides the complete v1 API implementation including:
    - URL scraping with various formats and options
    - Website crawling with monitoring capabilities
    - Batch scraping operations
    - Search functionality
    - Data extraction with LLM integration
    - Deep research capabilities
    - LLMs.txt generation
    
    This is used by the unified client to provide version-specific access
    through app.v1.method_name() patterns.
    a  Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.Napi_keyapi_urlreturnc                 *   |pt          j        d          | _        |pt          j        dd          | _        d| j        v r0| j        )t                              d           t          d          t                              d| j                    dS )	z
        Initialize the V1FirecrawlApp instance with API key, API URL.

        Args:
            api_key (Optional[str]): API key for authenticating with the Firecrawl API.
            api_url (Optional[str]): Base URL for the Firecrawl API.
        FIRECRAWL_API_KEYFIRECRAWL_API_URLhttps://api.firecrawl.devzapi.firecrawl.devNz%No API key provided for cloud servicezNo API key providedz)Initialized V1FirecrawlApp with API URL: )r   getenvrV  rW  r   r   
ValueErrordebug)selfrV  rW  s      r#   __init__zV1FirecrawlApp.__init__  s     @"),?"@"@]"),?A\"]"] $,..4<3GNNBCCC2333OOOPPPPPr2   rs   )rn   r\   include_tagsexclude_tagsonly_main_contentwait_forrt   ru   rv   skip_tls_verificationremove_base64_images	block_adsr   	parse_pdfrO   json_optionsrR   change_tracking_optionsmax_agestore_in_cachezero_data_retentionrJ   rn   rk   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rz   ri  rO   rj  rR   rk  rl  rm  rn  c                   |                      |d           |                                 }|dt           d}|r||d<   |r||d<   |r||d<   |r||d<   |||d	<   |r||d
<   |r||d<   |	r|	                    dd          |d<   |
|
|d<   |||d<   |||d<   |||d<   |r||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    dd          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    dd          |d<   |rd |D             |d<   |r1t          |t                    r|n|                    dd          |d<   |||d<   |||d<   |||d<   |                    |           d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   t          j        | j	         d||||dz  d z   nd!          }|j
        d"k    r	 |                                }|                    d#          rd$|v rt          d)i |d$         S d%|v rt          d&|d%                    t          d&|           # t          $ r t          d'          w xY w|                     |d(           dS )*aW  
        Scrape and extract content from a URL.

        Args:
          url (str): Target URL to scrape
          formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
          headers (Optional[Dict[str, str]]): Custom HTTP headers
          include_tags (Optional[List[str]]): HTML tags to include
          exclude_tags (Optional[List[str]]): HTML tags to exclude
          only_main_content (Optional[bool]): Extract main content only
          wait_for (Optional[int]): Wait for a specific element to appear
          timeout (Optional[int]): Request timeout (ms)
          location (Optional[LocationConfig]): Location configuration
          mobile (Optional[bool]): Use mobile user agent
          skip_tls_verification (Optional[bool]): Skip TLS verification
          remove_base64_images (Optional[bool]): Remove base64 images
          block_ads (Optional[bool]): Block ads
          proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
          extract (Optional[JsonConfig]): Content extraction settings
          json_options (Optional[JsonConfig]): JSON extraction settings
          actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
          change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
          zero_data_retention (Optional[bool]): Whether to delete data after scrape is done


        Returns:
          ScrapeResponse with:
          * Requested content formats
          * Page metadata
          * Extraction results
          * Success/error status

        Raises:
          Exception: If scraping fails
        
scrape_urlpython-sdk@rJ   r  rn   r\   ro   rp   Nrq   rr   rt   Tby_aliasexclude_noneru   rv   rw   rx   ry   r   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S Trs  
isinstancedict.0actions     r#   
<listcomp>z-V1FirecrawlApp.scrape_url.<locals>.<listcomp>  X      (U  (U  (U  CI*VT2J2J(}PVP[P[eix|P[P}P}  (U  (U  (Ur2   rR   r   r   r   zeroDataRetention
/v1/scrape     @@r  r\   rB   rt      r   r   r   Failed to scrape URL. Error: +Failed to parse Firecrawl response as JSON.z
scrape URLr1   )_validate_kwargs_prepare_headersversionrz  _ensure_schema_dictry  updaterequestspostrW  status_coderB   getr   r   r^  _handle_error)r`  rJ   rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   rk  rl  rm  rn  kwargs_headersscrape_paramsresponseresponse_jsons                               r#   rp  zV1FirecrawlApp.scrape_url  s   | 	fl333((** -G--
 
  	/'.M)$ 	/'.M)$ 	8+7M-( 	8+7M-((/@M+, 	0'/M)$ 	/'.M)$ 	X(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*% 	+%*M'" (1M*%..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$" 	~PZ[rtxPyPy  6~5L5L  @W  @\  @\  fj  y}  @\  @~  @~M12&-M(#%,:M.)*1DM-.V$$$%%-	*B%xS`ajSkGkGk151I1I-XaJbckJl1m1mM)$X.M))mM.J)x[hiv[wOwOw595M5Mm\iNjksNt5u5uM-(2 =|'''-4-@Wv%))d	
 
 
 3&&	O ( $$Y// UFm4K4K+DDmF.CDDD--#$\MRYDZ$\$\]]]#$SM$S$STTT O O O MNNNO x66666s   >L> .L> >M)r   r  r  r  rX   ru   rt   scrape_optionsr  r   r  r  r  rX   r  c                4   |                      |
d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |                    |
           |                    d          }t	          dd|i|}|                    d
d
          }dt
           |d<   |r||d<   t          j        | j         ddd| j	         i|          }|j
        dk    r	 |                                }|                    d          rd|v rt          di |S d|v rt          d|d                    t          d|           # t          $ r t          d          w xY w|                     |d           dS )aS  
        Search for content using Firecrawl.

        Args:
            query (str): Search query string
            limit (Optional[int]): Max results (default: 5)
            tbs (Optional[str]): Time filter (e.g. "qdr:d")
            filter (Optional[str]): Custom result filter
            lang (Optional[str]): Language code (default: "en")
            country (Optional[str]): Country code (default: "us") 
            location (Optional[str]): Geo-targeting
            timeout (Optional[int]): Request timeout in milliseconds
            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
            **kwargs: Additional keyword arguments for future compatibility

        Returns:
            SearchResponse: Response containing:
                * success (bool): Whether request succeeded
                * data (List[FirecrawlDocument]): Search results
                * warning (Optional[str]): Warning message if any
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If search fails or response cannot be parsed
        r   Nr   r  r  r  rX   ru   rt   Trs  r   integrationr  rq  r  
/v1/searchAuthorizationBearer r\   rB   r  r   r   r   zSearch failed. Error: r  r1   )r  rz  r  r  r
  r  r  r  rW  rV  r  rB   r  r   r^  r  )r`  r  r   r  r  r  rX   ru   rt   r  r  search_params_integrationfinal_paramsparams_dictr  r  s                    r#   r   zV1FirecrawlApp.search  sT   N 	fh///  %*M'"?#&M% &,M(#$(M&!'.M)$(0M*%'.M)$%-;-@-@$]a-@-b-bM/* 	V$$$$((77 &CCEC]CC"''D'II 7g 7 7H 	6)5K& =|'''$&>&>&>?
 
 
 3&&	O ( $$Y// NFm4K4K+<<m<<<--#$U]7=S$U$UVVV#$L]$L$LMMM O O O MNNNO x22222s   >8E% 7.E% %E?c                    |                                  }|                     | j         d|          }|j        dk    r	 |                                }|                    d          rd|v rt          d
i |S d|v rt          d|d                    t          d|           # t          $ r t          d          w xY w| 	                    |d           d	S )z1Get current credit usage and billing period (v1)./v1/team/credit-usager  r   r   r   z#Failed to get credit usage. Error: r  zget credit usageNr1   )
r  _get_requestrW  r  rB   r  r  r   r^  r  r`  r  r  r  s       r#   get_credit_usagezV1FirecrawlApp.get_credit_usage  s   ((**$$|222
 

 3&&	O ( $$Y// [Fm4K4K0AA=AAA--#$b-X_J`$b$bccc#$Y-$Y$YZZZ O O O MNNNO x);<<<<<   8B& 8.B& &C c                    |                                  }|                     | j         d|          }|j        dk    r	 |                                }|                    d          rd|v rt          d
i |S d|v rt          d|d                    t          d|           # t          $ r t          d          w xY w| 	                    |d           d	S )z0Get current token usage and billing period (v1)./v1/team/token-usager  r   r   r   z"Failed to get token usage. Error: r  zget token usageNr1   )
r  r  rW  r  rB   r  r"  r   r^  r  r  s       r#   get_token_usagezV1FirecrawlApp.get_token_usage  s   ((**$$|111
 

 3&&	O ( $$Y// ZFm4K4K/@@-@@@--#$aW^I_$a$abbb#$X$X$XYYY O O O MNNNO x):;;;;;r  F
by_api_keyc                    |                                  }| j         d|rdndz   }|                     ||          }|j        dk    r	 |                                }|                    d          rd|v rt          di |S d|v rt          d|d                    t          d|           # t          $ r t          d	          w xY w| 	                    |d
           dS )z!Get historical credit usage (v1).z /v1/team/credit-usage/historical?byApiKey=true r  r   r+  r   z.Failed to get historical credit usage. Error: r  zget credit usage historicalNr1   )
r  rW  r  r  rB   r  r*  r   r^  r  r`  r  r  rJ   r  r  s         r#   get_credit_usage_historicalz*V1FirecrawlApp.get_credit_usage_historical3  s.   ((**???WaCiCSCSgij$$S(333&&	O ( $$Y// fI4N4N:KK]KKK--#$mUbcjUk$m$mnnn#$dUb$d$deee O O O MNNNO x)FGGGGG   8B/ .B/ /C	c                    |                                  }| j         d|rdndz   }|                     ||          }|j        dk    r	 |                                }|                    d          rd|v rt          di |S d|v rt          d|d                    t          d|           # t          $ r t          d	          w xY w| 	                    |d
           dS )z Get historical token usage (v1).z/v1/team/token-usage/historicalr  r  r  r   r+  r   z-Failed to get historical token usage. Error: r  zget token usage historicalNr1   )
r  rW  r  r  rB   r  r0  r   r^  r  r  s         r#   get_token_usage_historicalz)V1FirecrawlApp.get_token_usage_historicalG  s.   ((**>>>V`BhBRBRfhi$$S(333&&	O ( $$Y// eI4N4N9JJMJJJ--#$lTabiTj$l$lmmm#$cTa$c$cddd O O O MNNNO x)EFFFFFr     )include_pathsexclude_paths	max_depthmax_discovery_depthr   allow_backward_linkscrawl_entire_domainrQ  ignore_sitemapr  r   deduplicate_similar_urlsignore_query_parametersregex_on_full_urlr   allow_subdomainsmax_concurrencyrn  poll_intervalidempotency_keyr  r  r  r  r  r  rQ  r  r   r  r  r  r   r  r  r  r  c                |   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||                    dd          |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |                    |           |                    d          }t	          d i |}|                    dd          }||d<   dt
           |d<   |r||d<   |                     |          }|                     | j         d||          }|j	        dk    rT	 |
                                                    d          }n#  t          d          xY w|                     |||          S |                     |d           dS )!a  
        Crawl a website starting from a URL.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            CrawlStatusResponse with:
            * Crawling status and progress
            * Crawled page contents
            * Success/error information

        Raises:
            Exception: If crawl fails
        	crawl_urlNr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   r   r  r  rJ   rq  r  	/v1/crawlr  r   r  start crawl jobr1   )r  rz  r  r  r   r  r  _post_requestrW  r  rB   r   _monitor_job_statusr  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  r  crawl_paramsr  r  r  r\   r  r   s                                 r#   r  zV1FirecrawlApp.crawl_url[  s   ~ 	fk222 $+8L($+8L( '0L$*0CL,-$)L!*0CL,--!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,a,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+&-<L)**0CL,-F####''66 %44|44"''D'II E 7g 7 7H 	6)5K& ''88%%&@&@&@+wWW3&&P]]__((..P NOOO++BGGGx):;;;;;s   'E; ;F)r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  c                   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||                    dd          |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |                    |           t          di |}|                    dd          }||d<   dt           |d<   |                     |          }|                     | j         d||          }|j        dk    r3	 t          di |
                                S #  t          d          xY w|                     |d           dS )a  
        Start an asynchronous crawl job.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlResponse with:
            * success - Whether crawl started successfully
            * id - Unique identifier for the crawl job
            * url - Status check URL for the crawl
            * error - Error message if start failed

        Raises:
            Exception: If crawl initiation fails
        async_crawl_urlNr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   r   r  rJ   rq  r  r  r  r  r  r1   )r  rz  r  r   r  r  r  rW  r  r   rB   r   r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  r  r  r  r\   r  s                              r#   r  zV1FirecrawlApp.async_crawl_url  s^   | 	f&7888 $+8L($+8L( '0L$*0CL,-$)L!*0CL,--!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,a,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+&-<L)**0CL,-F### %44|44"''D'II E 7g 7 7H ''88%%&@&@&@+wWW3&&P&99999P NOOOx):;;;;;s   7E E&r   c                    d| }|                                  }|                     | j         | |          }|j        dk    r	 |                                }n#  t          d          xY w|d         dk    r8d|v r3|d         }d|v r!t          |d                   dk    rn|                    d          }|st          	                    d	           n	 |                     ||          }|j        dk    r#t          
                    d
|j                    n	 |                                }	n#  t          d          xY w|                    |	                    dg                      |	}n4# t
          $ r'}
t          
                    d|
            Y d}
~
nd}
~
ww xY wd|v !||d<   |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          d}d|v r|d         |d<   d|v r|d         |d<   t          ddd|v rdndi|S |                     |d           dS )a$  
        Check the status and results of a crawl job.

        Args:
            id: Unique identifier for the crawl job

        Returns:
            V1CrawlStatusResponse containing:

            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of pages crawled
            * total - Total pages to crawl
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of crawled documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        
/v1/crawl/r  r  r   r]   r   r   r   Expected 'next' URL is missing.Failed to fetch next page: !Error during pagination request: Nr   r   r   r   r   r]   r   r   r   r   r   FTcheck crawl statusr1   )r  r  rW  r  rB   r   lenr  r   r   r   extendr   r  r`  r   endpointr\   r  status_datar   next_urlstatus_response	next_dataes              r#   check_crawl_statusz!V1FirecrawlApp.check_crawl_statusY  s   4 %$$''))$$%@h%@%@'JJ3&&P&mmooP NOOO8$33[((&v.D K//{6233q88!#.??6#:#:' ""NN+LMMM!".2.?.?'.R.RO.:cAA &-h?Kf-h-h i i i %`,;,@,@,B,B		`&/0^&_&_ _ KK	fb(A(ABBB*3KK( " " ""LL)PQ)P)PQQQ!EEEE"% !K//* +/K' &//(33$11(__[99*}==(__[99#// H +%%$/$8!$$#.v#6 (  !(K!7!7T  
 x)=>>>>>=   A A,AE6 !D6 5E6 6E.E6 6
F' F""F'c                    |                                  }|                     | j         d| d|          }|j        dk    r3	 t	          di |                                S #  t          d          xY w|                     |d           dS )aL  
        Returns information about crawl errors.

        Args:
            id (str): The ID of the crawl job

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
                - id (str): Error ID
                - timestamp (str): When the error occurred
                - url (str): URL that caused the error
                - error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If error check fails
        r  /errorsr  r  zcheck crawl errorsNr1   r  r  rW  r  r   rB   r   r  r`  r   r\   r  s       r#   check_crawl_errorsz!V1FirecrawlApp.check_crawl_errors  s    & ''))$$%K%K%K%K%KWUU3&&P,??x}}???P NOOOx)=>>>>>   A   A1c                    |                                  }|                     | j         d| |          }|j        dk    r)	 |                                S #  t          d          xY w|                     |d           dS )}  
        Cancel an asynchronous crawl job.

        Args:
            id (str): The ID of the crawl job to cancel

        Returns:
            Dict[str, Any] containing:
            * success (bool): Whether cancellation was successful
            * error (str, optional): Error message if cancellation failed

        Raises:
            Exception: If cancellation fails
        r  r  r  zcancel crawl jobN)r  _delete_requestrW  r  rB   r   r  r  s       r#   cancel_crawlzV1FirecrawlApp.cancel_crawl  s     ''))''4<(G(G2(G(GQQ3&&P}}&P NOOOx);<<<<<s   A A&V1CrawlWatcherc                     | j         |fi d|d|d|d|d|d|d|d|	d	|
d
|d|d|d|d|d|d|d|d|d||}|j        r|j        rt          |j        |           S t	          d          )aM  
        Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlWatcher: An instance to monitor the crawl job via WebSocket

        Raises:
            Exception: If crawl job fails to start
        r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  Crawl job failed to start)r  r   r   r  r   )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  crawl_responses                          r#   crawl_url_and_watchz"V1FirecrawlApp.crawl_url_and_watch  sM   r .-
 
 
'-
 (-
  i	

 !4 3
 %
 "6!5
 !4 3
 "6!5
 *>
 *>
 G
 &>%=
 %<$;
 0/
  %!
" .-#
$ ,O%
& !4 3'
( ,O+
 
. ! 	9n&7 	9!."3T:::7888r2   )r   r  include_subdomainssitemap_onlyr   rt   	use_indexru   r   r  r  r  c                :   |                      |
d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |                    |
           |                    d          }t	          di |}|                    d
d
          }||d<   dt
           |d<   |r||d<   t          j        | j         ddd| j	         i|          }|j
        dk    r	 |                                }|                    d          rd|v rt          di |S d|v rt          d|d                    t          d|           # t          $ r t          d          w xY w|                     |d           dS )a  
        Map and discover links from a URL.

        Args:
            url (str): Target URL to map
            search (Optional[str]): Filter pattern for URLs
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            include_subdomains (Optional[bool]): Include subdomain links
            sitemap_only (Optional[bool]): Only use sitemap.xml
            limit (Optional[int]): Maximum URLs to return
            timeout (Optional[int]): Request timeout in milliseconds
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1MapResponse: Response containing:
                * success (bool): Whether request succeeded
                * links (List[str]): Discovered URLs
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If mapping fails or response cannot be parsed
        map_urlNr   r   r   r   r   rt   r   Trs  ru   r  rJ   rq  r  /v1/mapr  r  r  r  r   rN   r   zMap failed. Error: r  mapr1   )r  rz  r  r  r   r  r  r  rW  rV  r  rB   r   r   r^  r  )r`  rJ   r   r  r  r  r   rt   r  ru   r  
map_paramsr  r  r  r  r  s                    r#   r  zV1FirecrawlApp.map_url7  sU   H 	fi000 
 #)Jx %*8J').@J*+#(4J}%"'Jw$+Jy! %.Jz"%-]]Dt]%T%TJz" 	&!!!!~~m44 #00Z00"''D'II E 7g 7 7H 	6)5K& =|$$$$&>&>&>?
 
 
 3&&	O ( $$Y// KG}4L4L(99=999--#$R-:P$R$RSSS#$I-$I$IJJJ O O O MNNNO x/////s   8E( :.E( (F)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  urls	rK   rL   rM   rl   rN   rP   rm   rO   rB   r   c                n   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |rd |D             |d<   ||                    d
d
          |d<   |||d<   |||d<   |                    |           t          d"i |}|                    d
d
          }||d<   dt           |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   |                     |          }|                     | j	         d||          }|j
        dk    rT	 |                                                    d          }n#  t          d           xY w|                     |||          S |                     |d!           dS )#aH  
        Batch scrape multiple URLs and monitor until completion.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeStatusResponse with:
            * Scraping status and progress
            * Scraped content for each URL
            * Success/error information

        Raises:
            Exception: If batch scrape fails
        batch_scrape_urlsNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S rw  rx  r{  s     r#   r~  z4V1FirecrawlApp.batch_scrape_urls.<locals>.<listcomp>  r  r2   rR   r   r   r  r  rq  r  /v1/batch/scraper  r   r  start batch scrape jobr1   )r  rz  r  ry  r  r   r  r  r  rW  r  rB   r  r   r  r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  r  r  r  r  r  r   s                                r#   r  z V1FirecrawlApp.batch_scrape_urls  sM   @ 	f&9::: '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$%*ZZDZ%Q%QM'"&.=M*+*1DM-. 	V$$$ &6666"''D'II"F 7g 7 7H##I(>#8{[dOeCeCe/3/G/GT]H^_gHh/i/iK	"8,K''K,F'8WbcpWqKqKq373K3KKXeLfgoLp3q3qK&x0 ''88%%&G&G&GV]^^3&&P]]__((..P NOOO++BGGGx)ABBBBBs   'K4 4L)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  c                ,   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |rd |D             |d<   ||                    d
d
          |d<   |||d<   |||d<   |                    |           t          d!i |}|                    d
d
          }||d<   dt           |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   |                     |          }|                     | j	         d||          }|j
        dk    r3	 t          d!i |                                S #  t          d          xY w|                     |d            dS )"a~  
        Initiate a batch scrape job asynchronously.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeResponse with:
            * success - Whether job started successfully
            * id - Unique identifier for the job
            * url - Status check URL
            * error - Error message if start failed

        Raises:
            Exception: If job initiation fails
        async_batch_scrape_urlsNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S rw  rx  r{  s     r#   r~  z:V1FirecrawlApp.async_batch_scrape_urls.<locals>.<listcomp>  r  r2   rR   r   r   r  r  rq  r  r  r  r  r  r1   )r  rz  r  ry  r  r   r  r  r  rW  r  r   rB   r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  r  r  r  r  s                              r#   r  z&V1FirecrawlApp.async_batch_scrape_urls!  s4   @ 	f&?@@@ '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$%*ZZDZ%Q%QM'"&.=M*+*1DM-. 	V$$$ &6666"''D'II"F 7g 7 7H##I(>#8{[dOeCeCe/3/G/GT]H^_gHh/i/iK	"8,K''K,F'8WbcpWqKqKq373K3KKXeLfgoLp3q3qK&x0 ''88%%&G&G&GV]^^3&&P,??x}}???P NOOOx)ABBBBBs   K* *K;)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  rn  r  c                   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |rd |D             |d<   ||                    d
d
          |d<   |||d<   |||d<   |                    |           t          d"i |}|                    d
d
          }||d<   dt           |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   |                     |          }|                     | j	         d||          }|j
        dk    re	 t          d"i |                                }|j        r|j        rt          |j        |           S t!          d          #  t!          d           xY w|                     |d!           dS )#a   
        Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket

        Raises:
            Exception: If batch scrape job fails to start
        batch_scrape_urls_and_watchNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S rw  rx  r{  s     r#   r~  z>V1FirecrawlApp.batch_scrape_urls_and_watch.<locals>.<listcomp>  r  r2   rR   r   r   r  r  rq  r  r  r   Batch scrape job failed to startr  r  r1   )r  rz  r  ry  r  r   r  r  r  rW  r  r   rB   r   r   r  r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  rn  r  r  r  r  r  r  r  s                               r#   r  z*V1FirecrawlApp.batch_scrape_urls_and_watch  sl   x 	f&CDDD '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$%*ZZDZ%Q%QM'"&.=M*+*1DM-. 	V$$$ &6666"''D'II"F 7g 7 7H##I(>#8{[dOeCeCe/3/G/GT]H^_gHh/i/iK	"8,K''K,F'8WbcpWqKqKq373K3KKXeLfgoLp3q3qK&x0 ''88%%&G&G&GV]^^3&&P!6!I!I!I!I!) Hn.? H).*;TBBB#$FGGGP NOOOx)ABBBBBs   A L L L-c                     d| }|                                  }|                     | j         | |          }|j        dk    r#	 |                                }n#  t          d          xY w|d         dk    r8d|v r3|d         }d|v r!t          |d                   dk    rn|                    d          }|st          	                    d	           n	 |                     ||          }|j        dk    r#t          
                    d
|j                    n	 |                                }	n#  t          d          xY w|                    |	                    dg                      |	}n4# t
          $ r'}
t          
                    d|
            Y d}
~
nd}
~
ww xY wd|v !||d<   t          di d|v rdnd|                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          d	S |                     |d           dS )a@  
        Check the status of a batch scrape job using the Firecrawl API.

        Args:
            id (str): The ID of the batch scrape job.

        Returns:
            V1BatchScrapeStatusResponse: The status of the batch scrape job.

        Raises:
            Exception: If the status check request fails.
        /v1/batch/scrape/r  r  r   r]   r   r   r   r  r  r  Nr   FTr   r   r   )	r   r   r   r]   r   r   r   r   r   zcheck batch scrape statusr1   )r  r  rW  r  rB   r   r  r  r   r   r   r  r   r  r  s              r#   check_batch_scrape_statusz(V1FirecrawlApp.check_batch_scrape_status9  s    ,r++''))$$%@h%@%@'JJ3&&P&mmooP NOOO8$33[((&v.D K//{6233q88!#.??6#:#:' ""NN+LMMM!".2.?.?'.R.RO.:cAA &-h?Kf-h-h i i i %`,;,@,@,B,B		`&/0^&_&_ _ KK	fb(A(ABBB*3KK( " " ""LL)PQ)P)PQQQ!EEEE"% !K//* +/K'. 
 
$+{$:$:55%//(33$11(__[99*}==(__[99#//#//$11
2 
2 
 
 
 x)DEEEEEr  c                    |                                  }|                     | j         d| d|          }|j        dk    r3	 t	          di |                                S #  t          d          xY w|                     |d           dS )aX  
        Returns information about batch scrape errors.

        Args:
            id (str): The ID of the crawl job.

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
              * id (str): Error ID
              * timestamp (str): When the error occurred
              * url (str): URL that caused the error
              * error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If the error check request fails
        r  r  r  r  zcheck batch scrape errorsNr1   r  r  s       r#   check_batch_scrape_errorsz(V1FirecrawlApp.check_batch_scrape_errorsw  s    & ''))$$%R%Rr%R%R%RT[\\3&&P,??x}}???P NOOOx)DEEEEEr  r*   re   rP  rQ  rR  rS  r   r*   re   rP  rR  rS  c                
   ddl }
|
                    dt          d           |                     |	d           |                                 }|s|st          d          |s|st          d          |r|                     |          }|pg ||||d	t                       d
}|r||d<   |r||d<   |r||d<   |                    |	           	 | 	                    | j
         d||          }|j        dk    r5	 |                                }n#  t          d          xY w|d         r|                    d          }|st          d          	 |                     | j
         d| |          }|j        dk    rn	 |                                }n#  t          d          xY w|d         dk    rt!          d"i |S |d         dv r!t          d|d          d|d                    n|                     |d           t%          j        d           t          d|d                    |                     |d           n/# t          $ r"}t          t)          |          d          d}~ww xY wt!          dd !          S )#a  
        Extract structured information from URLs.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If prompt/schema missing or extraction fails
        r   NThe extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.r  
stacklevelrO   #Either prompt or schema is required!Either urls or prompt is requiredrq  r  r   r  r  re   r  r*   r   r   /v1/extractr  r  r   r   )Job ID not returned from extract request.T/v1/extract/r   r]   r^   r   Extract job 	. Error: r   zextract-statusFailed to extract. Error:   FzInternal server error.r   r   r1   )warningswarnDeprecationWarningr  r  r^  r  r$   r  r  rW  r  rB   r   r  r  r  r  timesleepr0   )r`  r  r*   re   rP  rQ  rR  rS  r   r  r  r\   request_datar  r   job_idr  r  r  s                      r#   rO   zV1FirecrawlApp.extract  s\   P 	%  	 	
 	
 	
 	fi000'')) 	Df 	DBCCC 	BF 	B@AAA 	6--f55F JB"60'3KMM33
 
  	,%+L" 	9+8L( 	*$)L! 	F###)	*))<,,, H
 #s**T#==??DDT#$RSSS	? R!XXd^^F! U'(STTT&*.*;*;#|AAAA#+ + +6#==`.=.B.B.D.D`&/0^&_&_ _*84CC'8'G'G;'G'G G!,X!6:Q!Q!Q&/0u{8?T0u0u_jkr_s0u0u&v&v v "R !..@PQQQ
1#&& $$Pg$P$PQQQ""8Y7777 	* 	* 	*SVVS)))	* !6NOOOOsP   +I :D I D  AI >F I F$$I ?BI 
I1I,,I1r  c                    ddl }|                    dt          d           |                                 }	 |                     | j         d| |          }|j        dk    r3	 t          di |                                S #  t          d          xY w| 
                    |d	           dS # t          $ r"}t          t          |          d
          d}~ww xY w)a  
        Retrieve the status of an extract job.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            job_id (str): The ID of the extract job.

        Returns:
            ExtractResponse[Any]: The status of the extract job.

        Raises:
            ValueError: If there is an error retrieving the status.
        r   Nr	  r  r
  r  r  r  zget extract statusr  r1   )r  r  r  r  r  rW  r  r  rB   r   r  r^  r0   )r`  r  r  r\   r  r  s         r#   get_extract_statusz!V1FirecrawlApp.get_extract_status  s   $ 	%  	 	
 	
 	
 ''))
	*((DL)N)Nf)N)NPWXXH#s**T,??x}}???T#$RSSS""8-ABBBBB 	* 	* 	*SVVS)))	*s)   +B- #B BB- -
C7CCc                4   ddl }	|	                    dt          d           |                                 }
|}|r|                     |          }|||||dt
           d}|r||d<   |r||d	<   |r||d
<   	 |                     | j         d||
          }|j        dk    r3	 t          di |
                                S #  t          d          xY w|                     |d           dS # t          $ r"}t          t          |          d          d}~ww xY w)a  
        Initiate an asynchronous extract job.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (List[str]): URLs to extract information from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If job initiation fails
        r   Nr	  r  r
  rq  r  r*   r   r   r  r  r  zasync extractr  r1   )r  r  r  r  r  r  r  rW  r  r  rB   r   r  r^  r0   )r`  r  r*   re   rP  rQ  rR  rS  r   r  r\   r  r  r  s                 r#   async_extractzV1FirecrawlApp.async_extract;  s   N 	%  	 	
 	
 	
 '')) 	6--f55F "60'-G--
 
  	,%+L" 	9+8L( 	*$)L!
	*))T\*F*F*FV]^^H#s**T,??x}}???T#$RSSS""8_===== 	* 	* 	*SVVS)))	*s*   6*C+ !B? ?CC+ +
D5DDmax_urlsshow_full_textr6  experimental_streamr$  r%  r6  r&  c                r   t          ||||          }|                     |||||          }|j        r|j        st	          dddd          S |j        }	 |                     |          }	|	j        d	k    r|	S |	j        dk    r|	S |	j        d
k    rt	          dddd          S t          j        d           b)a  
        Generate LLMs.txt for a given URL and poll until completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            GenerateLLMsTextStatusResponse with:
            * Generated LLMs.txt content
            * Full version if requested
            * Generation status
            * Success/error information

        Raises:
            Exception: If generation fails
        r4  r5  r6  __experimental_streamr#  Fz#Failed to start LLMs.txt generationr^   r  r   r   r   r   Tr]   r  /LLMs.txt generation job terminated unexpectedlyr  )	r2  async_generate_llms_textr   r   rM  check_generate_llms_text_statusr   r  r  )
r`  rJ   r$  r%  r6  r&  paramsr  r  r   s
             r#   generate_llms_textz!V1FirecrawlApp.generate_llms_text  s   8 *'"5	
 
 
 00) 3 1 
 
  	x{ 	3;	    	99&AAF}++(**,..7!K# 	    JqMMM	r2   c                `   t          ||||          }|                                 }d|i|                    dd          }dt           |d<   	 |                     | j         d||          }	|	                                }
t          d|           t          d	|
           |
                    d
          r!	 t          di |
S #  t          d          xY w|                     |
d           n.# t          $ r!}t          t          |                    d}~ww xY wt          dd          S )a  
        Initiate an asynchronous LLMs.txt generation operation.

        Args:
            url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL.
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            GenerateLLMsTextResponse: A response containing:
            * success (bool): Whether the generation initiation was successful
            * id (str): The unique identifier for the generation job
            * error (str, optional): Error message if initiation failed

        Raises:
            Exception: If the generation job initiation fails.
        r(  rJ   Trs  rq  r  /v1/llmstxt	json_datar  r   r  zstart LLMs.txt generationNFInternal server errorr  r1   )r2  r  rz  r  r  rW  rB   r   r  rG  r   r  r^  r0   )r`  rJ   r$  r%  r6  r&  r.  r\   r2  reqr  r  s               r#   r,  z'V1FirecrawlApp.async_generate_llms_text  sp   6 *'"5	
 
 
 ''))CQ6;;4;#P#PQ	5G55	(	%$$%A%A%A9gVVCxxzzH+y)))*h'''||I&& JS5AAAAAS#$QRRR""8-HIIII 	% 	% 	%SVV$$$	% *)
 
 
 	
s+   A(C1 :C CC1 1
D;DDc                 
   |                                  }	 |                     | j         d| |          }|j        dk    rR	 |                                }t          di |S # t          $ r$}t          dt          |                     d}~ww xY w|j        dk    rt          d          |                     |d           n.# t          $ r!}t          t          |                    d}~ww xY wt          dd	d
d          S )a;  
        Check the status of a LLMs.txt generation operation.

        Args:
            id (str): The unique identifier of the LLMs.txt generation job to check status for.

        Returns:
            GenerateLLMsTextStatusResponse: A response containing:
            * success (bool): Whether the generation was successful
            * status (str): Status of generation ("processing", "completed", "failed")
            * data (Dict[str, str], optional): Generated text with fields:
              * llmstxt (str): Generated LLMs.txt content
              * llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            Exception: If the status check fails.
        /v1/llmstxt/r  zFFailed to parse Firecrawl response as GenerateLLMsTextStatusResponse: N  z!LLMs.txt generation job not foundz check LLMs.txt generation statusFr3  r^   r  r*  r1   )
r  r  rW  r  rB   rM  r   r0   r  r^  )r`  r   r\   r  r2  r  s         r#   r-  z.V1FirecrawlApp.check_generate_llms_text_status	  s>   ( ''))	%((DL)J)Jb)J)JGTTH#s**w (I;HHiHHH  w w w#$umpqrmsms$u$uvvvw%,, CDDD""8-OPPPP 	% 	% 	%SVV$$$	% 0E\emy{||||s5   +C A" "
B,BB3C 
C/C**C/c                 <    |rdd| j          |dS dd| j          dS )a$  
        Prepare the headers for API requests.

        Args:
            idempotency_key (Optional[str]): A unique key to ensure idempotency of requests.

        Returns:
            Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
        zapplication/jsonr  )Content-Typer  zx-idempotency-key)r9  r  )rV  )r`  r  s     r#   r  zV1FirecrawlApp._prepare_headers1	  sO      	 2!94<!9!9%4   /5t|55
 
 	
r2            ?r   retriesbackoff_factorc                     t          |          D ]]}t          j        |||d|v r|d         |d         dz  dz   nd          }|j        dk    rt	          j        |d|z  z             Z|c S |S )a^  
        Make a POST request with retries.

        Args:
            url (str): The URL to send the POST request to.
            data (Dict[str, Any]): The JSON data to include in the POST request.
            headers (Dict[str, str]): The headers to include in the POST request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the POST request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        rt   Nr  r  r    r  )ranger  r  r  r  r  )r`  rJ   r   r\   r<  r=  attemptr  s           r#   r  zV1FirecrawlApp._post_requestI	  s    . W~~ 	  	 G}S'qz  C  rC  rC  HL  MV  HW  HcPTU^P_bhPhklPlPl  im  o  o  oH#s**
>Q'\:;;;;r2   c                     t          |          D ]A}t          j        ||          }|j        dk    rt	          j        |d|z  z             >|c S |S )a	  
        Make a GET request with retries.

        Args:
            url (str): The URL to send the GET request to.
            headers (Dict[str, str]): The headers to include in the GET request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the GET request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        r\   r?  r  )r@  r  r  r  r  r  r`  rJ   r\   r<  r=  rA  r  s          r#   r  zV1FirecrawlApp._get_requesth	  se    * W~~ 	  	 G|C999H#s**
>Q'\:;;;;r2   c                     t          |          D ]A}t          j        ||          }|j        dk    rt	          j        |d|z  z             >|c S |S )a  
        Make a DELETE request with retries.

        Args:
            url (str): The URL to send the DELETE request to.
            headers (Dict[str, str]): The headers to include in the DELETE request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the DELETE request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        rC  r?  r  )r@  r  deleter  r  r  rD  s          r#   r  zV1FirecrawlApp._delete_request	  se    * W~~ 	  	 GsG<<<H#s**
>Q'\:;;;;r2   c                    	 | j          d| }|                     ||          }|j        dk    r:	 |                                }n#  t	          d          xY w|d         dk    rd|v r|d         }d|v rt          |d                   d	k    rns|                     |d         |          }	 |                                }n#  t	          d          xY w|                    |                    dg                      d|v ||d<   t          di |S t	          d
          |d         dv r%t          |d          }t          j        |           n.t	          d|d                    |                     |d           )a  
        Monitor the status of a crawl job until completion.

        Args:
            id (str): The ID of the crawl job.
            headers (Dict[str, str]): The headers to include in the status check requests.
            poll_interval (int): Seconds between status checks.

        Returns:
            CrawlStatusResponse: The crawl results if the job is completed successfully.

        Raises:
            Exception: If the job fails or an error occurs during status checks.
        Tr  r  r  r   r]   r   r   r   z,Crawl job completed but no data was returnedactivepausedpendingqueuedwaitingr   r  z)Crawl job failed or was stopped. Status: r  Nr1   )rW  r  r  rB   r   r  r  r  r   maxr  r  r  )r`  r   r\   r  rW  r  r  r   s           r#   r  z"V1FirecrawlApp._monitor_job_status	  s   &	J5555G"//AAO*c11T"1"6"6"8"8KKT#$RSSSx(K77,,*62$33";v#6771<< %.2.?.?F@SU\.].]O`.=.B.B.D.D`&/0^&_&_ _ KK(C(CDDD %33 /3F+4CC{CCC'(VWWW *.nnn"%mA"6"6MJ}----#$gP[\dPe$g$ghhh""?4HIII?	Js   A A-C Cr  r}  c                    	 |                                 }|                    dd          }|                    dd          }nd#  	 |j        dd         }|                                rd| }d|j         }nd	|j         }d
}n# t
          $ r d|j         }d
}Y nw xY wY nxY w|                     |j        |||          }t          j        	                    ||          )ah  
        Handle errors from API responses.

        Args:
            response (requests.Response): The response object from the API request.
            action (str): Description of the action that was being performed.

        Raises:
            Exception: An exception with a message containing the status code and error details from the response.
        r   No error message provided.details%No additional error details provided.Nr  z#Server returned non-JSON response: zFull response status: z+Server returned empty response with status zNo additional details availablez0Server returned unreadable response with status )r  )
rB   r  r   r   r  r^  _get_error_messager  
exceptions	HTTPError)r`  r  r}  r  error_messageerror_detailsresponse_textmessages           r#   r  zV1FirecrawlApp._handle_error	  s&   	B$MMOOM)--g7STTM)--i9`aaMM	B
B (dsd 3 &&(( F$Y-$Y$YM$SX=Q$S$SMM$hRZRf$h$hM$EM B B B iS[Sg i i AB ))(*>Wdee !++Gh+GGGs0   A A B$?BB$BB$BB$r  rV  rW  c                     |dk    rd| d| d| S |dk    rd| d| d| S |dk    rd| d	| d| S |d
k    rd| d| d| S |dk    rd| d| d| S d| d| d| d| S )a  
        Generate a standardized error message based on HTTP status code.
        
        Args:
            status_code (int): The HTTP status code from the response
            action (str): Description of the action that was being performed
            error_message (str): The error message from the API response
            error_details (str): Additional error details from the API response
            
        Returns:
            str: A formatted error message
        i  zPayment Required: Failed to z. z - i  z!Website Not Supported: Failed to i  zRequest Timeout: Failed to z as the request timed out. i  zConflict: Failed to z due to a conflict. r  z!Internal Server Error: Failed to zUnexpected error during z: Status code r1   r`  r  r}  rV  rW  s        r#   rS  z!V1FirecrawlApp._get_error_message	  s     #]&]]M]]m]]]CbvbbbbS`bbbCuuuTauufsuuuCg&ggmggXegggCbvbbbbS`bbbtfttKttS`ttertttr2   )r  
time_limitr$  analysis_promptrP  *_V1FirecrawlApp__experimental_stream_stepson_activity	on_sourcer\  r]  r^  r_  r`  c                   i }
|||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   t          di |
}
|                     ||||||          }|                    d	          rd
|vr|S |d
         }d}d}	 |                     |          }|r9d|v r5|d         |d         }|D ]} ||           t	          |d                   }|	r9d|v r5|d         |d         }|D ]} |	|           t	          |d                   }|d         dk    r|S |d         dk    r%t          d|                    d                     |d         dk    rnt          j        d           dddS a  
        Initiates a deep research operation on a given query and polls until completion.

        Args:
            query (str): Research query or topic to investigate
            max_depth (Optional[int]): Maximum depth of research exploration
            time_limit (Optional[int]): Time limit in seconds for research
            max_urls (Optional[int]): Maximum number of URLs to process
            analysis_prompt (Optional[str]): Custom prompt for analysis
            system_prompt (Optional[str]): Custom system prompt
            __experimental_stream_steps (Optional[bool]): Enable experimental streaming
            on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}
            on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}

        Returns:
            DeepResearchStatusResponse containing:
            * success (bool): Whether research completed successfully
            * status (str): Current state (processing/completed/failed)
            * error (Optional[str]): Error message if failed
            * id (str): Unique identifier for the research job
            * data (Any): Research findings and analysis
            * sources (List[Dict]): List of discovered sources
            * activities (List[Dict]): Research progress log
            * summaries (List[str]): Generated research summaries

        Raises:
            Exception: If research fails
        Nr   r;  r4  r=  r   __experimental_streamSteps)r  r\  r$  r]  rP  r   r   r   TrD  r  r   r]   r^   zDeep research failed. Error: r   r  r  Fz)Deep research job terminated unexpectedlyr  r1   )r9  async_deep_researchr  check_deep_research_statusr  r   r  r  )r`  r  r  r\  r$  r]  rP  r^  r_  r`  research_paramsr  r  last_activity_countlast_source_countr   new_activitiesactivitynew_sourcessources                       r#   deep_researchzV1FirecrawlApp.deep_research
  s6   P  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AAAA++!+' , 
 
 ||I&& 	$h*>*>O$	44V<<F @|v55!'!56I6J6J!K . * *HK))))&)&*>&?&?# ;Y&00$Y/0A0B0BC) & &FIf%%%%$'y(9$:$:!h;..!X-- U

7@S@S U UVVV!\11JqMMM-	0 !+VWWWr2   )r  r\  r$  r]  rP  r^  c                   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   t          di |}|                                 }	d|i|                    d	d	
          }
dt           |
d<   d|
v rG|
d         }|r=d|v r9t	          |d         d          r#|d                                         |
d         d<   	 |                     | j         d|
|	          }|j        dk    r)	 |	                                S #  t          d          xY w|                     |d           n.# t          $ r!}t          t          |                    d}~ww xY wdddS )  
        Initiates an asynchronous deep research operation.

        Args:
            query (str): Research query or topic to investigate
            max_depth (Optional[int]): Maximum depth of research exploration
            time_limit (Optional[int]): Time limit in seconds for research
            max_urls (Optional[int]): Maximum number of URLs to process
            analysis_prompt (Optional[str]): Custom prompt for analysis
            system_prompt (Optional[str]): Custom system prompt
            __experimental_stream_steps (Optional[bool]): Enable experimental streaming

        Returns:
            Dict[str, Any]: A response containing:
            * success (bool): Whether the research initiation was successful
            * id (str): The unique identifier for the research job
            * error (str, optional): Error message if initiation failed

        Raises:
            Exception: If the research initiation fails.
        Nr   r;  r4  r=  r   rc  r  Trs  rq  r  r   re   /v1/deep-researchr  r  zstart deep researchFr3  r  r1   )r9  r  rz  r  hasattrre   r  rW  r  rB   r   r  r^  r0   )r`  r  r  r\  r$  r]  rP  r^  rf  r\   r2  	json_optsr  r  s                 r#   rd  z"V1FirecrawlApp.async_deep_researchu
  s   >  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AAAA''))e^';';TX\';']']^	5G55	( I%%!-0I RX22wy?RT\7]7]25>x5H5O5O5Q5Q	-(2
	%))T\*L*L*LiY`aaH#s**S#==??*S#$QRRR""8-BCCCC 	% 	% 	%SVV$$$	% !+BCCCs*   *D+ ,D   DD+ +
E5EEc                    |                                  }	 |                     | j         d| |          }|j        dk    r)	 |                                S #  t          d          xY w|j        dk    rt          d          |                     |d           n.# t
          $ r!}t          t          |                    d}~ww xY wdd	d
S )  
        Check the status of a deep research operation.

        Args:
            id (str): The ID of the deep research operation.

        Returns:
            DeepResearchResponse containing:

            Status:
            * success - Whether research completed successfully
            * status - Current state (processing/completed/failed)
            * error - Error message if failed
            
            Results:
            * id - Unique identifier for the research job
            * data - Research findings and analysis
            * sources - List of discovered sources
            * activities - Research progress log
            * summaries - Generated research summaries

        Raises:
            Exception: If the status check fails.
        /v1/deep-research/r  r  r7  zDeep research job not foundzcheck deep research statusNFr3  r  )	r  r  rW  r  rB   r   r  r^  r0   )r`  r   r\   r  r  s        r#   re  z)V1FirecrawlApp.check_deep_research_status
  s    2 ''))	%((DL)P)PB)P)PRYZZH#s**S#==??*S#$QRRR%,, =>>>""8-IJJJJ 	% 	% 	%SVV$$$	% !+BCCCs)   +B A A''3B 
C%CCr  method_namec           	      &   |sdS h dh dh dh dh dh dh dh dd}|                     |t                                }t          |                                          |z
  }|r)t          d	| d
d                    |           d          dS )a  
        Validate additional keyword arguments before they are passed to the API.
        This provides early validation before the Pydantic model validation.

        Args:
            kwargs (Dict[str, Any]): Additional keyword arguments to validate
            method_name (str): Name of the method these kwargs are for

        Raises:
            ValueError: If kwargs contain invalid or unsupported parameters
        N>   r   rv   rR   rO   rn   rl  rt   ru   re  rh  r  rc  rb  rj  rd  rg  rf  rk  >	   r  r  r   r  rX   rt   ru   r  r  >   r   r   r  r  r  r  r  r  r  r  r  rQ  r  r  >   r   r   rt   r  r  r  r  >   r   r*   re   r  rS  rP  rR  rQ  >   r   r   rv   rR   rO   rn   r\   rt   r   ru   re  rh  rc  rb  rj  rd  rg  rf  )rp  r   r  r  rO   r  r  r  zUnsupported parameter(s) for z: z, zC. Please refer to the API documentation for the correct parameters.)r  setkeysr^  r   )r`  r  rv  method_paramsallowed_paramsunknown_paramss         r#   r  zV1FirecrawlApp._validate_kwargs
  s7     	FK K K |{{   }|| R  R  R"@ "@ "@(F (F (F,J ,J ,J%
 
2 '**;>> V[[]]++n< 	l  k[  k  kDIIVdLeLe  k  k  k  l  l  l	l 	lr2   c                     ||S t          |t                    rHt          |d          r|                                S t          |d          r|                                S t          |t
                    r  fd|                                D             S t          |t          t          f          r fd|D             S |S )zw
        Utility to ensure a schema is a dict, not a Pydantic model class. Recursively checks dicts and lists.
        Nmodel_json_schemare   c                 B    i | ]\  }}|                     |          S r1   r  )r|  kvr`  s      r#   
<dictcomp>z6V1FirecrawlApp._ensure_schema_dict.<locals>.<dictcomp>&  s-    NNNtq!At//22NNNr2   c                 :    g | ]}                     |          S r1   r  )r|  r  r`  s     r#   r~  z6V1FirecrawlApp._ensure_schema_dict.<locals>.<listcomp>(  s'    @@@AD,,Q//@@@r2   )	ry  r   rq  r~  re   rz  itemslisttupler`  re   s   ` r#   r  z"V1FirecrawlApp._ensure_schema_dict  s     >Mfd## 	'v233 '//111** '}}&fd## 	ONNNNv||~~NNNNftUm,, 	A@@@@@@@@r2   c                 
   |rt          |t          t          f          sdS |t                      }ddl} |j        |dt                    }||v rdS |                    |           	 t          |t                    r
d|v rt          |d         t                    r|d                             d          }t          |          dk    ru|d         d	k    ri|d
         dk    r]|d         }||k    r	 |
                    |           dS ||v r3|                     ||         |||          |
                    |           S |                                D ]2}	|                     |	|||          r |
                    |           dS 3nJt          |t                    r5|D ]2}
|                     |
|||          r |
                    |           dS 3|
                    |           n# |
                    |           w xY wdS )a  
        Check if an object contains a recursive reference to a specific definition.
        
        Args:
            obj: Object to check
            target_def_name: Name of the definition to check for recursion
            defs: Dictionary of definitions
            visited: Set of visited object keys to detect cycles
            
        Returns:
            True if recursive reference is found, False otherwise
        FNr   T)	sort_keysdefault$ref/r:  #r   $defs)ry  rz  r  rx  rB   dumpsr0   addsplitr  discard_contains_recursive_refvalues)r`  objtarget_def_namedefsvisitedrB   obj_keyref_pathdef_namevalueitems              r#   r  z&V1FirecrawlApp._contains_recursive_ref+  sT     	*S4,77 	5?eeG$*SD#>>>g5G	%#t$$ $S==ZFS%A%A="6{0055H8}}))hqkS.@.@Xa[T[E[E[#+B<#66#' OOG$$$$$ $t++#'#?#?XP_aegn#o#o OOG$$$$ !ZZ\\ $ $E33E?DRYZZ $# OOG$$$$$$$ C&& $ $ $D33D/4QXYY $# OOG$$$$$	$ OOG$$$$GOOG$$$$us%   )B	G) 
!G) /G) 4G) G) )H c                 v    |sdS |                                 D ]\  }}|                     |||          r dS  dS )z
        Check if $defs contain circular references.
        
        Args:
            defs: Dictionary of definitions to check
            
        Returns:
            True if circular references are found, False otherwise
        FT)r  r  )r`  r  r  	def_values       r#   _check_for_circular_defsz'V1FirecrawlApp._check_for_circular_defs]  sZ      	5#'::<< 	 	Hi++IxFF tt ur2   r   c                     |r"t          |t          t          f          rdk    r|S t                      t	          |          }|v r|S                     |           	 t          |t                    r#d|v rt          |d         t                    r|d                             d          }t          |          dk    rg|d         dk    r[|d         d	k    rO|d
         }|v rC 	                    t          |                   dz             
                    |           S |
                    |           S i }|                                D ]*\  }	}
|	d	k    r 	                    |
dz             ||	<   +|
                    |           S t          |t                    r& fd|D             
                    |           S 	 
                    |           n# 
                    |           w xY w|S )a|  
        Resolve $ref references in a JSON schema object.
        
        Args:
            obj: Object to resolve references in
            defs: Dictionary of definitions
            visited: Set to track visited objects and prevent infinite recursion
            depth: Current recursion depth
            
        Returns:
            Object with resolved references
        r3  Nr  r  r:  r   r  r   r  r  c           	      F    g | ]}                     |d z             S )r   )_resolve_refs)r|  r  r  depthr`  r  s     r#   r~  z0V1FirecrawlApp._resolve_refs.<locals>.<listcomp>  s1    [[[t**4w	JJ[[[r2   )ry  rz  r  rx  r   r  r0   r  r  r  r  r  )r`  r  r  r  r  obj_idr  r  resolvedr   r  s   ` ```      r#   r  zV1FirecrawlApp._resolve_refsp  sL     	*S4,77 	52::J?eeGCWJF	$#t$$ \S==ZFS%A%A="6{0055H8}}))hqkS.@.@Xa[T[E[E[#+B<#t++#'#5#5d4>6J6JDRY[`cd[d#e#e OOF####  OOF#### "%))++ X XJCg~~ $($6$6udGUUVY$W$WHSMM OOF####	 C&& \[[[[[[[WZ[[[ OOF####	\ OOF####GOOF####
s    &B4G1 0G1 AG1 %G1 1Hc                 v    |rt          |t                    s|S t                      fd |          S )z
        Normalize a schema for OpenAI compatibility by handling recursive references.
        
        Args:
            schema: Schema to normalize
            
        Returns:
            Normalized schema
        c                    t          | t                    s%t          | t                    r	fd| D             S | S t          |           }|
v r| S 
                    |           t          |           dv r
                    |           S dv r                    d          }i }                                D ]2\  }}t          |t                    rd|vr 	|          ||<   -|||<   3i |d	fd|                                D             i}
                    |           |S                     d          dk    rdv r                    d          d	u rd=                     d          dk    rad
v r]dv rYt          d
         t                    r;t          d         t                    r fdd
         D             }|r|d
<   nd
= nd
= t                                                    D ]S\  }}t          |t                    rd|vr 	|          |<   -t          |t                    r	fd|D             |<   T
                    |           S )Nc                 &    g | ]} |          S r1   r1   r|  r  normalize_objects     r#   r~  zYV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object.<locals>.<listcomp>  s%    CCCt,,T22CCCr2   r  r  c                 .    i | ]\  }}| |          S r1   r1   )r|  r  r  r  s      r#   r  zYV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object.<locals>.<dictcomp>  s.    5f5f5fQUQRTUa9I9I!9L9L5f5f5fr2   r   object
propertiesadditionalPropertiesTrequiredc                 (    g | ]}|d          v |S )r  r1   )r|  field
normalizeds     r#   r~  zYV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object.<locals>.<listcomp>  s5     &I &I &I&+z,/G&G&G ',&G&G&Gr2   c                 T    g | ]$}t          |t                    r |          n|%S r1   rx  r  s     r#   r~  zYV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object.<locals>.<listcomp>  s<    &t&t&tfjDRVAWAW'a'7'7'='='=]a&t&t&tr2   )	ry  rz  r  r   r  r  popr  r  )r  r  r  processed_restr   r  resultvalid_requiredr  r  r  s           @r#   r  zEV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object  s   c4(( c4(( DCCCCsCCCC
 WWF  
KKcJ ##'''!!*$$!~~g..!#","2"2"4"4 4 4JC"5$// 4F%4G4G.>.>u.E.Es++.3s++gNgG5f5f5f5fY]YcYcYeYe5f5f5fgg''' v&&(22
**566$>>56v&&(22j((
**z*5t<< 	/z,7>>	/&I &I &I &IJ9O &I &I &IN% 31?
:..&z22":. #:#3#3#5#566 u u
UeT** uvU/B/B&6&6u&=&=JsOOt,, u&t&t&t&tns&t&t&tJsOOOF###r2   ry  rz  rx  )r`  re   r  r  s     @@r#   _normalize_schema_for_openaiz+V1FirecrawlApp._normalize_schema_for_openai  s_      	Z55 	M%%>	 >	 >	 >	 >	 >	@  '''r2   c                 x    |rt          |t                    sdS t                      fd |           S )z
        Validate schema for OpenAI compatibility.
        
        Args:
            schema: Schema to validate
            
        Returns:
            True if schema is valid, False otherwise
        Tc                    t          | t                    sdS t          |           }|v rdS                     |           d| v r                    |           dS |                     d          dk    r6d| vr2d| vr.|                     d          du r                    |           dS |                                 D ]}t          |t                    r(d|vr$ |          r                    |            dS ?t          |t                    rB|D ]?}t          |t                    r(d|vr$ |          r                    |             dS @                    |           dS )	NFr  r   r  r  patternPropertiesr  T)ry  rz  r   r  r  r  r  r  )r  r  r  r  has_invalid_structurer  s       r#   r  zIV1FirecrawlApp._validate_schema_for_openai.<locals>.has_invalid_structure  s   c4(( uWWF  uKK}}'''u8++C''#3...//477'''t  , ,eT** 
,vU/B/B,,U33 $///#tt$  t,, , % , ,%dD11 ,fD6H6H44T:: , ' 7 7 7'+tttOOF###5r2   r  )r`  re   r  r  s     @@r#   _validate_schema_for_openaiz*V1FirecrawlApp._validate_schema_for_openai  sb      	Z55 	4%%#	 #	 #	 #	 #	 #	J )(0000r2   c                     |rt          |t                    sdS ddl} |j        |          }d|v pd|v pd|v }t	          |                    d          p|                    d                    }|p|S )	z
        Detect if a schema contains recursive references.
        
        Args:
            schema: Schema to analyze
            
        Returns:
            True if schema has recursive patterns, False otherwise
        Fr   N"$ref"#/$defs/z#/definitions/r  definitions)ry  rz  rB   r  r   r  )r`  re   rB   schema_stringhas_refshas_defss         r#   _detect_recursive_schemaz'V1FirecrawlApp._detect_recursive_schema)  s      	Z55 	5"
6**% .-'.- 	
 

7++Hvzz-/H/HII#8#r2   c                 N    |sdddS |                      |          rdddS dddS )z
        Select appropriate model based on schema complexity.
        
        Args:
            schema: Schema to analyze
            
        Returns:
            Dict with modelName and reason
        zgpt-4o-mini	no_schema)	modelNamereasonzgpt-4orecursive_schema_detectedsimple_schema)r  r  s     r#   _select_model_for_schemaz'V1FirecrawlApp._select_model_for_schemaA  sO      	G!.+FFF((00 	R!)5PQQQ*oFFFr2   c                    t          |t                    rh||v rc|                     ||                   }|rt          |t                    r|                    di           }ddl} |j        |          }|                    d          pd|v pd|v }|r`	 |                     ||          } |j        |          }	d|	v pd|	v }
|
s|}t          |t                    rd|v r|d= nT# t          $ r Y nHw xY w	 |                     ||          }t          |t                    rd|v r|d= n# t          $ r Y nw xY w|                     |          }| 	                    |          st          | j                  |||<   |S )a1  
        Process and validate a schema container (like extract or json_options).
        
        Args:
            schema_container: Dict containing schema
            schema_key: Key where schema is stored (default: "schema")
            
        Returns:
            Processed schema container
        r  r   Nr  r  )ry  rz  r  r  rB   r  r  r   r  r  r^  OPENAI_SCHEMA_ERROR_MESSAGE)r`  schema_container
schema_keyre   r  rB   r  has_any_refsresolved_schemaresolved_stringhas_remaining_refss              r#   _process_schema_with_validationz.V1FirecrawlApp._process_schema_with_validationS  s    &-- *	2*@P2P2P--.>z.JKKF   *VT22  zz'2.. *
6 2 2JJw'' 0-0-/    *.*<*<VT*J*J*4$*_*E*E-5-H-iJZiLi*1 4%4F)&$77 4Gv<M<M$*7O$   
!%!3!3FD!A!A%fd33 068I8I &w$    66v>>F33F;; C !ABBB+1Z(s%   AC( (
C54C592D, ,
D98D9NN)FNr:  r;  )Nr   )re   )^r+   r,   r-   r.   r  r   r0   ra  r   r	   r   r   r   rW   r   r   r   r   r   r   r   r   r   r   r   rc   r   r   rp  ri   r  r   r  r  r"  r  r*  r  r0  r  r[   r   r  r   r  r  r   r  r  r  r   r  r'   r   r  r   r  r  r  r  r  rO   r   r"  rM  r/  rG  r,  r-  r  r   r  Responser  r  r  r  r  rS  r   rB  rm  rd  re  r  r  r  r  r  r  r  r  r  r  r1   r2   r#   rU  rU    s        $	q  Q Q Qx} QX\ Q Q Q Q, mq04040404&*%*37%)4837(,OS(,.237 EIIM%)-1261U7 U7 U7U7 d7  ,g  $h  i  j	U7
 d38n-U7 #49-U7 #49-U7  (~U7 smU7 c]U7 /0U7 TNU7 $,D>U7 #+4.U7  ~U7  G$JKL!U7"  ~#U7$ l+%U7& #<0'U7( d57I=Zgiv  yG  IW  Yr  t  *  $@  A  B)U7* &..E%F+U7, c]-U7. %TN/U70 "*$1U72 *#.3U7 U7 U7 U7v $(!%$("&%)&*%*8<]3 ]3 ]3]3 C=	]3
 #]3 SM]3 3-]3 c]]3 sm]3 c]]3 %_5]3 *]3 ]3 ]3 ]3~="7 = = = =,<!5 < < < <,H Hd HGf H H H H(G GT GFd G G G G0 .2-1#'-1#/3.2/3)-489=3726,0#+/)-.2'()-/@< @< @<@<  S	*	@<
  S	*@< C=@< &c]@< }@< 'tn@< &d^@< 'tn@< !@< !1@< %_ 456@< #+4.@<  "*$!@<" $D>#@<$ }%@<& #4.'@<( "#)@<* &d^+@<,  }-@<. "#/@<2 
3@< @< @< @<L .2-1#'-1#/3.2/3)-489=3726,0#+/)-.2)--z< z< z<z<  S	*	z<
  S	*z< C=z< &c]z< }z< 'tnz< &d^z< 'tnz< !z< !1z< %_ 456z< #+4.z<  "*$!z<" $D>#z<$ }%z<& #4.'z<( "#)z<* &d^+z<, "#-z<0 
1z< z< z< z<xQ?S Q?-B Q? Q? Q? Q?f?S ?-B ? ? ? ?:=s =tCH~ = = = =: 2615'+15#'372637-18<=A7;6:04#'/3-126-1-S9 S9 S9S9 $DI.	S9
 $DI.S9  }S9 "*#S9 C=S9 #+4.S9 "*$S9 #+4.S9 %TNS9 %_5S9 eC$89:S9 '/tnS9  &.d^!S9"  (~#S9$ C=%S9& 'tn'S9( &c])S9* "*$+S9, &c]-S90 
1S9 S9 S9 S9r %)-115+/#'%*(,37[0 [0 [0[0 SM	[0
 %TN[0 !)[0 #4.[0 C=[0 c][0  ~[0 /0[0 '[0 [0 [0 [0B W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.'()-.2)-1KC KC KC3iKC $w  (Q   R  S  T	KC
 $sCx.)KC tCy)KC tCy)KC $D>KC 3-KC #KC +,KC KC  (~KC 'tnKC D>KC   FGH!KC" ,'#KC$ |,%KC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'KC( ')KC*  }+KC, "#-KC. &d^/KC0 "#1KC4 
%5KC KC KC KCb W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.)-)-.2/JC JC JC3iJC $w  (Q   R  S  T	JC
 $sCx.)JC tCy)JC tCy)JC $D>JC 3-JC #JC +,JC JC  (~JC 'tnJC D>JC   FGH!JC" ,'#JC$ |,%JC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JC( ')JC* "#+JC, "#-JC. &d^/JC2 
3JC JC JC JC` W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.)-.2)-/JC JC JC3iJC $w  (Q   R  S  T	JC
 $sCx.)JC tCy)JC tCy)JC $D>JC 3-JC #JC +,JC JC  (~JC 'tnJC D>JC   FGH!JC" ,'#JC$ |,%JC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JC( ')JC* "#+JC, &d^-JC. "#/JC2 
3JC JC JC JCX<FC <F4O <F <F <F <F|FC F4I F F F F> )-~P %)$(+/3805+0.2~P ~P ~P49%~P SM	~P
 SM~P $C=~P #+4.~P  (~~P #4.~P DcN+~P +3/~P ~P ~P ~P@%* %*1B31G %* %* %* %*R )-O* %)$(+/3805+0.2O* O* O*49%O* SM	O*
 SMO* $C=O* #+4.O*  (~O* #4.O* DcN+O* 8I7MO* O* O* O*j '+-1$(26C C CC sm	C
 %TNC D>C "*$C <\C C C CR '+-1$(268
 8
 8
8
 sm	8

 %TN8
 D>8
 "*$8
 <V8
 8
 8
 8
t$}# $}:Z $} $} $} $}P .2
 
%c]
6:38n
 
 
 
: $'  sCx. #s(^	
  " -5,=   F $'  #s(^ 	
 "
 -5,=   B $'  #s(^ 	
 "
 -5,=   :2J2J #s(^2J 	2J $9	2J 2J 2J 2Jh#H'#H #H !#H #H #H #HJuc u3 us ucf ukn u u u u< (,(,&*-1+/:>FJDH^X ^X ^X^X  }	^X
 !^X sm^X &c]^X $C=^X *2$^X "(DcN+;T+A"BC^X  $sCx.)94)? @A^X Nj^X ^X ^X ^XH (,(,&*-1+/:>ED ED EDED  }	ED
 !ED smED &c]ED $C=ED *2$ED DHS>ED ED ED EDN(DS (D5Q (D (D (D (DT.ltCH~ .lC .lD .l .l .l .lf  $0 0 0 0d  &0 0 0 0dO( O( O(b41 41 41l$ $ $0G G G G$6  6  6  6  6  6 r2   rU  c                       e Zd ZdZdedefdZddZddZd	ed
e	e
eef         gdf         ddfdZd	ede
eef         ddfdZde
eef         ddfdZdS )r  a  
    A class to watch and handle crawl job events via WebSocket connection.

    Attributes:
        id (str): The ID of the crawl job to watch
        app (V1FirecrawlApp): The V1FirecrawlApp instance
        data (List[Dict[str, Any]]): List of crawled documents/data
        status (str): Current status of the crawl job
        ws_url (str): WebSocket URL for the crawl job
        event_handlers (dict): Dictionary of event type to list of handler functions
    r   appc                     || _         || _        g | _        d| _        |j                            dd           d| | _        g g g d| _        d S )Nr   httpwsr  )doner   document)r   r  r   r   rW  replacews_urlevent_handlers)r`  r   r  s      r#   ra  zV1CrawlWatcher.__init__  sc    *,	 ,,VT::JJbJJ
 
r2   rX  Nc                    K   t          j        | j        ddd| j        j         fg          4 d{V 	 }|                     |           d{V  ddd          d{V  dS # 1 d{V swxY w Y   dS )zU
        Establishes WebSocket connection and starts listening for messages.
        Nr  r  )max_sizeadditional_headers
websocketsconnectr  r  rV  _listenr`  	websockets     r#   r  zV1CrawlWatcher.connect  s(      %K!02NDH<L2N2N OP
 
 
 	* 	* 	* 	* 	* 	* 	* 	* ,,y)))))))))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*s   A%%
A/2A/c                 ~   K   |2 3 d{V }t          j        |          }|                     |           d{V  76 dS )z
        Listens for incoming WebSocket messages and handles them.

        Args:
            websocket: The WebSocket connection object
        NrB   loads_handle_messager`  r  rY  msgs       r#   r  zV1CrawlWatcher._listen  q       ' 	, 	, 	, 	, 	, 	, 	,'*W%%C&&s++++++++++ 'YY   <
event_typehandlerc                 \    || j         v r"| j         |                             |           dS dS )z
        Adds an event handler function for a specific event type.

        Args:
            event_type (str): Type of event to listen for ('done', 'error', or 'document')
            handler (Callable): Function to handle the event
        N)r  append)r`  r  r  s      r#   add_event_listenerz!V1CrawlWatcher.add_event_listener  s<     ,,,
+227;;;;; -,r2   detailc                 R    || j         v r| j         |         D ]} ||           dS dS )z
        Dispatches an event to all registered handlers for that event type.

        Args:
            event_type (str): Type of event to dispatch
            detail (Dict[str, Any]): Event details/data to pass to handlers
        N)r  )r`  r  r  r  s       r#   dispatch_eventzV1CrawlWatcher.dispatch_event  sJ     ,,,.z:     -,   r2   r  c                   K   |d         dk    r2d| _         |                     d| j         | j        | j        d           dS |d         dk    r9d| _         |                     d| j         | j        |d         | j        d           dS |d         dk    rq|d	         d
         | _         | j                            |d	                             d	g                      | j        D ] }|                     d|| j        d           !dS |d         dk    rF| j                            |d	                    |                     d|d	         | j        d           dS dS )z
        Handles incoming WebSocket messages based on their type.

        Args:
            msg (Dict[str, Any]): The message to handle
        r   r  r]   r   r   r   r   r^   r   r   r   r   catchupr   r   r  r   r   Nr   r   r   r   r  r  r  r`  r  docs      r#   r  zV1CrawlWatcher._handle_message  }      v;&  %DK4;	Y]Y`(a(abbbbb[G##"DKDK]`ah]iquqx)y)yzzzzz[I%%f+h/DKIS[__VR88999y N N##JDG0L0LMMMMN N[J&&IS[)))
S[,P,PQQQQQ '&r2   rX  N)r+   r,   r-   r.   r0   rU  ra  r  r  r   r   r   r  r   r  r1   r2   r#   r  r    s       
 


3 

^ 

 

 

 

	* 	* 	* 	*	, 	, 	, 	,	<S 	<8T#s(^DTVZDZ;[ 	<`d 	< 	< 	< 	<
  
 d38n 
  
  
  
  
 Rc3h RD R R R R R Rr2   r  c            0           e Zd ZdZddedef fdZ	 	 	 dd	ed
edeeef         deeeef                  de	de
deeef         fdZ	 dd
edeeef         deeef         de	de
deeef         fdZ	 dd
edeeef         de	de
deeef         f
dZdej        deddfdZde	dedededef
dZdefdZdefdZ	 	 dd
edee         dee         ddfdZ	 	 dd ee         dee         dee         ddfd!Zddddddd"ddddddddddd#d
ed$eeed%                           deeeef                  d&eee                  d'eee                  d(ee         d)ee	         d*ee	         d+ee         d,ee         d-ee         d.ee         d/ee         d0eed1                  d2ee         d3ee         d4ee         d5eeee e!e"e#e$e%e&e'e(f	                           de)e         f&d6Z*ddddddd"ddddddddddd7dd8d ee         d$eeed9                           deeeef                  d&eee                  d'eee                  d(ee         d)ee	         d*ee	         d+ee         d,ee         d-ee         d.ee         d/ee         d0eed1                  d3ee         d4ee         d5eeee e!e"e#e$e%e&e'e(f	                           d:ee+         d;ee	         dee         de,f*d<Z-ddddddd"ddddddddddddd=d ee         d$eeed9                           deeeef                  d&eee                  d'eee                  d(ee         d)ee	         d*ee	         d+ee         d,ee         d-ee         d.ee         d/ee         d0eed1                  d3ee         d4ee         d5eeee e!e"e#e$e%e&e'e(f	                           d:ee+         d>ee         dee         de.f*d?Z/ddddddddddddddddd7dd@d
edAeee                  dBeee                  dCee	         dDee	         dEee	         dFee         dGee         dHee         dIee         dJee0         dKeeee1f                  dLee         dMee         dNee         dOee	         dPee         d;ee	         dee         de2f(dQZ3ddddddddddddddddd7dd@d
edAeee                  dBeee                  dCee	         dDee	         dEee	         dFee         dGee         dHee         dIee         dJee0         dKeeee1f                  dLee         dMee         dNee         dOee	         dPee         d;ee	         dee         de4f(dRZ5dSede2fdTZ6ddSedeeef         d;e	de2fdUZ7dddddd"dddVd
edWee         dIee         dXee         dYee         dEee	         d*ee	         d+ee         dee8         de9fdZZ:	 ddddd[d[d[dd\d eee                  d]ee         d^ee         d_ee         dHee         d`ee         daee         d:eeeef                  de;e         fdbZ<dSede,fdcZ=dSede>fddZ?dSede>fdeZ@dSedeeef         fdfZAdgede;e         fdhZB	 ddddd[d[d[dd\d eee                  d]ee         d^ee         d_ee         dHee         d`ee         daee         d:eeeef                  de;e         fdiZCddddjd
edkee	         dlee         dmee         deDf
dnZEdddddod
edkee	         dlee         dpee         dmee         deFfdqZGdSedeDfdrZHdddddddddsdtedCee	         duee	         dkee	         dvee         d_ee         dwee         dxeeIeeef         gdf                  dyeeIeeef         gdf                  deJfdzZKddddddd{dtedCee	         duee	         dkee	         dvee         d_ee         dwee         deeef         fd|ZLdSedeJfd}ZMddddddd"ddd~	dtedEee	         dee         dee         dee         dee         d+ee         d*ee	         dJee0         deeeeef         eNf                  deOfdZP xZQS )AsyncV1FirecrawlAppz
    Asynchronous version of V1FirecrawlApp that implements async methods using aiohttp.
    Provides non-blocking alternatives to all V1FirecrawlApp operations.
    r\  rV  rW  c                 N    t                                          ||           d S )N)rV  rW  superra  )r`  rV  rW  	__class__s      r#   ra  zAsyncV1FirecrawlApp.__init__  s&    ':::::r2   Nr:  r;  methodrJ   r\   r   r<  r=  rX  c           	        K   t          j                    4 d{V }t          |          D ]&}	 |                    ||||          4 d{V 	 }	|	j        dk    r3t          j        |d|z  z             d{V  	 ddd          d{V  c|	j        dk    r |                     |	d| d           d{V  |	                                 d{V cddd          d{V  c cddd          d{V  S # 1 d{V swxY w Y   # t           j	        $ r6}
||dz
  k    r|
t          j        |d|z  z             d{V  Y d}
~
 d}
~
ww xY wt          d	          # 1 d{V swxY w Y   dS )
a  
        Generic async request method with exponential backoff retry logic.

        Args:
            method (str): The HTTP method to use (e.g., "GET" or "POST").
            url (str): The URL to send the request to.
            headers (Dict[str, str]): Headers to include in the request.
            data (Optional[Dict[str, Any]]): The JSON data to include in the request body (only for POST requests).
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        N)r  rJ   r\   rB   r?  r  i,  zmake z requestr   zMax retries exceeded)aiohttpClientSessionr@  requestr   asyncior  r  rB   ClientErrorr   )r`  r  rJ   r\   r   r<  r=  sessionrA  r  r  s              r#   _async_requestz"AsyncV1FirecrawlApp._async_request  s5     6 (** 	4 	4 	4 	4 	4 	4 	4g >> I II&%3d  /     5 5 5 5 5 5 5 5!#?c11")-!w,0O"P"PPPPPPPP$5 5 5 5 5 5 5 5 5 5 5 5 5 $?c11"&"4"4X?Wv?W?W?W"X"XXXXXXXX%-]]__4444445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	45 5 5 5 5 5 5 5 5 5 5 5 5 5 5 * I I I'A+--!-!w,(GHHHHHHHHHHHHHHI 2333!	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4sw   E#D-C:=DE#AC:D%E#:
DDDDE#E+EE#EE##
E-0E-c                 F   K   |                      d|||||           d{V S )a  
        Make an async POST request with exponential backoff retry logic.

        Args:
            url (str): The URL to send the POST request to.
            data (Dict[str, Any]): The JSON data to include in the request body.
            headers (Dict[str, str]): Headers to include in the request.
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        POSTNr  )r`  rJ   r   r\   r<  r=  s         r#   _async_post_requestz'AsyncV1FirecrawlApp._async_post_request  s8      * ((gtWn]]]]]]]]]r2   c                 F   K   |                      d||d||           d{V S )a  
        Make an async GET request with exponential backoff retry logic.

        Args:
            url (str): The URL to send the GET request to.
            headers (Dict[str, str]): Headers to include in the request.
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        GETNr  )r`  rJ   r\   r<  r=  s        r#   _async_get_requestz&AsyncV1FirecrawlApp._async_get_request5  s8      ( ((WdG^\\\\\\\\\r2   r  r}  c                 F  K   	 |                                  d{V }|                    dd          }|                    dd          }n!#  t          j        d|j                   xY w|                     |j        |||           d{V }t          j        |          )aR  
        Handle errors from async API responses with detailed error messages.

        Args:
            response (aiohttp.ClientResponse): The response object from the failed request
            action (str): Description of the action that was being attempted

        Raises:
            aiohttp.ClientError: With a detailed error message based on the response status:
                - 402: Payment Required
                - 408: Request Timeout
                - 409: Conflict
                - 500: Internal Server Error
                - Other: Unexpected error with status code
        Nr   rP  rQ  rR  ?Failed to parse Firecrawl error response as JSON. Status code: )rB   r  r  r  r   _get_async_error_messager`  r  r}  
error_datarV  rW  rY  s          r#   r  z!AsyncV1FirecrawlApp._handle_errorK  s       	{'}}......J&NN74PQQM&NN96]^^MM	{%&yhphw&y&yzzz55hov}^kllllllll!'***   AA A)r  rV  rW  c                 6   K   |                      ||||          S a  
        Generate a standardized error message based on HTTP status code for async operations.
        
        Args:
            status_code (int): The HTTP status code from the response
            action (str): Description of the action that was being performed
            error_message (str): The error message from the API response
            error_details (str): Additional error details from the API response
            
        Returns:
            str: A formatted error message
        rS  r[  s        r#   r#  z,AsyncV1FirecrawlApp._get_async_error_messagef  "       &&{FM=YYYr2   c                    K   |                                  }|                     | j         d|           d{V }t          di |S )z8Get current credit usage and billing period (v1, async).r  Nr1   )r  r   rW  r  r`  r\   resps      r#   r  z$AsyncV1FirecrawlApp.get_credit_usageu  sq      '')),,|222
 
 
 
 
 
 
 
 %,,t,,,r2   c                    K   |                                  }|                     | j         d|           d{V }t          di |S )z7Get current token usage and billing period (v1, async).r  Nr1   )r  r   rW  r"  r,  s      r#   r  z#AsyncV1FirecrawlApp.get_token_usage~  sq      '')),,|111
 
 
 
 
 
 
 
 $++d+++r2   r.  r  AsyncV1CrawlWatcherc                    K   |                      |||           d{V }|                    d          rd|v rt          |d         |           S t          d          )a  
        Initiate an async crawl job and return an AsyncV1CrawlWatcher to monitor progress via WebSocket.

        Args:
          url (str): Target URL to start crawling from
          params (Optional[V1CrawlParams]): See V1CrawlParams model for configuration:
            URL Discovery:
            * includePaths - Patterns of URLs to include
            * excludePaths - Patterns of URLs to exclude
            * maxDepth - Maximum crawl depth
            * maxDiscoveryDepth - Maximum depth for finding new URLs
            * limit - Maximum pages to crawl

            Link Following:
            * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead
            * crawlEntireDomain - Follow parent directory links
            * allowExternalLinks - Follow external domain links  
            * ignoreSitemap - Skip sitemap.xml processing

            Advanced:
            * scrapeOptions - Page scraping configuration
            * webhook - Notification webhook settings
            * deduplicateSimilarURLs - Remove similar URLs
            * ignoreQueryParameters - Ignore URL parameters
            * regexOnFullURL - Apply regex to full URLs
          idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
          AsyncV1CrawlWatcher: An instance to monitor the crawl job via WebSocket

        Raises:
          Exception: If crawl job fails to start
        Nr   r   r  )r  r  r/  r   )r`  rJ   r.  r  r  s        r#   r  z'AsyncV1FirecrawlApp.crawl_url_and_watch  sv      L  $33CQQQQQQQQi(( 	9T^-C-C&~d';TBBB7888r2   r  c                    K   |                      |||           d{V }|                    d          rd|v rt          |d         |           S t          d          )a  
        Initiate an async batch scrape job and return an AsyncV1CrawlWatcher to monitor progress.

        Args:
            urls (List[str]): List of URLs to scrape
            params (Optional[V1ScrapeParams]): See V1ScrapeParams model for configuration:

              Content Options:
              * formats - Content formats to retrieve
              * includeTags - HTML tags to include
              * excludeTags - HTML tags to exclude
              * onlyMainContent - Extract main content only
              
              Request Options:
              * headers - Custom HTTP headers
              * timeout - Request timeout (ms)
              * mobile - Use mobile user agent
              * proxy - Proxy type
              
              Extraction Options:
              * extract - Content extraction config
              * jsonOptions - JSON extraction config
              * actions - Actions to perform
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            AsyncV1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket

        Raises:
            Exception: If batch scrape job fails to start
        Nr   r   r  )r  r  r/  r   )r`  r  r.  r  batch_responses        r#   r  z/AsyncV1FirecrawlApp.batch_scrape_urls_and_watch  sw      H  $;;D&/ZZZZZZZZi(( 	@T^-C-C&~d';TBBB>???r2   rs   )rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   rn   rk   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rz   ri  rO   rj  rR   c                  K   |                      |d           |                                 }|dt           d}|r||d<   |r||d<   |r||d<   |r||d<   |||d	<   |r||d
<   |r||d<   |	r|	                    dd          |d<   |
|
|d<   |||d<   |||d<   |||d<   |r||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    dd          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    dd          |d<   |rd |D             |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d}|                     | j         | ||           d{V }|                    d          rd|v rt          di |d         S d|v rt          d|d                    |                    dt          |                    }t          d|           ) aA  
        Scrape a single URL asynchronously.

        Args:
          url (str): Target URL to scrape
          formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
          headers (Optional[Dict[str, str]]): Custom HTTP headers
          include_tags (Optional[List[str]]): HTML tags to include
          exclude_tags (Optional[List[str]]): HTML tags to exclude
          only_main_content (Optional[bool]): Extract main content only
          wait_for (Optional[int]): Wait for a specific element to appear
          timeout (Optional[int]): Request timeout (ms)
          location (Optional[V1LocationConfig]): Location configuration
          mobile (Optional[bool]): Use mobile user agent
          skip_tls_verification (Optional[bool]): Skip TLS verification
          remove_base64_images (Optional[bool]): Remove base64 images
          block_ads (Optional[bool]): Block ads
          proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
          extract (Optional[V1JsonConfig]): Content extraction settings
          json_options (Optional[V1JsonConfig]): JSON extraction settings
          actions (Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]]): Actions to perform
          **kwargs: Additional parameters to pass to the API

        Returns:
            V1ScrapeResponse with:
            * success - Whether scrape was successful
            * markdown - Markdown content if requested
            * html - HTML content if requested
            * rawHtml - Raw HTML content if requested
            * links - Extracted links if requested
            * screenshot - Screenshot if requested
            * extract - Extracted data if requested
            * json - JSON data if requested
            * error - Error message if scrape failed

        Raises:
            Exception: If scraping fails
        rp  rq  rr  rn   r\   ro   rp   Nrq   rr   rt   Trs  ru   rv   rw   rx   ry   r   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S rw  rx  r{  s     r#   r~  z2AsyncV1FirecrawlApp.scrape_url.<locals>.<listcomp>L  r  r2   rR   r  r   r   r   r  r1   )r  r  r  rz  r  ry  r  rW  r  r   r   r0   )r`  rJ   rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   r  r  r  r  r  error_contents                            r#   rp  zAsyncV1FirecrawlApp.scrape_url  s%     z 	fl333((** -G--
 
  	/'.M)$ 	/'.M)$ 	8+7M-( 	8+7M-((/@M+, 	0'/M)$ 	/'.M)$ 	X(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*% 	+%*M'" (1M*%..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$%%-	*B%xS`ajSkGkGk151I1I-XaJbckJl1m1mM)$X.M))mM.J)x[hiv[wOwOw595M5Mm\iNjksNt5u5uM-(2 !11|'X''
 
 
 
 
 
 
 
 <<	"" 	Mv'9'9#77hv&6777  OHW<MOOPPP %LL#h--@@MKMKKLLLr2   r  )rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  r  r   r  c                ^  K   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |d |D             |d<   ||                    d
d
          |d<   |                    |           t          d i |}|                    d
d
          }||d<   dt           |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   |                     |          }|                     | j	         d||           d{V }|
                    d          rH	 |
                    d          }n#  t          d          xY w|                     |||           d{V S |                     |d           dS )!a  
        Asynchronously scrape multiple URLs and monitor until completion.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeStatusResponse with:
            * Scraping status and progress
            * Scraped content for each URL
            * Success/error information

        Raises:
            Exception: If batch scrape fails
        r  Nrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   c                 <    g | ]}|                     d d           S rw  )rz  r{  s     r#   r~  z9AsyncV1FirecrawlApp.batch_scrape_urls.<locals>.<listcomp>  s*    'l'l'lZ`TPT(U(U'l'l'lr2   rR   r   r  rq  r  r  r   r   r  r  r1   )r  rz  r  ry  r  r   r  r  r  rW  r  r   _async_monitor_job_statusr  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  r  r  r  r  r  r   s                              r#   r  z%AsyncV1FirecrawlApp.batch_scrape_urlsc  sN     z 	f&9::: '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-('l'ldk'l'l'lM)$%*ZZDZ%Q%QM'" 	V$$$ &6666"''D'II"F 7g 7 7H##I(>#8{[dOeCeCe/3/G/GT]H^_gHh/i/iK	"8,K''K,F'8WbcpWqKqKq373K3KKXeLfgoLp3q3qK&x0 ''8811|---
 
 
 
 
 
 
 
 <<	"" 	CP\\$''P NOOO77G]SSSSSSSSSx)ABBBBBs   K& &K7)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   rn  r  rn  c                V  K   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	                    d
d
          |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |}|                     |          }t          |t                    r"d|v r|                     |d                   |d<   t          |t                    r|n|                    d
d
          |d<   |rd |D             |d<   ||                    d
d
          |d<   |||d<   |                    |           t          d!i |}|                    d
d
          }||d<   dt           |d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   d|v r<|d         r4d|d         v r*|                     |d         d                   |d         d<   |                     |          }|                     | j	         d||           d{V }|
                    d          dk    r3	 t          d!i |                                S #  t          d          xY w|                     |d            d{V  dS )"a,  
        Initiate a batch scrape job asynchronously.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeResponse with:
            * success - Whether job started successfully
            * id - Unique identifier for the job
            * url - Status check URL
            * error - Error message if start failed

        Raises:
            Exception: If job initiation fails
        r  Nrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   c                 j    g | ]0}t          |t                    r|n|                    d d           1S rw  rx  r{  s     r#   r~  z?AsyncV1FirecrawlApp.async_batch_scrape_urls.<locals>.<listcomp>V  r  r2   rR   r   r  r  rq  r  r  r  r  r  r  r1   )r  rz  r  ry  r  r   r  r  r  rW  r  r   rB   r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   rn  r  r  r  r  r  r  s                             r#   r  z+AsyncV1FirecrawlApp.async_batch_scrape_urls  sh     | 	f&?@@@ '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(W(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w77G'4(( PX-@-@$($<$<WX=N$O$O!2<Wd2K2K'wwQXQ]Q]gkz~Q]QQM)$#33LAAL,-- Z(l2J2J)-)A)A,xBX)Y)YX&;ElTX;Y;Y  ,S<<_k_p_pz~  NR_p  `S  `SM-( 	U (U  (U  MT  (U  (U  (UM)$%*ZZDZ%Q%QM'"*1DM-. 	V$$$ &6666"''D'II"F 7g 7 7H##I(>#8{[dOeCeCe/3/G/GT]H^_gHh/i/iK	"8,K''K,F'8WbcpWqKqKq373K3KKXeLfgoLp3q3qK&x0 ''8811|---
 
 
 
 
 
 
 
 <<&&#--P,??x}}???P NOOO$$X/GHHHHHHHHHHHs   K9 9L
)r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  c                @  K   |                      |d           i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||                    dd          |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |                    |           t          di |}|                    dd          }||d<   dt           |d<   |                     |          }|                     | j         d||           d{V }|                    d          rH	 |                    d          }n#  t          d          xY w| 
                    |||           d{V S |                     |d           d{V  dS )a  
        Crawl a website starting from a URL.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlStatusResponse with:
            * Crawling status and progress
            * Crawled page contents
            * Success/error information

        Raises:
            Exception: If crawl fails
        r  Nr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   rJ   rq  r  r  r   r   r  r  r1   )r  rz  r  r   r  r  r  rW  r  r   r8  r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r\   r  r   s                              r#   r  zAsyncV1FirecrawlApp.crawl_urlz  s     v 	fk222 $+8L($+8L( '0L$*0CL,-$)L!*0CL,--!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,a,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+ 	F### %44|44"''D'II E 7g 7 7H''8811\
$
$
$k7< < < < < < < < <<	"" 	BP\\$''P NOOO77G]SSSSSSSSS$$X/@AAAAAAAAAAAs   ;E E"c                  K   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d<   |	|	|d	<   |
|
|d
<   ||                     dd          |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |                    |           t          di |}|                     dd          }||d<   dt           |d<   |                     |          }|                     | j         d||           d{V }|                    d          r!	 t          di |S #  t          d          xY w| 
                    |d           d{V  dS )a  
        Start an asynchronous crawl job.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlResponse with:
            * success - Whether crawl started successfully
            * id - Unique identifier for the crawl job
            * url - Status check URL for the crawl
            * error - Error message if start failed

        Raises:
            Exception: If crawl initiation fails
        Nr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   rJ   rq  r  r  r   r  r  r1   )rz  r  r   r  r  r  rW  r  r   r   r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r\   r  s                             r#   r  z#AsyncV1FirecrawlApp.async_crawl_url  sc     p  $+8L($+8L( '0L$*0CL,-$)L!*0CL,--!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,a,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+ 	F### %44|44"''D'II E 7g 7 7H ''8811\
$
$
$


 
 
 
 
 
 
 
 <<	"" 	BP&22222P NOOO$$X/@AAAAAAAAAAAs   %D1 1Er   c           
        K   |                                  }d| }|                     | j         | |           d{V }|                    d          dk    rd|v r|d         }d|v rt	          |d                   dk    rn}|                    d          }|st
                              d           nK|                     ||           d{V }|                    |                    dg                      |}d|v ||d<   t          |                    d          |                    d	          |                    d          |                    d
          |                    d          |                    d          d|v rdnd          }d|v r|                    d          |_	        d|v r|                    d          |_
        |S )a7  
        Check the status and results of an asynchronous crawl job.

        Args:
            id (str): Unique identifier for the crawl job

        Returns:
            V1CrawlStatusResponse containing:
            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of pages crawled
            * total - Total pages to crawl
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of crawled documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        r  Nr   r]   r   r   r   r  r   r   r   r   FT)r   r   r]   r   r   r   r   )r  r   rW  r  r  r   r   r  r   r   r   	r`  r   r\   r  r  r   r  r  r  s	            r#   r  z&AsyncV1FirecrawlApp.check_crawl_statusg  s     2 ''))$$$ 33|'X''
 
 
 
 
 
 
 

 ??8$$33$$"6*++;v.//144*v66H# 'HIII&*&=&=h&P&P P P P P P PIKK	fb 9 9:::"+K ++ '+F#(??8,,//'**!ook22#66!ook22(($33EE
 
 
 k!!(__W55HN[  'OOF33HMr2   c                   K   	 |                      | j         d| |           d{V }|                    d          dk    rd|v r|d         }d|v rt          |d                   dk    rn}|                    d          }|st                              d	           nK|                      ||           d{V }|                    |                    dg                      |}d|v ||d<   t          di |S t          d
          |                    d          dv r)t          j
        t          |d                     d{V  nt          d|d                    `)a  
        Monitor the status of an asynchronous job until completion.

        Args:
            id (str): The ID of the job to monitor
            headers (Dict[str, str]): Headers to include in status check requests
            poll_interval (int): Seconds between status checks (default: 2)

        Returns:
            V1CrawlStatusResponse: The job results if completed successfully

        Raises:
            Exception: If the job fails or an error occurs during status checks
        Tr  Nr   r]   r   r   r   r  z&Job completed but no data was returnedrH  r  z#Job failed or was stopped. Status: r1   )r   rW  r  r  r   r   r  r   r   r  r  rN  )r`  r   r\   r  r  r   r  r  s           r#   r8  z-AsyncV1FirecrawlApp._async_monitor_job_status  s     	_ $ 7 7<//2//! !      K
 x((K77[((&v.D K//{6233q88!#.??6#:#:' ""NN+LMMM!*.*A*A(G*T*T$T$T$T$T$T$T	IMM&"$=$=>>>&/ !K// +/K'0??;???#$LMMM**.nnnmCq$9$9:::::::::: ]kRZF[ ] ]^^^5	_r2   )r   r  r  r  r   rt   ru   r.  r   r  r  c                d  K   i }
|	r*|
                     |	                    dd                     |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d	<   ||                    dd          |
d
<   t          di |
}|                    dd          }||d<   dt           |d<   d}|                     | j         | |dd| j         i           d{V }|                    d          rd|v rt          di |S d|v rt          d|d                    t          d|           )a  
        Asynchronously map and discover links from a URL.

        Args:
          url (str): Target URL to map
          params (Optional[V1MapParams]): See V1MapParams model:
            Discovery Options:
            * search - Filter pattern for URLs
            * ignoreSitemap - Skip sitemap.xml
            * includeSubdomains - Include subdomain links
            * sitemapOnly - Only use sitemap.xml
            
            Limits:
            * limit - Max URLs to return
            * timeout - Request timeout (ms)

        Returns:
          V1MapResponse with:
          * Discovered URLs
          * Success/error status

        Raises:
          Exception: If mapping fails
        Trs  Nr   r   r   r   r   rt   ru   rJ   rq  r  r  r  r  rC  r   rN   r   zFailed to map URL. Error: r1   )
r  rz  r   r  r  rW  rV  r  r   r   )r`  rJ   r   r  r  r  r   rt   ru   r.  r  r  r  r  r  s                  r#   r  zAsyncV1FirecrawlApp.map_url  s     H 
 	Mfkk4dkKKLLL #)Jx %*8J').@J*+#(4J}%"'Jw$+Jy!%-]]Dt]%T%TJz" #00Z00"''D'II E 7g 7 7H 11|'X''$&>&>&>? 2 
 
 
 
 
 
 
 
 <<	"" 	Ew(':': ,,8,,,  L'9JLLMMMCCCDDDr2   Fr  r*   re   rP  rR  rS  c                \  K   ddl }	|	                    dt          d           |                                 }
|s|st	          d          |s|st	          d          |r|                     |          }|pg ||||dt                       d	}|r||d
<   |r||d<   |r||d<   |                     | j         d||
           d{V }|	                    d          r|	                    d          }|st          d          	 |                     | j         d| |
           d{V }|d         dk    rt          di |S |d         dv r!t          d|d          d|d                    t          j        d           d{V  t          d|	                    d                     )aQ  
        Asynchronously extract structured information from URLs.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration

        Returns:
          V1ExtractResponse with:
          * Structured data matching schema
          * Source information if requested
          * Success/error status

        Raises:
          ValueError: If prompt/schema missing or extraction fails
        r   Nr	  r  r
  r  r  rq  r  r*   r   r   r  r   r   r  Tr  r   r]   r  r  r  r   r  r1   )r  r  r  r  r^  r  r$   r  rW  r  r   r   r  r  r  )r`  r  r*   re   rP  rQ  rR  rS  r   r  r\   r  r  r  r  s                  r#   rO   zAsyncV1FirecrawlApp.extract!  s     N 	%  	 	
 	
 	
 '')) 	Df 	DBCCC 	BF 	B@AAA 	6--f55F JB"60'3KMM33
 
  	,%+L" 	9+8L( 	*$)L!11|(((
 
 
 
 
 
 
 
 <<	"" 	R\\$''F M KLLL'$($;$;|9999% %      
 x(K77,;;{;;; *.EEE#$i;x3H$i$iS^_fSg$i$ijjjmA&&&&&&&&&' Pg9N9NPPQQQr2   c           
      T  K   |                                  }d| }|                     | j         | |           d{V }|d         dk    rd|v r|d         }d|v rt          |d                   dk    rn}|                    d          }|st
                              d           nK|                     ||           d{V }|                    |                    dg                      |}d|v ||d<   t          |                    d          |                    d	          |                    d          |                    d
          |                    d          |                    d                    }d|v r|d         |d<   d|v r|d         |d<   dd|v rdndi|S )a2  
        Check the status of an asynchronous batch scrape job.

        Args:
            id (str): The ID of the batch scrape job

        Returns:
            V1BatchScrapeStatusResponse containing:
            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of URLs scraped
            * total - Total URLs to scrape
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of scraped documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        r  Nr   r]   r   r   r   r  r   r   r   r  r   r   FT)	r  r   rW  r  r  r   r   r  r   r>  s	            r#   r  z-AsyncV1FirecrawlApp.check_batch_scrape_status  s     2 ''))+r++ 33|'X''
 
 
 
 
 
 
 

 x K//$$"6*++;v.//144*v66H# 'HIII&*&=&=h&P&P P P P P P PIKK	fb 9 9:::"+K ++ '+F#.??8,,//'**!ook22#66!ook22((
 
 
 k!! +G 4HW[  *62HV ; 6 6uuD

 	
r2   c                 |   K   |                                  }|                     | j         d| d|           d{V S )a]  
        Get information about errors from an asynchronous batch scrape job.

        Args:
          id (str): The ID of the batch scrape job

        Returns:
          V1CrawlErrorsResponse containing:
            errors (List[Dict[str, str]]): List of errors with fields:
              * id (str): Error ID
              * timestamp (str): When the error occurred
              * url (str): URL that caused the error
              * error (str): Error message
          * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
          Exception: If error check fails
        r  r  Nr  r   rW  r`  r   r\   s      r#   r  z-AsyncV1FirecrawlApp.check_batch_scrape_errors  sh      & '')),,|99b999
 
 
 
 
 
 
 
 	
r2   c                 |   K   |                                  }|                     | j         d| d|           d{V S )aa  
        Get information about errors from an asynchronous crawl job.

        Args:
            id (str): The ID of the crawl job

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
                - id (str): Error ID
                - timestamp (str): When the error occurred
                - url (str): URL that caused the error
                - error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If error check fails
        r  r  NrD  rE  s      r#   r  z&AsyncV1FirecrawlApp.check_crawl_errors  sh      & '')),,|22r222
 
 
 
 
 
 
 
 	
r2   c                   K   |                                  }t          j                    4 d{V }|                    | j         d| |          4 d{V }|                                 d{V cddd          d{V  cddd          d{V  S # 1 d{V swxY w Y   	 ddd          d{V  dS # 1 d{V swxY w Y   dS )r  Nr  rC  )r  r  r  rF  rW  rB   )r`  r   r\   r  r  s        r#   r  z AsyncV1FirecrawlApp.cancel_crawl  s      ''))(** 	- 	- 	- 	- 	- 	- 	-g~~&E&E&E&Ew~WW - - - - - - -[c%]]__,,,,,,- - - - - - - - - - - - -	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-- - - - - - - - - - - - - - -	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-s5   (B<B2B<
B!	!B<$B!	%B<<
C	Cr  c                    K   |                                  }	 |                     | j         d| |           d{V S # t          $ r!}t	          t          |                    d}~ww xY w)a>  
        Check the status of an asynchronous extraction job.

        Args:
            job_id (str): The ID of the extraction job

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any
            * warning (Optional[str]): Warning message if any
            * sources (Optional[List[str]]): Source URLs if requested

        Raises:
            ValueError: If status check fails
        r  Nr  r   rW  r   r^  r0   )r`  r  r\   r  s       r#   r   z&AsyncV1FirecrawlApp.get_extract_status  s      $ ''))	%00<55V55          	% 	% 	%SVV$$$	%   %> 
A)A$$A)c          	        K   |                                  }	|s|st          d          |s|st          d          |r|                     |          }t          |pg ||||dt                     }
|r||
d<   |r||
d<   |r||
d<   	 |                     | j         d|
|	           d	{V S # t          $ r!}t          t          |                    d	}~ww xY w)
a  
        Initiate an asynchronous extraction job without waiting for completion.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If job initiation fails
        r  r  rq  r  r*   r   r   r  N)	r  r^  r  r  r  r  rW  r   r0   )r`  r  r*   re   rP  rQ  rR  rS  r   r\   r  r  s               r#   r"  z!AsyncV1FirecrawlApp.async_extract,  sc     D '')) 	Df 	DBCCC 	BF 	B@AAA 	6--f55F(3-$***
 
 
  	,%+L" 	9+8L( 	*$)L!	%11<,,,        
  	% 	% 	%SVV$$$	%s   	$B. .
C8CCr$  r%  r&  r$  r%  r&  c                  K   i }|||d<   |||d<   |||d<   |                      ||||           d{V }|                    d          rd|vr|S |d         }	 |                     |           d{V }|d	         d
k    r|S |d	         dk    r%t          d|                    d                     |d	         dk    rnt	          j        d           d{V  t          dddd          S )a  
        Generate LLMs.txt for a given URL and monitor until completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            V1GenerateLLMsTextStatusResponse containing:
            * success (bool): Whether generation completed successfully
            * status (str): Status of generation (processing/completed/failed)
            * data (Dict[str, str], optional): Generated text with fields:
                - llmstxt (str): Generated LLMs.txt content
                - llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            Exception: If generation fails
        Nr4  r5  r)  rL  r   r   Tr   r]   r^   z#LLMs.txt generation failed. Error: r   r  r  Fr+  r  r*  )r,  r  r-  r   r  r  rM  )	r`  rJ   r$  r%  r&  r.  r  r  r   s	            r#   r/  z&AsyncV1FirecrawlApp.generate_llms_textr  s     :  (F9%%3F>"*.AF*+66) 3	 7 
 
 
 
 
 
 
 
 ||I&& 	$h*>*>O$
	#??GGGGGGGGFh;..!X-- [fjjQXFYFY [ [\\\!\11-"""""""""
	# 0Ev  @H  TV  W  W  W  	Wr2   r#  r6  c                z  K   i }|||d<   |||d<   |||d<   t          ||||          }|                                 }d|i|                    dd          }d	t           |d
<   	 |                     | j         d||           d{V S # t          $ r!}	t          t          |	                    d}	~	ww xY w)a@  
        Initiate an asynchronous LLMs.txt generation job without waiting for completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            V1GenerateLLMsTextResponse containing:
            * success (bool): Whether job started successfully
            * id (str): Unique identifier for the job
            * error (str, optional): Error message if start failed

        Raises:
            ValueError: If job initiation fails
        Nr4  r5  r)  r(  rJ   Trs  rq  r  r1  )	r2  r  rz  r  r  rW  r   r^  r0   )
r`  rJ   r$  r%  r6  r&  r.  r\   r2  r  s
             r#   r,  z,AsyncV1FirecrawlApp.async_generate_llms_text  s&     6  (F9%%3F>"*.AF*+)'"5	
 
 
 ''))CQ6;;4;#P#PQ	5G55	(	%11<,,,        
  	% 	% 	%SVV$$$	%s   *$B 
B:B55B:c                    K   |                                  }	 |                     | j         d| |           d{V S # t          $ r!}t	          t          |                    d}~ww xY w)a  
        Check the status of an asynchronous LLMs.txt generation job.

        Args:
            id (str): The ID of the generation job

        Returns:
            V1GenerateLLMsTextStatusResponse containing:
            * success (bool): Whether generation completed successfully
            * status (str): Status of generation (processing/completed/failed)
            * data (Dict[str, str], optional): Generated text with fields:
                - llmstxt (str): Generated LLMs.txt content
                - llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            ValueError: If status check fails
        r6  NrI  r`  r   r\   r  s       r#   r-  z3AsyncV1FirecrawlApp.check_generate_llms_text_status  s      ( ''))	%00<11R11          	% 	% 	%SVV$$$	%rJ  )r  r\  r$  r]  rP  /_AsyncV1FirecrawlApp__experimental_stream_stepsr_  r`  r  r\  r]  rQ  r_  r`  c                  K   i }
|||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   t          di |
}
|                     ||||||           d{V }|                    d	          rd
|vr|S |d
         }d}d}	 |                     |           d{V }|r9d|v r5|d         |d         }|D ]} ||           t	          |d                   }|	r9d|v r5|d         |d         }|D ]} |	|           t	          |d                   }|d         dk    r|S |d         dk    r%t          d|                    d                     |d         dk    rnt          j        d           d{V  t          dd          S rb  )	r9  rd  r  re  r  r   r  r  rB  )r`  r  r  r\  r$  r]  rP  rQ  r_  r`  rf  r  r  rg  rh  r   ri  rj  rk  rl  s                       r#   rm  z!AsyncV1FirecrawlApp.deep_research  s     P  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AAAA11!+' 2 
 
 
 
 
 
 
 
 ||I&& 	$h*>*>O$	#::6BBBBBBBBF @|v55!'!56I6J6J!K . * *HK))))&)&*>&?&?# ;Y&00$Y/0A0B0BC) & &FIf%%%%$'y(9$:$:!h;..!X-- U

7@S@S U UVVV!\11-"""""""""-	#0 ,EAlmmmmr2   )r  r\  r$  r]  rP  rQ  c                  K   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   t          di |}|                                 }	d|i|                    d	d	
          }
dt           |
d<   	 |                     | j         d|
|	           d{V S # t          $ r!}t          t          |                    d}~ww xY w)ro  Nr   r;  r4  r=  r   rc  r  Trs  rq  r  rp  r1   )	r9  r  rz  r  r  rW  r   r^  r0   )r`  r  r  r\  r$  r]  rP  rQ  rf  r\   r2  r  s               r#   rd  z'AsyncV1FirecrawlApp.async_deep_researchc  sL     >  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AAAA''))e^';';TX\';']']^	5G55	(	%11<222        
  	% 	% 	%SVV$$$	%s   8$B 
C'CCc                    K   |                                  }	 |                     | j         d| |           d{V S # t          $ r!}t	          t          |                    d}~ww xY w)rt  ru  NrI  rP  s       r#   re  z.AsyncV1FirecrawlApp.check_deep_research_status  s      2 ''))	%00<77277          	% 	% 	%SVV$$$	%rJ  )	r   r  r  r  rX   ru   rt   r  r.  r  r  r  rX   c       	         >  K   i }|
rUt          |
t                    r|                    |
           n*|                    |
                    dd                     |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |||d
<   |	|	                    dd          |d<   |                    |           t          dd|i|}|                    dd          }dt           |d<   |                     | j         d|dd| j         i           d{V S )a  
        Asynchronously search for content using Firecrawl.

        Args:
            query (str): Search query string
            limit (Optional[int]): Max results (default: 5)
            tbs (Optional[str]): Time filter (e.g. "qdr:d")
            filter (Optional[str]): Custom result filter
            lang (Optional[str]): Language code (default: "en")
            country (Optional[str]): Country code (default: "us") 
            location (Optional[str]): Geo-targeting
            timeout (Optional[int]): Request timeout in milliseconds
            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
            params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
            **kwargs: Additional keyword arguments for future compatibility

        Returns:
            SearchResponse: Response containing:
                * success (bool): Whether request succeeded
                * data (List[FirecrawlDocument]): Search results
                * warning (Optional[str]): Warning message if any
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If search fails or response cannot be parsed
        Trs  Nr   r  r  r  rX   ru   rt   r   r  rq  r  r  r  r  r1   )ry  rz  r  r
  r  r  rW  rV  )r`  r  r   r  r  r  rX   ru   rt   r  r.  r  r  r  r  s                  r#   r   zAsyncV1FirecrawlApp.search  s     R  	T&$'' T$$V,,,,$$V[[$T[%R%RSSS %*M'"?#&M% &,M(#$(M&!'.M)$(0M*%'.M)$%-;-@-@$]a-@-b-bM/* 	V$$$ &CCEC]CC"''D'II 7g 7 7H--|'''6667
 
 
 
 
 
 
 
 	
r2   )r\  )Nr:  r;  r  r  )r  r  )Rr+   r,   r-   r.   r0   ra  r   r   r   r   r   r  r  r   r  ClientResponser  r#  r  r  r"  r  r   r  r   r   r  r	   r   rW   r   r   r   r   r   r   r   r   r   r   r   r   rp  r'   r   r  r   r  ri   r[   r   r  r   r  r  r8  r   r   r  r  rO   r  r   r  r  r  r   r"  rM  r/  rG  r,  r-  r   rB  rm  rd  re  r
  r  r   __classcell__r  s   @r#   r  r    s}        
; ; ;c ; ; ; ; ; ; .2$'+4 +4+4 +4 #s(^	+4
 4S>*+4 +4 "+4 -1cN+4 +4 +4 +4^ 7:^ ^^"&sCx.^;?S>^^.3^>B38n^ ^ ^ ^2 7:] ]]%)#s(^]].3]>B38n] ] ] ],+G,B +C +TX + + + +6Z# Zs Z[^ Zor Zwz Z Z Z Z-(= - - - -,'; , , , , /3-1	*9 *9*9 ]+*9 &c]	*9 7L	*9 *9 *9 *9^ 04-1	(@ (@s)(@ ^,(@ &c]	(@ 7L	(@ (@ (@ (@\ mq04040404&*%*37%)4837(,OS(,.237 EI)DM DM DMDM d7  ,g  $h  i  j	DM
 d38n-DM #49-DM #49-DM  (~DM smDM c]DM /0DM TNDM $,D>DM #+4.DM  ~DM  G$JKL!DM"  ~#DM$ l+%DM& #<0'DM( d57I=Zgiv  yG  IW  Yr  t  *  $@  A  B)DM* *#.+DM DM DM DMT W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.'()--HC HC HC3iHC $w  (Q   R  S  T	HC
 $sCx.)HC tCy)HC tCy)HC $D>HC 3-HC #HC +,HC HC  (~HC 'tnHC D>HC   FGH!HC" ,'#HC$ |,%HC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'HC( ')HC*  }+HC, "#-HC0 
%1HC HC HC HC^ W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*..2)--JI JI JI3iJI $w  (Q   R  S  T	JI
 $sCx.)JI tCy)JI tCy)JI $D>JI 3-JI #JI +,JI JI  (~JI 'tnJI D>JI   FGH!JI" ,'#JI$ |,%JI& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JI( ')JI* &d^+JI, "#-JI0 
1JI JI JI JI` .2-1#'-1#/3.2/3)-489=3726,0#+/'()-+uB uB uBuB  S	*	uB
  S	*uB C=uB &c]uB }uB 'tnuB &d^uB 'tnuB !uB !1uB %_ 456uB #+4.uB  "*$!uB" $D>#uB$ }%uB& #4.'uB(  })uB* "#+uB. 
/uB uB uB uBx .2-1#'-1#/3.2/3)-489=3726,0#+/'()-+sB sB sBsB  S	*	sB
  S	*sB C=sB &c]sB }sB 'tnsB &d^sB 'tnsB !sB !1sB %_ 456sB #+4.sB  "*$!sB" $D>#sB$ }%sB& #4.'sB(  })sB* "#+sB. 
/sB sB sB sBj@3 @3H @ @ @ @D)_ )_# )_S#X )__b )_  lA )_ )_ )_ )_^ !%)--1'+#!&/3(,KE KE KEKE 	KE
 !KE %TNKE tnKE }KE #KE +,KE %KE 2?KE KE KE KE^ )-dR %)$(+/3805+0.2dR dR dR49%dR SM	dR
 SMdR $C=dR #+4.dR  (~dR #4.dR DcN+dR 8I7MdR dR dR dRLB
# B
:U B
 B
 B
 B
H
# 
:O 
 
 
 
2
3 
3H 
 
 
 
2-S -T#s(^ - - - -(%s %7H7M % % % %: )-D% %)$(+/3805+0.2D% D% D%49%D% SM	D%
 SMD% $C=D% #+4.D%  (~D% #4.D% DcN+D% 8I7MD% D% D% D%T '+-126;W ;W ;W;W sm	;W
 %TN;W "*$;W <\;W ;W ;W ;WB '+-1$(265% 5% 5%5% sm	5%
 %TN5% D>5% "*$5% <V5% 5% 5% 5%n% %@` % % % %B (,(,&*-1+/:>FJDH^n ^n ^n^n  }	^n
 !^n sm^n &c]^n $C=^n *2$^n "(DcN+;T+A"BC^n  $sCx.)94)? @A^n Nj^n ^n ^n ^nH (,(,&*-1+/:>:% :% :%:%  }	:%
 !:% sm:% &c]:% $C=:% *2$:% DHS>:% :% :% :%x %3  %;W  %  %  %  %L $(!%$("&%)&*%*8<FJN
 N
 N
N
 C=	N

 #N
 SMN
 3-N
 c]N
 smN
 c]N
 %_5N
 U4S>>#ABCN
 *N
 N
 N
 N
 N
 N
 N
 N
r2   r  c            
            e Zd ZdZdedef fdZddZddZd	e	ee
f         ddfd
Zdej        deddfdZdededededef
dZ xZS )r/  zQ
    Async version of V1CrawlWatcher that properly handles async operations.
    r   r  c                 L    t                                          ||           d S r  r  )r`  r   r  r  s      r#   ra  zAsyncV1CrawlWatcher.__init__  s#    S!!!!!r2   rX  Nc                    K   t          j        | j        dd| j        j         fg          4 d{V 	 }|                     |           d{V  ddd          d{V  dS # 1 d{V swxY w Y   dS )z[
        Establishes async WebSocket connection and starts listening for messages.
        r  r  )r  Nr  r  s     r#   r  zAsyncV1CrawlWatcher.connect  s%      %K!02NDH<L2N2N OP
 
 
 	* 	* 	* 	* 	* 	* 	* 	* ,,y)))))))))		* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*s   A$$
A.1A.c                 ~   K   |2 3 d{V }t          j        |          }|                     |           d{V  76 dS )z
        Listens for incoming WebSocket messages and handles them asynchronously.

        Args:
            websocket: The WebSocket connection object
        Nr  r  s       r#   r  zAsyncV1CrawlWatcher._listen"  r  r  r  c                   K   |d         dk    r2d| _         |                     d| j         | j        | j        d           dS |d         dk    r9d| _         |                     d| j         | j        |d         | j        d           dS |d         dk    rq|d	         d
         | _         | j                            |d	                             d	g                      | j        D ] }|                     d|| j        d           !dS |d         dk    rF| j                            |d	                    |                     d|d	         | j        d           dS dS )z
        Handles incoming WebSocket messages based on their type asynchronously.

        Args:
            msg (Dict[str, Any]): The message to handle
        r   r  r]   r  r   r^   r  r  r   r   r  r  Nr  r  s      r#   r  z#AsyncV1CrawlWatcher._handle_message-  r	  r2   r  r}  c                 P  K   	 |                                  d{V }|                    dd          }|                    dd          }n!#  t          j        d|j                   xY w| j                            |j        |||           d{V }t          j        |          )z9
        Handle errors from async API responses.
        Nr   rP  rQ  rR  r"  )rB   r  r  r  r   r  r#  r$  s          r#   r  z!AsyncV1CrawlWatcher._handle_errorC  s      	{'}}......J&NN74PQQM&NN96]^^MM	{%&yhphw&y&yzzz 99(/6S`bopppppppp!'***r&  r  rV  rW  c                 6   K   |                      ||||          S r(  r)  r[  s        r#   r#  z,AsyncV1CrawlWatcher._get_async_error_messageS  r*  r2   r
  )r+   r,   r-   r.   r0   r  ra  r  r  r   r   r  r  rV  r  r   r#  rW  rX  s   @r#   r/  r/    s"        "3 "%8 " " " " " "* * * *	, 	, 	, 	,Rc3h RD R R R R,+G,B +C +TX + + + + Z# Zs Z[^ Zor Zwz Z Z Z Z Z Z Z Zr2   r/  )Tr.   loggingr   r  typingr   r   r   r   r   r   r	   r
   r   rB   r   r   r  rF   r  r  r  	getLoggerr   Loggerr/   r$   r  r%   	BaseModelr'   r4   r9   r=   rI   rW   r[   rc   ri   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r
  r  r  r  r  r"  r$  r*  r-  r0  r2  r9  r@  rB  rG  rI  rM  rU  r  r  r/  r1   r2   r#   <module>re     s<      				  X X X X X X X X X X X X X X X X X X X X X X        				        +'+K88 8 8 8
 
 
 +--GCLLJ! ! ! ! !X' ! ! !
( ( ( ( (H. ( ( (    h(   
C C C C C8- C C C: : : : :(,gaj : : : * * * * *x) * * *
U U U U Uh( U U U    h0   $ $ $ $ $h( $ $ $(# # # # #8% # # #" " " " "+ " " "    H&   
    H&   
    H&   
# # # # #X' # # #    X'        2   
" " " " "($ " " "( ( ( ( (X' ( ( (+ + + + +8% + + +. . . . ._ . . .         *1-wqz      , , , , ,H. , , ,	$ 	$ 	$ 	$ 	$("4 	$ 	$ 	$+ + + + +H& + + +(         h(      	$ 	$ 	$ 	$ 	$H. 	$ 	$ 	$    8%       H.   
	0 	0 	0 	0 	0($ 	0 	0 	0         H&      
4 
4 
4 
4 
4h( 
4 
4 
4
& 
& 
& 
& 
&*GAJ 
& 
& 
&
4 
4 
4 
4 
4X' 
4 
4 
4         x)      - - - - -* - - -    H.   - - - - -x) - - -    8-       H$6   1 1 1 1 1h&8 1 1 1    8#5   0 0 0 0 0X%7 0 0 01 1 1 1 1x1 1 1 1	6 	6 	6 	6 	68- 	6 	6 	6         X/          8#5            !3      & & & & &8+= & & &    x'9            x)      + + + + +h( + + +b*  b*  b*  b*  b*  b*  b*  b* JUYR YR YR YR YR YR YR YRvh
 h
 h
 h
 h
. h
 h
 h
T9OZ OZ OZ OZ OZ. OZ OZ OZ OZ OZr2   