
    i(                         d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
 ddlZddlmZmZmZ ddlmZmZmZ ddlmZ e	d         Z G d	 d
          ZdS )z
Async WebSocket watcher with async iterator interface for v2 jobs.

Usage:
    async for snapshot in AsyncWatcher(client, job_id, kind="crawl"):
        print(snapshot.status)
    N)AsyncIteratorDictListLiteralOptional)ConnectionClosedConnectionClosedOKConnectionClosedError   )BatchScrapeJobCrawlJobDocument)normalize_document_input)crawlbatchc                       e Zd Zdddddedededed	ee         d
dfdZd
e	e         fdZ
d
efdZd
e	e         fdZd ZdefdZd Zdddededeee                  fdZdS )AsyncWatcherr      N)kindpoll_intervaltimeoutclientjob_idr   r   r   returnc                v   || _         || _        || _        || _        t	          dt          |                    | _        t          |dd           }|-t          |dd           | _        t          |dd           | _	        n,t          |dd           | _        t          |dd           | _	        d| _
        g | _        d S )N        http_clientapi_urlapi_keyscraping)_client_job_id_kind_timeoutmaxfloat_poll_intervalgetattr_api_url_api_key_status_data)selfr   r   r   r   r   r   s          d/home/agentuser/.hermes/hermes-agent/venv/lib/python3.11/site-packages/firecrawl/v2/watcher_async.py__init__zAsyncWatcher.__init__   s     
%(eM.B.B%C%CfmT::"#KDAADM#KDAADMM $FIt<<DM#FIt<<DM&!#


    c                 *    |                                  S N)_iterater-   s    r.   	__aiter__zAsyncWatcher.__aiter__4   s    }}r0   c                     | j         st          d          | j                             dd                              ddd          }| j        dk    r| d| j         S | d	| j         S )
Nz)API URL is required for WebSocket watcherzhttps://zwss://zhttp://zws://r   r   z
/v2/crawl/z/v2/batch/scrape/)r)   
ValueErrorreplacer#   r"   )r-   ws_bases     r.   _build_ws_urlzAsyncWatcher._build_ws_url7   s    } 	JHIII-''
H==EEiQXZ[\\:  77777::DL:::r0   c           
     >  K   |                                  }g }| j        r|                    dd| j         f           	 t          j        |d |          4 d {V }| j        r-t          j                                                    | j        z   nd }	 | 	                                 d {V }|W V  |j
        dv r	 d d d           d {V  d S n# t          $ r Y nw xY w	 	 |Tt          d|t          j                                                    z
            }|dk    rt          | j        |          nd}n| j        }t          j        |                                |           d {V }nn# t          j        $ r |                                  d {V }	|	"|	W V  |	j
        dv rY d d d           d {V  d S |=t          j                                                    |k    rY d d d           d {V  d S Y t&          t(          t*          f$ r t          j                    | j        pd	z   }	 	 | 	                                 d {V }	|	W V  |	j
        dv rY d d d           d {V  d S n## t          $ r Y Y d d d           d {V  d S w xY wt          j                    |k    rY d d d           d {V  d S t          j        d
           d {V  w xY w	 t/          j        |          }
n# t          $ r Y  w xY w|
                    d          }|dk    rYd| _        | j        dk    rt9          ddddd d g           W V  nt;          ddddd d g           W V  	 d d d           d {V  d S |dk    rj|
                    di           }|                    d| j                  | _        |                    dg           pg }| j                            |           n|dk    rE|
                    d          }tA          |tB                    r| j                            |           n|dk    rd| _        |
                    di           pi }|                    dg           pg }tA          |tD                    r6|r4|D ]1}tA          |tB                    r| j                            |           2| #                    d|| j                  W V  	 d d d           d {V  d S |
                    d|
          }|                    d|
                    d| j                            }| #                    ||          }|W V  |dv r	 d d d           d {V  d S # 1 d {V swxY w Y   d S # t          $ r t          j                    | j        pd	z   }	 	 | 	                                 d {V }	|	W V  |	j
        dv rY d S n# t          $ r Y Y d S w xY wt          j                    |k    rY d S t          j        d
           d {V  tw xY w)NAuthorizationzBearer )max_sizeadditional_headers)	completedfailed	cancelledTr   r   )r      r   typeerrorr@   r   statusr?   totalcredits_used
expires_atnextdatacatchuprK   rF   documentdoner?   )rF   payloaddocs_override)rF   rO   )$r:   r*   append
websocketsconnectr$   asyncioget_event_looptime_fetch_job_statusrF   	Exceptionr%   minr'   wait_forrecvTimeoutError_safe_fetchr	   r   r
   sleepjsonloadsgetr+   r#   r   r   r,   extend
isinstancedictlist_make_snapshot)r-   uriheaders_list	websocketdeadlinepre	remainingr   msgjobbodymsg_typeddocs_indocraw_payloadrO   
status_strsnapshots                      r.   r3   zAsyncWatcher._iterate?   sX
       ""= 	N2KDM2K2K LMMMe	'!)#Q]^^^ V V V V V V VbkNRme713388::T]JJae $ 6 6 8 8888888CIIIIz%IIIV V V V V V V V V V V V V V J    DK3#/(+CG<R<T<T<Y<Y<[<[1[(\(\IMVYZ]]c$*=y&I&I&I`cGG&*&9G$+$4Y^^5E5Ew$W$W$WWWWWWW"/ 	! 	! 	!$($4$4$6$6666666?"%IIII"z-QQQ &3V V V V V V V V V V V V V V4 $/G4J4L4L4Q4Q4S4SW_4_4_"7V V V V V V V V V V V V V V8 !.0@BWX 3 3 3#'9;;$-2E2#F
3',0,B,B,D,D&D&D&D&D&D&D&)				#&:1U#U#U$*KV V V V V V V V V V V V V VH $V#, ' ' ' &OV V V V V V V V V V V V V VL'#y{{h66 &SV V V V V V V V V V V V V VT #*-"2"22222222
33!#z#$ ! ! ! !  $xx//H7**'/:00"*(aq_`mqx|  DF  #G  #G  #G  G  G  G  G  G"0AUVefsw  C  JL  #M  #M  #M  M  M  M  MqV V V V V V V V V V V V V Vr "Y.. HHVR00'(uuXt|'D'D"#%%"3"3"9r
))'2222!Z//"hhv..%c400 3 J--c222!V++'2&*hhvr&:&:&@b"-//&""="="C%gt44 ; ;'. ; ;#-c4#8#8 !;$(J$5$5c$:$:$:"11kimis1tttttt]V V V V V V V V V V V V V Vb #hhvt44G!(Xtxx$,7W7W!X!XJ#22*g2VVH"NNNN!%IIImV V V V V V V V V V V V V VKV V V V V V V V V V V V V V V Vn  	' 	' 	'y{{dm&9r:H
' $ 6 6 8 8888888CIIIIz%III J    FFF9;;(**FFmA&&&&&&&&&
'	'sZ   U? 7U,(C>U? U,
CU,CU,$BE1/U,19K*U,+U? >+K)U,*U? =U, 4K5(I2U,U? 1K2
J<K=U,>U? JK,U,-U?  KU, K54U,5
L?U,LA$U,'U? :E$U,U? 1A&U,U? *U,,
U66U? 9U6:U? ?(X((WX
W#X"W##X Xc                    K   | j         dk    r|                     d           d {V S |                     d           d {V S )Nr   get_crawl_statusget_batch_scrape_status)r#   _call_status_methodr4   s    r.   rW   zAsyncWatcher._fetch_job_status   sa      :  112DEEEEEEEEE--.GHHHHHHHHHr0   method_namec                 @  K   t          | j        |d           }|d	  || j                  }n# t          $ r d }Y nw xY w|t	          j        |          r| d {V S |S t          j        || j                   d {V S t          | j        dd           }|wt          ||d           }|d	  || j                  }n# t          $ r d }Y nw xY w|t	          j        |          r| d {V S |S t          j        || j                   d {V S t          d|           )Nv2zClient does not expose )	r(   r!   r"   	TypeErrorinspectisawaitablerT   	to_threadRuntimeError)r-   r{   methresultr}   s        r.   rz   z AsyncWatcher._call_status_method   s     t|[$77dl++   !&v.. (!'<<<<<<' *4>>>>>>>>> T\4..>2{D11D"!T$,//FF  " " "!FFF"%*622 ,%+||||||+!M$.tT\BBBBBBBBBB[BBCCCs   - <<+B< <C
Cc                 ^   K   	 |                                   d {V S # t          $ r Y d S w xY wr2   )rW   rX   r4   s    r.   r]   zAsyncWatcher._safe_fetch   sQ      	//111111111 	 	 	44	s    
,,)rP   rF   rO   rP   c          
         g }||n|                     dg           pg }|D ]E}t          |t                    r.t          |          }|                    t          d
i |           F| j        dk    rxt          ||                     dd          |                     dd          |                     dd          |                     d          |                     d          |	          S t          ||                     dd          |                     dd          |                     d          |                     d          |                     d          |	          S )NrK   r   r?   r   rG   creditsUsed	expiresAtrJ   rE    )	ra   rc   rd   r   rQ   r   r#   r   r   )r-   rF   rO   rP   docssource_docsrs   rq   s           r.   rf   zAsyncWatcher._make_snapshot   s_   '4'@mmgkkRXZ\F]F]Fcac 	+ 	+C#t$$ +,S11HMMqMM***:  !++k155kk'1--$[[::";;{33[[((    kk+q11++gq)) ]33{{;//V$$
 
 
 	
r0   )__name__
__module____qualname__objectstrJobKindintr   r/   r   r5   r:   r3   rW   rz   r]   r   r   rf   r   r0   r.   r   r      sd         !%$ $ $$ $
 $ $ #$ 
$ $ $ $6=0    ;s ; ; ; ;l'f 5 l' l' l' l'\I I I
DS D D D D@   cg 
 
 
 
d 
8TXY]T^K_ 
 
 
 
 
 
r0   r   )__doc__rT   r   r_   rV   typingr   r   r   r   r   rR   websockets.exceptionsr   r	   r
   typesr   r   r   utils.normalizer   r   r   r   r0   r.   <module>r      s         ? ? ? ? ? ? ? ? ? ? ? ? ? ?     ] ] ] ] ] ] ] ] ] ] 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5
"
#Z
 Z
 Z
 Z
 Z
 Z
 Z
 Z
 Z
 Z
r0   