
    i7                         d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZ ddlZddlmZmZmZ ddlmZ ed         Ze	eef         Z G d d	          ZdS )
z
WebSocket-based watcher for v2 jobs (crawl and batch), mirroring v1 behavior.

Usage:
    watcher = client.watcher(job_id, kind="crawl")
    watcher.add_listener(lambda status: print(status.status))
    watcher.start()
    N)CallableListOptionalLiteralUnionDictAny   )CrawlJobBatchScrapeJobDocument)normalize_document_input)crawlbatchc                      e Zd Z	 	 	 ddededededee         d	dfd
Zde	e
gdf         d	dfdZde
d	dfdZdede	eeef         gdf         d	dfdZdedeeef         d	dfdZd	efdZddZd	efdZddZddZddZdS )Watcherr      Nclientjob_idkindpoll_intervaltimeoutreturnc                 `   || _         || _        || _        || _        || _        g | _        d | _        t          j                    | _	        t          |dd           }t          |dd           | _        t          |dd           | _        d| _        g | _        g g g d| _        d| _        d| _        d S )Nhttp_clientapi_urlapi_keyscraping)doneerrordocumentF)_client_job_id_kind_timeout_poll_interval
_listeners_thread	threadingEvent_stopgetattr_api_url_api_keystatusdata_event_handlers
_sent_done_sent_error)selfr   r   r   r   r   r   s          ^/home/agentuser/.hermes/hermes-agent/venv/lib/python3.11/site-packages/firecrawl/v2/watcher.py__init__zWatcher.__init__   s     
+;=37_&&
fmT::'.{It'L'L'.{It'L'L &*,	S
 S

 !&!&    callbackc                 :    | j                             |           d S N)r'   append)r4   r8   s     r5   add_listenerzWatcher.add_listener:   s    x(((((r7   r/   c                 l    t          | j                  D ]}	  ||           # t          $ r Y w xY wd S r:   )listr'   	Exception)r4   r/   cbs      r5   _emitzWatcher._emit=   sY    t'' 	 	B6



   	 	s   $
11
event_typehandlerc                 \    || j         v r"| j         |                             |           d S d S r:   )r1   r;   )r4   rB   rC   s      r5   add_event_listenerzWatcher.add_event_listenerE   s:    --- ,33G<<<<< .-r7   detailc                 t    || j         v r,| j         |         D ] }	  ||           # t          $ r Y w xY wd S d S r:   )r1   r?   )r4   rB   rF   rC   s       r5   dispatch_eventzWatcher.dispatch_eventI   sm    ---/
;  GFOOOO    D	 .- s   &
33c                     | j         st          d          | j                             dd                              ddd          }| j        dk    r| d| j         S | d	| j         S )
Nz)API URL is required for WebSocket watcherzhttps://zwss://zhttp://zws://r
   r   z
/v2/crawl/z/v2/batch/scrape/)r-   
ValueErrorreplacer$   r#   )r4   ws_bases     r5   _build_ws_urlzWatcher._build_ws_urlQ   s    } 	JHIII-''
H==EEiQXZ[\\:  77777::DL:::r7   c                 b  K   |                                  }g }| j        r|                    dd| j         f           	 t          j        |d |          4 d {V }| j        r-t          j                                                    | j        z   nd }| j	        
                                	s|Nt          d|t          j                                                    z
            }t          | j        p||          }n	| j        pd}	 t          j        |                                |           d {V }n# t          j        $ r  |                                  d {V rY 	n	Y t          j        $ r Y nt&          $ r | j	        
                                s4|                                  d {V rZY d d d           d {V  | j        dk    r9| j        s4|                     d| j        | j        | j        d	           d
| _        d S d S d S |t          j                                                    |k    rZY d d d           d {V  | j        dk    r9| j        s4|                     d| j        | j        | j        d	           d
| _        d S d S d S t          j        | j        pd           d {V  | j	        
                                4Y d d d           d {V  | j        dk    r9| j        s4|                     d| j        | j        | j        d	           d
| _        d S d S d S w xY w	 t5          j        |          }n# t&          $ r Y w xY w|                    d          }	|	dk    rd| _        |                     d| j        | j        |                    d          | j        d           d
| _        | j        dk    rt?          ddddd d g           }
ntA          ddddd d g           }
| !                    |
           nd|	dk    r|                    di           }|                    d| j                  | _        |                    dg           }| j        "                    |           |D ] }|                     d|| j        d           !n|	dk    rd|                    d          }tG          |tH                    r8| j                            |           |                     d|| j        d           n|	dk    rd| _        |                    di           pi }|                    dg           pg }tG          |tJ                    r6|r4|D ]1}tG          |tH                    r| j                            |           2|                     d| j        | j        | j        d	           d
| _        g }| j        D ]E}tG          |tH                    r.tM          |          }|                    tO          di |           F| j        dk    ryt?          d|                    dd          |                    dd          |                    dd          |                    d          |                    d          |          }
nxtA          d|                    dd          |                    dd          |                    dd          |                    d          |                    d          |          }
| !                    |
           nN|                    d|          }tG          |tH                    rd|v pd|v }|s|                    d|                    d| j                            }| j        dk    rcg }|                    dg           D ]E}tG          |tH                    r.tM          |          }|                    tO          di |           Ft?          ||                    dd          |                    dd          |                    dd          |                    d          |                    d          |          }
| !                    |
           |dv rr|dk    r2| j        s+|                     d|| j        | j        d	           d
| _        |dk    r2| j        s+|                     d|| j        | j        d	           d
| _        n{n_g }|                    dg           D ]E}tG          |tH                    r.tM          |          }|                    tO          di |           FtA          ||                    dd          |                    dd          |                    d          |                    d          |                    d          |          }
| !                    |
           |dv rq|dk    r2| j        s+|                     d|| j        | j        d	           d
| _        |dk    r2| j        s+|                     d|| j        | j        d	           d
| _        n| j	        
                                		 d d d           d {V  n# 1 d {V swxY w Y   n# t&          $ r Y nw xY w| j        dk    r9| j        s4|                     d| j        | j        | j        d	           d
| _        d S d S d S # | j        dk    r7| j        s0|                     d| j        | j        | j        d	           d
| _        w xY w)NAuthorizationzBearer )max_sizeadditional_headersg           r   	completedr   r/   r0   idTr   typer    failed)r/   r0   r    rV   r   r   )r/   rT   totalcredits_used
expires_atnextr0   catchupr0   r/   r!   )r0   rV   rY   creditsUsed	expiresAtr\   rT   rX   	cancelled )(rM   r.   r;   
websocketsconnectr%   asyncioget_event_looptimer+   is_setmaxminr&   wait_forrecvTimeoutError_poll_status_onceCancelledErrorr?   r/   r2   rH   r0   r#   sleepjsonloadsgetr3   r$   r   r   rA   extend
isinstancedictr>   r   r   )r4   uriheaders_list	websocketdeadline	remainingr   msgbodymsg_typejobddocs_indocraw_payloaddocspayloadhas_status_field
status_strs                      r5   _run_wszWatcher._run_wsY   s       ""= 	N2KDM2K2K LMMMh	'!)#Q]^^^ `" `" `" `" `" `" `"bkNRme713388::T]JJae*++-- ^"+$'X8N8P8P8U8U8W8W-W$X$X	"%d&9&FY	"R"R"&"5":$+$4Y^^5E5Ew$W$W$WWWWWWW"/ % % %!%!7!7!9!9999999 %!E$H"1   $   "&*"3"3"5"5 J%)%;%;%=%======= ' &/`" `" `" `" `" `" `" `" `" `" `" `"J {k))$/)##Ft{DI]a]i,j,jkkk"& *)))[  (38N8P8P8U8U8W8W[c8c8c &3`" `" `" `" `" `" `" `" `" `" `" `"J {k))$/)##Ft{DI]a]i,j,jkkk"& *)))W #*-0C0Hq"I"IIIIIIII #'*"3"3"5"5 J 7`" `" `" `" `" `" `" `" `" `" `" `"J {k))$/)##Ft{DI]a]i,j,jkkk"& *)))e!#z#$ ! ! ! !  $xx//H7**&.++G&*k$(I%)XXg%6%6"&,	6 6    ,0(:00"*(aq_`mqx|  DF  #G  #G  #GCC"0AUVefsw  C  JL  #M  #M  #MC

3!Y.. HHVR00&'eeHdk&B&B"#%%"3"3	((111#* _ _C //
SPTP\<]<]^^^^_!Z//"hhv..%c400 _ I,,S111 //
SPTP\<]<]^^^!V++&1&*hhvr&:&:&@b"-//&""="="C%gt44 : :'. : :#-c4#8#8 !:$(I$4$4S$9$9$9++Ft{TXT]eieq4r4rsss*./1#'9 ; ;C)#t44 ;$<S$A$A $HMMqMM : : ::00"*'2*5//+q*I*I&1oogq&A&A-8__]A-N-N+6??;+G+G%0__V%<%<%)# # #CC #1'2*5//+q*I*I&1oogq&A&A-8__]A-N-N+6??;+G+G%0__V%<%<%)# # #C 

3"hhvt44G(27D(A(A(YhRYFY'p_gko_o$+ ! !(Xtxx$+7V7V!W!WJzW,,!#*;;vr#:#: ; ;C)#t44 ;$<S$A$A $HMMqMM : : :&#-&-kk+q&A&A")++gq"9"9)0]A)F)F'.{{;'?'?!(V!4!4!%   

3%)MMM)[888 $ 3 3Fz[_[dlplx<y<y z z z26)X55d>N5 $ 3 3G
\`\emqmy=z=z { { {37 0! N  "#*;;vr#:#: ; ;C)#t44 ;$<S$A$A $HMMqMM : : :,#-&-kk+q&A&A")++gq"9"9)0])C)C'.{{;'?'?!(V!4!4!%   

3%)MMM)[888 $ 3 3Fz[_[dlplx<y<y z z z26)X55d>N5 $ 3 3G
\`\emqmy=z=z { { {37 0!} *++-- ^" ^"`" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `" `"B  	 	 	D	 {k))$/)##Ft{DI]a]i,j,jkkk"& *)))t{k))$/)##Ft{DI]a]i,j,jkkk"&&&&&s    j B*j .D86j 8)L*!j &L*4j 7>L*5j 6j +L*:j ;j ;L*j j )L**j .Mj 
Mj M\j .j  
j

j j
j k* 
jk* jk* *Al.c                 x  K   	 | j         dk    r+t          j        | j        j        | j                   d{V }n*t          j        | j        j        | j                   d{V }n# t          $ r Y dS w xY w|j        | _        | 	                    |           |j        dv r|j        dk    rA| j
        s:|                     d|j        d |j        D             | j        d           d	| _
        |j        d
k    rA| j        s:|                     d|j        d |j        D             | j        d           d	| _        d	S dS )z9Poll job status over HTTP once. Returns True if terminal.r   NFr`   rT   r   c                 6    g | ]}|                                 S rb   
model_dump.0r   s     r5   
<listcomp>z-Watcher._poll_status_once.<locals>.<listcomp>  s!    KmKmKm_`ALLNNKmKmKmr7   rU   TrX   r    c                 6    g | ]}|                                 S rb   r   r   s     r5   r   z-Watcher._poll_status_once.<locals>.<listcomp>  s!    LnLnLn`aQ\\^^LnLnLnr7   )r$   re   	to_threadr"   get_crawl_statusr#   get_batch_scrape_statusr?   r/   rA   r2   rH   r0   r3   )r4   r   s     r5   rn   zWatcher._poll_status_once	  s     	zW$$&-&78UW[Wc&d&d d d d d d d,3,=dl>bdhdp,q,q&q&q&q&q&q&q 	 	 	55	 j

3:===z[(((##FszKmKmdgdlKmKmKmuy  vB  -C  -C  D  D  D"&zX%%d.>%##G
LnLnehemLnLnLnvz  wC  .D  .D  E  E  E#' 4us   A A% %
A32A3c                 R    t          j        |                                            d S r:   )re   runr   r4   s    r5   _loopzWatcher._loop  s     DLLNN#####r7   c                     | j         r| j                                         rd S | j                                         t	          j        | j        d          | _         | j                                          d S )NT)targetdaemon)r(   is_aliver+   clearr)   Threadr   startr   s    r5   r   zWatcher.start"  sm    < 	DL1133 	F
 'tz$GGGr7   c                     | j                                          | j        r| j                            d           d S d S )Nr
   rS   )r+   setr(   joinr   s    r5   stopzWatcher.stop)  sG    
< 	)La(((((	) 	)r7   )r   r   N)r   N)__name__
__module____qualname__objectstrJobKindintr   r6   r   JobTyper<   rA   r   r	   rE   rH   rM   r   boolrn   r   r   r   rb   r7   r5   r   r      s       
  !%' '' ' 	'
 ' #' 
' ' ' '@)Xwio%> )4 ) ) ) )G     =S =8T#s(^DTVZDZ;[ =`d = = = = d38n     ;s ; ; ; ;n' n' n' n'`    ,$ $ $ $   ) ) ) ) ) )r7   r   )__doc__re   rq   r)   typingr   r   r   r   r   r   r	   rc   typesr   r   r   utils.normalizer   r   r   r   rb   r7   r5   <module>r      s           F F F F F F F F F F F F F F F F F F     5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 "
#
.(
)S) S) S) S) S) S) S) S) S) S)r7   