U
    \Sh`                     @  s  d Z ddlmZ ddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5 errddlm6Z6m7Z7m8Z8m9Z9 ddl-m:Z: ddl;m<Z< ddl=m>Z> e,?e@ZAeBdZCddiZDdZEdZFdZGG dd de%ZHG dd  d e)ZId!d"d#d$d"d%d&d'd(ZJG d)d* d*eZKG d+d, d,ZLG d-d. d.eZMG d/d0 d0eZNG d1d2 d2eZOd"d3d4d5d6d7ZPd8d"d9d:d;d<ZQG d=d> d>eZRd"d"d?d9d@dAdBZSG dCdD dDeZTd!d"dEdFdGdHZUd!dId%dJdKdLZVd!dMdNdOdPZWdS )Qz$The CheckExternalLinksBuilder class.    )annotationsN)datetimetimezone)parsedate_to_datetime)
HTMLParser)path)PriorityQueueQueue)Thread)TYPE_CHECKING
NamedTuplecast)unquoteurlparseurlsplit
urlunparse)nodes)ConnectionError	HTTPErrorSSLErrorTooManyRedirects)DummyBuilder)__)SphinxPostTransform)
encode_uriloggingrequests)darkgray	darkgreenpurplered	turquoise)get_node_line)AnyCallable	GeneratorIterator)Response)Sphinx)Configz([a-z]+:)?//Acceptz/text/html,application/xhtml+xml;q=0.9,*/*;q=0.8   g      N@c                   @  sp   e Zd ZdZdZedZddddZdddd	Zd
ddddZ	dddddZ
dddddddddZdS )CheckExternalLinksBuilderz+
    Checks for broken external links.
    	linkcheckzCLook for any errors in the above output or in %(outdir)s/output.txtNonereturnc                 C  s   d| _ i | _td d S )Nr   g      @)broken_hyperlinks
hyperlinkssocketsetdefaulttimeout)self r6   L/root/rtd-docs/venv/lib/python3.8/site-packages/sphinx/builders/linkcheck.pyinit8   s    zCheckExternalLinksBuilder.initc              
   C  s   t | j}td t| jd}t| jd}t|ddd@| _t|ddd$| _	|
| jD ]}| | q`W 5 Q R X W 5 Q R X | jrd| j_d S )N z
output.txtzoutput.jsonwzutf-8)encodingr+   )HyperlinkAvailabilityCheckerconfigloggerinfor   joinZoutdiropentxt_outfilejson_outfilecheckr2   process_resultr1   appZ
statuscode)r5   checkerZoutput_textZoutput_jsonresultr6   r6   r7   finish>   s    

 z CheckExternalLinksBuilder.finishCheckResult)rH   r0   c                 C  s  | j |jd}||j|j|j|j|jd}| | |jdkrDd S |jdkr\|jdkr\d S |jrxt	j
d|j|jdd |jd	kr|jrt	
td
|j d |j  nt	
td
|j  n |jdkrt	
td|j  | d|j||j|j n|jdkr$t	
td|j |j  n|jdkr| jjsD| jjrht	jtd|j|j|j|jfd n"t	
td|j td|j   | d|j||j|jd |j  |  jd7  _n|jdkrz2dtfdtfdtfdtfdtfd|j \}}W n  tk
r   dt }}Y nX ||d< | jjr`t	jd|j d | d |j |j|jfd n*t	
|d|j |d| d |j   | d| |j||j|jd |j  ntd |j d S )!NF)filenamelinenostatuscodeurir?   	uncheckedworkingoldz(%16s: line %4d) T)Znonlignoredz
-ignored- z: localz
-local-   z
ok        brokenzbroken link: %s (%s))locationz
broken    z - r+   
redirectedZpermanentlyz
with Foundzwith See OtherZtemporarily)i-  i.  i/  i3  i4  zwith unknown codetextz
redirect  z to zredirected zUnknown status %s.)envdoc2pathdocnamerL   rM   rN   rO   messagewrite_linkstatr>   r?   r   write_entryr   rF   quietZwarningiserrorwarningr   r    r1   r   r!   KeyErrorr=   linkcheck_allowed_redirects
ValueError)r5   rH   rK   ZlinkstatrX   colorr6   r6   r7   rE   L   sx      


 

"

 z(CheckExternalLinksBuilder.process_resultdict)datar0   c                 C  s"   | j t| | j d d S )N
)rC   writejsondumps)r5   rf   r6   r6   r7   r]      s    z(CheckExternalLinksBuilder.write_linkstatstrint)whatr[   rK   linerO   r0   c              
   C  s(   | j | d| d| d| d d S )N:z: [z] rg   )rB   rh   )r5   rm   r[   rK   rn   rO   r6   r6   r7   r^      s    z%CheckExternalLinksBuilder.write_entryN)__name__
__module____qualname____doc__namer   epilogr8   rI   rE   r]   r^   r6   r6   r6   r7   r,   0   s   8r,   c                   @  s$   e Zd ZdZdZdddddZdS )	HyperlinkCollector)r-      r#   r.   )kwargsr0   c           	      K  s   t t| jj}|j}| jj}| jt	j
D ]&}d|kr*|d }t| j|||| q*| jt	jD ]0}|d d}|r`d|kr`t| j|||| q`| jt	jD ],}|d}|rd|krt| j|||| qd S )NZrefuri
candidates?://source)r   r,   rF   builderr2   rY   r[   Zdocumentfindallr   	reference_add_uriimagegetraw)	r5   rx   r}   r2   r[   ZrefnoderO   ZimgnodeZrawnoder6   r6   r7   run   s    
zHyperlinkCollector.runN)rp   rq   rr   ZbuildersZdefault_priorityr   r6   r6   r6   r7   rv      s   rv   r(   rk   znodes.Elementdict[str, Hyperlink]r.   )rF   rO   noder2   r[   r0   c                 C  s`   |  d| }r|}zt|}W n tk
r8   d}Y nX ||kr\t||| j||||< d S )Nlinkcheck-process-uri)Zemit_firstresultr"   rc   	HyperlinkrY   rZ   )rF   rO   r   r2   r[   ZnewurirL   r6   r6   r7   r      s    
r   c                   @  s.   e Zd ZU ded< ded< ded< ded< dS )r   rk   rO   r[   docpathrl   rL   Nrp   rq   rr   __annotations__r6   r6   r6   r7   r      s   
r   c                   @  sX   e Zd ZdddddZdddd	d
ZddddZddddZdddddZdS )r<   r)   r.   )r=   r0   c                 C  sD   || _ i | _t | _g | _t | _|j| _t	t
tj| j j| _d S N)r=   rate_limitsr	   rqueueworkersr   wqueuelinkcheck_workersnum_workerslistmaprecompilelinkcheck_ignore	to_ignore)r5   r=   r6   r6   r7   __init__   s    z%HyperlinkAvailabilityChecker.__init__r   z"Generator[CheckResult, None, None])r2   r0   c                 c  s   |    d}| D ]H}| |jr@t|j|j|jdddV  q| jt	t
|d |d7 }qd}||k r| j V  |d7 }qb|   d S )Nr   rS   r9   Fr+   )invoke_threadsvaluesis_ignored_urirO   rJ   r[   rL   r   putCheckRequestCHECK_IMMEDIATELYr   r   shutdown_threads)r5   r2   Ztotal_links	hyperlinkdoner6   r6   r7   rD      s       

z"HyperlinkAvailabilityChecker.checkr/   c                 C  s>   t | jD ].}t| j| j| j| j}|  | j	| q
d S r   )
ranger    HyperlinkAvailabilityCheckWorkerr=   r   r   r   startr   append)r5   Z_ithreadr6   r6   r7   r      s     z+HyperlinkAvailabilityChecker.invoke_threadsc                 C  s.   | j   | jD ]}| j ttd d qd S NF)r   r@   r   r   r   r   )r5   Z_workerr6   r6   r7   r      s    

z-HyperlinkAvailabilityChecker.shutdown_threadsrk   bool)rO   r0   c                   s   t  fdd| jD S )Nc                 3  s   | ]}|  V  qd S r   match).0patrO   r6   r7   	<genexpr>   s     z>HyperlinkAvailabilityChecker.is_ignored_uri.<locals>.<genexpr>)anyr   )r5   rO   r6   r   r7   r      s    z+HyperlinkAvailabilityChecker.is_ignored_uriN)rp   rq   rr   r   rD   r   r   r   r6   r6   r6   r7   r<      s
   r<   c                   @  s   e Zd ZU ded< ded< dS )r   float
next_checkzHyperlink | Noner   Nr   r6   r6   r6   r7   r      s   
r   c                   @  s>   e Zd ZU ded< ded< ded< ded< ded< ded< d	S )
rJ   rk   rO   r[   rl   rL   rM   r\   rN   Nr   r6   r6   r6   r7   rJ      s   
rJ   c                      s   e Zd ZdZdddddd fdd	Zdd
ddZdddddddZddddddZddddddZddddddZ	  Z
S )r   z;A worker class for checking the availability of hyperlinks.r)   zQueue[CheckResult]zQueue[CheckRequest]zdict[str, RateLimit]r.   )r=   r   r   r   r0   c                   s   || _ || _|| _tttj|j| _tttj|j	| _
tttj|j| _dd |jD | _|j| _|j| _|j| _|  |j| _|j| _|j| _|j| _|j| _|j| _t | _ t! j"dd d S )Nc                 S  s   g | ]\}}t ||fqS r6   )r   r   )r   pattern	auth_infor6   r6   r7   
<listcomp>  s     z=HyperlinkAvailabilityCheckWorker.__init__.<locals>.<listcomp>T)daemon)#r   r   r   r   r   r   r   linkcheck_anchors_ignoreanchors_ignore linkcheck_anchors_ignore_for_urlanchors_ignore_for_urllinkcheck_exclude_documentsdocuments_excludelinkcheck_authauthlinkcheck_timeouttimeoutlinkcheck_request_headersrequest_headerslinkcheck_anchorscheck_anchorsrb   allowed_redirectslinkcheck_retriesretrieslinkcheck_rate_limit_timeoutrate_limit_timeout
user_agent
tls_verifytls_cacertsr   Z_Session_sessionsuperr   )r5   r=   r   r   r   	__class__r6   r7   r      s6    
z)HyperlinkAvailabilityCheckWorker.__init__r/   c              	   C  s   | j  \}}|d kr"| j  q|\}}}}|d kr8qt|j}z| j| j}W n tk
rf   Y nX |t		 krt	
t | j t||d | j   q | |||\}}	}
|dkrttd| td  n| jt|||||	|
 | j   q d S )NFrate-limitedz-rate limited-   z | sleeping...)r   r   r   closer   netlocr   r   ra   timesleepQUEUE_POLL_SECSr   r   	task_done_checkr>   r?   r   r   rJ   )r5   r   r   rO   r[   Z_docpathrL   r   rM   r?   rN   r6   r6   r7   r      s,    



z$HyperlinkAvailabilityCheckWorker.runrk   r   ztuple[str, str, int])r[   rO   r   r0   c           
      C  s   | j D ].}||r| d|j d}d|df  S qt|dksL|drPdS |dst|rhdS t|j}t	t
||rdS d	S d
\}}}t| jD ]"}	| ||\}}}|dkr qq|||fS )Nz	 matched z! from linkcheck_exclude_documentsrS   r   )#zmailto:ztel:)rP   r9   r   )zhttp:zhttps:rQ   r9   r   )rU   r9   r   )r9   r9   r   rU   )r   r   r   len
startswithuri_rer   dirnamer   existsr@   r   r   
_check_uri)
r5   r[   rO   r   Zdoc_matcherr?   src_dirrM   rN   _r6   r6   r7   r   C  s(    




z'HyperlinkAvailabilityCheckWorker._checkr   zIterator[tuple[Callable, dict]])r   anchorr0   c                 c  s0   |r|s| j jddifV  | j jddifV  d S )Nallow_redirectsTstream)r   headr   )r5   r   r   r6   r6   r7   _retrieval_methodsc  s    z3HyperlinkAvailabilityCheckWorker._retrieval_methods)rO   r   r0   c                 C  sV  | d\}}}|rT|rT| jD ]}||rd} qTq| jD ]}||r<d} qTq<z|d W n tk
r~   t|}Y nX | jD ]\}}||r qqd }t|| j	}	d}
d}d }}| 
| j|D ]\}}z|f |||	| jd|| j| j| jfd<}| jr<|jr<|r<t||s<ttd|dW 5 Q R X |j}|jr`|jd jnd }|jd	}|j }|  ~W  qW q tk
r } zd
t|df W Y   S d }~X Y q ttfk
r } zt|}
W Y qW 5 d }~X Y q tk
r } zt|}
|dkr,W Y z dS |dkr~| || }rh| j !t"||d W Y > dS d
|
df W Y (  S |dkrW Y  dS W Y qW 5 d }~X Y q tk
r } zd
t|df W Y   S d }~X Y qX qd
|
dfS t#|j$}| j%&|d  |'d|'dks0t(||| j)r4dS |d k	rHd||fS d|dfS d S )Nr   r9   asciir   )urlr   headersr   )Z_user_agentZ	_tls_infozAnchor z
 not foundzRetry-AfterrU   r   i  )rQ   unauthorizedr   i  F)r   r9   r   i  )rS   zservice unavailabler   /r   rW   )*	partitionr   r   r   encodeUnicodeErrorr   r   _get_request_headersr   r   r   r   r   r   r   okcontains_anchor	Exceptionr   status_codehistoryr   r   r   raise_for_statusr   rk   r   r   r   
limit_rater   r   r   r   r   r   poprstrip_allowed_redirectr   )r5   rO   r   Zreq_url	delimiterr   rexr   r   r   error_messager   response_urlretry_afterZretrieval_methodrx   responseZredirect_status_codeerrr   r   r6   r6   r7   r   j  s    




 

&


*



z+HyperlinkAvailabilityCheckWorker._check_urizfloat | None)r   r   r0   c           
      C  s  t }d }|r~zt|}W nX tk
rp   zt|}W n ttfk
rL   Y n X t|}|ttj	 
 }Y nX t | }t|j}|d kr| j}z| j| }W n tk
r   t }Y n,X |j}	d|	 }||  kr|	krn n|}||krd S t | }t||| j|< |S )Ng       @)DEFAULT_DELAYr   rc   r   	TypeErrorr   	timestampnowr   utctotal_secondsr   r   r   r   r   ra   delay	RateLimit)
r5   r   r   r	  r   Zuntilr   Z	max_delayZ
rate_limitZlast_wait_timer6   r6   r7   r     s:    



z+HyperlinkAvailabilityCheckWorker.limit_rate)rp   rq   rr   rs   r   r   r   r   r   r   __classcell__r6   r6   r   r7   r      s   !# rr   zdict[str, dict[str, str]]zdict[str, str])rO   r   r0   c                 C  sX   t | }|j d|j |j d|j d| df}|D ]}||kr6t||   S q6i S )Nr{   r   *)r   schemer   DEFAULT_REQUEST_HEADERS)rO   r   r   ry   ur6   r6   r7   r     s    r   r'   r   )r  r   r0   c                 C  sT   t t|}| jdddD ]*}t|tr0| }|| |jr qFq|  |jS )z<Determine if an anchor is contained within an HTTP response.i   T)
chunk_sizedecode_unicode)	AnchorCheckParserr   iter_content
isinstancebytesdecodefeedfoundr   )r  r   parserchunkr6   r6   r7   r     s    

r   c                      s:   e Zd ZdZddd fddZddddd	d
Z  ZS )r  z9Specialised HTML parser that looks for a specific anchor.rk   r.   )search_anchorr0   c                   s   t    || _d| _d S r   )r   r   r  r  )r5   r  r   r6   r7   r   '  s    
zAnchorCheckParser.__init__r#   )tagattrsr0   c                 C  s.   |D ]$\}}|dkr|| j krd| _ q*qd S )N)idrt   T)r  r  )r5   r  r  keyvaluer6   r6   r7   handle_starttag-  s    z!AnchorCheckParser.handle_starttag)rp   rq   rr   rs   r   r!  r  r6   r6   r   r7   r  $  s   r  z&dict[re.Pattern[str], re.Pattern[str]])r   new_urlr   r0   c                   s   t  fdd| D S )Nc                 3  s&   | ]\}}| o|  V  qd S r   r   )r   Zfrom_urlZto_urlr"  r   r6   r7   r   6  s   z$_allowed_redirect.<locals>.<genexpr>)r   items)r   r"  r   r6   r#  r7   r   4  s    r   c                   @  s   e Zd ZU ded< ded< dS )r
  r   r	  r   Nr   r6   r6   r6   r7   r
  =  s   
r
  z
str | None)rF   rO   r0   c                 C  sH   t |}|jdkrD|jrD|jd}|sDd|j }t|j|dS dS )zRewrite anchor name of the hyperlink to github.com

    The hyperlink anchors in github.com are dynamically generated.  This rewrites
    them before checking and makes them comparable.
    z
github.comzuser-content-)fragmentN)r   hostnamer%  r   r   _replace)rF   rO   parsedprefixedr%  r6   r6   r7   rewrite_github_anchorB  s    r*  r)   )rF   r=   r0   c                 C  s   t | jj D ]v\}}z\zt|| jjt|< W n: tjk
rp } zt	t
d|j|j W 5 d}~X Y nX W 5 | jj| X qdS )zFCompile patterns in linkcheck_allowed_redirects to the regexp objects.z=Failed to compile regex in linkcheck_allowed_redirects: %r %sN)r   r=   rb   r$  r   r   r   errorr>   r`   r   r   msg)rF   r=   r   r   excr6   r6   r7   #compile_linkcheck_allowed_redirectsQ  s    
 r.  zdict[str, Any])rF   r0   c                 C  s   |  t | t | dg d | dg d | di d | dg d | di d | ddd | d	d dttg | d
dd | ddd | ddgd | dddttf | ddd | 	d | j
dtdd ddddS )Nr   Fr   rb   r   r   r   r+   r   r      r   Tr   z^!r   r6   r   g     r@r   zconfig-initedrw   )prioritybuiltin)versionZparallel_read_safeZparallel_write_safe)Zadd_builderr,   Zadd_post_transformrv   Zadd_config_valuerl   r   tupler   Z	add_eventconnectr.  )rF   r6   r6   r7   setup^  s(    


r5  )Xrs   
__future__r   ri   r   r3   r   r   r   email.utilsr   html.parserr   osr   queuer   r	   	threadingr
   typingr   r   r   urllib.parser   r   r   r   Zdocutilsr   Zrequests.exceptionsr   r   r   r   Zsphinx.builders.dummyr   Zsphinx.localer   Z!sphinx.transforms.post_transformsr   Zsphinx.utilr   r   r   Zsphinx.util.consoler   r   r   r    r!   Zsphinx.util.nodesr"   r#   r$   r%   r&   r'   Zsphinx.applicationr(   Zsphinx.configr)   	getLoggerrp   r>   r   r   r  r   r   r  r,   rv   r   r   r<   r   rJ   r   r   r   r  r   r
  r*  r.  r5  r6   r6   r6   r7   <module>   sh   

 ]0	  		