�
�A�[c@`s�dd
lm
Z
mZmZ
ddlmZ
ddlmZ
ddlm	Z
mZ
e
e_	ee_dg
Z
def
d��YZdef
d	��YZd
ef
d��YZdS(
i(tabsolute_importtdivisiontunicode_literals(
tstr(
turllib(tparsetrequestuRobotFileParsertRobotFileParsercB`sbeZ
dZd
d�
Zd�Zd�Zd�Zd�Zd�Zd�Z	d	�Z
d
�ZRS(us This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    ucC`s>g|_d|_t|_t|_|j|
�

d
|_dS(Ni(tentriestNonet
default_entrytFalsetdisallow_allt	allow_alltset_urltlast_checked(tselfturl((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyt__init__s
	
	
	
	


c


C`s|jS(
u�Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        (
R(
R((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pytmtime&sc
C`sd
dl}
|
j�|_
dS(uYSets the time the robots.txt file was last fetched to the
        current time.

        iN(ttimeR(RR((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pytmodified/s
cC`s2|
|_t
jj|
�
d
d!\|_|_dS(u,Sets the URL referring to a robots.txt file.i
iN(RRRturlparsethosttpath(RR((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR7s	
c
C`s�ytj
j|j�
}
WnOtjjk
rj
}
|jdkrLt|_q�|jdkr�t|_	q�n)X|
j
�}|j|jd�
j
��

dS(u4Reads the robots.txt URL and feeds it to the parser.i�
i�
i�
uutf-8N(i�
i�
(RRturlopenRterrort	HTTPErrortcodetTrueRR
treadRtdecodet
splitlines(Rt
fterrtraw((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR<s






cC`sAd
|
jkr-|j
dkr=|
|_
q=n|jj|
�

dS(Nu
*(t
useragentsR
R	Rtappend(Rtentry((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyt
_add_entryIs
cC`sd
}t�}x�
|
D]�
}|sn|dkr@t�}d
}qn|dkrn|j
|�

t�}d
}qnn|jd�
}|d
kr�|| }n|j�}|s�qn|jdd�}t|�
dkr|d
j�j�|d
<tjj	|dj��
|d<|d
dkr^
|dkrA
|j
|�

t�}n|j
j|d�

d}q�
|d
dkr�
|d
kr�
|jjt
|dt��

d}q�
q�
|d
dkr�
|d
kr�
|jjt
|dt��

d}q�
q�
qqW|dkr|j
|�

nd	S(
u�Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        ii
iu
#u
:u
user-agentudisallowuallowN(tEntryR'tfindtstriptsplittlentlowerRRtunquoteR$R%t	rulelinestRuleLineRR(RtlineststateR&tlinet
i((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyRRsJ

	



	
	



	










 






	









cC`s�|jr
t
S|jrtStjjtjj|�
�
}tjjd
d
|j	|j
|j|jf�
}tjj
|�
}|s�d}nx-|jD]"}|j|
�
r�|j|�
Sq�W|jr�|jj|�
StS(u=using the parsed robots.txt decide if useragent can fetch urluu
/(RRR
RRRRR.t
urlunparseRtparamstquerytfragmenttquoteRt
applies_tot	allowanceR
(Rt	useragentRt
parsed_urlR&((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyt	can_fetch�s 	

	





	


	
c
C`s-d
jg|j
D]}
t|
�
d^q�
S(Nuu

(tjoinRR(RR&((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyt__str__�s
(t__name__t
__module__t__doc__RRRRRR'RR>R@(((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyRs					
			3	R0c
B`s)eZ
dZd
�Zd�Zd�ZRS(uoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.cC`s>|
d
kr|rt}nt
jj|
�
|_||_dS(Nu(RRRR9RR;(RRR;((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR�s
	

cC`s|jd
kp|
j
|j�
S(Nu
*(Rt
startswith(Rtfilename((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR:�s
c

C`s|jrd
pdd|j
S(NuAllowuDisallowu: (R;R(
R((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR@�s
(RARBRCRR:R@(((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR0�s
		R(c
B`s2eZ
dZd
�Zd�Zd�Zd�ZRS(u?An entry has one or more user-agents and zero or more rulelinesc

C`sg|_g|_
dS(
N(R$R/(
R((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR�s
	
c
C`sjg}
x'|jD]}|
j
d
|dg�

qWx*|jD]}|
j
t|�
dg�

q:Wdj|
�
S(NuUser-agent: u

u(R$textendR/RR?(RtrettagentR3((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR@�s





cC`s]|
jd
�
dj
�}
x=|jD]2}|dkr9tS|j
�}||
kr#tSq#WtS(u2check if this entry applies to the specified agentu
/iu
*(R+R-R$RR(RR<RH((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR:�s





cC`s.x'|jD]}|j
|
�
r
|jSq
WtS(
uZPreconditions:
        - our agent applies to this entry
        - filename is URL decoded(R/R:R;R(RRER3((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR;�s


(RARBRCRR@R:R;(((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyR(�s


			
N(t
__future__RR
Rtfuture.builtinsRtfuture.backportsRtfuture.backports.urllibRt_parseRt_requestt__all__tobjectRR0R((((sT/opt/alt/python27/lib/python2.7/site-packages/future/backports/urllib/robotparser.pyt<module>
s


	
		�