-í
ç¶<c       sã     d  Z    d k Z d k Z d k Z  d g Z  d a  d „  Z  d f  d „  ƒ  YZ ’ d f  d „  ƒ  YZ   d f  d	 „  ƒ  YZ	 Æ d
 e i
 f d „  ƒ  YZ Ý d „  Z è d „  Z e d j o e ƒ  n d S(   s@   robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PYTHON 2.0 OPEN SOURCE LICENSE

    The robots.txt Exclusion Protocol is implemented as specified in
    http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
Ns   RobotFileParseri    c    s     t  o  |  GHn d  S(   N(   s   debugs   msg(   s   msg(    (    s!   /usr/lib/python2.2/robotparser.pys   _debug s   
 c      sn    t  Z  d d „ Z  d „  Z ! d „  Z % d „  Z ) d „  Z 8 d „  Z y d „  Z ‹ d	 „  Z	 RS(
   Ns    c    sG     g  |  _  d |  _  d |  _  |  i | ƒ  d |  _ d  S(   Ni    (   s   selfs   entriess   disallow_alls	   allow_alls   set_urls   urls   last_checked(   s   selfs   url(    (    s!   /usr/lib/python2.2/robotparser.pys   __init__ s
   c    s     |  i Sd  S(   N(   s   selfs   last_checked(   s   self(    (    s!   /usr/lib/python2.2/robotparser.pys   mtime s   c    s%   ! " d  k  } # | i  ƒ  |  _ d  S(   N(   s   times   selfs   last_checked(   s   selfs   time(    (    s!   /usr/lib/python2.2/robotparser.pys   modified! s   c    s8   % & | |  _  ' t i | ƒ d d !\ |  _ |  _ d  S(   Ni   i   (   s   urls   selfs   urlparses   hosts   path(   s   selfs   url(    (    s!   /usr/lib/python2.2/robotparser.pys   set_url% s   c    sñ   ) * t  ƒ  } + | i |  i ƒ } , | i ƒ  } - | i |  _ . |  i d j p |  i d j o / d |  _	 0 t
 d ƒ nl 1 |  i d j o 2 d |  _ 3 t
 d ƒ n< 4 |  i d j o | o! 5 t
 d ƒ 6 |  i | ƒ n d  S(	   Ni‘  i“  i   s   disallow alli  s	   allow alliÈ   s   parse lines(   s	   URLopeners   openers   opens   selfs   urls   fs	   readliness   liness   errcodes   disallow_alls   _debugs	   allow_alls   parse(   s   selfs   openers   fs   lines(    (    s!   /usr/lib/python2.2/robotparser.pys   read) s   #c    sI  8 ; < d } = d } > t ƒ  } @ xÝ| D@ ]Ò} A | i ƒ  } B | d } C | oz D | d j o* E t d | ƒ H t ƒ  } I d } n= J | d j o, K |  i	 i
 | ƒ L t ƒ  } M d } n n O | i d ƒ } P | d j o Q | |  } n R | i ƒ  } S | o
 T q. n U | i d d ƒ } V t | ƒ d j o’W | d i ƒ  i ƒ  | d <X | d i ƒ  | d <Y | d d j oh Z | d j o4 [ t d | ƒ ^ |  i	 i
 | ƒ _ t ƒ  } n ` | i i
 | d ƒ a d } nß b | d d	 j oR c | d j o d t d
 | ƒ n* g | i i
 t | d d ƒ ƒ h d } ny i | d d j oI j | d j o k t d
 | ƒ n! n | i i
 t | d d ƒ ƒ n p t d | | d f ƒ n s t d | | f ƒ q. Wt | d j o u |  i	 i
 | ƒ n v t d t |  ƒ ƒ d S(   s   parse the input lines from a robot.txt file.
           We allow that a user-agent: line is not preceded by
           one or more blank lines.i    i   s]   line %d: warning: you should insert allow: or disallow: directives below any user-agent: linei   s   #s   :s
   user-agentsP   line %d: warning: you should insert a blank line before any user-agent directives   disallowsH   line %d: error: you must insert a user-agent: directive before this lines   allows    line %d: warning: unknown key %ss!   line %d: error: malformed line %ss   Parsed rules:
%sN(   s   states
   linenumbers   Entrys   entrys   liness   lines   strips   _debugs   selfs   entriess   appends   finds   is   splits   lens   lowers
   useragentss	   ruleliness   RuleLines   str(   s   selfs   liness   is
   linenumbers   states   entrys   line(    (    s!   /usr/lib/python2.2/robotparser.pys   parse8 s`   		
 	
 $c    s¿   y z { t  d | | f ƒ } |  i o ~ d Sn  |  i o € d Sn ƒ t i t i | ƒ d ƒ p d } „ x; |  i	 D„ ]- } … | i | ƒ o † | i | ƒ Sn q€ Wˆ d Sd S(   s=   using the parsed robots.txt decide if useragent can fetch urls<   Checking robot.txt allowance for:
  user agent: %s
  url: %si    i   i   s   /N(   s   _debugs	   useragents   urls   selfs   disallow_alls	   allow_alls   urllibs   quotes   urlparses   entriess   entrys
   applies_tos	   allowance(   s   selfs	   useragents   urls   entry(    (    s!   /usr/lib/python2.2/robotparser.pys	   can_fetchy s   & 	c    sH   ‹ Œ d }  x+ |  i D ] } Ž | t | ƒ d } q W | Sd  S(   Ns    s   
(   s   rets   selfs   entriess   entrys   str(   s   selfs   entrys   ret(    (    s!   /usr/lib/python2.2/robotparser.pys   __str__‹ s
   	 	(
   s   __name__s
   __module__s   __init__s   mtimes   modifieds   set_urls   reads   parses	   can_fetchs   __str__(    (    (    s!   /usr/lib/python2.2/robotparser.pys   RobotFileParser s   	As   RuleLinec      s8   ’ t  Z d  Z ” • d „  Z ™ d „  Z œ d „  Z RS(   sh   A rule line is a single "Allow:" (allowance==1) or "Disallow:"
       (allowance==0) followed by a path.c    s(   • – t  i | ƒ |  _ — | |  _ d  S(   N(   s   urllibs   quotes   paths   selfs	   allowance(   s   selfs   paths	   allowance(    (    s!   /usr/lib/python2.2/robotparser.pys   __init__• s   c    s-   ™ š |  i d j p t i |  i | ƒ Sd  S(   Ns   *(   s   selfs   paths   res   matchs   filename(   s   selfs   filename(    (    s!   /usr/lib/python2.2/robotparser.pys
   applies_to™ s   c    s*   œ  |  i o d p d d |  i Sd  S(   Ns   Allows   Disallows   : (   s   selfs	   allowances   path(   s   self(    (    s!   /usr/lib/python2.2/robotparser.pys   __str__œ s   (   s   __name__s
   __module__s   __doc__s   __init__s
   applies_tos   __str__(    (    (    s!   /usr/lib/python2.2/robotparser.pys   RuleLine’ s   s   Entryc      sD     t  Z d  Z ¡ ¢ d „  Z ¦ d „  Z ® d „  Z ¼ d „  Z RS(   s?   An entry has one or more user-agents and zero or more rulelinesc    s   ¢ £ g  |  _ ¤ g  |  _ d  S(   N(   s   selfs
   useragentss	   rulelines(   s   self(    (    s!   /usr/lib/python2.2/robotparser.pys   __init__¢ s   c    sw   ¦ § d } ¨ x) |  i D¨ ] } © | d | d } q Wª x+ |  i Dª ] } « | t | ƒ d } qH W¬ | Sd  S(   Ns    s   User-agent: s   
(   s   rets   selfs
   useragentss   agents	   ruleliness   lines   str(   s   selfs   lines   agents   ret(    (    s!   /usr/lib/python2.2/robotparser.pys   __str__¦ s   	 	 	c    s›   ® ¯ ± | i d ƒ d i ƒ  } ² xh |  i D² ]Z } ³ | d j o µ d Sn ¶ | i ƒ  } ¸ t i t i | ƒ | ƒ o ¹ d Sn q/ Wº d Sd S(   s2   check if this entry applies to the specified agents   /i    s   *i   N(	   s	   useragents   splits   lowers   selfs
   useragentss   agents   res   searchs   escape(   s   selfs	   useragents   agent(    (    s!   /usr/lib/python2.2/robotparser.pys
   applies_to® s    	c    sk   ¼ ¿ À xT |  i DÀ ]F } Á t | t | ƒ | i f ƒ Â | i | ƒ o Ã | i Sn q WÄ d Sd S(   sZ   Preconditions:
        - our agent applies to this entry
        - filename is URL decodedi   N(   s   selfs	   ruleliness   lines   _debugs   filenames   strs	   allowances
   applies_to(   s   selfs   filenames   line(    (    s!   /usr/lib/python2.2/robotparser.pys	   allowance¼ s    	(   s   __name__s
   __module__s   __doc__s   __init__s   __str__s
   applies_tos	   allowance(    (    (    s!   /usr/lib/python2.2/robotparser.pys   Entry  s
   s	   URLopenerc      s2   Æ t  Z Ç d „  Z Í d „  Z Ò e d „ Z RS(   Nc    sH   Ç È t  t i i |  f | ƒ É d |  _ Ê d |  _ Ë d |  _ d  S(   NiÈ   i    i
   (	   s   applys   urllibs   FancyURLopeners   __init__s   selfs   argss   errcodes   triess   maxtries(   s   selfs   args(    (    s!   /usr/lib/python2.2/robotparser.pys   __init__Ç s   c    s8   Í Î | |  _  Ï t i i |  | | | Ð | | ƒ Sd  S(   N(	   s   errcodes   selfs   urllibs   FancyURLopeners   http_error_defaults   urls   fps   errmsgs   headers(   s   selfs   urls   fps   errcodes   errmsgs   headers(    (    s!   /usr/lib/python2.2/robotparser.pys   http_error_defaultÍ s   c    s’   Ò Ó |  i d 7_ Ô |  i |  i j o& Õ |  i | | d Ö d × | ƒ Sn Ø t i i	 |  | | | Ù | | | ƒ } Ú d |  _ Û | Sd  S(   Ni   iô  s)   Internal Server Error: Redirect Recursioni    (   s   selfs   triess   maxtriess   http_error_defaults   urls   fps   headerss   urllibs   FancyURLopeners   http_error_302s   errcodes   errmsgs   datas   result(   s   selfs   urls   fps   errcodes   errmsgs   headerss   datas   result(    (    s!   /usr/lib/python2.2/robotparser.pys   http_error_302Ò s   (   s   __name__s
   __module__s   __init__s   http_error_defaults   Nones   http_error_302(    (    (    s!   /usr/lib/python2.2/robotparser.pys	   URLopenerÆ s   	c    sT   Ý Þ | o ß d } n
 á d } â |  | j o ã d GHn å d | GHæ Hd  S(   Ns   access denieds   access alloweds   faileds   ok (%s)(   s   bs   acs   a(   s   as   bs   ac(    (    s!   /usr/lib/python2.2/robotparser.pys   _checkÝ s   	c     s×  è é ê t  ƒ  }  ë d a î |  i d ƒ ï |  i ƒ  ò t |  i d d ƒ d ƒ ô t |  i d d ƒ d ƒ ö t |  i d ÷ d ƒ ø d ƒ ù t |  i d	 ú d ƒ û d ƒ ü t |  i d
 ý d ƒ þ d ƒ  t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ t |  i d d ƒ d ƒ |  i d ƒ |  i ƒ  t |  i d d ƒ d ƒ d  S(   Ni   s"   http://www.musi-cal.com/robots.txts   *s   http://www.musi-cal.com/s    i    s   CherryPickerSEs?   http://www.musi-cal.com/cgi-bin/event-search?city=San+Franciscos   CherryPickerSE/1.0s   CherryPickerSE/1.5s   ExtractorPros   http://www.musi-cal.com/blubbas   extractorpros   toolpak/1.1s   spams   http://www.musi-cal.com/searchs#   http://www.musi-cal.com/Musician/mes   http://www.lycos.com/robots.txts   Mozillas   http://www.lycos.com/search(   s   RobotFileParsers   rps   debugs   set_urls   reads   _checks	   can_fetch(   s   rp(    (    s!   /usr/lib/python2.2/robotparser.pys   _testè s4   		
	
	
s   __main__(   s   __doc__s   res   urlparses   urllibs   __all__s   debugs   _debugs   RobotFileParsers   RuleLines   Entrys   FancyURLopeners	   URLopeners   _checks   _tests   __name__(   s	   URLopeners   _debugs   __all__s   _tests   urllibs   urlparses   res   _checks   RuleLines   Entrys   RobotFileParser(    (    s!   /usr/lib/python2.2/robotparser.pys   ? s   	|&'