0hhb>SSKrSSKrSSKrSSKrSSKrSSKJr SSKJ r SSKJ r SSK J r J r "SS5r"SS \\R5r"S S \\R5r"S S \\R5r"SS\\R5r"SS\5r"SS\\R5r"SS\\R5r"SS\5r"SS\\R5r"SS\\R5r"SS\\R5r"SS\\R5r"S S!\5r"S"S#\\R5r"S$S%\\R5r"S&S'\\R5r"S(S)\\R5r "S*S+\\R5r!"S,S-\\R5r"\RF"\RHS.5"S/S055r%S1r&"S2S3\%\R5r'"S4S5\%\R5r(\RR"5"S6S7\R55r*\+S8:Xa\RX"5 gg)9N)support) socket_helper)threading_helper)BaseHTTPRequestHandler HTTPServercZ^\rSrSrSrSr/r/rSrU4Sjr Sr Sr Sr S r S rS rU=r$) BaseRobotTest test_robotparserNc>[TU]5 UR(dSUlUR(dSUlggN)super__init_subclass__goodtest_good_urlsbad test_bad_urls)cls __class__s >/opt/python-3.13.8/usr/lib/python3.13/test/test_robotparser.pyrBaseRobotTest.__init_subclass__s1 !#xx!%C ww $C c[R"UR5R5n[R R 5UlURRU5 gr) ioStringIO robots_txt readlinesurllib robotparserRobotFileParserparserparse)selfliness rsetUpBaseRobotTest.setUpsE DOO,668((88:  % rcV[U[5(aUup!X!4$URU4$r) isinstancetupleagentr$urlr+s rget_agent_and_urlBaseRobotTest.get_agent_and_url s, c5 ! !JE: zz3rcURHXnURU5up!URXS9 URURR X!55 SSS5 MZ g!,(df  Ml=fN)r-r+)rr.subTest assertTruer" can_fetchr,s rrBaseRobotTest.test_good_urls&sZ99C//4JE#3 5 5e AB4333 +A** A9 cURHXnURU5up!URXS9 URURR X!55 SSS5 MZ g!,(df  Ml=fr1)rr.r2 assertFalser"r4r,s rrBaseRobotTest.test_bad_urls,s\88C//4JE#3  !6!6u!BC4333r6clURURR5UR5 gr) assertEqualr" site_mapsr$s rtest_site_mapsBaseRobotTest.test_site_maps2s# ..0$..Ar)r")__name__ __module__ __qualname____firstlineno__rr+rrr<rr&r.rrr>__static_attributes__ __classcell__)rs@rr r sCJ E D CI%!  C D BBrr c(\rSrSrSrSS/r/SQrSrg)UserAgentWildcardTest6zUser-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html / /test.html)/cyberworld/map/index.htmlz/tmp/xxx /foo.htmlNr@rArBrCrrrrDrMrrrGrG6sJ  D ACrrGc&\rSrSrSr/SQrS/rSrg)CrawlDelayAndCustomAgentTestAz# robots.txt for http://www.example.com/ User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: )rIrJ) cybermapperrKrKrMNrNrMrrrPrPAs J ND ' (CrrPc.\rSrSrSrSS/rS/rSS/rSrg ) SitemapTestRa# robots.txt for http://www.example.com/ User-agent: * Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml Sitemap: http://www.google.com/hostednews/sitemap_index.xml Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space rIrJrKz7http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xmlz2http://www.google.com/hostednews/sitemap_index.xmlrMN) r@rArBrCrrrr<rDrMrrrTrTRs+ J  D ' (CJEGIrrTc$\rSrSrSr/r/SQrSrg)RejectAllRobotsTestcz(# go away User-agent: * Disallow: / )rKrI/tmp/rMNrNrMrrrWrWcsJ D 6CrrWc"\rSrSrSrSrSrSrg)BaseRequestRateTestmNcURnURUR-GHnURU5up2UR X#S9 UR UR U5UR 5 URU5nUR X@R5 URbURU[RR5 UR URURR5 UR URURR5 SSS5 GM g!,(df  GM3=fr1)r"rrr.r2r; crawl_delay request_rateassertIsInstancerr RequestRaterequestsseconds)r$r"r-r+parsed_request_rates rtest_request_rate%BaseRequestRateTest.test_request_rateqs99txx'C//4JE#3  !3!3E!:DrsG &):'B'BTBM8+<+<B)=(2C2C)"G-!2!2G"7-):):7-8'):): .#68I8I.$-5- ]H,=,=   M8+<+<   (M83D3D ( M8+<+<  5 (M8+<+< (,mX->->," M83D3D H(-):):H(V )*H,=,= )A=(*;*;A4  ,## #0K3X5F5FKB<$O>O<(   "'4h'''4#'4R Z MMOr