o HXîh‰+ã@s(ddlZddlZddlZddlZddlZddlmZddlm Z ddlm Z ddl m Z m Z Gdd„dƒZGdd „d eejƒZGd d „d eejƒZGd d „d eejƒZGdd„deejƒZGdd„deƒZGdd„deejƒZGdd„deejƒZGdd„deƒZGdd„deejƒZGdd„deejƒZGdd„deejƒZGdd„deejƒZGd d!„d!eƒZGd"d#„d#eejƒZGd$d%„d%eejƒZGd&d'„d'eejƒZGd(d)„d)eejƒZ Gd*d+„d+eejƒZ!Gd,d-„d-eejƒZ"Gd.d/„d/e ƒZ#Gd0d1„d1ejƒZ$Gd2d3„d3ejƒZ%e&d4kre '¡dSdS)5éN)Úsupport)Ú socket_helper)Úthreading_helper)ÚBaseHTTPRequestHandlerÚ HTTPServerc@sHeZdZdZdZgZgZdZdd„Zdd„Z dd „Z d d „Z d d „Z dS)Ú BaseRobotTestÚZtest_robotparserNcCs,t |j¡ ¡}tj ¡|_|j |¡dS©N) ÚioÚStringIOÚ robots_txtÚ readlinesÚurllibÚ robotparserÚRobotFileParserÚparserÚparse)ÚselfÚlines©rú?/opt/python-3.10.19/usr/lib/python3.10/test/test_robotparser.pyÚsetUps zBaseRobotTest.setUpcCs$t|tƒr |\}}||fS|j|fSr )Ú isinstanceÚtupleÚagent©rÚurlrrrrÚget_agent_and_urls  zBaseRobotTest.get_agent_and_urlc Có`|jD]*}| |¡\}}|j||d| |j ||¡¡Wdƒn1s(wYqdS©N)rr)ÚgoodrÚsubTestÚ assertTruerÚ can_fetchrrrrÚtest_good_urlsó ÿ€þzBaseRobotTest.test_good_urlsc Crr)Úbadrr!Ú assertFalserr#rrrrÚ test_bad_urls$r%zBaseRobotTest.test_bad_urlscCs| |j ¡|j¡dSr )Ú assertEqualrÚ site_maps©rrrrÚtest_site_maps*ózBaseRobotTest.test_site_maps) Ú__name__Ú __module__Ú __qualname__r rr r&r*rrr$r(r,rrrrr s rc@s eZdZdZddgZgd¢ZdS)ÚUserAgentWildcardTestz•User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html ú/ú /test.html)ú/cyberworld/map/index.htmlz/tmp/xxxú /foo.htmlN©r.r/r0r r r&rrrrr1.s r1c@seZdZdZgd¢ZdgZdS)ÚCrawlDelayAndCustomAgentTestzå# robots.txt for http://www.example.com/ User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: )r2r3)Z cybermapperr4r4Nr6rrrrr79s r7c@s&eZdZdZddgZdgZddgZdS)Ú SitemapTesta# robots.txt for http://www.example.com/ User-agent: * Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml Sitemap: http://www.google.com/hostednews/sitemap_index.xml Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space r2r3r4z7http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xmlz2http://www.google.com/hostednews/sitemap_index.xmlN)r.r/r0r r r&r*rrrrr8Js ÿr8c@seZdZdZgZgd¢ZdS)ÚRejectAllRobotsTestz(# go away User-agent: * Disallow: / )r4r2ú/tmp/Nr6rrrrr9[s r9c@seZdZdZdZdd„ZdS)ÚBaseRequestRateTestNc CsÂ|j}|j|jD]U}| |¡\}}|j||d=| | |¡|j¡| |¡}| ||j¡|jdurO| |t j j ¡| |j |jj ¡| |j |jj ¡Wdƒn1sYwYq dSr)rr r&rr!r)Ú crawl_delayÚ request_rateZassertIsInstancerrÚ RequestRateZrequestsZseconds)rrrrZparsed_request_raterrrÚtest_request_rateis0  þþþ€ò€þz%BaseRequestRateTest.test_request_rate)r.r/r0r=r<r?rrrrr;es r;c@seZdZdZdgZdS)Ú EmptyFileTestrz/fooN)r.r/r0r r rrrrr@s r@c@s4eZdZdZdZej dd¡ZdZ dgZ gd¢Z dS) ÚCrawlDelayAndRequestRateTestz’User-agent: figtree Crawl-delay: 3 Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html Disallow: /%7ejoe/index.html Úfigtreeé éé)rBr5)ú/tmpz /tmp.htmlú /tmp/a.htmlú /a%3cd.htmlú /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r.r/r0r rrrr>r=r<r r&rrrrrA†s  rAc@óeZdZdZdS)ÚDifferentAgentTestzFigTree Robot libwww-perl/5.04N©r.r/r0rrrrrrK˜órKc@s"eZdZdZdgZgd¢ZdZdS)ÚInvalidRequestRateTestzUser-agent: * Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html Crawl-delay: 3 Request-rate: 9/banana rF)r:rGrHrIz /a/b.htmlz/%7Ejoe/index.htmlrEN)r.r/r0r r r&r<rrrrrNœs  rNc@seZdZdZdgZgZdS)ÚInvalidCrawlDelayTestz2User-Agent: * Disallow: /. Crawl-delay: pears r5Nr6rrrrrO¬srOc@ó eZdZdZdZdgZdgZdS)ÚAnotherInvalidRequestRateTestzeUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Request-rate: whale/banana Ú Googlebotú/folder1/myfile.htmlú/folder1/anotherfile.htmlN©r.r/r0r rr r&rrrrrQ¸s  rQc@seZdZdZdZdgZdS)ÚUserAgentOrderingTestzMUser-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / rRz/something.jpgN)r.r/r0r rr&rrrrrVÅs rVc@rJ)ÚUserAgentGoogleMobileTestzGooglebot-MobileNrLrrrrrWÔrMrWc@rP)ÚGoogleURLOrderingTestzJUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Z googlebotrSrTNrUrrrrrXØs  rXc@óeZdZdZdgZdgZdS)ÚDisallowQueryStringTestz2User-agent: * Disallow: /some/path?name=value ú /some/pathz/some/path?name=valueNr6rrrrrZås rZc@rY)ÚUseFirstUserAgentWildcardTestzNUser-agent: * Disallow: /some/path User-agent: * Disallow: /another/path z /another/pathr[Nr6rrrrr\ïs r\c@rY)ÚEmptyQueryStringTestz>User-agent: * Allow: /some/path? Disallow: /another/path? z /some/path?z/another/path?Nr6rrrrr]üs r]c@s0eZdZdZej dd¡ZdZddgZ dgZ dS) ÚDefaultEntryTestzOUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ rEéér2r3r4N) r.r/r0r rrr>r=r<r r&rrrrr^s  r^c@seZdZdZdZdd„ZdS)ÚStringFormattingTestzÆUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: /some/path zxUser-agent: cybermapper Disallow: /some/path User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/cCs| t|jƒ|j¡dSr )r)ÚstrrÚexpected_outputr+rrrÚtest_string_formatting*r-z+StringFormattingTest.test_string_formattingN)r.r/r0r rcrdrrrrras rac@seZdZdd„Zdd„ZdS)Ú RobotHandlercCs| dd¡dS)Ni“zForbidden access)Z send_errorr+rrrÚdo_GET0szRobotHandler.do_GETcGsdSr r)rÚformatÚargsrrrÚ log_message3szRobotHandler.log_messageN)r.r/r0rfrirrrrre.s rec@s*eZdZdd„Zdd„Zejdd„ƒZdS)ÚPasswordProtectedSiteTestCasecCsP| tjj¡ttjdftƒ|_t j d|jj ddid|_ d|j _ |j  ¡dS)NrzHTTPServer servingZ poll_intervalg{®Gáz„?)ÚnameÚtargetÚkwargsT)Z addCleanuprZrequestÚ urlcleanuprrÚHOSTreÚserverÚ threadingÚThreadZ serve_foreverÚtÚdaemonÚstartr+rrrr9súz#PasswordProtectedSiteTestCase.setUpcCs"|j ¡|j ¡|j ¡dSr )rpÚshutdownrsÚjoinZ server_closer+rrrÚtearDownIs  z&PasswordProtectedSiteTestCase.tearDowncCs\|jj}dtjdt|dƒ}|d}tj ¡}| |¡|  ¡|  |  d|¡¡dS)Nzhttp://ú:r`z /robots.txtÚ*) rpZserver_addressrrorbrrrZset_urlÚreadr'r#)rÚaddrrZ robots_urlrrrrÚtestPasswordProtectedSiteNs  z7PasswordProtectedSiteTestCase.testPasswordProtectedSiteN)r.r/r0rrxrZ reap_threadsr}rrrrrj7s rjc@sFeZdZdZd e¡Zedd„ƒZdd„Zdd„Z d d „Z d d „Z d S)ÚNetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtcCsVt d¡t |j¡tj |j¡|_ |j   ¡WdƒdS1s$wYdS)NÚnetwork) rZrequiresrZtransient_internetÚbase_urlrrrr rr{)ÚclsrrrÚ setUpClass^s  "þzNetworkTestCase.setUpClasscCs&d |j|tj |¡dsd¡Sd¡S)Nz{}{}{}r`r2r)rgr€ÚosÚpathÚsplitext)rr„rrrres ÿÿzNetworkTestCase.urlcCsV| |jj¡| |jj¡| |j ¡d¡| |j d¡¡| |j d¡¡dS)Nrrz)r'rÚ disallow_allÚ allow_allZ assertGreaterÚmtimer<r=r+rrrÚ test_basicjs zNetworkTestCase.test_basiccCs˜| |j d| d¡¡¡| |j d|j¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d|j¡¡dS)NrzZ elsewhereZNutchZbrianZwebstats)r"rr#rr'r€r+rrrÚtest_can_fetchqs zNetworkTestCase.test_can_fetchcCsftj | d¡¡}| ¡| |j¡| |j¡|  |  ¡d¡|  |  d¡¡|  |  d¡¡dS)Nz i-robot.txtrrz)rrrrr{r"r‡r'r†r)rˆZ assertIsNoner<r=)rrrrrÚ test_read_404ys  zNetworkTestCase.test_read_404N) r.r/r0r€rgr Ú classmethodr‚rr‰rŠr‹rrrrr~Ys   r~Ú__main__)(r rƒrqZunittestZurllib.robotparserrÚtestrZ test.supportrrZ http.serverrrrZTestCaser1r7r8r9r;r@rArKrNrOrQrVrWrXrZr\r]r^rarerjr~r.ÚmainrrrrÚsF   "          " ) ÿ