mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-25 10:54:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix Issue6325 - robotparse to honor urls with query strings.

Browse Source
This commit is contained in:
Senthil Kumaran 2010-07-28 16:27:56 +00:00
parent 96a60ae90c
commit 3f8ab965f7
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Lib/test/test_robotparser.py
View File

@ -205,6 +205,17 @@ bad = ['/folder1/anotherfile.html']
RobotTest(13, doc, good, bad, agent="googlebot")
# 14. For issue #6325 (query string support)
doc = """
User-agent: *
Disallow: /some/path?name=value
"""
good = ['/some/path']
bad = ['/some/path?name=value']
RobotTest(14, doc, good, bad)
class NetworkTestCase(unittest.TestCase):

6
Lib/urllib/robotparser.py
View File

@ -129,8 +129,10 @@ class RobotFileParser:
return True
# search for given user agent matches
# the first match counts
url = urllib.parse.quote(
urllib.parse.urlparse(urllib.parse.unquote(url))[2])
parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
url = urllib.parse.urlunparse(('','',parsed_url.path,
parsed_url.params,parsed_url.query, parsed_url.fragment))
url = urllib.parse.quote(url)
if not url:
url = "/"
for entry in self.entries: