tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Added in support for '-location' in httlib2 responses

Browse Source
This commit is contained in:
Joe Gregorio 2006-11-07 13:19:42 -05:00
parent 56a447e1be
commit daec4769c7
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
planet/httplib2/__init__.py
View File

@ -16,7 +16,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)",
"Jonathan Feinberg", "Jonathan Feinberg",
"Blair Zajac"] "Blair Zajac"]
__license__ = "MIT" __license__ = "MIT"
__version__ = "$Rev: 208 $" __version__ = "$Rev: 209 $"
import re import re
import md5 import md5
@ -637,6 +637,7 @@ class Http:
response['location'] = urlparse.urljoin(absolute_uri, location) response['location'] = urlparse.urljoin(absolute_uri, location)
if response.status == 301 and method in ["GET", "HEAD"]: if response.status == 301 and method in ["GET", "HEAD"]:
response['-x-permanent-redirect-url'] = response['location'] response['-x-permanent-redirect-url'] = response['location']
response['-location'] = absolute_uri
_updateCache(headers, response, content, self.cache, cachekey) _updateCache(headers, response, content, self.cache, cachekey)
if headers.has_key('if-none-match'): if headers.has_key('if-none-match'):
del headers['if-none-match'] del headers['if-none-match']
@ -645,6 +646,7 @@ class Http:
if response.has_key('location'): if response.has_key('location'):
location = response['location'] location = response['location']
old_response = copy.deepcopy(response) old_response = copy.deepcopy(response)
old_response['-location'] = absolute_uri
redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method
(response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1)
response.previous = old_response response.previous = old_response
@ -652,6 +654,7 @@ class Http:
raise RedirectLimit( _("Redirected more times than rediection_limit allows.")) raise RedirectLimit( _("Redirected more times than rediection_limit allows."))
elif response.status in [200, 203] and method == "GET": elif response.status in [200, 203] and method == "GET":
# Don't cache 206's since we aren't going to handle byte range requests # Don't cache 206's since we aren't going to handle byte range requests
response['-location'] = absolute_uri
_updateCache(headers, response, content, self.cache, cachekey) _updateCache(headers, response, content, self.cache, cachekey)
return (response, content) return (response, content)

2
planet/spider.py
View File

@ -144,7 +144,7 @@ def spiderFeed(feed, only_if_new=0, content=None, resp_headers=None):
# httplib2 was used to get the content, so prepare a # httplib2 was used to get the content, so prepare a
# proper object to pass to feedparser. # proper object to pass to feedparser.
f = StringIO(content) f = StringIO(content)
setattr(f, 'url', feed) setattr(f, 'url', resp_headers.get('-location', feed))
if resp_headers: if resp_headers:
if resp_headers.has_key('content-encoding'): if resp_headers.has_key('content-encoding'):
del resp_headers['content-encoding'] del resp_headers['content-encoding']