tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Refactor http status implementation

Browse Source
This commit is contained in:
Sam Ruby 2006-09-24 22:30:23 -04:00
parent 4b0cd8d5d9
commit c7c2a2019d
1 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
planet/spider.py
View File

@ -116,9 +116,6 @@ def spiderFeed(feed):
data = feedparser.parse(feed_info.feed.get('planet_http_location',feed), data = feedparser.parse(feed_info.feed.get('planet_http_location',feed),
etag=feed_info.feed.get('planet_http_etag',None), modified=modified) etag=feed_info.feed.get('planet_http_etag',None), modified=modified)
# if read failed, retain cached information
if not data.version and feed_info.version: data.feed = feed_info.feed
# capture http status # capture http status
if not data.has_key("status"): if not data.has_key("status"):
if data.has_key("entries") and len(data.entries)>0: if data.has_key("entries") and len(data.entries)>0:
@ -127,7 +124,6 @@ def spiderFeed(feed):
data.status = 408 data.status = 408
else: else:
data.status = 500 data.status = 500
data.feed['planet_http_status'] = str(data.status)
# process based on the HTTP status code # process based on the HTTP status code
log = planet.logger log = planet.logger
@ -138,18 +134,19 @@ def spiderFeed(feed):
data.feed['planet_http_location'] = data.url data.feed['planet_http_location'] = data.url
elif data.status == 304: elif data.status == 304:
return log.info("Feed %s unchanged", feed) return log.info("Feed %s unchanged", feed)
elif data.status >= 400: elif data.status == 410:
feed_info.update(data.feed)
data.feed = feed_info
if data.status == 410:
log.info("Feed %s gone", feed) log.info("Feed %s gone", feed)
elif data.status == 408: elif data.status == 408:
log.warning("Feed %s timed out", feed) log.warning("Feed %s timed out", feed)
else: elif data.status >= 400:
log.error("Error %d while updating feed %s", data.status, feed) log.error("Error %d while updating feed %s", data.status, feed)
else: else:
log.info("Updating feed %s", feed) log.info("Updating feed %s", feed)
# if read failed, retain cached information
if not data.version and feed_info.version: data.feed = feed_info.feed
data.feed['planet_http_status'] = str(data.status)
# capture etag and last-modified information # capture etag and last-modified information
if data.has_key('headers'): if data.has_key('headers'):
if data.has_key('etag') and data.etag: if data.has_key('etag') and data.etag: