tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Warn on duplicate subscriptions

Browse Source
This commit is contained in:
Sam Ruby 2007-11-04 20:46:25 -05:00
parent 26d407c709
commit cb2ae87034
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
planet/spider.py
View File

@ -405,6 +405,7 @@ def spiderPlanet(only_if_new = False):
fetch_queue.put(item=(None, None)) fetch_queue.put(item=(None, None))
# Process the results as they arrive # Process the results as they arrive
feeds_seen = {}
while fetch_queue.qsize() or parse_queue.qsize() or threads: while fetch_queue.qsize() or parse_queue.qsize() or threads:
while parse_queue.qsize() == 0 and threads: while parse_queue.qsize() == 0 and threads:
time.sleep(0.1) time.sleep(0.1)
@ -428,9 +429,16 @@ def spiderPlanet(only_if_new = False):
else: else:
data = feedparser.FeedParserDict({'version': None, data = feedparser.FeedParserDict({'version': None,
'headers': feed.headers, 'entries': [], 'feed': {}, 'headers': feed.headers, 'entries': [], 'feed': {},
'bozo': 0, 'status': int(feed.headers.status)}) 'href': feed.url, 'bozo': 0,
'status': int(feed.headers.status)})
writeCache(uri, feed_info, data) id = data.feed.get('id', data.href)
if not feeds_seen.has_key(id):
writeCache(uri, feed_info, data)
feeds_seen[id] = uri
else:
log.warn('Duplicate subscription: %s and %s' %
(uri, feeds_seen[id]))
except Exception, e: except Exception, e:
import sys, traceback import sys, traceback