tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Fine tune duplicate detection

Browse Source
This commit is contained in:
Sam Ruby 2007-11-04 21:00:54 -05:00
parent cb2ae87034
commit 73a5dcaaf9
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
planet/spider.py
View File

@ -432,7 +432,10 @@ def spiderPlanet(only_if_new = False):
'href': feed.url, 'bozo': 0,
'status': int(feed.headers.status)})
id = data.feed.get('id', data.href)
id = data.feed.get('id', None)
if not id and hasattr(data, 'href'): id=data.href
if not id: id=uri
if not feeds_seen.has_key(id):
writeCache(uri, feed_info, data)
feeds_seen[id] = uri