tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

ETags for reading lists

Browse Source
This commit is contained in:
Sam Ruby 2006-09-03 13:12:54 -04:00
parent c610c93720
commit 88bdbe930e
3 changed files with 37 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
planet/__init__.py
View File

@ -39,12 +39,12 @@ def setTimeout(timeout):
try:
from planet import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(timeout)
logger.debug("Socket timeout set to %d seconds", timeout)
logger.info("Socket timeout set to %d seconds", timeout)
except ImportError:
import socket
if hasattr(socket, 'setdefaulttimeout'):
logger.debug("timeoutsocket not found, using python function")
socket.setdefaulttimeout(timeout)
logger.debug("Socket timeout set to %d seconds", timeout)
logger.info("Socket timeout set to %d seconds", timeout)
else:
logger.error("Unable to set timeout to %d seconds", timeout)

40
planet/config.py
View File

@ -28,6 +28,7 @@ Todo:
import os, sys, re
from ConfigParser import ConfigParser
from urlparse import urljoin
parser = ConfigParser()
@ -169,16 +170,44 @@ def load(config_file):
for list in reading_lists:
cache_filename = filename(config.cache_lists_directory(), list)
try:
import urllib, StringIO
import urllib2, StringIO
# read once to verify
data=StringIO.StringIO(urllib.urlopen(list).read())
# retrieve list options (e.g., etag, last-modified) from cache
options = {}
try:
cached_config = ConfigParser()
cached_config.read(cache_filename)
for option in cached_config.options(list):
options[option] = cached_config.get(list,option)
except:
pass
cached_config = ConfigParser()
cached_config.add_section(list)
for key, value in options.items():
cached_config.set(list, key, value)
# read list
base = urljoin('file:', os.path.abspath(os.path.curdir))
request = urllib2.Request(urljoin(base + '/', list))
if options.has_key("etag"):
request.add_header('If-None-Match', options['etag'])
if options.has_key("last-modified"):
request.add_header('If-Modified-Since',
options['last-modified'])
response = urllib2.urlopen(request)
if response.headers.has_key('etag'):
cached_config.set(list, 'etag', response.headers['etag'])
if response.headers.has_key('last-modified'):
cached_config.set(list, 'last-modified',
response.headers['last-modified'])
# convert to config.ini
data=StringIO.StringIO(response.read())
if content_type(list).find('opml')>=0:
opml.opml2config(data, cached_config)
elif content_type(list).find('foaf')>=0:
foaf.foaf2config(data, cached_config)
if not cached_config.sections(): raise Exception
if cached_config.sections() in [[], [list]]: raise Exception
# write to cache
cache = open(cache_filename, 'w')
@ -196,7 +225,6 @@ def load(config_file):
except:
log.exception("Unable to read %s readinglist", list)
continue
# planet.foaf.foaf2config(data, list, config)
def cache_sources_directory():
if parser.has_option('Planet', 'cache_sources_directory'):
@ -217,7 +245,7 @@ def feed():
for template_file in template_files:
name = os.path.splitext(os.path.basename(template_file))[0]
if name.find('atom')>=0 or name.find('rss')>=0:
return urlparse.urljoin(link(), name)
return urljoin(link(), name)
def feedtype():
if parser.has_option('Planet', 'feedtype'):

2
tests/test_rlists.py
View File

@ -45,5 +45,5 @@ class ReadingListTest(unittest.TestCase):
feeds = [split(feed)[1] for feed in parser.sections()]
feeds.sort()
self.assertEqual(['testfeed0.atom', 'testfeed1a.atom',
self.assertEqual(['opml.xml', 'testfeed0.atom', 'testfeed1a.atom',
'testfeed2.atom', 'testfeed3.rss'], feeds)