diff --git a/docs/config.html b/docs/config.html index 70c6710..06d3083 100644 --- a/docs/config.html +++ b/docs/config.html @@ -90,6 +90,10 @@ number of days will be marked as inactive
log_level
One of DEBUG, INFO, WARNING, ERROR or CRITICAL
+
log_format
+
format string to +use for logging output. Note: this configuration value is processed +raw
feed_timeout
Number of seconds to wait for any given feed
new_feed_items
diff --git a/planet.py b/planet.py index d4d5976..62fb7ac 100755 --- a/planet.py +++ b/planet.py @@ -50,7 +50,7 @@ if __name__ == "__main__": if verbose: import planet - planet.getLogger('DEBUG') + planet.getLogger('DEBUG',config.log_format()) if not offline: from planet import spider diff --git a/planet/__init__.py b/planet/__init__.py index 444b30b..6be34ed 100644 --- a/planet/__init__.py +++ b/planet/__init__.py @@ -9,7 +9,7 @@ config.__init__() from ConfigParser import ConfigParser from urlparse import urljoin -def getLogger(level): +def getLogger(level, format): """ get a logger with the specified log level """ global logger if logger: return logger @@ -19,7 +19,7 @@ def getLogger(level): except: import compat_logging as logging - logging.basicConfig() + logging.basicConfig(format=format) logging.getLogger().setLevel(logging.getLevelName(level)) logger = logging.getLogger("planet.runner") try: diff --git a/planet/compat_logging/__init__.py b/planet/compat_logging/__init__.py index 3bd0c6d..3b83493 100644 --- a/planet/compat_logging/__init__.py +++ b/planet/compat_logging/__init__.py @@ -1090,7 +1090,7 @@ Logger.manager = Manager(Logger.root) BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" -def basicConfig(): +def basicConfig(format=BASIC_FORMAT): """ Do basic configuration for the logging system by creating a StreamHandler with a default Formatter and adding it to the @@ -1098,7 +1098,7 @@ def basicConfig(): """ if len(root.handlers) == 0: hdlr = StreamHandler() - fmt = Formatter(BASIC_FORMAT) + fmt = Formatter(format) hdlr.setFormatter(fmt) root.addHandler(hdlr) diff --git a/planet/config.py b/planet/config.py index f9fab41..5c8ffe3 100644 --- a/planet/config.py +++ b/planet/config.py @@ -43,6 +43,8 @@ def __init__(): if section and parser.has_option(section, option): return parser.get(section, option) elif parser.has_option('Planet', option): + if option == 'log_format': + return parser.get('Planet', option, raw=True) return parser.get('Planet', option) else: return default @@ -88,6 +90,7 @@ def __init__(): define_planet('link', '') define_planet('cache_directory', "cache") define_planet('log_level', "WARNING") + define_planet('log_format', "%(levelname)s:%(name)s:%(message)s") define_planet('feed_timeout', 20) define_planet('date_format', "%B %d, %Y %I:%M %p") define_planet('new_date_format', "%B %d, %Y") @@ -123,7 +126,7 @@ def load(config_file): import config, planet from planet import opml, foaf - log = planet.getLogger(config.log_level()) + log = planet.getLogger(config.log_level(),config.log_format()) # Theme support theme = config.output_theme() @@ -146,10 +149,11 @@ def load(config_file): # complete search list for theme directories dirs += [os.path.join(theme_dir,dir) for dir in - config.template_directories()] + config.template_directories() if dir not in dirs] # merge configurations, allowing current one to override theme template_files = config.template_files() + parser.set('Planet','template_files','') parser.read(config_file) for file in config.bill_of_materials(): if not file in bom: bom.append(file) @@ -334,7 +338,8 @@ def filters(section=None): def planet_options(): """ dictionary of planet wide options""" - return dict(map(lambda opt: (opt, parser.get('Planet',opt)), + return dict(map(lambda opt: (opt, + parser.get('Planet', opt, raw=(opt=="log_format"))), parser.options('Planet'))) def feed_options(section): diff --git a/planet/shell/__init__.py b/planet/shell/__init__.py index bd0e005..8a6fcfa 100644 --- a/planet/shell/__init__.py +++ b/planet/shell/__init__.py @@ -6,7 +6,7 @@ logged_modes = [] def run(template_file, doc, mode='template'): """ select a template module based on file extension and execute it """ - log = planet.getLogger(planet.config.log_level()) + log = planet.getLogger(planet.config.log_level(),planet.config.log_format()) if mode == 'template': dirs = planet.config.template_directories() diff --git a/planet/spider.py b/planet/spider.py index bf94d48..ce473ee 100644 --- a/planet/spider.py +++ b/planet/spider.py @@ -31,6 +31,8 @@ def filename(directory, filename): filename=filename.encode('idna') except: pass + if isinstance(filename,unicode): + filename=filename.encode('utf-8') filename = re_url_scheme.sub("", filename) filename = re_slash.sub(",", filename) filename = re_initial_cruft.sub("", filename) @@ -307,7 +309,7 @@ def spiderFeed(feed, only_if_new=0): def spiderPlanet(only_if_new = False): """ Spider (fetch) an entire planet """ - log = planet.getLogger(config.log_level()) + log = planet.getLogger(config.log_level(),config.log_format()) planet.setTimeout(config.feed_timeout()) global index diff --git a/planet/splice.py b/planet/splice.py index e95d57f..e50f927 100644 --- a/planet/splice.py +++ b/planet/splice.py @@ -9,7 +9,7 @@ from planet import idindex def splice(): """ Splice together a planet from a cache of entries """ import planet - log = planet.getLogger(config.log_level()) + log = planet.getLogger(config.log_level(),config.log_format()) log.info("Loading cached data") cache = config.cache_directory() @@ -97,7 +97,7 @@ def splice(): def apply(doc): output_dir = config.output_dir() if not os.path.exists(output_dir): os.makedirs(output_dir) - log = planet.getLogger(config.log_level()) + log = planet.getLogger(config.log_level(),config.log_format()) # Go-go-gadget-template for template_file in config.template_files(): diff --git a/runtests.py b/runtests.py index 3ebe331..1e866cb 100755 --- a/runtests.py +++ b/runtests.py @@ -23,7 +23,7 @@ modules = map(fullmodname, glob.glob(os.path.join('tests', 'test_*.py'))) # enable warnings import planet -planet.getLogger("WARNING") +planet.getLogger("WARNING",None) # load all of the tests into a suite try: diff --git a/tests/capture.py b/tests/capture.py index 1e577b6..0ea3473 100755 --- a/tests/capture.py +++ b/tests/capture.py @@ -18,7 +18,7 @@ os.chdir(sys.path[0]) # copy spider output to splice input import planet from planet import spider, config -planet.getLogger('CRITICAL') +planet.getLogger('CRITICAL',None) config.load('tests/data/spider/config.ini') spider.spiderPlanet() diff --git a/tests/test_idindex.py b/tests/test_idindex.py index d27cf18..0de818d 100644 --- a/tests/test_idindex.py +++ b/tests/test_idindex.py @@ -8,6 +8,15 @@ class idIndexTest(unittest.TestCase): def tearDown(self): idindex.destroy() + def test_unicode(self): + from planet.spider import filename + index = idindex.create() + iri = 'http://www.\xe8\xa9\xb9\xe5\xa7\x86\xe6\x96\xaf.com/' + index[filename('', iri)] = 'data' + index[filename('', iri.decode('utf-8'))] = 'data' + index[filename('', u'1234')] = 'data' + index.close() + def test_index_spider(self): import test_spider config.load(test_spider.configfile) diff --git a/tests/test_spider.py b/tests/test_spider.py index 01e730a..1230c23 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -13,7 +13,7 @@ class SpiderTest(unittest.TestCase): def setUp(self): # silence errors planet.logger = None - planet.getLogger('CRITICAL') + planet.getLogger('CRITICAL',None) try: os.makedirs(workdir) diff --git a/tests/test_themes.py b/tests/test_themes.py index 4f171ed..de70c59 100644 --- a/tests/test_themes.py +++ b/tests/test_themes.py @@ -4,7 +4,7 @@ import unittest from planet import config from os.path import split -class ConfigTest(unittest.TestCase): +class ThemesTest(unittest.TestCase): def setUp(self): config.load('tests/data/config/themed.ini') @@ -17,7 +17,8 @@ class ConfigTest(unittest.TestCase): # administrivia def test_template(self): - self.assertTrue('index.html.xslt' in config.template_files()) + self.assertEqual(1, len([1 for file in config.template_files() + if file == 'index.html.xslt'])) def test_feeds(self): feeds = config.subscriptions()