diff --git a/docs/config.html b/docs/config.html
index 70c6710..06d3083 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -90,6 +90,10 @@ number of days will be marked as inactive
log_level
One of DEBUG
, INFO
, WARNING
, ERROR
or CRITICAL
+log_format
+format string to
+use for logging output. Note: this configuration value is processed
+raw
feed_timeout
Number of seconds to wait for any given feed
new_feed_items
diff --git a/planet.py b/planet.py
index d4d5976..62fb7ac 100755
--- a/planet.py
+++ b/planet.py
@@ -50,7 +50,7 @@ if __name__ == "__main__":
if verbose:
import planet
- planet.getLogger('DEBUG')
+ planet.getLogger('DEBUG',config.log_format())
if not offline:
from planet import spider
diff --git a/planet/__init__.py b/planet/__init__.py
index 444b30b..6be34ed 100644
--- a/planet/__init__.py
+++ b/planet/__init__.py
@@ -9,7 +9,7 @@ config.__init__()
from ConfigParser import ConfigParser
from urlparse import urljoin
-def getLogger(level):
+def getLogger(level, format):
""" get a logger with the specified log level """
global logger
if logger: return logger
@@ -19,7 +19,7 @@ def getLogger(level):
except:
import compat_logging as logging
- logging.basicConfig()
+ logging.basicConfig(format=format)
logging.getLogger().setLevel(logging.getLevelName(level))
logger = logging.getLogger("planet.runner")
try:
diff --git a/planet/compat_logging/__init__.py b/planet/compat_logging/__init__.py
index 3bd0c6d..3b83493 100644
--- a/planet/compat_logging/__init__.py
+++ b/planet/compat_logging/__init__.py
@@ -1090,7 +1090,7 @@ Logger.manager = Manager(Logger.root)
BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s"
-def basicConfig():
+def basicConfig(format=BASIC_FORMAT):
"""
Do basic configuration for the logging system by creating a
StreamHandler with a default Formatter and adding it to the
@@ -1098,7 +1098,7 @@ def basicConfig():
"""
if len(root.handlers) == 0:
hdlr = StreamHandler()
- fmt = Formatter(BASIC_FORMAT)
+ fmt = Formatter(format)
hdlr.setFormatter(fmt)
root.addHandler(hdlr)
diff --git a/planet/config.py b/planet/config.py
index f9fab41..5c8ffe3 100644
--- a/planet/config.py
+++ b/planet/config.py
@@ -43,6 +43,8 @@ def __init__():
if section and parser.has_option(section, option):
return parser.get(section, option)
elif parser.has_option('Planet', option):
+ if option == 'log_format':
+ return parser.get('Planet', option, raw=True)
return parser.get('Planet', option)
else:
return default
@@ -88,6 +90,7 @@ def __init__():
define_planet('link', '')
define_planet('cache_directory', "cache")
define_planet('log_level', "WARNING")
+ define_planet('log_format', "%(levelname)s:%(name)s:%(message)s")
define_planet('feed_timeout', 20)
define_planet('date_format', "%B %d, %Y %I:%M %p")
define_planet('new_date_format', "%B %d, %Y")
@@ -123,7 +126,7 @@ def load(config_file):
import config, planet
from planet import opml, foaf
- log = planet.getLogger(config.log_level())
+ log = planet.getLogger(config.log_level(),config.log_format())
# Theme support
theme = config.output_theme()
@@ -146,10 +149,11 @@ def load(config_file):
# complete search list for theme directories
dirs += [os.path.join(theme_dir,dir) for dir in
- config.template_directories()]
+ config.template_directories() if dir not in dirs]
# merge configurations, allowing current one to override theme
template_files = config.template_files()
+ parser.set('Planet','template_files','')
parser.read(config_file)
for file in config.bill_of_materials():
if not file in bom: bom.append(file)
@@ -334,7 +338,8 @@ def filters(section=None):
def planet_options():
""" dictionary of planet wide options"""
- return dict(map(lambda opt: (opt, parser.get('Planet',opt)),
+ return dict(map(lambda opt: (opt,
+ parser.get('Planet', opt, raw=(opt=="log_format"))),
parser.options('Planet')))
def feed_options(section):
diff --git a/planet/shell/__init__.py b/planet/shell/__init__.py
index bd0e005..8a6fcfa 100644
--- a/planet/shell/__init__.py
+++ b/planet/shell/__init__.py
@@ -6,7 +6,7 @@ logged_modes = []
def run(template_file, doc, mode='template'):
""" select a template module based on file extension and execute it """
- log = planet.getLogger(planet.config.log_level())
+ log = planet.getLogger(planet.config.log_level(),planet.config.log_format())
if mode == 'template':
dirs = planet.config.template_directories()
diff --git a/planet/spider.py b/planet/spider.py
index bf94d48..ce473ee 100644
--- a/planet/spider.py
+++ b/planet/spider.py
@@ -31,6 +31,8 @@ def filename(directory, filename):
filename=filename.encode('idna')
except:
pass
+ if isinstance(filename,unicode):
+ filename=filename.encode('utf-8')
filename = re_url_scheme.sub("", filename)
filename = re_slash.sub(",", filename)
filename = re_initial_cruft.sub("", filename)
@@ -307,7 +309,7 @@ def spiderFeed(feed, only_if_new=0):
def spiderPlanet(only_if_new = False):
""" Spider (fetch) an entire planet """
- log = planet.getLogger(config.log_level())
+ log = planet.getLogger(config.log_level(),config.log_format())
planet.setTimeout(config.feed_timeout())
global index
diff --git a/planet/splice.py b/planet/splice.py
index e95d57f..e50f927 100644
--- a/planet/splice.py
+++ b/planet/splice.py
@@ -9,7 +9,7 @@ from planet import idindex
def splice():
""" Splice together a planet from a cache of entries """
import planet
- log = planet.getLogger(config.log_level())
+ log = planet.getLogger(config.log_level(),config.log_format())
log.info("Loading cached data")
cache = config.cache_directory()
@@ -97,7 +97,7 @@ def splice():
def apply(doc):
output_dir = config.output_dir()
if not os.path.exists(output_dir): os.makedirs(output_dir)
- log = planet.getLogger(config.log_level())
+ log = planet.getLogger(config.log_level(),config.log_format())
# Go-go-gadget-template
for template_file in config.template_files():
diff --git a/runtests.py b/runtests.py
index 3ebe331..1e866cb 100755
--- a/runtests.py
+++ b/runtests.py
@@ -23,7 +23,7 @@ modules = map(fullmodname, glob.glob(os.path.join('tests', 'test_*.py')))
# enable warnings
import planet
-planet.getLogger("WARNING")
+planet.getLogger("WARNING",None)
# load all of the tests into a suite
try:
diff --git a/tests/capture.py b/tests/capture.py
index 1e577b6..0ea3473 100755
--- a/tests/capture.py
+++ b/tests/capture.py
@@ -18,7 +18,7 @@ os.chdir(sys.path[0])
# copy spider output to splice input
import planet
from planet import spider, config
-planet.getLogger('CRITICAL')
+planet.getLogger('CRITICAL',None)
config.load('tests/data/spider/config.ini')
spider.spiderPlanet()
diff --git a/tests/test_idindex.py b/tests/test_idindex.py
index d27cf18..0de818d 100644
--- a/tests/test_idindex.py
+++ b/tests/test_idindex.py
@@ -8,6 +8,15 @@ class idIndexTest(unittest.TestCase):
def tearDown(self):
idindex.destroy()
+ def test_unicode(self):
+ from planet.spider import filename
+ index = idindex.create()
+ iri = 'http://www.\xe8\xa9\xb9\xe5\xa7\x86\xe6\x96\xaf.com/'
+ index[filename('', iri)] = 'data'
+ index[filename('', iri.decode('utf-8'))] = 'data'
+ index[filename('', u'1234')] = 'data'
+ index.close()
+
def test_index_spider(self):
import test_spider
config.load(test_spider.configfile)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 01e730a..1230c23 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -13,7 +13,7 @@ class SpiderTest(unittest.TestCase):
def setUp(self):
# silence errors
planet.logger = None
- planet.getLogger('CRITICAL')
+ planet.getLogger('CRITICAL',None)
try:
os.makedirs(workdir)
diff --git a/tests/test_themes.py b/tests/test_themes.py
index 4f171ed..de70c59 100644
--- a/tests/test_themes.py
+++ b/tests/test_themes.py
@@ -4,7 +4,7 @@ import unittest
from planet import config
from os.path import split
-class ConfigTest(unittest.TestCase):
+class ThemesTest(unittest.TestCase):
def setUp(self):
config.load('tests/data/config/themed.ini')
@@ -17,7 +17,8 @@ class ConfigTest(unittest.TestCase):
# administrivia
def test_template(self):
- self.assertTrue('index.html.xslt' in config.template_files())
+ self.assertEqual(1, len([1 for file in config.template_files()
+ if file == 'index.html.xslt']))
def test_feeds(self):
feeds = config.subscriptions()