config: log_format

only run templates once
ensure idindexes are strings, not unicode

docs/config.html

@@ -90,6 +90,10 @@ number of days will be marked as inactive</dd>
 
 <dt>log_level</dt>
 <dd>One of <code>DEBUG</code>, <code>INFO</code>, <code>WARNING</code>, <code>ERROR</code> or <code>CRITICAL</code></dd>
+<dt><ins>log_format</ins></dt>
+<dd><a href="http://docs.python.org/lib/node422.html">format string</a> to
+use for logging output.  Note: this configuration value is processed
+<a href="http://docs.python.org/lib/ConfigParser-objects.html">raw</a></dd>
 <dt>feed_timeout</dt>
 <dd>Number of seconds to wait for any given feed</dd>
 <dt><del>new_feed_items</del></dt>

planet.py

@@ -50,7 +50,7 @@ if __name__ == "__main__":
 
     if verbose:
         import planet
-        planet.getLogger('DEBUG')
+        planet.getLogger('DEBUG',config.log_format())
 
     if not offline:
         from planet import spider

planet/__init__.py

@@ -9,7 +9,7 @@ config.__init__()
 from ConfigParser import ConfigParser
 from urlparse import urljoin
 
-def getLogger(level):
+def getLogger(level, format):
     """ get a logger with the specified log level """
     global logger
     if logger: return logger
@@ -19,7 +19,7 @@ def getLogger(level):
     except:
         import compat_logging as logging
 
-    logging.basicConfig()
+    logging.basicConfig(format=format)
     logging.getLogger().setLevel(logging.getLevelName(level))
     logger = logging.getLogger("planet.runner")
     try:

planet/compat_logging/__init__.py

@@ -1090,7 +1090,7 @@ Logger.manager = Manager(Logger.root)
 
 BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s"
 
-def basicConfig():
+def basicConfig(format=BASIC_FORMAT):
     """
     Do basic configuration for the logging system by creating a
     StreamHandler with a default Formatter and adding it to the
@@ -1098,7 +1098,7 @@ def basicConfig():
     """
     if len(root.handlers) == 0:
         hdlr = StreamHandler()
-        fmt = Formatter(BASIC_FORMAT)
+        fmt = Formatter(format)
         hdlr.setFormatter(fmt)
         root.addHandler(hdlr)
 

planet/config.py

@@ -43,6 +43,8 @@ def __init__():
         if section and parser.has_option(section, option):
             return parser.get(section, option)
         elif parser.has_option('Planet', option):
+            if option == 'log_format':
+                return parser.get('Planet', option, raw=True)
             return parser.get('Planet', option)
         else:
             return default
@@ -88,6 +90,7 @@ def __init__():
     define_planet('link', '')
     define_planet('cache_directory', "cache")
     define_planet('log_level', "WARNING")
+    define_planet('log_format', "%(levelname)s:%(name)s:%(message)s")
     define_planet('feed_timeout', 20)
     define_planet('date_format', "%B %d, %Y %I:%M %p")
     define_planet('new_date_format', "%B %d, %Y")
@@ -123,7 +126,7 @@ def load(config_file):
 
     import config, planet
     from planet import opml, foaf
-    log = planet.getLogger(config.log_level())
+    log = planet.getLogger(config.log_level(),config.log_format())
 
     # Theme support
     theme = config.output_theme()
@@ -146,10 +149,11 @@ def load(config_file):
 
                 # complete search list for theme directories
                 dirs += [os.path.join(theme_dir,dir) for dir in 
-                    config.template_directories()]
+                    config.template_directories() if dir not in dirs]
 
                 # merge configurations, allowing current one to override theme
                 template_files = config.template_files()
+                parser.set('Planet','template_files','')
                 parser.read(config_file)
                 for file in config.bill_of_materials():
                     if not file in bom: bom.append(file)
@@ -334,7 +338,8 @@ def filters(section=None):
 
 def planet_options():
     """ dictionary of planet wide options"""
-    return dict(map(lambda opt: (opt, parser.get('Planet',opt)),
+    return dict(map(lambda opt: (opt,
+        parser.get('Planet', opt, raw=(opt=="log_format"))),
         parser.options('Planet')))
 
 def feed_options(section):

planet/shell/__init__.py

@@ -6,7 +6,7 @@ logged_modes = []
 
 def run(template_file, doc, mode='template'):
     """ select a template module based on file extension and execute it """
-    log = planet.getLogger(planet.config.log_level())
+    log = planet.getLogger(planet.config.log_level(),planet.config.log_format())
 
     if mode == 'template':
         dirs = planet.config.template_directories()

planet/spider.py

@@ -31,6 +31,8 @@ def filename(directory, filename):
                 filename=filename.encode('idna')
     except:
         pass
+    if isinstance(filename,unicode):
+        filename=filename.encode('utf-8')
     filename = re_url_scheme.sub("", filename)
     filename = re_slash.sub(",", filename)
     filename = re_initial_cruft.sub("", filename)
@@ -307,7 +309,7 @@ def spiderFeed(feed, only_if_new=0):
 
 def spiderPlanet(only_if_new = False):
     """ Spider (fetch) an entire planet """
-    log = planet.getLogger(config.log_level())
+    log = planet.getLogger(config.log_level(),config.log_format())
     planet.setTimeout(config.feed_timeout())
 
     global index

planet/splice.py

@@ -9,7 +9,7 @@ from planet import idindex
 def splice():
     """ Splice together a planet from a cache of entries """
     import planet
-    log = planet.getLogger(config.log_level())
+    log = planet.getLogger(config.log_level(),config.log_format())
 
     log.info("Loading cached data")
     cache = config.cache_directory()
@@ -97,7 +97,7 @@ def splice():
 def apply(doc):
     output_dir = config.output_dir()
     if not os.path.exists(output_dir): os.makedirs(output_dir)
-    log = planet.getLogger(config.log_level())
+    log = planet.getLogger(config.log_level(),config.log_format())
 
     # Go-go-gadget-template
     for template_file in config.template_files():

runtests.py

@@ -23,7 +23,7 @@ modules = map(fullmodname, glob.glob(os.path.join('tests', 'test_*.py')))
 
 # enable warnings
 import planet
-planet.getLogger("WARNING")
+planet.getLogger("WARNING",None)
 
 # load all of the tests into a suite
 try:

tests/capture.py

@@ -18,7 +18,7 @@ os.chdir(sys.path[0])
 # copy spider output to splice input
 import planet
 from planet import spider, config
-planet.getLogger('CRITICAL')
+planet.getLogger('CRITICAL',None)
 
 config.load('tests/data/spider/config.ini')
 spider.spiderPlanet()

tests/test_idindex.py

@@ -8,6 +8,15 @@ class idIndexTest(unittest.TestCase):
     def tearDown(self):
         idindex.destroy()
 
+    def test_unicode(self):
+        from planet.spider import filename
+        index = idindex.create()
+        iri = 'http://www.\xe8\xa9\xb9\xe5\xa7\x86\xe6\x96\xaf.com/'
+        index[filename('', iri)] = 'data'
+        index[filename('', iri.decode('utf-8'))] = 'data'
+        index[filename('', u'1234')] = 'data'
+        index.close()
+        
     def test_index_spider(self):
         import test_spider
         config.load(test_spider.configfile)

tests/test_spider.py

@@ -13,7 +13,7 @@ class SpiderTest(unittest.TestCase):
     def setUp(self):
         # silence errors
         planet.logger = None
-        planet.getLogger('CRITICAL')
+        planet.getLogger('CRITICAL',None)
 
         try:
              os.makedirs(workdir)

tests/test_themes.py

@@ -4,7 +4,7 @@ import unittest
 from planet import config
 from os.path import split
 
-class ConfigTest(unittest.TestCase):
+class ThemesTest(unittest.TestCase):
     def setUp(self):
         config.load('tests/data/config/themed.ini')
 
@@ -17,7 +17,8 @@ class ConfigTest(unittest.TestCase):
     # administrivia
 
     def test_template(self):
-        self.assertTrue('index.html.xslt' in config.template_files())
+        self.assertEqual(1, len([1 for file in config.template_files()
+            if file == 'index.html.xslt']))
 
     def test_feeds(self):
         feeds = config.subscriptions()