From 7dd301cdf48089bbfb686ef6c9d8b6f2004772b3 Mon Sep 17 00:00:00 2001 From: Joe Gregorio Date: Wed, 27 Dec 2006 01:25:06 -0500 Subject: [PATCH 1/2] Moved creation of cache directory so that it is created consistently --- planet/config.py | 2 ++ planet/spider.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/planet/config.py b/planet/config.py index cc12ecc..8958623 100644 --- a/planet/config.py +++ b/planet/config.py @@ -288,6 +288,8 @@ def http_cache_directory(): if parser.has_option('Planet', 'http_cache_directory'): os.path.join(cache_directory(), parser.get('Planet', 'http_cache_directory')) + else: + return os.path.join(cache_directory(), "cache") def cache_sources_directory(): if parser.has_option('Planet', 'cache_sources_directory'): diff --git a/planet/spider.py b/planet/spider.py index 6630be9..ab09e80 100644 --- a/planet/spider.py +++ b/planet/spider.py @@ -355,11 +355,13 @@ def spiderPlanet(only_if_new = False): parse_queue = Queue() threads = {} - if int(config.spider_threads()): - http_cache = config.http_cache_directory() - if http_cache and not os.path.exists(http_cache): - os.makedirs(http_cache, 0700) + http_cache = config.http_cache_directory() + # Should this be done in config? + if http_cache and not os.path.exists(http_cache): + os.makedirs(http_cache) + + if int(config.spider_threads()): # Start all the worker threads for i in range(int(config.spider_threads())): threads[i] = Thread(target=httpThread, From 51c80375199f4db6db168c96c1649b5b87266791 Mon Sep 17 00:00:00 2001 From: Joe Gregorio Date: Wed, 27 Dec 2006 01:30:32 -0500 Subject: [PATCH 2/2] Fixed unit tests, which now reflect the http cache directory being created. --- tests/test_spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 7bc24ed..ecf5986 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -83,7 +83,7 @@ class SpiderTest(unittest.TestCase): files = glob.glob(workdir+"/*") # verify that exactly eight files + 1 source dir were produced - self.assertEqual(13, len(files)) + self.assertEqual(14, len(files)) # verify that the file names are as expected self.assertTrue(os.path.join(workdir,