tsr/planet
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Pull in Joe's fix for to consistently create the http cache directory

Browse Source
This commit is contained in:
Sam Ruby 2006-12-27 13:38:25 -05:00
commit 9d7627781c
3 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
planet/config.py
View File

@ -294,6 +294,8 @@ def http_cache_directory():
if parser.has_option('Planet', 'http_cache_directory'): if parser.has_option('Planet', 'http_cache_directory'):
os.path.join(cache_directory(), os.path.join(cache_directory(),
parser.get('Planet', 'http_cache_directory')) parser.get('Planet', 'http_cache_directory'))
else:
return os.path.join(cache_directory(), "cache")
def cache_sources_directory(): def cache_sources_directory():
if parser.has_option('Planet', 'cache_sources_directory'): if parser.has_option('Planet', 'cache_sources_directory'):

10
planet/spider.py
View File

@ -355,11 +355,13 @@ def spiderPlanet(only_if_new = False):
parse_queue = Queue() parse_queue = Queue()
threads = {} threads = {}
if int(config.spider_threads()): http_cache = config.http_cache_directory()
http_cache = config.http_cache_directory() # Should this be done in config?
if http_cache and not os.path.exists(http_cache): if http_cache and not os.path.exists(http_cache):
os.makedirs(http_cache, 0700) os.makedirs(http_cache)
if int(config.spider_threads()):
# Start all the worker threads # Start all the worker threads
for i in range(int(config.spider_threads())): for i in range(int(config.spider_threads())):
threads[i] = Thread(target=httpThread, threads[i] = Thread(target=httpThread,

2
tests/test_spider.py
View File

@ -83,7 +83,7 @@ class SpiderTest(unittest.TestCase):
files = glob.glob(workdir+"/*") files = glob.glob(workdir+"/*")
# verify that exactly eight files + 1 source dir were produced # verify that exactly eight files + 1 source dir were produced
self.assertEqual(13, len(files)) self.assertEqual(14, len(files))
# verify that the file names are as expected # verify that the file names are as expected
self.assertTrue(os.path.join(workdir, self.assertTrue(os.path.join(workdir,