Load configuration only once
This commit is contained in:
parent
5d7bd64342
commit
c610c93720
@ -41,14 +41,17 @@ if __name__ == "__main__":
|
|||||||
else:
|
else:
|
||||||
config_file = arg
|
config_file = arg
|
||||||
|
|
||||||
|
from planet import config
|
||||||
|
config.load(config_file)
|
||||||
|
|
||||||
if verbose:
|
if verbose:
|
||||||
import planet
|
import planet
|
||||||
planet.getLogger('DEBUG')
|
planet.getLogger('DEBUG')
|
||||||
|
|
||||||
if not offline:
|
if not offline:
|
||||||
from planet import spider
|
from planet import spider
|
||||||
spider.spiderPlanet(config_file)
|
spider.spiderPlanet()
|
||||||
|
|
||||||
from planet import splice
|
from planet import splice
|
||||||
doc = splice.splice(config_file)
|
doc = splice.splice()
|
||||||
splice.apply(doc.toxml('utf-8'))
|
splice.apply(doc.toxml('utf-8'))
|
||||||
|
@ -180,9 +180,8 @@ def spiderFeed(feed):
|
|||||||
write(output, cache_file)
|
write(output, cache_file)
|
||||||
os.utime(cache_file, (mtime, mtime))
|
os.utime(cache_file, (mtime, mtime))
|
||||||
|
|
||||||
def spiderPlanet(configFile):
|
def spiderPlanet():
|
||||||
""" Spider (fetch) an entire planet """
|
""" Spider (fetch) an entire planet """
|
||||||
config.load(configFile)
|
|
||||||
log = planet.getLogger(config.log_level())
|
log = planet.getLogger(config.log_level())
|
||||||
planet.setTimeout(config.feed_timeout())
|
planet.setTimeout(config.feed_timeout())
|
||||||
|
|
||||||
|
@ -5,10 +5,9 @@ import planet, config, feedparser, reconstitute, shell
|
|||||||
from reconstitute import createTextElement, date
|
from reconstitute import createTextElement, date
|
||||||
from spider import filename
|
from spider import filename
|
||||||
|
|
||||||
def splice(configFile):
|
def splice():
|
||||||
""" Splice together a planet from a cache of entries """
|
""" Splice together a planet from a cache of entries """
|
||||||
import planet
|
import planet
|
||||||
config.load(configFile)
|
|
||||||
log = planet.getLogger(config.log_level())
|
log = planet.getLogger(config.log_level())
|
||||||
|
|
||||||
log.info("Loading cached data")
|
log.info("Loading cached data")
|
||||||
|
@ -8,12 +8,13 @@ from planet import spider, config
|
|||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
config.load(sys.argv[1])
|
||||||
|
|
||||||
if len(sys.argv) == 2:
|
if len(sys.argv) == 2:
|
||||||
# spider all feeds
|
# spider all feeds
|
||||||
spider.spiderPlanet(sys.argv[1])
|
spider.spiderPlanet()
|
||||||
elif len(sys.argv) > 2:
|
elif len(sys.argv) > 2:
|
||||||
# spider selected feeds
|
# spider selected feeds
|
||||||
config.load(sys.argv[1])
|
|
||||||
for feed in sys.argv[2:]:
|
for feed in sys.argv[2:]:
|
||||||
spider.spiderFeed(feed)
|
spider.spiderFeed(feed)
|
||||||
else:
|
else:
|
||||||
|
@ -5,14 +5,13 @@ Main program to run just the splice portion of planet
|
|||||||
|
|
||||||
import os.path
|
import os.path
|
||||||
import sys
|
import sys
|
||||||
from planet import splice
|
from planet import splice, config
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
||||||
if len(sys.argv) == 2 and os.path.isfile(sys.argv[1]):
|
if len(sys.argv) == 2 and os.path.isfile(sys.argv[1]):
|
||||||
# at the moment, we don't have template support, so we cheat and
|
config.load(sys.argv[1])
|
||||||
# simply insert a XSLT processing instruction
|
doc = splice.splice()
|
||||||
doc = splice.splice(sys.argv[1])
|
|
||||||
splice.apply(doc.toxml('utf-8'))
|
splice.apply(doc.toxml('utf-8'))
|
||||||
else:
|
else:
|
||||||
print "Usage:"
|
print "Usage:"
|
||||||
|
@ -66,7 +66,8 @@ class SpiderTest(unittest.TestCase):
|
|||||||
self.test_spiderFeed()
|
self.test_spiderFeed()
|
||||||
|
|
||||||
def test_spiderPlanet(self):
|
def test_spiderPlanet(self):
|
||||||
spiderPlanet(configfile)
|
config.load(configfile)
|
||||||
|
spiderPlanet()
|
||||||
files = glob.glob(workdir+"/*")
|
files = glob.glob(workdir+"/*")
|
||||||
|
|
||||||
# verify that exactly eight files + 1 source dir were produced
|
# verify that exactly eight files + 1 source dir were produced
|
||||||
|
@ -1,14 +1,15 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from planet.splice import splice
|
from planet.splice import splice, config
|
||||||
|
|
||||||
configfile = 'tests/data/splice/config.ini'
|
configfile = 'tests/data/splice/config.ini'
|
||||||
|
|
||||||
class SpliceTest(unittest.TestCase):
|
class SpliceTest(unittest.TestCase):
|
||||||
|
|
||||||
def test_splice(self):
|
def test_splice(self):
|
||||||
doc = splice(configfile)
|
config.load(configfile)
|
||||||
|
doc = splice()
|
||||||
self.assertEqual(12,len(doc.getElementsByTagName('entry')))
|
self.assertEqual(12,len(doc.getElementsByTagName('entry')))
|
||||||
self.assertEqual(4,len(doc.getElementsByTagName('planet:source')))
|
self.assertEqual(4,len(doc.getElementsByTagName('planet:source')))
|
||||||
self.assertEqual(16,len(doc.getElementsByTagName('planet:name')))
|
self.assertEqual(16,len(doc.getElementsByTagName('planet:name')))
|
||||||
|
Loading…
x
Reference in New Issue
Block a user