v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-18 15:20:09 -07:00
Code Issues Releases Wiki Activity

prefetch_dir

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-11-03 13:34:48 +03:00
parent bdc115db2d
commit 08b916dc65
1 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
pol/server.py
View File

@ -225,11 +225,12 @@ class Site(resource.Resource):
feed_regexp = re.compile('^/feed1?/(\d{1,10})$') feed_regexp = re.compile('^/feed1?/(\d{1,10})$')
def __init__(self, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None): def __init__(self, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None, prefetch_dir=None):
self.db_creds = db_creds self.db_creds = db_creds
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
self.user_agent = user_agent self.user_agent = user_agent
self.limiter = limiter self.limiter = limiter
self.prefetch_dir = prefetch_dir
self.feed = Feed(db_creds) self.feed = Feed(db_creds)
self.downloader = Downloader(self.feed, debug, snapshot_dir, stat_tool, memon) self.downloader = Downloader(self.feed, debug, snapshot_dir, stat_tool, memon)
@ -238,7 +239,7 @@ class Site(resource.Resource):
sresponse = self.tryLocalPage(url) sresponse = self.tryLocalPage(url)
if sresponse: if sresponse:
if selector_defer: if selector_defer:
selector_defer.callback(sresponse) reactor.callLater(0, selector_defer.callback, sresponse)
else: else:
self.downloader.writeResponse(request, sresponse, feed_config) self.downloader.writeResponse(request, sresponse, feed_config)
else: else:
@ -265,13 +266,14 @@ class Site(resource.Resource):
d.addErrback(self.downloader.downloadError, request=request, url=url, feed_config=feed_config, selector_defer=selector_defer) d.addErrback(self.downloader.downloadError, request=request, url=url, feed_config=feed_config, selector_defer=selector_defer)
def tryLocalPage(self, url): def tryLocalPage(self, url):
m = md5(url).hexdigest() if self.prefetch_dir:
domain = urlparse(url).netloc m = md5(url).hexdigest()
try: domain = urlparse(url).netloc
with open('/home/taroved/pages/' + m + '.' + domain) as f: try:
return pickle.load(f) with open(self.prefetch_dir + '/' + m + '.' + domain) as f:
except IOError: return pickle.load(f)
return None except IOError:
pass
return None return None
def render_GET(self, request): def render_GET(self, request):
@ -306,7 +308,7 @@ class Site(resource.Resource):
class Server(object): class Server(object):
def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None): def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None, prefetch_dir=None):
self.port = port self.port = port
self.db_creds = db_creds self.db_creds = db_creds
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
@ -315,10 +317,11 @@ class Server(object):
self.limiter = limiter self.limiter = limiter
self.memon = memon self.memon = memon
self.stat_tool=stat_tool self.stat_tool=stat_tool
self.prefetch_dir = prefetch_dir
self.log_handler = LogHandler() self.log_handler = LogHandler()
self.site = Site(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.memon, self.stat_tool) self.site = Site(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.memon, self.stat_tool, self.prefetch_dir)
def requestSelector(self, url=None, feed_config=None): def requestSelector(self, url=None, feed_config=None):
d = defer.Deferred() d = defer.Deferred()