v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-18 07:10:09 -07:00
Code Issues Releases Wiki Activity

prem

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-10-04 15:13:03 -04:00
parent 6e9884e34a
commit a6e75a22a3
1 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pol/server.py
View File

@ -8,7 +8,7 @@ import re
from lxml import etree from lxml import etree
from twisted.web import server, resource from twisted.web import server, resource
from twisted.internet import reactor, endpoints from twisted.internet import reactor, endpoints, defer
from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError, HTTPConnectionPool from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError, HTTPConnectionPool
from twisted.web.server import NOT_DONE_YET from twisted.web.server import NOT_DONE_YET
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
@ -34,12 +34,12 @@ log = Logger()
class Downloader(object): class Downloader(object):
def __init__(self, feed, debug, snapshot_dir='/tmp', stat_tool=None, mem_mon=None): def __init__(self, feed, debug, snapshot_dir='/tmp', stat_tool=None, memon=None):
self.feed = feed self.feed = feed
self.debug = debug self.debug = debug
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
self.stat_tool = stat_tool self.stat_tool = stat_tool
self.mem_mon = mem_mon self.memon = memon
def html2json(self, el): def html2json(self, el):
return [ return [
@ -142,7 +142,7 @@ class Downloader(object):
request.write(self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage()))) request.write(self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage())))
sys.stderr.write('\n'.join([str(datetime.utcnow()), request.uri, url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()])) sys.stderr.write('\n'.join([str(datetime.utcnow()), request.uri, url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()]))
request.finish() request.finish()
try: try:
feed_id = feed_config and feed_config['id'] feed_id = feed_config and feed_config['id']
s_url = None s_url = None
@ -192,13 +192,13 @@ class Downloader(object):
request.write(response_str) request.write(response_str)
request.finish() request.finish()
self.run_mem_mon() self.run_memon()
def run_mem_mon(self): def run_memon(self):
if self.mem_mon: if self.memon:
d = defer.Deferred() d = defer.Deferred()
reactor.callLater(0, d.callback, None) reactor.callLater(0, d.callback, None)
d.addCallback(self.mem_mon.show_diff) d.addCallback(self.memon.show_diff)
d.addErrback(lambda err: print("Memory Monitor error: %s\nPGC traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) d.addErrback(lambda err: print("Memory Monitor error: %s\nPGC traceback: %s" % (err.getErrorMessage(), err.getTraceback())))
@ -207,14 +207,14 @@ class Site(resource.Resource):
feed_regexp = re.compile('^/feed1?/(\d{1,10})$') feed_regexp = re.compile('^/feed1?/(\d{1,10})$')
def __init__(self, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, mem_mon=None, stat_tool=None): def __init__(self, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None):
self.db_creds = db_creds self.db_creds = db_creds
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
self.user_agent = user_agent self.user_agent = user_agent
self.limiter = limiter self.limiter = limiter
self.feed = Feed(db_creds) self.feed = Feed(db_creds)
self.downloader = Downloader(self.feed, debug, snapshot_dir, stat_tool, mem_mon) self.downloader = Downloader(self.feed, debug, snapshot_dir, stat_tool, memon)
def startRequest(self, request, url, feed_config = None): def startRequest(self, request, url, feed_config = None):
agent = BrowserLikeRedirectAgent( agent = BrowserLikeRedirectAgent(
@ -237,7 +237,7 @@ class Site(resource.Resource):
) )
print('Request <GET %s> started' % (url,)) print('Request <GET %s> started' % (url,))
d.addCallback(self.downloader.downloadStarted, request=request, url=url, feed_config=feed_config) d.addCallback(self.downloader.downloadStarted, request=request, url=url, feed_config=feed_config)
d.addErrback(self.downloader.downloadError, request=request, url=url) d.addErrback(self.downloader.downloadError, request=request, url=url, feed_config=feed_config)
def render_GET(self, request): def render_GET(self, request):
''' '''
@ -271,17 +271,18 @@ class Site(resource.Resource):
class Server(object): class Server(object):
def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, mem_mon=None): def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None):
self.port = port self.port = port
self.db_creds = db_creds self.db_creds = db_creds
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
self.user_agent = user_agent self.user_agent = user_agent
self.debug = debug self.debug = debug
self.limiter = limiter self.limiter = limiter
self.mem_mon = mem_mon self.memon = memon
self.stat_tool=stat_tool
self.log_handler = LogHandler() self.log_handler = LogHandler()
def run(self): def run(self):
endpoints.serverFromString(reactor, "tcp:%s" % self.port).listen(server.Site(Site(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.mem_mon))) endpoints.serverFromString(reactor, "tcp:%s" % self.port).listen(server.Site(Site(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.memon, self.stat_tool)))
reactor.run() reactor.run()