v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-31 21:40:09 -07:00
Code Issues Releases Wiki Activity

sanitize

Browse Source
This commit is contained in:
Alexandr Nesterenko 2018-02-07 22:48:35 +03:00
parent 94b7e53a40
commit 6f97665b68
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pol/server.py
View File

@ -205,7 +205,7 @@ class Downloader(object):
self.writeResponse(sresponse) self.writeResponse(sresponse)
self.run_memon() self.run_memon()
def writeResponse(self, sresponse, response_str='PolitePol: Local page processing is failed'): def writeResponse(self, sresponse): #, response_str='PolitePol: Local page processing is failed'
sresponse = HttpCompressionMiddleware().process_response(Request(sresponse.url), sresponse, None) sresponse = HttpCompressionMiddleware().process_response(Request(sresponse.url), sresponse, None)
sresponse = DecompressionMiddleware().process_response(None, sresponse, None) sresponse = DecompressionMiddleware().process_response(None, sresponse, None)
@ -216,6 +216,8 @@ class Downloader(object):
response_str = self.prepare_response_str(sresponse.selector, sresponse.headers, sresponse.body_as_unicode(), sresponse.url, ip) response_str = self.prepare_response_str(sresponse.selector, sresponse.headers, sresponse.body_as_unicode(), sresponse.url, ip)
if self.feed_config: if self.feed_config:
response_headers = {b"Content-Type": b'text/xml; charset=utf-8'} response_headers = {b"Content-Type": b'text/xml; charset=utf-8'}
else: # images and such
response_str = sresponse.body
for k, v in response_headers.items(): for k, v in response_headers.items():
self.request.setHeader(k, v) self.request.setHeader(k, v)
@ -346,7 +348,7 @@ class Site(resource.Resource):
class Server(object): class Server(object):
def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None, prefetch_dir=None, feed=None, site=None): def __init__(self, port, db_creds, snapshot_dir, user_agent, debug=False, limiter=None, memon=None, stat_tool=None, prefetch_dir=None, feed=None, sitecls=None, downloadercls=None):
self.port = port self.port = port
self.db_creds = db_creds self.db_creds = db_creds
self.snapshot_dir = snapshot_dir self.snapshot_dir = snapshot_dir
@ -359,7 +361,10 @@ class Server(object):
self.log_handler = LogHandler() self.log_handler = LogHandler()
self.site = site or Site(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.memon, self.stat_tool, self.prefetch_dir, feed) if not sitecls:
sitecls = Site
self.site = sitecls(self.db_creds, self.snapshot_dir, self.user_agent, self.debug, self.limiter, self.memon, self.stat_tool, self.prefetch_dir, feed, downloadercls=downloadercls)
def requestSelector(self, url=None, feed_config=None): def requestSelector(self, url=None, feed_config=None):
d = defer.Deferred() d = defer.Deferred()