v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-28 03:50:08 -07:00
Code Issues Releases Wiki Activity

Merge branch 'encoding' of github.com:taroved/pol into exception_200

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-07-21 10:46:47 -07:00
commit 839d220ae7
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
downloader.py
View File

@ -7,6 +7,7 @@ from twisted.internet import reactor, endpoints
from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError
from twisted.web.server import NOT_DONE_YET
from twisted.web.http_headers import Headers
from twisted.web.html import escape
twisted_headers = Headers
from scrapy.http.response.text import TextResponse
@ -130,6 +131,9 @@ def downloadDone(response_str, request, response, feed_config):
request.write(response_str)
request.finish()
def error_html(msg):
return "<html><body>%s</body></html" % escape(msg).replace("\n", "<br/>\n")
def downloadError(error, request=None, url=None, response=None, feed_config=None):
# read for details: https://stackoverflow.com/questions/29423986/twisted-giving-twisted-web-client-partialdownloaderror-200-ok
if error.type is PartialDownloadError and error.value.status == '200':
@ -140,7 +144,7 @@ def downloadError(error, request=None, url=None, response=None, feed_config=None
request.write('Downloader error: ' + error.getErrorMessage())
request.write('Traceback: ' + error.getTraceback())
else:
request.write('Something wrong. Contact us by email: politepol.com@gmail.com . Geek comment: ' + error.getErrorMessage())
request.write(error_html('Something wrong. Contact us by email: politepol.com@gmail.com \n Scary mantra: ' + error.getErrorMessage()))
sys.stderr.write('\n'.join([str(datetime.now()), url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()]))
request.finish()