diff --git a/downloader.py b/downloader.py index a49e827..9de458e 100644 --- a/downloader.py +++ b/downloader.py @@ -7,6 +7,7 @@ from twisted.internet import reactor, endpoints from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError from twisted.web.server import NOT_DONE_YET from twisted.web.http_headers import Headers +from twisted.web.html import escape twisted_headers = Headers from scrapy.http.response.text import TextResponse @@ -130,6 +131,9 @@ def downloadDone(response_str, request, response, feed_config): request.write(response_str) request.finish() +def error_html(msg): + return "%s\n") + def downloadError(error, request=None, url=None, response=None, feed_config=None): # read for details: https://stackoverflow.com/questions/29423986/twisted-giving-twisted-web-client-partialdownloaderror-200-ok if error.type is PartialDownloadError and error.value.status == '200': @@ -140,7 +144,7 @@ def downloadError(error, request=None, url=None, response=None, feed_config=None request.write('Downloader error: ' + error.getErrorMessage()) request.write('Traceback: ' + error.getTraceback()) else: - request.write('Something wrong. Contact us by email: politepol.com@gmail.com . Geek comment: ' + error.getErrorMessage()) + request.write(error_html('Something wrong. Contact us by email: politepol.com@gmail.com \n Scary mantra: ' + error.getErrorMessage())) sys.stderr.write('\n'.join([str(datetime.now()), url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()])) request.finish()