v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-29 04:20:09 -07:00
Code Issues Releases Wiki Activity

invalid headers: no chanked or content-length

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-07-20 14:21:50 -07:00
parent 615215ff16
commit b3140827b9
1 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
downloader.py
View File

@ -4,7 +4,7 @@ from datetime import datetime
from twisted.web import server, resource from twisted.web import server, resource
from twisted.internet import reactor, endpoints from twisted.internet import reactor, endpoints
from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError
from twisted.web.server import NOT_DONE_YET from twisted.web.server import NOT_DONE_YET
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
twisted_headers = Headers twisted_headers = Headers
@ -106,15 +106,15 @@ def buildScrapyResponse(response, body, url):
respcls = responsetypes.from_args(headers=headers, url=url) respcls = responsetypes.from_args(headers=headers, url=url)
return respcls(url=url, status=status, headers=headers, body=body) return respcls(url=url, status=status, headers=headers, body=body)
def downloadStarted(response, response_ref): def downloadStarted(response, request, url, feed_config):
response_ref.append(response) # seve the response reference d = readBody(response)
d.addCallback(downloadDone, request=request, response=response, feed_config=feed_config)
d.addErrback(downloadError, request=request, url=url, response=response, feed_config=feed_config)
return response return response
def downloadDone(response_str, request, response_ref, feed_config): def downloadDone(response_str, request, response, feed_config):
response = response_ref.pop() # get the response reference
url = response.request.absoluteURI url = response.request.absoluteURI
print 'Response <%s> ready (%s bytes)' % (url, len(response_str)) print 'Response <%s> ready (%s bytes)' % (url, len(response_str))
response = buildScrapyResponse(response, response_str, url) response = buildScrapyResponse(response, response_str, url)
@ -130,7 +130,12 @@ def downloadDone(response_str, request, response_ref, feed_config):
request.write(response_str) request.write(response_str)
request.finish() request.finish()
def downloadError(error, request=None, url=None): def downloadError(error, request=None, url=None, response=None, feed_config=None):
# read for details: https://stackoverflow.com/questions/29423986/twisted-giving-twisted-web-client-partialdownloaderror-200-ok
if error.type is PartialDownloadError and error.value.status == '200':
downloadDone(error.value.response, request, response, feed_config)
return
if DEBUG: if DEBUG:
request.write('Downloader error: ' + error.getErrorMessage()) request.write('Downloader error: ' + error.getErrorMessage())
request.write('Traceback: ' + error.getTraceback()) request.write('Traceback: ' + error.getTraceback())
@ -158,9 +163,7 @@ class Downloader(resource.Resource):
) )
print 'Request <GET %s> started' % (url,) print 'Request <GET %s> started' % (url,)
response_ref = [] response_ref = []
d.addCallback(downloadStarted, response_ref) d.addCallback(downloadStarted, request=request, url=url, feed_config=feed_config)
d.addCallback(readBody)
d.addCallback(downloadDone, request=request, response_ref=response_ref, feed_config=feed_config)
d.addErrback(downloadError, request=request, url=url) d.addErrback(downloadError, request=request, url=url)
def render_POST(self, request): def render_POST(self, request):