v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-29 12:30:09 -07:00
Code Issues Releases Wiki Activity

disable caching

Browse Source
This commit is contained in:
Al Ne 2018-07-06 21:08:04 +03:00
parent 0d8ec7a7b5
commit 33afd5f9ce
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pol/server.py
View File

@ -108,7 +108,10 @@ class Downloader(object):
tree = selector.root.getroottree() tree = selector.root.getroottree()
file_name = self._saveResponse(headers, url, tree) if self.snapshot_dir:
file_name = self._saveResponse(headers, url, tree)
else:
file_name = 'DISABLED'
# set base url to html document # set base url to html document
head = tree.xpath("//head") head = tree.xpath("//head")
@ -135,7 +138,7 @@ class Downloader(object):
)) ))
body[0].append(script) body[0].append(script)
return (etree.tostring(tree, method='html'), file_name) return etree.tostring(tree, method='html')
def buildScrapyResponse(self, response, body, url): def buildScrapyResponse(self, response, body, url):
status = response.code status = response.code
@ -241,7 +244,7 @@ class Downloader(object):
if self.stat_tool: if self.stat_tool:
self.stat_tool.trace(ip=ip, feed_id=self.feed_config['id'], post_cnt=post_cnt, new_post_cnt=new_post_cnt) self.stat_tool.trace(ip=ip, feed_id=self.feed_config['id'], post_cnt=post_cnt, new_post_cnt=new_post_cnt)
else: else:
response_str, file_name = self.setBaseAndRemoveScriptsAndMore(selector, headers, url) response_str = self.setBaseAndRemoveScriptsAndMore(selector, headers, url)
if self.stat_tool: if self.stat_tool:
self.stat_tool.trace(ip=ip, feed_id=0, post_cnt=0, new_post_cnt=0, url=url) self.stat_tool.trace(ip=ip, feed_id=0, post_cnt=0, new_post_cnt=0, url=url)
return response_str return response_str