v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-16 22:30:09 -07:00
Code Issues Releases Wiki Activity

gc

Browse Source
This commit is contained in:
Your Name 2017-09-15 15:46:54 +00:00
parent 2bdfc6e0a3
commit acde8d3d8d
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
downloader.py
View File

@ -2,6 +2,7 @@ import json
import time, sys import time, sys
from hashlib import md5 from hashlib import md5
from datetime import datetime from datetime import datetime
import gc
from twisted.web import server, resource from twisted.web import server, resource
from twisted.internet import reactor, endpoints, defer from twisted.internet import reactor, endpoints, defer
@ -45,6 +46,16 @@ def check_feed_request_time_limit(url):
r.set(url, int(time.time())) r.set(url, int(time.time()))
return 0 return 0
GC_PERIOD_SECONDS = 3 * 60 * 60 # 3 hours
def periodical_garbage_collect():
tm = int(time.time())
if tm - periodical_garbage_collect.time >= GC_PERIOD_SECONDS:
print('GC: the number of unreachable objects: %s' % gc.collect())
periodical_garbage_collect.time = tm
periodical_garbage_collect.time = int(time.time())
agent = BrowserLikeRedirectAgent( agent = BrowserLikeRedirectAgent(
Agent(reactor, Agent(reactor,
contextFactory=ScrapyClientContextFactory(), # skip certificate verification contextFactory=ScrapyClientContextFactory(), # skip certificate verification
@ -157,6 +168,8 @@ def downloadDone(response_str, request, response, feed_config):
request.write(response_str) request.write(response_str)
request.finish() request.finish()
periodical_garbage_collect()
def error_html(msg): def error_html(msg):
return "<html><body>%s</body></html" % escape(msg).replace("\n", "<br/>\n") return "<html><body>%s</body></html" % escape(msg).replace("\n", "<br/>\n")