v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-16 06:10:09 -07:00
Code Issues Releases Wiki Activity

gc

Browse Source
This commit is contained in:
Your Name 2017-09-15 15:46:54 +00:00
parent 2bdfc6e0a3
commit acde8d3d8d
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
downloader.py
View File

@ -2,6 +2,7 @@ import json
import time, sys
from hashlib import md5
from datetime import datetime
import gc
from twisted.web import server, resource
from twisted.internet import reactor, endpoints, defer
@ -45,6 +46,16 @@ def check_feed_request_time_limit(url):
r.set(url, int(time.time()))
return 0
GC_PERIOD_SECONDS = 3 * 60 * 60 # 3 hours
def periodical_garbage_collect():
tm = int(time.time())
if tm - periodical_garbage_collect.time >= GC_PERIOD_SECONDS:
print('GC: the number of unreachable objects: %s' % gc.collect())
periodical_garbage_collect.time = tm
periodical_garbage_collect.time = int(time.time())
agent = BrowserLikeRedirectAgent(
Agent(reactor,
contextFactory=ScrapyClientContextFactory(), # skip certificate verification
@ -156,6 +167,8 @@ def downloadDone(response_str, request, response, feed_config):
request.write(response_str)
request.finish()
periodical_garbage_collect()
def error_html(msg):
return "<html><body>%s</body></html" % escape(msg).replace("\n", "<br/>\n")