v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-31 21:40:09 -07:00
Code Issues Releases Wiki Activity

mem leak in progress

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-09-18 20:19:43 +03:00
parent 82517931a8
commit 3f1fec7cfa
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
downloader.py
View File

@ -7,7 +7,7 @@ import gc
from twisted.web import server, resource from twisted.web import server, resource
from twisted.internet import reactor, endpoints, defer from twisted.internet import reactor, endpoints, defer
from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError from twisted.web.client import Agent, BrowserLikeRedirectAgent, readBody, PartialDownloadError, HTTPConnectionPool
from twisted.web.server import NOT_DONE_YET from twisted.web.server import NOT_DONE_YET
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.html import escape from twisted.web.html import escape
@ -122,6 +122,9 @@ def stats_str(stat):
return None #s # not changed return None #s # not changed
def pgc(none): # periodical_garbage_collect def pgc(none): # periodical_garbage_collect
global pool
pool.closeCachedConnections()
tm = int(time.time()) tm = int(time.time())
if tm - pgc.time >= GC_PERIOD_SECONDS: if tm - pgc.time >= GC_PERIOD_SECONDS:
print('GC: COLLECTED: %s' % gc.collect()) print('GC: COLLECTED: %s' % gc.collect())
@ -197,18 +200,23 @@ pgc.prev_size = None
pgc.hist_ids = [] pgc.hist_ids = []
pgc.ids = [] pgc.ids = []
pgc.id_types = [ pgc.id_types = [
#"<type 'function'>", "<type 'instance'>",
#"<type 'instancemethod'>",
#"<type 'cell'>", #"<type 'cell'>",
"<class 'twisted.logger._logger.Logger'>", #"<class 'twisted.logger._logger.Logger'>",
#"<type 'tuple'>" #"<type 'tuple'>"
] ]
pgc.time = int(time.time()) pgc.time = int(time.time())
pool = HTTPConnectionPool(reactor, persistent=False)
pool.cachedConnectionTimeout = 3
agent = BrowserLikeRedirectAgent( agent = BrowserLikeRedirectAgent(
Agent(reactor, Agent(reactor,
contextFactory=ScrapyClientContextFactory(), # skip certificate verification contextFactory=ScrapyClientContextFactory(), # skip certificate verification
connectTimeout=10), connectTimeout=10,
pool=pool),
redirectLimit=5 redirectLimit=5
) )
@ -391,6 +399,10 @@ class Downloader(resource.Resource):
else: # neither page and feed else: # neither page and feed
return 'Url is required' return 'Url is required'
from twisted.logger import globalLogPublisher
globalLogPublisher.addObserver(print)
port = sys.argv[1] if len(sys.argv) >= 2 else 1234 port = sys.argv[1] if len(sys.argv) >= 2 else 1234
endpoints.serverFromString(reactor, "tcp:%s" % port).listen(server.Site(Downloader())) endpoints.serverFromString(reactor, "tcp:%s" % port).listen(server.Site(Downloader()))