v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-16 14:20:10 -07:00
Code Issues Releases Wiki Activity

remove debug code

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-09-21 19:53:19 +03:00
parent ec0f9c5ea4
commit 9e451d12e8
2 changed files with 65 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
downloader.py
View File

@ -46,8 +46,10 @@ def print_log(event):
if 'isError' in event and event['isError']: if 'isError' in event and event['isError']:
sys.stdout.write(bcolors.FAIL + formatEventAsClassicLogText(event) + bcolors.ENDC) sys.stdout.write(bcolors.FAIL + formatEventAsClassicLogText(event) + bcolors.ENDC)
sys.stderr.write(formatEventAsClassicLogText(event)) sys.stderr.write(formatEventAsClassicLogText(event))
sys.stderr.flush()
else: else:
sys.stdout.write(formatEventAsClassicLogText(event)) sys.stdout.write(formatEventAsClassicLogText(event))
sys.stdout.flush()
globalLogBeginner.beginLoggingTo([print_log], discardBuffer=True, redirectStandardIO=False) # requred, discardBuffer gets rid of the LimitedHistoryLogObserver, redirectStandardIO will loop print action globalLogBeginner.beginLoggingTo([print_log], discardBuffer=True, redirectStandardIO=False) # requred, discardBuffer gets rid of the LimitedHistoryLogObserver, redirectStandardIO will loop print action
@ -184,7 +186,9 @@ def downloadDone(response_str, request, response, feed_config):
request.write(response_str) request.write(response_str)
request.finish() request.finish()
#run_pgc()
def run_pgc():
d = defer.Deferred() d = defer.Deferred()
reactor.callLater(0, d.callback, None) reactor.callLater(0, d.callback, None)
d.addCallback(pgc) d.addCallback(pgc)

95
mlm.py
View File

@ -14,7 +14,7 @@ class bcolors:
BOLD = '\033[1m' BOLD = '\033[1m'
UNDERLINE = '\033[4m' UNDERLINE = '\033[4m'
GC_PERIOD_SECONDS = 3 * 60 * 60 # 3 hours GC_PERIOD_SECONDS = 1#3 * 60 * 60 # 3 hours
def is_hist_obj(tpe, _str_or_o): def is_hist_obj(tpe, _str_or_o):
for t in pgc.id_types: for t in pgc.id_types:
@ -24,26 +24,32 @@ def is_hist_obj(tpe, _str_or_o):
return True return True
return False return False
class Stat:
def __init__(self, count, size, objects):
self.count = count
self.size = size
self.objects = objects
def get_gc_stats(): def get_gc_stats():
go = {} go = {}
for o in gc.garbage: for o in gc.garbage:
tpe = str(type(o)) tpe = type_str(o)
if tpe not in go: if tpe not in go:
go[tpe] = [1, sys.getsizeof(o), []] go[tpe] = Stat(1, sys.getsizeof(o), [])
else: else:
go[tpe][0] += 1 go[tpe].count += 1
go[tpe][1] += sys.getsizeof(o) go[tpe].size += sys.getsizeof(o)
go[tpe][2].append([id(o), str(o)]) go[tpe].objects.append((id(o), str(o)))
allo = {} allo = {}
for o in gc.get_objects(): for o in gc.get_objects():
tpe = str(type(o)) tpe = type_str(o)
if tpe not in allo: if tpe not in allo:
allo[tpe] = [1, sys.getsizeof(o), []] allo[tpe] = Stat(1, sys.getsizeof(o), [])
else: else:
allo[tpe][0] += 1 allo[tpe].count += 1
allo[tpe][1] += sys.getsizeof(o) allo[tpe].size += sys.getsizeof(o)
if is_hist_obj(tpe, o): if is_hist_obj(tpe, o):
allo[tpe][2].append([id(o), str(o)[:180]]) allo[tpe].objects.append((id(o), str(o)[:180]))
return [go, allo] return [go, allo]
def stats_str(stat): def stats_str(stat):
@ -51,14 +57,14 @@ def stats_str(stat):
prev_diff = [0, 0] prev_diff = [0, 0]
if tpe in pgc.prev_stats: if tpe in pgc.prev_stats:
prev_diff[0] = count - pgc.prev_stats[tpe][0] prev_diff[0] = count - pgc.prev_stats[tpe].count
prev_diff[1] = size - pgc.prev_stats[tpe][1] prev_diff[1] = size - pgc.prev_stats[tpe].size
first_diff = [0, 0] first_diff = [0, 0]
if tpe in pgc.first_stats: if tpe in pgc.first_stats:
first_diff[0] = count - pgc.first_stats[tpe][0] first_diff[0] = count - pgc.first_stats[tpe].count
first_diff[1] = size - pgc.first_stats[tpe][1] first_diff[1] = size - pgc.first_stats[tpe].size
prev_count_sigh = '' prev_count_sigh = ''
if prev_diff[0] > 0: if prev_diff[0] > 0:
@ -79,8 +85,10 @@ def stats_str(stat):
s = "%s: %s,%s%s,%s%s %s,%s%s,%s%s" % (tpe, count, prev_count_sigh, prev_diff[0], first_count_sigh, first_diff[0], size, prev_size_sigh, prev_diff[1], first_size_sigh, first_diff[1]) s = "%s: %s,%s%s,%s%s %s,%s%s,%s%s" % (tpe, count, prev_count_sigh, prev_diff[0], first_count_sigh, first_diff[0], size, prev_size_sigh, prev_diff[1], first_size_sigh, first_diff[1])
if prev_diff[0] != 0 or prev_diff[1] != 0 or first_diff[0] != 0 or first_diff[1] != 0: if prev_diff[0] != 0 or prev_diff[1] != 0 or first_diff[0] != 0 or first_diff[1] != 0:
if prev_diff[0] != 0 or prev_diff[1] != 0: if prev_diff[0] > 0 or prev_diff[1] > 0:
return bcolors.WARNING + s + bcolors.ENDC return bcolors.WARNING + s + bcolors.ENDC
elif prev_diff[0] < 0 or prev_diff[1] < 0:
return bcolors.OKGREEN + s + bcolors.ENDC
else: else:
return bcolors.OKBLUE + s + bcolors.ENDC return bcolors.OKBLUE + s + bcolors.ENDC
else: else:
@ -96,7 +104,7 @@ def pgc(none): # periodical_garbage_collect
go, allo = get_gc_stats() go, allo = get_gc_stats()
log.info("GC: GARBAGE OBJECTS STATS (%s)" % len(go)) log.info("GC: GARBAGE OBJECTS STATS (%s)" % len(go))
for tpe, stats in sorted(go.iteritems(), key=lambda t: t[0]): for tpe, stats in sorted(go.iteritems(), key=lambda t: t[0]):
log.info("GC: %s: %s, %s" % (tpe, stats[0]. stats[1])) log.info("GC: %s: %s, %s" % (tpe, stats.count. stats.size))
log.info("GC: ALL OBJECTS STATS (%s)" % len(allo)) log.info("GC: ALL OBJECTS STATS (%s)" % len(allo))
@ -105,10 +113,11 @@ def pgc(none): # periodical_garbage_collect
size = 0 size = 0
cur_ids = [] cur_ids = []
cur_values = []
for tpe, stats in sorted(allo.iteritems(), key=lambda t: t[0]): for tpe, stats in sorted(allo.iteritems(), key=lambda t: t[0]):
scount = stats[0] scount = stats.count
ssize = stats[1] ssize = stats.size
objects = stats[2] objects = stats.objects
sstr = stats_str([tpe, scount, ssize]) sstr = stats_str([tpe, scount, ssize])
if sstr: if sstr:
log.info("GC: %s" % sstr) log.info("GC: %s" % sstr)
@ -116,24 +125,25 @@ def pgc(none): # periodical_garbage_collect
for _id, _str in objects: for _id, _str in objects:
if is_hist_obj(tpe, _str): if is_hist_obj(tpe, _str):
cur_ids.append(_id) cur_ids.append(_id)
cur_values.append(_str)
if not pgc.first_size: if not pgc.first_size:
pgc.first_size = size pgc.first_size = size
pgc.prev_size = size pgc.prev_size = size
log.info('GC: ALL OBJECT SIZE: %s,%s,%s' % (size, size - pgc.prev_size, size - pgc.first_size)) log.info('GC: ALL OBJECT SIZE: %s,%s,%s' % (size, size - pgc.prev_size, size - pgc.first_size))
if pgc.ids: if pgc.ids:
new_ids = []
for tpe_filter in pgc.id_types: for tpe_filter in pgc.id_types:
#import pdb;pdb.set_trace() #import pdb;pdb.set_trace()
if type(tpe_filter) is str: if type(tpe_filter) is str:
tpe = tpe_filter tpe = tpe_filter
else: else:
tpe = tpe_filter[0] tpe = tpe_filter[0]
objects = allo[tpe][2] objects = allo[tpe].objects
count = 0 count = 0
new_ids = []
for _id, _str in objects: for _id, _str in objects:
if is_hist_obj(tpe, _str): if is_hist_obj(tpe, _str) and _id not in pgc.ids and (not pgc.filter_by_value or _str not in pgc.values):
log.info('GC new obj %s(%s): %s' % (tpe, _id, _str)) log.info('GC new obj %s(%s): {str!r}' % (tpe, _id), str=_str)
count += 1 count += 1
new_ids.append(_id) new_ids.append(_id)
log.info('GC new obj %s: %s items' % (tpe, count)) log.info('GC new obj %s: %s items' % (tpe, count))
@ -146,13 +156,14 @@ def pgc(none): # periodical_garbage_collect
tpe = tpe_filter tpe = tpe_filter
else: else:
tpe = tpe_filter[0] tpe = tpe_filter[0]
objects = allo[tpe][2] objects = allo[tpe].objects
count = 0 count = 0
for _id, _str in objects: for _id, _str in objects:
if _id in ids and is_hist_obj(tpe, _str): if _id in ids and is_hist_obj(tpe, _str) and (not pgc.filter_by_value or _str not in pgc.values):
log.info('GC %s new obj %s(%s): %s' % (step, tpe, _id, _str)) log.info('GC %s new obj %s(%s): {str!r}' % (step, tpe, _id), str = _str)
count += 1 count += 1
step_ids.append(_id) step_ids.append(_id)
break
log.info('GC %s new obj %s: %s items' % (step, tpe, count)) log.info('GC %s new obj %s: %s items' % (step, tpe, count))
step -= 1 step -= 1
ids[:] = [] #clear list ids[:] = [] #clear list
@ -160,12 +171,14 @@ def pgc(none): # periodical_garbage_collect
if step_ids: if step_ids:
pgc.oldest_id = step_ids[-1] pgc.oldest_id = step_ids[-1]
pgc.hist_ids.insert(0, new_ids) pgc.hist_ids.insert(0, new_ids)
pgc.hist_ids[:] = pgc.hist_ids[0:3]
log.info('GC oldest id %s' % pgc.oldest_id) log.info('GC oldest id %s' % pgc.oldest_id)
if pgc.oldest_id: #if pgc.oldest_id:
print_obj_id_refs(pgc.oldest_id) # print_obj_id_refs(pgc.oldest_id)
pgc.ids = cur_ids pgc.ids = cur_ids
pgc.values = cur_values
pgc.prev_stats = allo pgc.prev_stats = allo
pgc.prev_size = size pgc.prev_size = size
@ -187,20 +200,20 @@ def print_obj_id_refs(o_id):
#print_obj_ref(0, (get_obj_by_id(o_id),)) #print_obj_ref(0, (get_obj_by_id(o_id),))
o = get_obj_by_id(o_id) o = get_obj_by_id(o_id)
refs = gc.get_referrers(o) refs = gc.get_referrers(o)
log.info('gc oldest obj cnt:%s %s(%s): %s' % (len(refs), str(type(o)), id(o), str(o)[:500])) log.info('gc oldest obj cnt:%s %s(%s): {str!r}' % (len(refs), str(type(o)), id(o)), {'str': str(o)[:500]})
#import types #import types
first = True first = True
for r in refs: for r in refs:
# import pdb;pdb.set_trace() # import pdb;pdb.set_trace()
log.info('gc oldest %s ref cnt:%s %s(%s): %s' % ('*', -1, str(type(r)), id(r), str(r)[:500].replace(hex(o_id), bcolors.WARNING + str(hex(o_id)) + bcolors.ENDC))) log.info('gc oldest %s ref cnt:%s %s(%s): {str!r}' % ('*', -1, str(type(r)), id(r)), {'str': str(r)[:500].replace(hex(o_id), bcolors.WARNING + str(hex(o_id)) + bcolors.ENDC)})
if first and type(r) is dict: if first and type(r) is dict:
refs2 = gc.get_referrers(r) refs2 = gc.get_referrers(r)
for r2 in refs2: for r2 in refs2:
log.info('gc oldest %s ref cnt:%s %s(%s): %s' % ('**', -2, str(type(r2)), id(r2), str(r2)[:500].replace(hex(id(r)), bcolors.WARNING + str(hex(id(r))) + bcolors.ENDC))) log.info('gc oldest %s ref cnt:%s %s(%s): {str!r}' % ('**', -2, str(type(r2)), id(r2)), {'str': str(r2)[:500].replace(hex(id(r)), bcolors.WARNING + str(hex(id(r))) + bcolors.ENDC)})
if str(type(r2)) == "<type 'collections.deque'>": if str(type(r2)) == "<type 'collections.deque'>":
refs3= gc.get_referrers(r2) refs3= gc.get_referrers(r2)
for r3 in refs3: for r3 in refs3:
log.info('gc oldest %s ref cnt:%s %s(%s): %s' % ('**', -3, str(type(r3)), id(r3), str(r3)[:500].replace(hex(id(r2)), bcolors.WARNING + str(hex(id(r2))) + bcolors.ENDC))) log.info('gc oldest %s ref cnt:%s %s(%s): {str!r}' % ('**', -3, str(type(r3)), id(r3)), {'str': str(r3)[:500].replace(hex(id(r2)), bcolors.WARNING + str(hex(id(r2))) + bcolors.ENDC)})
first = False first = False
@ -212,13 +225,25 @@ pgc.prev_size = None
pgc.oldest_id = None pgc.oldest_id = None
pgc.hist_ids = [] pgc.hist_ids = []
def type_str(obj):
switcher = {
"<type 'instance'>": lambda o: str(type(o)) + '(' + o.__class__.__name__ + ')',
"<type 'instancemethod'>": lambda o: str(type(o)) + '(' + str(o) + ')'
}
func = switcher.get(str(type(obj)), lambda o: str(type(o)))
return func(obj)
pgc.ids = [] pgc.ids = []
pgc.id_types = [ pgc.id_types = [
#["<type 'instance'>", "<twisted.web.client._HTTP11ClientFactory instance"], #["<type 'instance'>", "<twisted.web.client._HTTP11ClientFactory instance"],
#"<type 'instancemethod'>", #"<type 'instancemethod'>",
#"<type 'list'>", #"<type 'instance'>(DelayedCall)",
#"<class 'twisted.logger._logger.Logger'>", #"<class 'twisted.logger._logger.Logger'>",
#"<type 'tuple'>" #"<type 'list'>"
] ]
pgc.values = []
pgc.filter_by_value = False
pgc.time = int(time.time()) pgc.time = int(time.time())