mon lib + mysql fix

downloader.py

@@ -29,7 +29,6 @@ import re
 from feed import getFeedData, buildFeed
 
 from settings import DOWNLOADER_USER_AGENT, FEED_REQUEST_PERIOD_LIMIT, DEBUG, SNAPSHOT_DIR
-from mlm import pgc
 
 
 class bcolors:
@@ -71,14 +70,14 @@ def check_feed_request_time_limit(url):
     return 0
 
 
-pool = HTTPConnectionPool(reactor, persistent=False)
+#pool = HTTPConnectionPool(reactor, persistent=False)
-pool.cachedConnectionTimeout = 3
+#pool.cachedConnectionTimeout = 3
 
 agent = BrowserLikeRedirectAgent(
             Agent(reactor,
                 contextFactory=ScrapyClientContextFactory(), # skip certificate verification
-                connectTimeout=10,
+                connectTimeout=10),
-                pool=pool),
+                #pool=pool),
             redirectLimit=5
         )
 
@@ -188,28 +187,22 @@ def downloadDone(response_str, request, response, feed_config):
     request.finish()
     run_pgc()
 
-from pympler import summary, muppy, tracker, refbrowser
+from pympler import tracker
 import gc
 #sum = None
 tr = tracker.SummaryTracker()
-iterator = 0
+MON_PERIOD_SECONDS = 5#3 * 60 * 60 # 3 hours
+mon_time = None
 def mon(none):
-    global pool
+    global mon_time
-    pool.closeCachedConnections()
+    tm = int(time.time())
-    #gc.collect()
+    if not mon_time or tm - mon_time >= MON_PERIOD_SECONDS:
+        #global pool
+        #pool.closeCachedConnections()
+        gc.collect()
         global tr
         tr.print_diff()
-    global iterator
+    mon_time = tm
-    iterator += 1
-    if iterator % 4 == 0:
-        global reactor
-        ib = refbrowser.InteractiveBrowser(reactor)
-        ib.main()
-        #cb = refbrowser.ConsoleBrowser(reactor, maxdepth=2, str_func=output_function)
-        #cb.print_tree()
-
-def output_function(o):
-    return str(type(o))
 
 def run_pgc():
     d = defer.Deferred()

feed.py

@@ -9,8 +9,10 @@ from feedgenerator import Rss201rev2Feed, Enclosure
 import datetime
 
 import MySQLdb
+from contextlib import closing
 from settings import DATABASES, DOWNLOADER_USER_AGENT
 
+
 url_hash_regexp = re.compile('(#.*)?$')
 
 POST_TIME_DISTANCE = 15 # minutes, RSS Feed Reader skip same titles created in 10 min interval
@@ -18,7 +20,7 @@ POST_TIME_DISTANCE = 15 # minutes, RSS Feed Reader skip same titles created in 1
 FIELD_IDS = {'title': 1, 'description': 2, 'link': 3}
 
 def save_post(conn, created, feed_id, post_fields):
-    cur = conn.cursor()
+    with conn as cur:
         cur.execute("""insert into frontend_post (md5sum, created, feed_id)
                         values (%s, %s, %s)""", (post_fields['md5'], created, feed_id))
         print(cur._last_executed)
@@ -44,11 +46,10 @@ def fill_time(feed_id, items):
 
     #fetch dates from db
     fetched_dates = {}
-    db = get_conn()
+    with closing(get_conn()) as conn:
-    with db:
+        with conn as cur:
             quoted_hashes = ','.join(["'%s'" % (i['md5']) for i in items])
 
-        cur = db.cursor()
             cur.execute("""select p.md5sum, p.created, p.id
                            from frontend_post p
                            where p.md5sum in (%s)
@@ -60,6 +61,7 @@ def fill_time(feed_id, items):
                 created = row[1]
                 post_id = row[2]
                 fetched_dates[md5hash] = created
+
         cur_time = datetime.datetime.utcnow()
         new_posts = []
         for item in items:
@@ -67,7 +69,7 @@ def fill_time(feed_id, items):
                 item['time'] = fetched_dates[item['md5']]
             else:
                 item['time'] = cur_time
-            save_post(db, cur_time, feed_id, item)
+                save_post(conn, cur_time, feed_id, item)
                 cur_time -= datetime.timedelta(minutes=POST_TIME_DISTANCE)
 
 
@@ -152,9 +154,9 @@ def buildFeed(response, feed_config):
 def getFeedData(request, feed_id):
     # get url, xpathes
     feed = {}
-    db = get_conn()
+
-    with db:
+    with closing(get_conn()) as conn:
-        cur = db.cursor()
+        with conn as cur:
             cur.execute("""select f.uri, f.xpath, fi.name, ff.xpath, fi.required from frontend_feed f
                            right join frontend_feedfield ff on ff.feed_id=f.id
                            left join frontend_field fi on fi.id=ff.field_id