feed name

frontend/frontend/migrations/0007_auto_20180211_0027.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2018-02-11 00:27
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('frontend', '0006_feed_owner'),
+    ]
+
+    operations = [
+        migrations.RunSQL("ALTER TABLE `frontend_feed` ADD COLUMN `js` ENUM('Y','N') DEFAULT 'N' NOT NULL;"
+                          "ALTER TABLE `frontend_feed` ALTER COLUMN `js` DROP DEFAULT;",
+                          state_operations=[
+                              migrations.AddField(
+                                model_name='feed',
+                                name='js',
+                                field=models.CharField(choices=[(b'N', b'No javascript'), (b'Y', b'With javascript')], default=b'N', max_length=1),
+                              )
+                          ]),
+        migrations.AddField(
+            model_name='feed',
+            name='name',
+            field=models.CharField(max_length=255, null=True),
+        ),
+    ]

pol/db.py

@@ -1,7 +1,10 @@
 import MySQLdb
+import MySQLdb.cursors
 
 
-def get_conn(creds):
+def get_conn(creds, dict_result=False):
-    db = MySQLdb.connect(host=creds['HOST'], port=int(creds['PORT']), user=creds['USER'], passwd=creds['PASSWORD'], db=creds['NAME'], init_command='SET NAMES utf8mb4')
+    cursor = MySQLdb.cursors.DictCursor if dict_result else MySQLdb.cursors.Cursor
+    db = MySQLdb.connect(host=creds['HOST'], port=int(creds['PORT']), user=creds['USER'], passwd=creds['PASSWORD'],
+                         db=creds['NAME'], init_command='SET NAMES utf8mb4', cursorclass=cursor)
     db.autocommit(True)
     return db

pol/feed.py

@@ -110,13 +110,16 @@ class Feed(object):
             if required_count == required_found:
                 items.append(item)
 
-        title = selector.xpath('//title/text()').extract()
+        title = selector.xpath('//title/text()').extract_first()
+
+        if not feed_config['name'] and title:
+            self.update_feed_name(feed_config['id'], title.encode('utf-8'))
 
         #build feed
         feed = Rss201rev2Feed(
-            title = title[0] if title else 'Polite Pol: ' + feed_config['uri'],
+            title = title if title else 'PolitePol: ' + feed_config['uri'],
             link=feed_config['uri'],
-            description="Generated by PolitePol.com.\n"+\
+            description="Generated by PolitePol.\n"+\
                 "Source page: " + feed_config['uri'],
             language="en",
         )
@@ -140,27 +143,32 @@ class Feed(object):
             )
         return [feed.writeString('utf-8'), len(items), new_post_cnt]
 
+    def update_feed_name(self, feed_id, name):
+        with closing(get_conn(self.db_creds)) as conn, conn as cur:
+            cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id))
+
     def getFeedData(self, feed_id):
         # get url, xpathes
         feed = {}
 
-        with closing(get_conn(self.db_creds)) as conn:
+        with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur:
-            with conn as cur:
+            cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
-                cur.execute("""select f.uri, f.xpath, fi.name, ff.xpath, fi.required from frontend_feed f
+                           from frontend_feed f
-                               right join frontend_feedfield ff on ff.feed_id=f.id
+                           right join frontend_feedfield ff on ff.feed_id=f.id
-                               left join frontend_field fi on fi.id=ff.field_id
+                           left join frontend_field fi on fi.id=ff.field_id
-                               where f.id=%s""", (feed_id,))
+                           where f.id=%s""", (feed_id,))
-                rows = cur.fetchall()
+            rows = cur.fetchall()
 
-                for row in rows:
+            for row in rows:
-                    if not feed:
+                if not feed:
-                        feed['id'] = feed_id
+                    feed['id'] = feed_id
-                        feed['uri'] = row[0]
+                    feed['name'] = row['feed_name']
-                        feed['xpath'] = row[1]
+                    feed['uri'] = row['uri']
-                        feed['fields'] = {}
+                    feed['xpath'] = row['feed_xpath']
-                        feed['required'] = {}
+                    feed['fields'] = {}
-                    feed['fields'][row[2]] = row[3]
+                    feed['required'] = {}
-                    feed['required'][row[2]] = row[4]
+                feed['fields'][row['name']] = row['xpath']
+                feed['required'][row['name']] = row['required']
 
         if feed:
             return [feed['uri'], feed]

pol/server.py

@@ -181,7 +181,7 @@ class Downloader(object):
                         )
                 else:
                     sys.stderr.write('\n'.join(
-                        [str(datetime.utcnow()), self.request.uri, url, 'Downloader error: ' + error.getErrorMessage(),
+                        [str(datetime.utcnow()), self.request.uri, self.url, 'Downloader error: ' + error.getErrorMessage(),
                          'Traceback: ' + error.getTraceback()]))
             except:
                 traceback.print_exc(file=sys.stdout)