From 6d037ba78484cfe1f90f06854c5e767f2a9a59e3 Mon Sep 17 00:00:00 2001 From: Alexandr Nesterenko Date: Tue, 13 Feb 2018 19:06:49 +0300 Subject: [PATCH] feed name --- .../migrations/0007_auto_20180211_0027.py | 29 ++++++++++++ pol/db.py | 7 ++- pol/feed.py | 46 +++++++++++-------- pol/server.py | 2 +- 4 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 frontend/frontend/migrations/0007_auto_20180211_0027.py diff --git a/frontend/frontend/migrations/0007_auto_20180211_0027.py b/frontend/frontend/migrations/0007_auto_20180211_0027.py new file mode 100644 index 0000000..bd1846c --- /dev/null +++ b/frontend/frontend/migrations/0007_auto_20180211_0027.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-11 00:27 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('frontend', '0006_feed_owner'), + ] + + operations = [ + migrations.RunSQL("ALTER TABLE `frontend_feed` ADD COLUMN `js` ENUM('Y','N') DEFAULT 'N' NOT NULL;" + "ALTER TABLE `frontend_feed` ALTER COLUMN `js` DROP DEFAULT;", + state_operations=[ + migrations.AddField( + model_name='feed', + name='js', + field=models.CharField(choices=[(b'N', b'No javascript'), (b'Y', b'With javascript')], default=b'N', max_length=1), + ) + ]), + migrations.AddField( + model_name='feed', + name='name', + field=models.CharField(max_length=255, null=True), + ), + ] diff --git a/pol/db.py b/pol/db.py index 947015b..6e76cf7 100755 --- a/pol/db.py +++ b/pol/db.py @@ -1,7 +1,10 @@ import MySQLdb +import MySQLdb.cursors -def get_conn(creds): - db = MySQLdb.connect(host=creds['HOST'], port=int(creds['PORT']), user=creds['USER'], passwd=creds['PASSWORD'], db=creds['NAME'], init_command='SET NAMES utf8mb4') +def get_conn(creds, dict_result=False): + cursor = MySQLdb.cursors.DictCursor if dict_result else MySQLdb.cursors.Cursor + db = MySQLdb.connect(host=creds['HOST'], port=int(creds['PORT']), user=creds['USER'], passwd=creds['PASSWORD'], + db=creds['NAME'], init_command='SET NAMES utf8mb4', cursorclass=cursor) db.autocommit(True) return db diff --git a/pol/feed.py b/pol/feed.py index ab3e45f..0074187 100755 --- a/pol/feed.py +++ b/pol/feed.py @@ -110,13 +110,16 @@ class Feed(object): if required_count == required_found: items.append(item) - title = selector.xpath('//title/text()').extract() + title = selector.xpath('//title/text()').extract_first() + + if not feed_config['name'] and title: + self.update_feed_name(feed_config['id'], title.encode('utf-8')) #build feed feed = Rss201rev2Feed( - title = title[0] if title else 'Polite Pol: ' + feed_config['uri'], + title = title if title else 'PolitePol: ' + feed_config['uri'], link=feed_config['uri'], - description="Generated by PolitePol.com.\n"+\ + description="Generated by PolitePol.\n"+\ "Source page: " + feed_config['uri'], language="en", ) @@ -140,27 +143,32 @@ class Feed(object): ) return [feed.writeString('utf-8'), len(items), new_post_cnt] + def update_feed_name(self, feed_id, name): + with closing(get_conn(self.db_creds)) as conn, conn as cur: + cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id)) + def getFeedData(self, feed_id): # get url, xpathes feed = {} - with closing(get_conn(self.db_creds)) as conn: - with conn as cur: - cur.execute("""select f.uri, f.xpath, fi.name, ff.xpath, fi.required from frontend_feed f - right join frontend_feedfield ff on ff.feed_id=f.id - left join frontend_field fi on fi.id=ff.field_id - where f.id=%s""", (feed_id,)) - rows = cur.fetchall() + with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur: + cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required + from frontend_feed f + right join frontend_feedfield ff on ff.feed_id=f.id + left join frontend_field fi on fi.id=ff.field_id + where f.id=%s""", (feed_id,)) + rows = cur.fetchall() - for row in rows: - if not feed: - feed['id'] = feed_id - feed['uri'] = row[0] - feed['xpath'] = row[1] - feed['fields'] = {} - feed['required'] = {} - feed['fields'][row[2]] = row[3] - feed['required'][row[2]] = row[4] + for row in rows: + if not feed: + feed['id'] = feed_id + feed['name'] = row['feed_name'] + feed['uri'] = row['uri'] + feed['xpath'] = row['feed_xpath'] + feed['fields'] = {} + feed['required'] = {} + feed['fields'][row['name']] = row['xpath'] + feed['required'][row['name']] = row['required'] if feed: return [feed['uri'], feed] diff --git a/pol/server.py b/pol/server.py index 7629ce3..6777c49 100755 --- a/pol/server.py +++ b/pol/server.py @@ -181,7 +181,7 @@ class Downloader(object): ) else: sys.stderr.write('\n'.join( - [str(datetime.utcnow()), self.request.uri, url, 'Downloader error: ' + error.getErrorMessage(), + [str(datetime.utcnow()), self.request.uri, self.url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()])) except: traceback.print_exc(file=sys.stdout)