diff --git a/frontend/frontend/migrations/0006_feed_owner.py b/frontend/frontend/migrations/0006_feed_owner.py deleted file mode 100644 index 4138cbb..0000000 --- a/frontend/frontend/migrations/0006_feed_owner.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-10 22:54 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('frontend', '0005_auto_20180131_1949'), - ] - - operations = [ - migrations.AddField( - model_name='feed', - name='owner', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/frontend/frontend/migrations/0007_auto_20180211_0027.py b/frontend/frontend/migrations/0007_auto_20180211_0027.py deleted file mode 100644 index bd1846c..0000000 --- a/frontend/frontend/migrations/0007_auto_20180211_0027.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-11 00:27 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('frontend', '0006_feed_owner'), - ] - - operations = [ - migrations.RunSQL("ALTER TABLE `frontend_feed` ADD COLUMN `js` ENUM('Y','N') DEFAULT 'N' NOT NULL;" - "ALTER TABLE `frontend_feed` ALTER COLUMN `js` DROP DEFAULT;", - state_operations=[ - migrations.AddField( - model_name='feed', - name='js', - field=models.CharField(choices=[(b'N', b'No javascript'), (b'Y', b'With javascript')], default=b'N', max_length=1), - ) - ]), - migrations.AddField( - model_name='feed', - name='name', - field=models.CharField(max_length=255, null=True), - ), - ] diff --git a/frontend/frontend/migrations/0008_auto_20180215_1445.py b/frontend/frontend/migrations/0008_auto_20180215_1445.py index 7259617..e3b5123 100644 --- a/frontend/frontend/migrations/0008_auto_20180215_1445.py +++ b/frontend/frontend/migrations/0008_auto_20180215_1445.py @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('frontend', '0007_auto_20180211_0027'), + ('frontend', '0005_auto_20180131_1949'), ] operations = [ diff --git a/frontend/frontend/setup_tool_ext.py b/frontend/frontend/setup_tool_ext.py index af0710c..f082ef8 100644 --- a/frontend/frontend/setup_tool_ext.py +++ b/frontend/frontend/setup_tool_ext.py @@ -31,52 +31,52 @@ def build_xpath_results(selectors, file_name): if post_elems: for elem in post_elems: selected_required = True - selected_link = True extracted_post = {} - for name, xpath in field_xpathes.iteritems(): - if not (name in field_results): - field_results[name] = {} + for field_id, xpath_required in field_xpathes.iteritems(): + xpath, required = xpath_required + if not (field_id in field_results): + field_results[field_id] = {} xpath = xpath.strip() try: extracts = elem.xpath(xpath).extract() - if name == 'link': - if not extracts: - selected_link = False - else: - extracted_post[name] = u''.join(extracts) + if not required: + if extracts: + extracted_post[field_id] = u''.join(extracts) else: if not extracts: selected_required = False else: - extracted_post[name] = u''.join(extracts) + extracted_post[field_id] = u''.join(extracts) except ValueError as ex: success = False - field_results[name]['error'] = ex.message + field_results[field_id]['error'] = ex.message if selected_required: - for name, xpath in field_xpathes.iteritems(): - if name == 'link': - if selected_link: - if 'count' in field_results[name]: - field_results[name]['count'] += 1 + for field_id, xpath_required in field_xpathes.iteritems(): + xpath, required = xpath_required + if not required: + if field_id in extracted_post: + if 'count' in field_results[field_id]: + field_results[field_id]['count'] += 1 else: - field_results[name]['count'] = 1 + field_results[field_id]['count'] = 1 else: - if 'count' in field_results[name]: - field_results[name]['count'] += 1 + if 'count' in field_results[field_id]: + field_results[field_id]['count'] += 1 else: - field_results[name]['count'] = 1 + field_results[field_id]['count'] = 1 extracted_posts.append(extracted_post) else: - for name, xpath in field_xpathes.iteritems(): + for field_id, xpath_required in field_xpathes.iteritems(): + xpath, required = xpath_required xpath = xpath.strip() try: doc.xpath(xpath).extract() except ValueError as ex: - if not (name in field_results): - field_results[name] = {} - field_results[name]['error'] = ex.message + if not (field_id in field_results): + field_results[field_id] = {} + field_results[field_id]['error'] = ex.message success = False diff --git a/frontend/frontend/templates/frontend/setup.html b/frontend/frontend/templates/frontend/setup.html index 6fb17fc..4c83b9e 100644 --- a/frontend/frontend/templates/frontend/setup.html +++ b/frontend/frontend/templates/frontend/setup.html @@ -80,9 +80,9 @@
- + + @@ -91,9 +91,9 @@
- + + @@ -102,9 +102,9 @@
- + + diff --git a/frontend/frontend/views.py b/frontend/frontend/views.py index 9e324e0..e1bc63b 100644 --- a/frontend/frontend/views.py +++ b/frontend/frontend/views.py @@ -94,6 +94,10 @@ def _get_link_xpath(title_xpath): xpath = title_xpath[:len(title_xpath)-len('/child::node()')] return xpath +'/ancestor-or-self::node()/@href' +_BASIC_TITLE_ID=1 +_BASIC_DESCRIPTION_ID=2 +_BASIC_LINK_ID=3 + def _create_feed(url, xpathes, edited=False): feed_xpath = xpathes[0] item_xpathes = xpathes[1] @@ -104,11 +108,11 @@ def _create_feed(url, xpathes, edited=False): fields = Field.objects.all() for field in fields: - if field.name == 'link' and 'title' in item_xpathes and not edited: - ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes['title'])) + if field.id == _BASIC_LINK_ID and _BASIC_TITLE_ID in item_xpathes and not edited: + ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes[_BASIC_TITLE_ID][0])) ff.save() - elif field.name in item_xpathes: - ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.name]) + elif field.id in item_xpathes: + ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.id][0]) ff.save() return feed.id @@ -126,6 +130,15 @@ def setup_create_feed(request): return HttpResponseBadRequest('html is invalid') xpathes = build_xpathes_for_items(item_names, html_json) + + field_xpathes = {} + required = True + if 'title' in xpathes[1]: + field_xpathes[_BASIC_TITLE_ID] = [xpathes[1]['title'], required] + if 'description' in xpathes[1]: + field_xpathes[_BASIC_DESCRIPTION_ID] = [xpathes[1]['description'], required] + xpathes[1] = field_xpathes + feed_id = _create_feed(url, xpathes) return HttpResponse(reverse('preview', args=(feed_id,))) @@ -141,16 +154,18 @@ def _validate_selectors(selectors): if not isinstance(item_xpathes, dict): return False + item_xpathes = {int(field_id): xpath for field_id, xpath in item_xpathes.iteritems()} + fields = Field.objects.all() item_xpathes_out = {} for field in fields: - if field.name in item_xpathes: - if not isinstance(item_xpathes[field.name], basestring): + if field.id in item_xpathes: + if not isinstance(item_xpathes[field.id], basestring): return False else: - item_xpathes_out[field.name] = item_xpathes[field.name] + item_xpathes_out[field.id] = [item_xpathes[field.id], field.required] return [feed_xpath, item_xpathes_out] def setup_validate_selectors(request): diff --git a/pol/feed.py b/pol/feed.py index 87f7269..3d0acf3 100755 --- a/pol/feed.py +++ b/pol/feed.py @@ -113,9 +113,6 @@ class Feed(object): title = selector.xpath('//title/text()').extract_first() - if not feed_config['name'] and title: - self.update_feed_name(feed_config['id'], title.encode('utf-8')) - #build feed feed = Rss201rev2Feed( title = title if title else 'PolitePol: ' + feed_config['uri'], @@ -144,16 +141,12 @@ class Feed(object): ) return [feed.writeString('utf-8'), len(items), new_post_cnt] - def update_feed_name(self, feed_id, name): - with closing(get_conn(self.db_creds)) as conn, conn as cur: - cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id)) - def getFeedData(self, feed_id): # get url, xpathes feed = {} with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur: - cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required + cur.execute("""select f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required from frontend_feed f right join frontend_feedfield ff on ff.feed_id=f.id left join frontend_field fi on fi.id=ff.field_id @@ -163,7 +156,6 @@ class Feed(object): for row in rows: if not feed: feed['id'] = feed_id - feed['name'] = row['feed_name'] feed['uri'] = row['uri'] feed['xpath'] = row['feed_xpath'].decode('utf-8') feed['fields'] = {} diff --git a/pol/server.py b/pol/server.py index 41cd3ed..6e312de 100755 --- a/pol/server.py +++ b/pol/server.py @@ -189,7 +189,9 @@ class Downloader(object): self.request.write('Downloader error: ' + error.getErrorMessage()) self.request.write('Traceback: ' + error.getTraceback()) else: - self.request.write(self.error_html('

PolitePol says: "Something wrong"

Try to refresh page or contact us by email: politepol.com@gmail.com\n(Help us to improve our service with your feedback)

Scary mantra: %s

' % escape(error.getErrorMessage()))) + err_message = self.error_html('

PolitePol says: "Something wrong"

Try to refresh page or contact us by email: politepol.com@gmail.com\n(Help us to improve our service with your feedback)

Scary mantra: %s

' % escape(error.getErrorMessage())) + self.request.write(err_message) + self.request.finish() def downloadStarted(self, response):