fixes

frontend/frontend/migrations/0006_feed_owner.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.10 on 2018-02-10 22:54
-from __future__ import unicode_literals
-
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('frontend', '0005_auto_20180131_1949'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='feed',
-            name='owner',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
-        ),
-    ]

frontend/frontend/migrations/0007_auto_20180211_0027.py

@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.10 on 2018-02-11 00:27
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('frontend', '0006_feed_owner'),
-    ]
-
-    operations = [
-        migrations.RunSQL("ALTER TABLE `frontend_feed` ADD COLUMN `js` ENUM('Y','N') DEFAULT 'N' NOT NULL;"
-                          "ALTER TABLE `frontend_feed` ALTER COLUMN `js` DROP DEFAULT;",
-                          state_operations=[
-                              migrations.AddField(
-                                model_name='feed',
-                                name='js',
-                                field=models.CharField(choices=[(b'N', b'No javascript'), (b'Y', b'With javascript')], default=b'N', max_length=1),
-                              )
-                          ]),
-        migrations.AddField(
-            model_name='feed',
-            name='name',
-            field=models.CharField(max_length=255, null=True),
-        ),
-    ]

frontend/frontend/migrations/0008_auto_20180215_1445.py

@@ -8,7 +8,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('frontend', '0007_auto_20180211_0027'),
+        ('frontend', '0005_auto_20180131_1949'),
     ]
 
     operations = [

frontend/frontend/setup_tool_ext.py

@@ -31,52 +31,52 @@ def build_xpath_results(selectors, file_name):
         if post_elems:
             for elem in post_elems:
                 selected_required = True
-                selected_link = True
                 extracted_post = {}
-                for name, xpath in field_xpathes.iteritems():
+                for field_id, xpath_required in field_xpathes.iteritems():
-                    if not (name in field_results):
+                    xpath, required = xpath_required
-                        field_results[name] = {}
+                    if not (field_id in field_results):
+                        field_results[field_id] = {}
                     xpath = xpath.strip()
                     try:
                         extracts = elem.xpath(xpath).extract()
-                        if name == 'link':
+                        if not required:
-                            if not extracts:
+                            if extracts:
-                                selected_link = False
+                                extracted_post[field_id] = u''.join(extracts)
-                            else:
-                                extracted_post[name] = u''.join(extracts)
                         else:
                             if not extracts:
                                 selected_required = False
                             else:
-                                extracted_post[name] = u''.join(extracts)
+                                extracted_post[field_id] = u''.join(extracts)
                     except ValueError as ex:
                         success = False
-                        field_results[name]['error'] = ex.message
+                        field_results[field_id]['error'] = ex.message
 
                 if selected_required:
-                    for name, xpath in field_xpathes.iteritems():
+                    for field_id, xpath_required in field_xpathes.iteritems():
-                        if name == 'link':
+                        xpath, required = xpath_required
-                            if selected_link:
+                        if not required:
-                                if 'count' in field_results[name]:
+                            if field_id in extracted_post:
-                                    field_results[name]['count'] += 1
+                                if 'count' in field_results[field_id]:
+                                    field_results[field_id]['count'] += 1
                                 else:
-                                    field_results[name]['count'] = 1
+                                    field_results[field_id]['count'] = 1
                         else:
-                            if 'count' in field_results[name]:
+                            if 'count' in field_results[field_id]:
-                                field_results[name]['count'] += 1
+                                field_results[field_id]['count'] += 1
                             else:
-                                field_results[name]['count'] = 1
+                                field_results[field_id]['count'] = 1
 
                     extracted_posts.append(extracted_post)
             else:
-                for name, xpath in field_xpathes.iteritems():
+                for field_id, xpath_required in field_xpathes.iteritems():
+                    xpath, required = xpath_required
                     xpath = xpath.strip()
                     try:
                         doc.xpath(xpath).extract()
                     except ValueError as ex:
-                        if not (name in field_results):
+                        if not (field_id in field_results):
-                            field_results[name] = {}
+                            field_results[field_id] = {}
-                        field_results[name]['error'] = ex.message
+                        field_results[field_id]['error'] = ex.message
                         success = False
 
 

frontend/frontend/templates/frontend/setup.html

@@ -80,9 +80,9 @@
 						<div class="control-group info">
 								<label class="control-label">{% trans 'setup.title_xpath' %}:</label>
 								<div class="controls">
-										<select disabled>
+										<select id="ste-title-required" disabled>
-												<option selected="selected">{% trans 'setup.Required' %}</option>
+												<option selected="selected" value="1">{% trans 'setup.Required' %}</option>
-												<option>{% trans 'setup.Optional' %}</option>
+												<option value="0">{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-title" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
 										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
@@ -91,9 +91,9 @@
 						<div class="control-group info">
               <label class="control-label">{% trans 'setup.link_xpath' %}:</label>
 								<div class="controls">
-										<select style="width:100px" disabled>
+										<select id="ste-link-required" style="width:100px" disabled>
-												<option>{% trans 'setup.Required' %}</option>
+												<option value="1">{% trans 'setup.Required' %}</option>
-												<option selected="selected">{% trans 'setup.Optional' %}</option>
+												<option selected="selected" value="0">{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-link" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
 										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
@@ -102,9 +102,9 @@
 						<div class="control-group info">
               <label class="control-label">{% trans 'setup.description_xpath' %}:</label>
 								<div class="controls">
-										<select style="width:100px" disabled>
+										<select id="ste-description-required" style="width:100px" disabled>
-												<option selected="selected">{% trans 'setup.Required' %}</option>
+												<option selected="selected" value="1">{% trans 'setup.Required' %}</option>
-												<option>{% trans 'setup.Optional' %}</option>
+												<option value="0">{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-description" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
 										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>

frontend/frontend/views.py

@@ -94,6 +94,10 @@ def _get_link_xpath(title_xpath):
         xpath = title_xpath[:len(title_xpath)-len('/child::node()')]
         return xpath +'/ancestor-or-self::node()/@href'
 
+_BASIC_TITLE_ID=1
+_BASIC_DESCRIPTION_ID=2
+_BASIC_LINK_ID=3
+
 def _create_feed(url, xpathes, edited=False):
     feed_xpath = xpathes[0]
     item_xpathes = xpathes[1]
@@ -104,11 +108,11 @@ def _create_feed(url, xpathes, edited=False):
     fields = Field.objects.all()
 
     for field in fields:
-        if field.name == 'link' and 'title' in item_xpathes and not edited:
+        if field.id == _BASIC_LINK_ID and _BASIC_TITLE_ID in item_xpathes and not edited:
-            ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes['title']))
+            ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes[_BASIC_TITLE_ID][0]))
             ff.save()
-        elif field.name in item_xpathes:
+        elif field.id in item_xpathes:
-            ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.name])
+            ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.id][0])
             ff.save()
 
     return feed.id
@@ -126,6 +130,15 @@ def setup_create_feed(request):
             return HttpResponseBadRequest('html is invalid')
 
         xpathes = build_xpathes_for_items(item_names, html_json)
+
+        field_xpathes = {}
+        required = True
+        if 'title' in xpathes[1]:
+            field_xpathes[_BASIC_TITLE_ID] = [xpathes[1]['title'], required]
+        if 'description' in xpathes[1]:
+            field_xpathes[_BASIC_DESCRIPTION_ID] = [xpathes[1]['description'], required]
+        xpathes[1] = field_xpathes
+
         feed_id = _create_feed(url, xpathes)
 
         return HttpResponse(reverse('preview', args=(feed_id,)))
@@ -141,16 +154,18 @@ def _validate_selectors(selectors):
     if not isinstance(item_xpathes, dict):
         return False
 
+    item_xpathes = {int(field_id): xpath for field_id, xpath in item_xpathes.iteritems()}
+
     fields = Field.objects.all()
 
     item_xpathes_out = {}
 
     for field in fields:
-        if field.name in item_xpathes:
+        if field.id in item_xpathes:
-            if not isinstance(item_xpathes[field.name], basestring):
+            if not isinstance(item_xpathes[field.id], basestring):
                 return False
             else:
-                item_xpathes_out[field.name] = item_xpathes[field.name]
+                item_xpathes_out[field.id] = [item_xpathes[field.id], field.required]
     return [feed_xpath, item_xpathes_out]
 
 def setup_validate_selectors(request):

pol/feed.py

@@ -113,9 +113,6 @@ class Feed(object):
 
         title = selector.xpath('//title/text()').extract_first()
 
-        if not feed_config['name'] and title:
-            self.update_feed_name(feed_config['id'], title.encode('utf-8'))
-
         #build feed
         feed = Rss201rev2Feed(
             title = title if title else 'PolitePol: ' + feed_config['uri'],
@@ -144,16 +141,12 @@ class Feed(object):
             )
         return [feed.writeString('utf-8'), len(items), new_post_cnt]
 
-    def update_feed_name(self, feed_id, name):
-        with closing(get_conn(self.db_creds)) as conn, conn as cur:
-            cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id))
-
     def getFeedData(self, feed_id):
         # get url, xpathes
         feed = {}
 
         with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur:
-            cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
+            cur.execute("""select f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
                            from frontend_feed f
                            right join frontend_feedfield ff on ff.feed_id=f.id
                            left join frontend_field fi on fi.id=ff.field_id
@@ -163,7 +156,6 @@ class Feed(object):
             for row in rows:
                 if not feed:
                     feed['id'] = feed_id
-                    feed['name'] = row['feed_name']
                     feed['uri'] = row['uri']
                     feed['xpath'] = row['feed_xpath'].decode('utf-8')
                     feed['fields'] = {}

pol/server.py

@@ -189,7 +189,9 @@ class Downloader(object):
                 self.request.write('Downloader error: ' + error.getErrorMessage())
                 self.request.write('Traceback: ' + error.getTraceback())
             else:
-                self.request.write(self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage())))
+                err_message = self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage()))
+                self.request.write(err_message)
+
             self.request.finish()
 
     def downloadStarted(self, response):