v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-16 14:20:10 -07:00
Code Issues Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Al Ne 2019-06-30 20:41:49 +03:00
parent 019c56bb82
commit 79c0bc467e
8 changed files with 60 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
frontend/frontend/migrations/0006_feed_owner.py
View File

@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.10 on 2018-02-10 22:54
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('frontend', '0005_auto_20180131_1949'),
]
operations = [
migrations.AddField(
model_name='feed',
name='owner',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
),
]

29
frontend/frontend/migrations/0007_auto_20180211_0027.py
View File

@ -1,29 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.10 on 2018-02-11 00:27
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('frontend', '0006_feed_owner'),
]
operations = [
migrations.RunSQL("ALTER TABLE `frontend_feed` ADD COLUMN `js` ENUM('Y','N') DEFAULT 'N' NOT NULL;"
"ALTER TABLE `frontend_feed` ALTER COLUMN `js` DROP DEFAULT;",
state_operations=[
migrations.AddField(
model_name='feed',
name='js',
field=models.CharField(choices=[(b'N', b'No javascript'), (b'Y', b'With javascript')], default=b'N', max_length=1),
)
]),
migrations.AddField(
model_name='feed',
name='name',
field=models.CharField(max_length=255, null=True),
),
]

2
frontend/frontend/migrations/0008_auto_20180215_1445.py
View File

@ -8,7 +8,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('frontend', '0007_auto_20180211_0027'),
('frontend', '0005_auto_20180131_1949'),
]
operations = [

48
frontend/frontend/setup_tool_ext.py
View File

@ -31,52 +31,52 @@ def build_xpath_results(selectors, file_name):
if post_elems:
for elem in post_elems:
selected_required = True
selected_link = True
extracted_post = {}
for name, xpath in field_xpathes.iteritems():
if not (name in field_results):
field_results[name] = {}
for field_id, xpath_required in field_xpathes.iteritems():
xpath, required = xpath_required
if not (field_id in field_results):
field_results[field_id] = {}
xpath = xpath.strip()
try:
extracts = elem.xpath(xpath).extract()
if name == 'link':
if not extracts:
selected_link = False
else:
extracted_post[name] = u''.join(extracts)
if not required:
if extracts:
extracted_post[field_id] = u''.join(extracts)
else:
if not extracts:
selected_required = False
else:
extracted_post[name] = u''.join(extracts)
extracted_post[field_id] = u''.join(extracts)
except ValueError as ex:
success = False
field_results[name]['error'] = ex.message
field_results[field_id]['error'] = ex.message
if selected_required:
for name, xpath in field_xpathes.iteritems():
if name == 'link':
if selected_link:
if 'count' in field_results[name]:
field_results[name]['count'] += 1
for field_id, xpath_required in field_xpathes.iteritems():
xpath, required = xpath_required
if not required:
if field_id in extracted_post:
if 'count' in field_results[field_id]:
field_results[field_id]['count'] += 1
else:
field_results[name]['count'] = 1
field_results[field_id]['count'] = 1
else:
if 'count' in field_results[name]:
field_results[name]['count'] += 1
if 'count' in field_results[field_id]:
field_results[field_id]['count'] += 1
else:
field_results[name]['count'] = 1
field_results[field_id]['count'] = 1
extracted_posts.append(extracted_post)
else:
for name, xpath in field_xpathes.iteritems():
for field_id, xpath_required in field_xpathes.iteritems():
xpath, required = xpath_required
xpath = xpath.strip()
try:
doc.xpath(xpath).extract()
except ValueError as ex:
if not (name in field_results):
field_results[name] = {}
field_results[name]['error'] = ex.message
if not (field_id in field_results):
field_results[field_id] = {}
field_results[field_id]['error'] = ex.message
success = False

18
frontend/frontend/templates/frontend/setup.html
View File

@ -80,9 +80,9 @@
<div class="control-group info">
<label class="control-label">{% trans 'setup.title_xpath' %}:</label>
<div class="controls">
<select disabled>
<option selected="selected">{% trans 'setup.Required' %}</option>
<option>{% trans 'setup.Optional' %}</option>
<select id="ste-title-required" disabled>
<option selected="selected" value="1">{% trans 'setup.Required' %}</option>
<option value="0">{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-title" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
@ -91,9 +91,9 @@
<div class="control-group info">
<label class="control-label">{% trans 'setup.link_xpath' %}:</label>
<div class="controls">
<select style="width:100px" disabled>
<option>{% trans 'setup.Required' %}</option>
<option selected="selected">{% trans 'setup.Optional' %}</option>
<select id="ste-link-required" style="width:100px" disabled>
<option value="1">{% trans 'setup.Required' %}</option>
<option selected="selected" value="0">{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-link" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
@ -102,9 +102,9 @@
<div class="control-group info">
<label class="control-label">{% trans 'setup.description_xpath' %}:</label>
<div class="controls">
<select style="width:100px" disabled>
<option selected="selected">{% trans 'setup.Required' %}</option>
<option>{% trans 'setup.Optional' %}</option>
<select id="ste-description-required" style="width:100px" disabled>
<option selected="selected" value="1">{% trans 'setup.Required' %}</option>
<option value="0">{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-description" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>

29
frontend/frontend/views.py
View File

@ -94,6 +94,10 @@ def _get_link_xpath(title_xpath):
xpath = title_xpath[:len(title_xpath)-len('/child::node()')]
return xpath +'/ancestor-or-self::node()/@href'
_BASIC_TITLE_ID=1
_BASIC_DESCRIPTION_ID=2
_BASIC_LINK_ID=3
def _create_feed(url, xpathes, edited=False):
feed_xpath = xpathes[0]
item_xpathes = xpathes[1]
@ -104,11 +108,11 @@ def _create_feed(url, xpathes, edited=False):
fields = Field.objects.all()
for field in fields:
if field.name == 'link' and 'title' in item_xpathes and not edited:
ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes['title']))
if field.id == _BASIC_LINK_ID and _BASIC_TITLE_ID in item_xpathes and not edited:
ff = FeedField(feed=feed, field=field, xpath= _get_link_xpath(item_xpathes[_BASIC_TITLE_ID][0]))
ff.save()
elif field.name in item_xpathes:
ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.name])
elif field.id in item_xpathes:
ff = FeedField(feed=feed, field=field, xpath=item_xpathes[field.id][0])
ff.save()
return feed.id
@ -126,6 +130,15 @@ def setup_create_feed(request):
return HttpResponseBadRequest('html is invalid')
xpathes = build_xpathes_for_items(item_names, html_json)
field_xpathes = {}
required = True
if 'title' in xpathes[1]:
field_xpathes[_BASIC_TITLE_ID] = [xpathes[1]['title'], required]
if 'description' in xpathes[1]:
field_xpathes[_BASIC_DESCRIPTION_ID] = [xpathes[1]['description'], required]
xpathes[1] = field_xpathes
feed_id = _create_feed(url, xpathes)
return HttpResponse(reverse('preview', args=(feed_id,)))
@ -141,16 +154,18 @@ def _validate_selectors(selectors):
if not isinstance(item_xpathes, dict):
return False
item_xpathes = {int(field_id): xpath for field_id, xpath in item_xpathes.iteritems()}
fields = Field.objects.all()
item_xpathes_out = {}
for field in fields:
if field.name in item_xpathes:
if not isinstance(item_xpathes[field.name], basestring):
if field.id in item_xpathes:
if not isinstance(item_xpathes[field.id], basestring):
return False
else:
item_xpathes_out[field.name] = item_xpathes[field.name]
item_xpathes_out[field.id] = [item_xpathes[field.id], field.required]
return [feed_xpath, item_xpathes_out]
def setup_validate_selectors(request):

10
pol/feed.py
View File

@ -113,9 +113,6 @@ class Feed(object):
title = selector.xpath('//title/text()').extract_first()
if not feed_config['name'] and title:
self.update_feed_name(feed_config['id'], title.encode('utf-8'))
#build feed
feed = Rss201rev2Feed(
title = title if title else 'PolitePol: ' + feed_config['uri'],
@ -144,16 +141,12 @@ class Feed(object):
)
return [feed.writeString('utf-8'), len(items), new_post_cnt]
def update_feed_name(self, feed_id, name):
with closing(get_conn(self.db_creds)) as conn, conn as cur:
cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id))
def getFeedData(self, feed_id):
# get url, xpathes
feed = {}
with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur:
cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
cur.execute("""select f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
from frontend_feed f
right join frontend_feedfield ff on ff.feed_id=f.id
left join frontend_field fi on fi.id=ff.field_id
@ -163,7 +156,6 @@ class Feed(object):
for row in rows:
if not feed:
feed['id'] = feed_id
feed['name'] = row['feed_name']
feed['uri'] = row['uri']
feed['xpath'] = row['feed_xpath'].decode('utf-8')
feed['fields'] = {}

4
pol/server.py
View File

@ -189,7 +189,9 @@ class Downloader(object):
self.request.write('Downloader error: ' + error.getErrorMessage())
self.request.write('Traceback: ' + error.getTraceback())
else:
self.request.write(self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage())))
err_message = self.error_html('<h1>PolitePol says: "Something wrong"</h1> <p><b>Try to refresh page or contact us by email: <a href="mailto:politepol.com@gmail.com">politepol.com@gmail.com</a></b>\n(Help us to improve our service with your feedback)</p> <p><i>Scary mantra: %s</i></p>' % escape(error.getErrorMessage()))
self.request.write(err_message)
self.request.finish()
def downloadStarted(self, response):