v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-21 00:20:22 -07:00
Code Issues Releases Wiki Activity

xpath editor in progress

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-08-02 21:11:47 +00:00
parent e846b63c97
commit 84c6b96838
5 changed files with 104 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +1,2 @@
settings.py
*.pyc

75
frontend/frontend/setup_tool_ext.py Normal file
View File

@ -0,0 +1,75 @@
from hashlib import md5
from scrapy.selector import Selector
from .settings import SNAPSHOT_DIR
def build_xpath_results(selectors, file_name):
feed_xpath = selectors[0]
field_xpathes = selectors[1]
fpath = "%s/%s" % (SNAPSHOT_DIR, file_name)
with open(fpath) as f:
data = f.read()
html = data.decode('utf-8').split('\n\n', 1)[1]
feed_result = None
field_results = {}
post_elems = None
try:
doc = Selector(text=html)
post_elems = doc.xpath(feed_xpath)
feed_result = {'count': len(post_elems)}
if post_elems:
for elem in post_elems:
selected_required = True
selected_link = True
for name, xpath in field_xpathes.iteritems():
if not (name in field_results):
field_results[name] = {}
# import pdb;pdb.set_trace()
xpath = xpath.strip()
try:
extracts = elem.xpath(xpath).extract()
if name == 'link':
if not extracts:
selected_link = False
else:
if not extracts:
selected_required = False
except ValueError as ex:
field_results[name]['error'] = ex.message
for name, xpath in field_xpathes.iteritems():
if selected_required:
if name == 'link':
if selected_link:
if 'count' in field_results[name]:
field_results[name]['count'] += 1
else:
field_results[name]['count'] = 1
else:
if 'count' in field_results[name]:
field_results[name]['count'] += 1
else:
field_results[name]['count'] = 1
else:
for name, xpath in field_xpathes.iteritems():
xpath = xpath.strip()
try:
doc.xpath(xpath).extract()
except ValueError as ex:
if not (name in field_results):
field_results[name] = {}
field_results[name]['error'] = ex.message
except ValueError as ex:
feed_result = {'error': ex.message}
return [feed_result, field_results]

6
frontend/frontend/static/frontend/assets/js/setup-tool-ext.js
View File

@ -40,13 +40,14 @@ var _active = false;
function updateSelector(name, messages) {
var control_group = $('#ste-'+ name).parent().parent();
var help_text = control_group.find('.help-inline');
if ('error' in messages) {
control_group.removeClass('info').addClass('error');
control_group.find('.help-inline').text(messages['error']);
help_text.text(messages['error']);
}
else {
control_group.removeClass('error').addClass('info');
control_group.find('.help-inline').text(messages['count']);
help_text.text(help_text.attr('count-tpl').replace('%s', messages['count']));
}
}
@ -108,6 +109,7 @@ window.ET = {
'init': init_tool,
'check': check_pathes,
'updateUI': updateUI,
'updateUIMessages': updateUIMessages,
'getUIConfig': getUIConfig,
'active': active
};

28
frontend/frontend/static/frontend/assets/js/setup-tool.js
View File

@ -406,20 +406,34 @@ function onCreateButtonClick() {
//freeze UI
loader(true);
createFeed().then(function(data){
if (typeof(data) == 'string')
window.location.href = data; // feed_page_url
else
ET.updateUI(data);
}, function(error){
if (ET.active()) {
ET.updateUIMessages(JSON.parse(data));
//unfreez UI
loader(false);
}
else
window.location.href = data; // feed_page_url
}, function(error){
console.log('Server error: '+ error);
//unfreez UI
loader(false);
});
}
}
function createFeed() {
if (!ET.active()) {
var selectors = null;
if (ET.active()) {
selectors = ET.getUIConfig();
selectors[0] = selectors[0].trim();
for (var name in selectors[1]) {
var xpath = selectors[1][name];
if (xpath.trim().length == 0)
delete selectors[1][name];
}
}
else {
// gather selected tag-ids
var name_ids = {};
selected_any = gatherSelectedTagIds(name_ids);
@ -431,7 +445,7 @@ function createFeed() {
type: 'POST',
url: ET.active() ? "/setup_create_feed_ext" :"/setup_create_feed",
data: JSON.stringify(ET.active()
? { selectors: ET.getUIConfig(), snapshot_time: snapshot_time, url:$('#create').data('page-url') }
? { selectors: selectors, snapshot_time: snapshot_time, url:$('#create').data('page-url') }
: { html: iframeHtmlJson, names: name_ids, url:$('#create').data('page-url') }
),
contentType: "application/json; charset=utf-8",

8
frontend/frontend/templates/frontend/setup.html
View File

@ -63,7 +63,7 @@
<label class="control-label"><a href="{{ xpath_link }}" target="_blank">{% trans 'setup.item_xpath' %}</a>:</label>
<div class="controls">
<input type="text" id="ste-parent" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline" selected-text="{% trans 'setup.Selected' %}"></span>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
</div>
</div>
<div class="control-group info">
@ -74,7 +74,7 @@
<option>{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-title" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline">{% trans 'setup.Selected' %}</span>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
</div>
</div>
<div class="control-group info">
@ -85,7 +85,7 @@
<option selected="selected">{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-link" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline">{% trans 'setup.Selected' %}</span>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
</div>
</div>
<div class="control-group info">
@ -96,7 +96,7 @@
<option>{% trans 'setup.Optional' %}</option>
<select>
<input type="text" id="ste-description" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
<span class="help-inline">{% trans 'setup.Selected' %}</span>
<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
</div>
</div>
</form>