xpath editor in progress

.gitignore

@@ -1 +1,2 @@
 settings.py
+*.pyc

frontend/frontend/setup_tool_ext.py

@@ -0,0 +1,75 @@
+from hashlib import md5
+
+from scrapy.selector import Selector
+
+from .settings import SNAPSHOT_DIR
+
+
+def build_xpath_results(selectors, file_name):
+    feed_xpath = selectors[0]
+    field_xpathes = selectors[1]
+
+    fpath = "%s/%s" % (SNAPSHOT_DIR, file_name)
+
+    with open(fpath) as f:
+        data = f.read()
+
+    html = data.decode('utf-8').split('\n\n', 1)[1]
+
+    feed_result = None
+    field_results = {}
+
+    post_elems = None
+    try:
+        doc = Selector(text=html)
+        post_elems = doc.xpath(feed_xpath)
+        feed_result = {'count': len(post_elems)}
+
+        if post_elems:
+            for elem in post_elems:
+                selected_required = True
+                selected_link = True
+                for name, xpath in field_xpathes.iteritems():
+                    if not (name in field_results):
+                        field_results[name] = {}
+                    # import pdb;pdb.set_trace()
+                    xpath = xpath.strip()
+                    try:
+                        extracts = elem.xpath(xpath).extract()
+                        if name == 'link':
+                            if not extracts:
+                                selected_link = False
+                        else:
+                            if not extracts:
+                                selected_required = False
+                    except ValueError as ex:
+                        field_results[name]['error'] = ex.message
+
+                for name, xpath in field_xpathes.iteritems():
+                    if selected_required:
+                        if name == 'link':
+                            if selected_link:
+                                if 'count' in field_results[name]:
+                                    field_results[name]['count'] += 1
+                                else:
+                                    field_results[name]['count'] = 1
+                        else:
+                            if 'count' in field_results[name]:
+                                field_results[name]['count'] += 1
+                            else:
+                                field_results[name]['count'] = 1
+            else:
+                for name, xpath in field_xpathes.iteritems():
+                    xpath = xpath.strip()
+                    try:
+                        doc.xpath(xpath).extract()
+                    except ValueError as ex:
+                        if not (name in field_results):
+                            field_results[name] = {}
+                        field_results[name]['error'] = ex.message
+
+
+    except ValueError as ex:
+        feed_result = {'error': ex.message}
+
+    return [feed_result, field_results]

frontend/frontend/static/frontend/assets/js/setup-tool-ext.js

@@ -40,13 +40,14 @@ var _active = false;
 
 function updateSelector(name, messages) {
     var control_group = $('#ste-'+ name).parent().parent();
+    var help_text = control_group.find('.help-inline');
     if ('error' in messages) {
         control_group.removeClass('info').addClass('error');
-        control_group.find('.help-inline').text(messages['error']);
+        help_text.text(messages['error']);
     }
     else {
         control_group.removeClass('error').addClass('info');
-        control_group.find('.help-inline').text(messages['count']);
+        help_text.text(help_text.attr('count-tpl').replace('%s', messages['count']));
     }
 }
 
@@ -108,6 +109,7 @@ window.ET = {
     'init': init_tool,
     'check': check_pathes,
     'updateUI': updateUI,
+    'updateUIMessages': updateUIMessages,
     'getUIConfig': getUIConfig,
     'active': active
 };

frontend/frontend/static/frontend/assets/js/setup-tool.js

@@ -406,20 +406,34 @@ function onCreateButtonClick() {
         //freeze UI
         loader(true);
         createFeed().then(function(data){
-            if (typeof(data) == 'string')
+            if (ET.active()) {
-				window.location.href = data; // feed_page_url
+				ET.updateUIMessages(JSON.parse(data));
-			else
-				ET.updateUI(data);
-        }, function(error){
                 //unfreez UI
                 loader(false);
+            }
+			else
+				window.location.href = data; // feed_page_url
+        }, function(error){
             console.log('Server error: '+ error);
+            //unfreez UI
+            loader(false);
         });
     }
 }
 
 function createFeed() {
-    if (!ET.active()) {
+    var selectors = null;
+    if (ET.active()) {
+        selectors = ET.getUIConfig();
+        selectors[0] = selectors[0].trim();
+        
+        for (var name in selectors[1]) {
+            var xpath = selectors[1][name];
+            if (xpath.trim().length == 0)
+                delete selectors[1][name];
+        }
+    }
+    else {
         // gather selected tag-ids
         var name_ids = {};
         selected_any = gatherSelectedTagIds(name_ids);
@@ -431,7 +445,7 @@ function createFeed() {
                 type: 'POST',
                 url: ET.active() ? "/setup_create_feed_ext" :"/setup_create_feed",
                 data: JSON.stringify(ET.active()
-                                     ? { selectors: ET.getUIConfig(), snapshot_time: snapshot_time, url:$('#create').data('page-url') }
+                                     ? { selectors: selectors, snapshot_time: snapshot_time, url:$('#create').data('page-url') }
                                      : { html: iframeHtmlJson, names: name_ids, url:$('#create').data('page-url') }
                                     ),
                 contentType: "application/json; charset=utf-8",

frontend/frontend/templates/frontend/setup.html

@@ -63,7 +63,7 @@
               <label class="control-label"><a href="{{ xpath_link }}" target="_blank">{% trans 'setup.item_xpath' %}</a>:</label>
 								<div class="controls">
 										<input type="text" id="ste-parent" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
-                    <span class="help-inline" selected-text="{% trans 'setup.Selected' %}"></span>
+                    <span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
 								</div>
 						</div>
 						<div class="control-group info">
@@ -74,7 +74,7 @@
 												<option>{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-title" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
-										<span class="help-inline">{% trans 'setup.Selected' %}</span>
+										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
 								</div>
 						</div>
 						<div class="control-group info">
@@ -85,7 +85,7 @@
 												<option selected="selected">{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-link" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
-										<span class="help-inline">{% trans 'setup.Selected' %}</span>
+										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
 								</div>
 						</div>
 						<div class="control-group info">
@@ -96,7 +96,7 @@
 												<option>{% trans 'setup.Optional' %}</option>
 										<select>
 										<input type="text" id="ste-description" class="input-xxlarge" placeholder="{% trans 'setup.skipped' %}" autocomplete="off"/>
-										<span class="help-inline">{% trans 'setup.Selected' %}</span>
+										<span class="help-inline" count-tpl="{% trans 'setup.Selected' %}"></span>
 								</div>
 						</div>
         </form>