v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-16 14:20:10 -07:00
Code Issues Releases Wiki Activity

user agent + feed1

Browse Source
This commit is contained in:
Alexandr Nesterenko 2016-07-24 06:58:17 -07:00
parent 8a4eedb4ef
commit 06f57064ac
6 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
downloader.py
View File

@ -17,6 +17,9 @@ import re
from feed import startFeedRequest from feed import startFeedRequest
from settings import DOWNLOADER_USER_AGENT
def getPageFactory(url, contextFactory=None, *args, **kwargs): def getPageFactory(url, contextFactory=None, *args, **kwargs):
""" """
Download a web page as a string. Download a web page as a string.
@ -96,14 +99,14 @@ def downloadError(error, request=None, page_factory=None):
class Downloader(resource.Resource): class Downloader(resource.Resource):
isLeaf = True isLeaf = True
feed_regexp = re.compile('^/feed/(\d+)$') feed_regexp = re.compile('^/feed1?/(\d+)$')
def startRequest(self, request, url): def startRequest(self, request, url):
page_factory = getPageFactory(url, page_factory = getPageFactory(url,
headers={ headers={
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Encoding': 'gzip, deflate, sdch',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' 'User-Agent': DOWNLOADER_USER_AGENT
}, },
redirectLimit=5, redirectLimit=5,
timeout=10 timeout=10

4
feed.py
View File

@ -19,7 +19,7 @@ from feedgenerator import Rss201rev2Feed, Enclosure
import datetime import datetime
import MySQLdb import MySQLdb
from settings import DATABASES from settings import DATABASES, DOWNLOADER_USER_AGENT
def _getPageFactory(url, contextFactory=None, *args, **kwargs): def _getPageFactory(url, contextFactory=None, *args, **kwargs):
@ -137,7 +137,7 @@ def startFeedRequest(request, feed_id):
headers={ headers={
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Encoding': 'gzip, deflate, sdch',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' 'User-Agent': DOWNLOADER_USER_AGENT
}, },
redirectLimit=5, redirectLimit=5,
timeout=10 timeout=10

2
frontend/frontend/settings.py.example
View File

@ -164,3 +164,5 @@ PIPELINE_JS = {
DOWNLOADER_PAGE_URL = '/downloader?url=' DOWNLOADER_PAGE_URL = '/downloader?url='
FEED_PAGE_URL = '/feed/' FEED_PAGE_URL = '/feed/'
FEED1_PAGE_URL = '/feed1/'
DOWNLOADER_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36'

2
frontend/frontend/templates/frontend/preview.html
View File

@ -28,7 +28,7 @@
<script type="text/javascript"> <script type="text/javascript">
function tryGetFeed() { function tryGetFeed() {
$.getFeed({ $.getFeed({
url: '{{ feed_url }}', url: '{{ feed1_url }}',
success: function (feed) { success: function (feed) {
if ('items' in feed) { if ('items' in feed) {
$('#preview').empty() $('#preview').empty()

5
frontend/frontend/views.py
View File

@ -9,7 +9,7 @@ from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
from .forms import IndexForm from .forms import IndexForm
from .settings import DOWNLOADER_PAGE_URL, FEED_PAGE_URL from .settings import DOWNLOADER_PAGE_URL, FEED_PAGE_URL, FEED1_PAGE_URL
from .setup_tool import get_selection_tag_ids, build_xpathes_for_items from .setup_tool import get_selection_tag_ids, build_xpathes_for_items
from .models import Feed, Field, FeedField from .models import Feed, Field, FeedField
@ -106,12 +106,11 @@ def setup_create_feed(request):
return HttpResponse(reverse('preview', args=(feed_id,))) return HttpResponse(reverse('preview', args=(feed_id,)))
def preview(request, feed_id): def preview(request, feed_id):
#import pdb; pdb.set_trace()
if request.method == 'GET': if request.method == 'GET':
return render(request, 'frontend/preview.html', return render(request, 'frontend/preview.html',
{ {
'feed_url': FEED_PAGE_URL + feed_id, 'feed_url': FEED_PAGE_URL + feed_id,
'feed1_url': FEED1_PAGE_URL + feed_id,
}) })
return HttpResponseBadRequest('Only GET method supported') return HttpResponseBadRequest('Only GET method supported')

3
nginx/default.site-example
View File

@ -17,4 +17,7 @@ server {
location /feed { location /feed {
proxy_pass http://127.0.0.1:1234; proxy_pass http://127.0.0.1:1234;
} }
location /feed1 {
proxy_pass http://127.0.0.1:1234;
}
} }