diff --git a/downloader.py b/downloader.py index 389e3e7..280b4e2 100644 --- a/downloader.py +++ b/downloader.py @@ -17,6 +17,9 @@ import re from feed import startFeedRequest +from settings import DOWNLOADER_USER_AGENT + + def getPageFactory(url, contextFactory=None, *args, **kwargs): """ Download a web page as a string. @@ -96,14 +99,14 @@ def downloadError(error, request=None, page_factory=None): class Downloader(resource.Resource): isLeaf = True - feed_regexp = re.compile('^/feed/(\d+)$') + feed_regexp = re.compile('^/feed1?/(\d+)$') def startRequest(self, request, url): page_factory = getPageFactory(url, headers={ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate, sdch', - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' + 'User-Agent': DOWNLOADER_USER_AGENT }, redirectLimit=5, timeout=10 diff --git a/feed.py b/feed.py index bfeda47..70d906d 100644 --- a/feed.py +++ b/feed.py @@ -19,7 +19,7 @@ from feedgenerator import Rss201rev2Feed, Enclosure import datetime import MySQLdb -from settings import DATABASES +from settings import DATABASES, DOWNLOADER_USER_AGENT def _getPageFactory(url, contextFactory=None, *args, **kwargs): @@ -137,7 +137,7 @@ def startFeedRequest(request, feed_id): headers={ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate, sdch', - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' + 'User-Agent': DOWNLOADER_USER_AGENT }, redirectLimit=5, timeout=10 diff --git a/frontend/frontend/settings.py.example b/frontend/frontend/settings.py.example index bf481a0..f70d6fb 100644 --- a/frontend/frontend/settings.py.example +++ b/frontend/frontend/settings.py.example @@ -164,3 +164,5 @@ PIPELINE_JS = { DOWNLOADER_PAGE_URL = '/downloader?url=' FEED_PAGE_URL = '/feed/' +FEED1_PAGE_URL = '/feed1/' +DOWNLOADER_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' diff --git a/frontend/frontend/templates/frontend/preview.html b/frontend/frontend/templates/frontend/preview.html index 515a57e..6e616fd 100644 --- a/frontend/frontend/templates/frontend/preview.html +++ b/frontend/frontend/templates/frontend/preview.html @@ -28,7 +28,7 @@