from twisted.web import server, resource from twisted.internet import reactor, endpoints from twisted.web.client import HTTPClientFactory, _makeGetterFactory from twisted.web.server import NOT_DONE_YET from scrapy.http.response.text import TextResponse from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware from scrapy.selector import Selector from scrapy.http import Headers from scrapy.responsetypes import responsetypes import w3lib.url import w3lib.html from lxml import etree from feedgenerator import Rss201rev2Feed, Enclosure import datetime import MySQLdb from settings import DATABASES def _getPageFactory(url, contextFactory=None, *args, **kwargs): """ Download a web page as a string. Download a page. Return a deferred, which will callback with a page (as a string) or errback with a description of the error. See L{HTTPClientFactory} to see what extra arguments can be passed. """ return _makeGetterFactory( url, HTTPClientFactory, contextFactory=contextFactory, *args, **kwargs) def _buildScrapyResponse(page_factory, body): status = int(page_factory.status) headers = Headers(page_factory.response_headers) respcls = responsetypes.from_args(headers=headers, url=page_factory.url) return respcls(url=page_factory.url, status=status, headers=headers, body=body) def element_to_string(element): s = [element.text] if element.text else [] for sub_element in element: s.append(etree.tostring(sub_element)) if element.tail: s.append(element.tail) return ''.join(s) def _build_link(html, doc_url, url): base_url = w3lib.html.get_base_url(html, doc_url) return w3lib.url.urljoin_rfc(base_url, url) def _buildFeed(response, feed_config): tree = response.selector._root.getroottree() # get data from html items = [] for node in tree.xpath(feed_config['xpath']): item = {} title_link = None for field_name in ['title', 'description']: if field_name in feed_config['fields']: element = node.xpath(feed_config['fields'][field_name]) if element: item[field_name] = element_to_string(element[0]) # get item link if field_name == 'title': anchor = element[0].xpath('ancestor-or-self::node()[name()="a"]') if anchor and anchor[0].get('href'): title_link = _build_link(response.body_as_unicode(), feed_config['uri'], anchor[0].get('href')) if len(item) == len(feed_config['fields']): # all fields are required item['title_link'] = title_link items.append(item) #build feed feed = Rss201rev2Feed( title='Polite Pol: ' + feed_config['uri'], link=feed_config['uri'], description="Generated by PolitePol.com.\n"+\ "Url: " + feed_config['uri'], language="en", ) for item in items: feed.add_item( title=item['title'] if 'title' in item else '', link = item['title_link'] if 'title_link' in item else feed_config['uri'], description=item['description'] if 'description' in item else '', #enclosure=Enclosure(fields[4], "32000", "image/jpeg") if 4 in fields else None, #"Image" pubdate=datetime.datetime.now() ) return feed.writeString('utf-8') def _downloadDone(response_str, request=None, page_factory=None, feed_config=None): response = _buildScrapyResponse(page_factory, response_str) response = DecompressionMiddleware().process_response(None, response, None) if (isinstance(response, TextResponse)): response_str = _buildFeed(response, feed_config) request.setHeader(b"Content-Type", b'text/xml') request.write(response_str) request.finish() def _downloadError(error, request=None, page_factory=None): request.write('Downloader error: ' + error.getErrorMessage()) request.write('Traceback: ' + error.getTraceback()) request.finish() def startFeedRequest(request, feed_id): # get url, xpathes creds = DATABASES['default'] db = MySQLdb.connect(host=creds['HOST'], port=int(creds['PORT']), user=creds['USER'], passwd=creds['PASSWORD'], db=creds['NAME']) feed = {} with db: cur = db.cursor() cur.execute("""select f.uri, f.xpath, fi.name, ff.xpath from frontend_feed f right join frontend_feedfield ff on ff.feed_id=f.id left join frontend_field fi on fi.id=ff.field_id where f.id=%s""", (feed_id,)) rows = cur.fetchall() for row in rows: if not feed: feed['uri'] = row[0] feed['xpath'] = row[1] feed['fields'] = {} feed['fields'][row[2]] = row[3] if feed: page_factory = _getPageFactory(feed['uri'], headers={ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate, sdch', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36' }, redirectLimit=5, timeout=10 ) d = page_factory.deferred d.addCallback(_downloadDone, request=request, page_factory=page_factory, feed_config=feed) d.addErrback(_downloadError, request=request, page_factory=page_factory) else: Request.write('Feed generator error: config of feed is empty') request.finish() return