diff --git a/frontend/frontend/models.py b/frontend/frontend/models.py index 5e3332b..6c57ddc 100644 --- a/frontend/frontend/models.py +++ b/frontend/frontend/models.py @@ -26,4 +26,4 @@ class Post(models.Model): class PostField(models.Model): post = models.ForeignKey(Post, on_delete=models.CASCADE) field = models.ForeignKey(Field, on_delete=models.CASCADE) - text = models.TextField(max_length=64*1024-1) + text = models.TextField(max_length=64*1024-1) \ No newline at end of file diff --git a/pol/log.py b/pol/log.py index 2d79f12..b6f92b0 100755 --- a/pol/log.py +++ b/pol/log.py @@ -14,20 +14,20 @@ class bcolors: class LogHandler(object): - """Handler of twisted log meaasges""" + """Handler of twisted log meaasges""" - def __init__(self, stat_tool=None): - self.stat_tool = stat_tool - # requred, discardBuffer gets rid of the LimitedHistoryLogObserver, redirectStandardIO will loop print action - globalLogBeginner.beginLoggingTo([self.print_log], discardBuffer=True, redirectStandardIO=False) + def __init__(self, stat_tool=None): + self.stat_tool = stat_tool + # requred, discardBuffer gets rid of the LimitedHistoryLogObserver, redirectStandardIO will loop print action + globalLogBeginner.beginLoggingTo([self.print_log], discardBuffer=True, redirectStandardIO=False) - def print_log(event): - if 'isError' in event and event['isError']: - sys.stdout.write(bcolors.FAIL + formatEventAsClassicLogText(event) + bcolors.ENDC) - sys.stderr.write(formatEventAsClassicLogText(event)) - sys.stderr.flush() - else: - sys.stdout.write(formatEventAsClassicLogText(event)) - sys.stdout.flush() + def print_log(self, event): + if 'isError' in event and event['isError']: + sys.stdout.write(bcolors.FAIL + formatEventAsClassicLogText(event) + bcolors.ENDC) + sys.stderr.write(formatEventAsClassicLogText(event)) + sys.stderr.flush() + else: + sys.stdout.write(formatEventAsClassicLogText(event)) + sys.stdout.flush() diff --git a/test.py b/test.py new file mode 100644 index 0000000..8dc6db4 --- /dev/null +++ b/test.py @@ -0,0 +1,8 @@ +from tests.test_downloader import MFTests + + +def main(): + ts = MFTests() + ts.test_log_handler() + +main() \ No newline at end of file diff --git a/tests.py b/tests.py deleted file mode 100644 index 959e382..0000000 --- a/tests.py +++ /dev/null @@ -1,151 +0,0 @@ -from lxml import etree -import sys -import requests -from datetime import datetime - -from feed import element_to_unicode - -def tostring(el, fields): - if fields: - s = '' - for f in fields: - for e in el.xpath(f+'/text()'): - s += e - return s - else: - etree.tostring(el) - -def element_to_string(element, fields=None): - if isinstance(element, basestring): # attribute - return element - - s = [element.text] if element.text else [] - if fields: - return tostring(element, fields).encode('utf-8') - else: - for sub_element in element: - s.append(etree.tostring(sub_element)) - return ''.join(s) - -def test1_get_inner_html(): - root = etree.fromstring('12345') - assert element_to_unicode(root, 'utf-8') == u'12345' -ids = [1,54,100,131,134,140,146,159,162,166,168,175,176,183,189,190,192,204,205,226,230,236,244,251,253,260,261,263,271,272,273,275,277,279,280,308,311,312,313,315,316,317,318,327,332,333,334,335,337,338,340,347,350,352,354,355,356,357,358,359,360,361,362,363,369,371,373,376,385,399,402,405,406,410,411,412,422,427,448,467,470,471,472,473,477,479,481,512,514,519,522,523,524,526,527,528,529,532,533,536,538,547,557,587,592,597,598,599,600,606,607,608,615,616,617,618,628,629,641,642,643,645,646,647,648,649,653,658,660,673,676,678,680,681,683,685,704,709,710,717,718,719,728,730,732,735,744,745,746,749,757,758,759,772,776,777,778,779,783,784,785,786,789,790,791,792,793,794,795,797,798,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,835,836,839,840,842,843,844,845,846,847,848,849,850,851,852,853,854,855,861,862,863,864,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,889,890,891,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,923,924,926,927,928,929,930,931,933,934,935,936,937,938,939,940,941,942,943,944,947,948,949,950] -domain = "politepol.com" - -def parse_feed0(text): - ch = etree.fromstring(text).xpath('/rss/channel') - title = ch[0].xpath('title')[0].text - link = ch[0].xpath('link')[0].text - items = ch[0].xpath('item') - return [title, link, items] - -def parse_feed(text): - ch = etree.fromstring(text.encode('utf-8')).xpath('/rss/channel') - title = ch[0].xpath('title')[0].text - link = ch[0].xpath('link')[0].text - items = ch[0].xpath('item') - return [title, link, items] - -def crawl(extention): - number = 0 - for id in ids: - print "ID: %s (%s of %s) %s" % (id, number, len(ids), datetime.utcnow()) - r = requests.get("http://%s/feed/%s" % (domain, id)) - #r.encoding = 'utf-8' - text = r.text.encode('utf-8') - with open("tests/%s.%s" % (id, extention), 'w') as f: - f.write(text) - title, link, items = parse_feed0(text) - print "Title: %s" % title - print "Link: %s" % link - print "Items count: %s" % len(items) - number += 1 - -def diff(ext1, ext2, fields): - diff = [] - number = 0 - for id in ids: - print "ID: %s" % (id,) - text1 = None - with open("tests/%s.%s" % (id, ext1), 'r') as f: - text1 = f.read().decode('utf-8') - text2 = None - with open("tests/%s.%s" % (id, ext2), 'r') as f: - text2 = f.read().decode('utf-8') - - if text1 == text2: - print "Identical" - else: - diff.append(id) - posts_diff = 0 - with open("tests/%s.diff" % (id,), 'w') as f: - title1, link, items1 = parse_feed(text1) - title2, link, items2 = parse_feed(text2) - if title1 != title2: - print "Different titles" - f.write("<<<<<<<<<<<<<<< Different titles >>>>>>>>>>>>>>>\n") - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext1)) - f.write(title1.encode('utf-8') + "\n") - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext2)) - f.write(title2.encode('utf-8') + "\n") - posts1 = {} - posts2 = {} - if len(items1) != len(items2): - print "Different post count: %s vs %s" % (len(items1), len(items2)) - f.write("<< Different posts count: %s.%s:%s vs %s.%s:%s >>\n" % (id, ext1, len(items1), id, ext2, len(items2))) - for post in items1: - posts1[element_to_string(post, fields)] = True - for post in items2: - posts2[element_to_string(post, fields)] = True - - for post in items1: - if not (element_to_string(post, fields) in posts2): - posts_diff += 1 - f.write("<<<<<<<<<<<<<<< Different posts (%s) >>>>>>>>>>>>>>>\n" % posts_diff) - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext1)) - #import pdb;pdb.set_trace() - f.write(element_to_string(post, fields) + "\n") - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext2)) - f.write("*** Not found ***\n") - - for post in items2: - if not (element_to_string(post, fields) in posts1): - posts_diff += 1 - f.write("<<<<<<<<<<<<<<< Different posts (%s) >>>>>>>>>>>>>>>\n" % posts_diff) - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext1)) - f.write("*** Not found ***\n") - f.write(">>>>>>>>>>>>>>> %s.%s <<<<<<<<<<<<<\n" % (id, ext2)) - f.write(element_to_string(post, fields) + "\n") - print "Content of files %s.%s and %s.%s is different. Diff: %s.diff" % (id, ext1, id, ext2, id) - if posts_diff > 0: - print "Different feeds: %s" % posts_diff - number += 1 - if diff > 0: - print "Different feed ids: %s" % str(diff) - -print "Example of usage: python tests.py crawl before politepol.com" -print str(sys.argv) -if len(sys.argv) == 1: - test1_get_inner_html() -elif len(sys.argv) > 2: - if sys.argv[1] == 'crawl': - if len(sys.argv) == 4: - domain = sys.argv[3] - crawl(sys.argv[2]) - else: - raise Exception("Invalid argument count for crawl") - elif sys.argv[1] == 'diff': - if len(sys.argv) >= 4: - fields = None - if len(sys.argv) == 5: - fields = sys.argv[4].split(',') - diff(sys.argv[2], sys.argv[3], fields) - else: - raise Exception("Invalid argument count for diff") - else: - raise Exception("Unsupported operation %s" % sys.argv[1]) -else: - raise Exception("Invaid argument count") - -print 'All tests are OK' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..7b83e9d --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,30 @@ +from __future__ import print_function + +from twisted.internet import reactor, defer +from twisted.logger import Logger + +from pol.log import LogHandler + + +class MFTests(object): + + def __init__(self): + self.log = Logger() + pass + + def stop_callback(self, none): + reactor.stop() + pass + + def test_log_handler(self): + handler = LogHandler() + self.log.info('Test msg with {parameter} is OK', parameter="value") + self.log.error('Test error with {parameter} is OK', parameter="value", isError=True) + + d = defer.Deferred() + reactor.callLater(0, d.callback, None) + d.addCallback(self.stop_callback) + d.addErrback(lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) + + reactor.run() +