v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-29 12:30:09 -07:00
Code Issues Releases Wiki Activity

stats

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-09-23 18:01:25 -04:00
parent 91a33febb9
commit fcc773d434
3 changed files with 46 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
downloader.py
View File

@ -1,6 +1,6 @@
from __future__ import print_function from __future__ import print_function
import json import json
import time, sys import time, sys, traceback
from hashlib import md5 from hashlib import md5
from datetime import datetime from datetime import datetime
@ -63,15 +63,21 @@ def get_ip_id(ip, cur):
def save_stat(stat): def save_stat(stat):
with closing(get_conn()) as conn: try:
with conn as cur: with closing(get_conn()) as conn:
ip_id = get_ip_id(stat.ip, cur) with conn as cur:
cur.execute("""insert into requests (ip_id, feed_id, post_cnt, new_post_cnt) ip_id = get_ip_id(stat.ip, cur)
values (%s, %s, %s, %s)""", (ip_id, stat.feed_id, stat.post_cnt, stat.new_post_cnt)) cur.execute("""insert into requests (ip_id, feed_id, post_cnt, new_post_cnt)
stat_id = cur.lastrowid values (%s, %s, %s, %s)""", (ip_id, stat.feed_id, stat.post_cnt, stat.new_post_cnt))
if not stat.feed_id: stat_id = cur.lastrowid
cur.execute("insert into request_urls (url, request_id) values (%s, %s)", (stat.url.encode('utf-8'), stat_id)) if not stat.feed_id:
cur.execute("insert into request_urls (url, request_id) values (%s, %s)", (stat.url.encode('utf-8')[:2000], stat_id))
if stat.ex_msg:
cur.execute("""insert into request_fails (request_id, ex_msg, ex_callstack)
values (%s, %s, %s)""", (stat_id, stat.ex_msg[0:2000], stat.ex_callstack[:2000]))
except:
traceback.print_exc(file=sys.stdout)
def print_log(event): def print_log(event):
if 'isError' in event and event['isError']: if 'isError' in event and event['isError']:
@ -212,11 +218,12 @@ def downloadDone(response_str, request, response, feed_config):
response = DecompressionMiddleware().process_response(None, response, None) response = DecompressionMiddleware().process_response(None, response, None)
if (isinstance(response, TextResponse)): if (isinstance(response, TextResponse)):
ip = request.getHeader('x-real-ip') or request.client.host
if feed_config: if feed_config:
[response_str, post_cnt, new_post_cnt] = buildFeed(response, feed_config) [response_str, post_cnt, new_post_cnt] = buildFeed(response, feed_config)
request.setHeader(b"Content-Type", b'text/xml; charset=utf-8') request.setHeader(b"Content-Type", b'text/xml; charset=utf-8')
log.debug('Stat: ip={request.ip} feed_id={request.feed_id} new_post_cnt={request.post_cnt} new_post_cnt={request.new_post_cnt}', request=RequestStat( log.info('Stat: ip={request.ip} feed_id={request.feed_id} post_cnt={request.post_cnt} new_post_cnt={request.new_post_cnt}', request=RequestStat(
ip=request.client.host, ip=ip,
feed_id=feed_config['id'], feed_id=feed_config['id'],
post_cnt=post_cnt, post_cnt=post_cnt,
new_post_cnt=new_post_cnt new_post_cnt=new_post_cnt
@ -225,8 +232,8 @@ def downloadDone(response_str, request, response, feed_config):
) )
else: else:
response_str, file_name = setBaseAndRemoveScriptsAndMore(response, url) response_str, file_name = setBaseAndRemoveScriptsAndMore(response, url)
log.debug('Stat: ip={request.ip} url={request.url}', request=RequestStat( log.info('Stat: ip={request.ip} url={request.url}', request=RequestStat(
ip=request.client.host, ip=ip,
feed_id=0, feed_id=0,
post_cnt=0, post_cnt=0,
new_post_cnt=0, new_post_cnt=0,
@ -274,6 +281,7 @@ def downloadError(error, request=None, url=None, response=None, feed_config=None
d.addCallback(downloadDone, request=request, response=response, feed_config=feed_config) d.addCallback(downloadDone, request=request, response=response, feed_config=feed_config)
d.addErrback(downloadError, request=request, url=url, response=response, feed_config=feed_config) d.addErrback(downloadError, request=request, url=url, response=response, feed_config=feed_config)
return return
if DEBUG: if DEBUG:
request.write('Downloader error: ' + error.getErrorMessage()) request.write('Downloader error: ' + error.getErrorMessage())
request.write('Traceback: ' + error.getTraceback()) request.write('Traceback: ' + error.getTraceback())
@ -281,6 +289,26 @@ def downloadError(error, request=None, url=None, response=None, feed_config=None
request.write(error_html('Something wrong. Contact us by email: politepol.com@gmail.com \n Scary mantra: ' + error.getErrorMessage())) request.write(error_html('Something wrong. Contact us by email: politepol.com@gmail.com \n Scary mantra: ' + error.getErrorMessage()))
sys.stderr.write('\n'.join([str(datetime.utcnow()), request.uri, url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()])) sys.stderr.write('\n'.join([str(datetime.utcnow()), request.uri, url, 'Downloader error: ' + error.getErrorMessage(), 'Traceback: ' + error.getTraceback()]))
request.finish() request.finish()
try:
feed_id = feed_config and feed_config['id']
s_url = None
if not feed_id:
feed_id = 0
s_url = url
log.info('Stat: ip={request.ip} feed_id={request.feed_id} url="{request.url}" error="{request.ex_msg}"', request=RequestStat(
ip = request.getHeader('x-real-ip') or request.client.host,
feed_id = feed_id,
post_cnt=0,
new_post_cnt=0,
url=s_url,
ex_msg=error.getErrorMessage(),
ex_callstack=error.getTraceback()
),
stat=True
)
except:
traceback.print_exc(file=sys.stdout)
class Downloader(resource.Resource): class Downloader(resource.Resource):
@ -337,4 +365,4 @@ port = sys.argv[1] if len(sys.argv) >= 2 else 1234
endpoints.serverFromString(reactor, "tcp:%s" % port).listen(server.Site(Downloader())) endpoints.serverFromString(reactor, "tcp:%s" % port).listen(server.Site(Downloader()))
print('Server starting at http://localhost:%s' % port) print('Server starting at http://localhost:%s' % port)
reactor.run() reactor.run()

3
feed.py
View File

@ -36,7 +36,7 @@ def save_post(conn, created, feed_id, post_fields):
def fill_time(feed_id, items): def fill_time(feed_id, items):
if not items: if not items:
return [] return 0
new_post_cnt = 0 new_post_cnt = 0
for item in items: for item in items:
@ -132,7 +132,6 @@ def buildFeed(response, feed_config):
"Source page url: " + feed_config['uri'], "Source page url: " + feed_config['uri'],
language="en", language="en",
) )
new_post_cnt = fill_time(feed_config['id'], items) new_post_cnt = fill_time(feed_config['id'], items)
for item in items: for item in items:

3
nginx/default.site-example
View File

@ -11,13 +11,16 @@ server {
} }
location /downloader { location /downloader {
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://127.0.0.1:1234; proxy_pass http://127.0.0.1:1234;
} }
location /feed { location /feed {
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://127.0.0.1:1234; proxy_pass http://127.0.0.1:1234;
} }
location /feed1 { location /feed1 {
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://127.0.0.1:1234; proxy_pass http://127.0.0.1:1234;
} }
} }