v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-21 08:30:24 -07:00
Code Issues Releases Wiki Activity

remove try catch

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-07-19 14:05:52 -07:00
parent ed50a76498
commit 27de6c7363
1 changed files with 36 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
feed.py
View File

@ -31,46 +31,43 @@ def save_post(conn, created, feed_id, post_fields):
print(cur._last_executed) print(cur._last_executed)
def fill_time(feed_id, items): def fill_time(feed_id, items):
try: if not items:
if not items: return []
return [] for item in items:
for item in items: #create md5
#create md5 h = md5('')
h = md5('') for key in ['title', 'description', 'link']:
for key in ['title', 'description', 'link']: if key in item:
if key in item: h.update(item[key].encode('utf-8'))
h.update(item[key].encode('utf-8')) item['md5'] = h.hexdigest()
item['md5'] = h.hexdigest()
#fetch dates from db #fetch dates from db
fetched_dates = {} fetched_dates = {}
db = get_conn() db = get_conn()
with db: with db:
quoted_hashes = ','.join(["'%s'" % (i['md5']) for i in items]) quoted_hashes = ','.join(["'%s'" % (i['md5']) for i in items])
cur = db.cursor() cur = db.cursor()
cur.execute("""select p.md5sum, p.created, p.id cur.execute("""select p.md5sum, p.created, p.id
from frontend_post p from frontend_post p
where p.md5sum in (%s) where p.md5sum in (%s)
and p.feed_id=%s""" % (quoted_hashes, feed_id,)) and p.feed_id=%s""" % (quoted_hashes, feed_id,))
rows = cur.fetchall() rows = cur.fetchall()
print(cur._last_executed) print(cur._last_executed)
for row in rows: for row in rows:
md5hash = row[0] md5hash = row[0]
created = row[1] created = row[1]
post_id = row[2] post_id = row[2]
fetched_dates[md5hash] = created fetched_dates[md5hash] = created
cur_time = datetime.datetime.utcnow() cur_time = datetime.datetime.utcnow()
new_posts = [] new_posts = []
for item in items: for item in items:
if item['md5'] in fetched_dates: if item['md5'] in fetched_dates:
item['time'] = fetched_dates[item['md5']] item['time'] = fetched_dates[item['md5']]
else: else:
item['time'] = cur_time item['time'] = cur_time
save_post(db, cur_time, feed_id, item) save_post(db, cur_time, feed_id, item)
cur_time -= datetime.timedelta(minutes=POST_TIME_DISTANCE) cur_time -= datetime.timedelta(minutes=POST_TIME_DISTANCE)
except Exception as ex:
sys.stderr.write('\n'.join([str(datetime.datetime.now()), "Feed exception:" +str(ex)]))
def decode(text, encoding): # it's strange but true def decode(text, encoding): # it's strange but true
@ -90,7 +87,7 @@ def element_to_unicode(element, encoding):
def _build_link(html, doc_url, url): def _build_link(html, doc_url, url):
base_url = w3lib.html.get_base_url(html, doc_url) base_url = w3lib.html.get_base_url(html, doc_url)
return w3lib.url.urljoin_rfc(base_url, url) return w3lib.url.urljoin_rfc(base_url, url).decode('utf-8')
def buildFeed(response, feed_config): def buildFeed(response, feed_config):
response.selector.remove_namespaces() response.selector.remove_namespaces()