v/pol
1
0
Fork 0
mirror of https://github.com/taroved/pol synced 2025-05-29 12:30:09 -07:00
Code Issues Releases Wiki Activity

posts time saving

Browse Source
This commit is contained in:
Alexandr Nesterenko 2017-07-12 14:20:58 -07:00
parent c3b4a6835d
commit acc4b6fd85
1 changed files with 22 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
feed.py
View File

@ -17,61 +17,56 @@ POST_TIME_DISTANCE = 15 # minutes
FIELD_IDS = {'title': 1, 'description': 2, 'title_link': 3} FIELD_IDS = {'title': 1, 'description': 2, 'title_link': 3}
def save_post(conn, md5sum, created, feed_id, post_fields): def save_post(conn, created, feed_id, post_fields):
cur = conn.cursor() cur = conn.cursor()
try: cur.execute("""insert into frontend_post (md5sum, created, feed_id)
#import pdb;pdb.set_trace() values (%s, %s, %s)""", (post_fields['md5'], created, feed_id))
cur.execute("""insert into frontend_post (md5sum, created, feed_id) print(cur._last_executed)
values (%s, %s, %s)""", (md5sum.hexdigest(), created, feed_id))
finally:
print(cur._last_executed)
post_id = conn.insert_id() post_id = conn.insert_id()
for key in ['title', 'description', 'title_link']: for key in ['title', 'description', 'title_link']:
if key in post_fields: if key in post_fields:
try: cur.execute("""insert into frontend_postfield (field_id, post_id, `text`)
cur.execute("""insert into frontend_postfield (field_id, post_id, `text`) values (%s, %s, %s)""", (FIELD_IDS[key], post_id, post_fields[key].encode('utf-8')))
values (%s, %s, %s)""", (FIELD_IDS[key], post_id, post_fields[key].encode('utf-8'))) print(cur._last_executed)
finally:
print(cur._last_executed)
def fill_dates(feed_id, items): def fill_time(feed_id, items):
if not items: if not items:
return [] return []
hashes = {}
for item in items: for item in items:
#create md5 #create md5
h = md5('') h = md5('')
for key in ['title', 'description', 'title_link']: for key in ['title', 'description', 'title_link']:
if key in item: if key in item:
h.update(item[key].encode('utf-8')) h.update(item[key].encode('utf-8'))
hashes[h] = item item['md5'] = h.hexdigest()
#fetch dates from db #fetch dates from db
fetched_dates = {} fetched_dates = {}
db = get_conn() db = get_conn()
with db: with db:
quoted_hashes = ','.join(["'%s'" % (h.hexdigest()) for h in hashes]) quoted_hashes = ','.join(["'%s'" % (i['md5']) for i in items])
cur = db.cursor() cur = db.cursor()
cur.execute("""select p.md5sum, p.created, p.id cur.execute("""select p.md5sum, p.created, p.id
from frontend_post p from frontend_post p
where p.md5sum in (%s) where p.md5sum in (%s)
and p.id=%s""" % (quoted_hashes, feed_id,)) and p.feed_id=%s""" % (quoted_hashes, feed_id,))
rows = cur.fetchall() rows = cur.fetchall()
print(cur._last_executed) print(cur._last_executed)
for row in rows: for row in rows:
md5hash = row[0] md5hash = row[0]
created = row[1] created = row[1]
post_id = row[2] post_id = row[2]
fetched_dates[md5hash] = datetime.datetime.fromtimestamp(int(created)) fetched_dates[md5hash] = created
cur_time = datetime.datetime.now() cur_time = datetime.datetime.utcnow()
new_posts = [] new_posts = []
for h in hashes: for item in items:
if h in fetched_dates: if item['md5'] in fetched_dates:
hashes[h]['time'] = fetched_date[h] item['time'] = fetched_dates[item['md5']]
else: else:
hashes[h]['time'] = cur_time item['time'] = cur_time
save_post(db, h, cur_time, feed_id, hashes[h]) save_post(db, cur_time, feed_id, item)
cur_time -= datetime.timedelta(minutes=POST_TIME_DISTANCE) cur_time -= datetime.timedelta(minutes=POST_TIME_DISTANCE)
def element_to_string(element): def element_to_string(element):
@ -120,11 +115,12 @@ def buildFeed(response, feed_config):
language="en", language="en",
) )
fill_dates(feed_config['id'], items) fill_time(feed_config['id'], items)
for item in items: for item in items:
title = item['title'] if 'title' in item else '' title = item['title'] if 'title' in item else ''
desc = item['description'] if 'description' in item else '' desc = item['description'] if 'description' in item else ''
time = item['time']
if 'title_link' in item: if 'title_link' in item:
link = item['title_link'] link = item['title_link']
else: else:
@ -134,7 +130,7 @@ def buildFeed(response, feed_config):
link = link, link = link,
description = desc, description = desc,
#enclosure=Enclosure(fields[4], "32000", "image/jpeg") if 4 in fields else None, #"Image" #enclosure=Enclosure(fields[4], "32000", "image/jpeg") if 4 in fields else None, #"Image"
pubdate=datetime.datetime.now() pubdate = time
) )
return feed.writeString('utf-8') return feed.writeString('utf-8')