fix: 修复yichens_spider_v4.py两处bug
1. post_time加入ON CONFLICT UPDATE SET - 未来重爬时可更新post_time 2. author_username提取从user_link改为<b>标签 - 修复用户名抓不到的问题
This commit is contained in:
parent
9a6f0de418
commit
c330d3f0c2
|
|
@ -87,6 +87,7 @@ def save_post(post):
|
|||
reply_count = EXCLUDED.reply_count,
|
||||
view_count = EXCLUDED.view_count,
|
||||
price = EXCLUDED.price,
|
||||
post_time = EXCLUDED.post_time,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
cur.execute(sql, (
|
||||
|
|
@ -462,10 +463,11 @@ def parse_userinfo_block(userinfo_div):
|
|||
if dispuser_match:
|
||||
user_id = dispuser_match.group(1)
|
||||
|
||||
# username
|
||||
# username - from <b> tag inside userinfo div
|
||||
username = None
|
||||
if user_link:
|
||||
username = user_link.get_text(strip=True)
|
||||
b_tag = userinfo_div.find('b')
|
||||
if b_tag:
|
||||
username = b_tag.get_text(strip=True)
|
||||
|
||||
# transaction_level
|
||||
level_match = re.search(r'交易等级[::]\s*([^\n]+)', text)
|
||||
|
|
|
|||
Loading…
Reference in New Issue