caibotmi
/
CoolBotDataSys
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 修复yichens_spider_v4.py两处bug 

1. post_time加入ON CONFLICT UPDATE SET - 未来重爬时可更新post_time
2. author_username提取从user_link改为<b>标签 - 修复用户名抓不到的问题
This commit is contained in:
caibotmini 2026-04-05 10:13:13 +08:00
parent 9a6f0de418
commit c330d3f0c2
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
crawlers/yichens_spider_v4.py
View File

@ -87,6 +87,7 @@ def save_post(post):
reply_count = EXCLUDED.reply_count, reply_count = EXCLUDED.reply_count,
view_count = EXCLUDED.view_count, view_count = EXCLUDED.view_count,
price = EXCLUDED.price, price = EXCLUDED.price,
post_time = EXCLUDED.post_time,
updated_at = CURRENT_TIMESTAMP updated_at = CURRENT_TIMESTAMP
""" """
cur.execute(sql, ( cur.execute(sql, (
@ -462,10 +463,11 @@ def parse_userinfo_block(userinfo_div):
if dispuser_match: if dispuser_match:
user_id = dispuser_match.group(1) user_id = dispuser_match.group(1)
# username # username - from <b> tag inside userinfo div
username = None username = None
if user_link: b_tag = userinfo_div.find('b')
username = user_link.get_text(strip=True) if b_tag:
username = b_tag.get_text(strip=True)
# transaction_level # transaction_level
level_match = re.search(r'交易等级[：:]\s*([^\n]+)', text) level_match = re.search(r'交易等级[：:]\s*([^\n]+)', text)