diff --git a/scripts/crown_extract_incremental.py b/scripts/crown_extract_incremental.py index 6f157a4..c00a6bd 100644 --- a/scripts/crown_extract_incremental.py +++ b/scripts/crown_extract_incremental.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -冠字号增量提取脚本 - 只提取新增帖子的藏品数据 +冠字号增量提取脚本 V3 - 修复版 """ import sys sys.path.insert(0, '/root/coolbot-data') @@ -111,31 +111,32 @@ def main(): conn = psycopg2.connect(**DB_CONFIG) cur = conn.cursor() - # 增量:只选还没有在 collections 表中的帖子 + # 用 NOT EXISTS 替代 NOT IN,避免 TEXT/INTEGER 类型转换问题 cur.execute(""" - SELECT id, post_id, title, content, post_type, - author_username, price, price_unit, url, crawled_at - FROM yichens_posts - WHERE category IN ('龙钞', '马钞', '蛇钞', '其他') - AND id NOT IN ( - SELECT DISTINCT CAST(post_id AS INTEGER) - FROM collections - WHERE post_id IS NOT NULL + SELECT p.id, p.post_id, p.title, p.content, p.post_type, + p.author_username, p.price, p.price_unit, p.url, p.crawled_at + FROM yichens_posts p + WHERE p.category IN ('龙钞', '马钞', '蛇钞', '其他') + AND NOT EXISTS ( + SELECT 1 FROM collections c + WHERE c.post_id = CAST(p.id AS TEXT) ) - ORDER BY id - LIMIT 500 + ORDER BY p.id + LIMIT 1000 """) posts = cur.fetchall() print(f'待处理新帖子: {len(posts)} 条') new_count = 0 skip_count = 0 + total_codes = 0 for (pid, post_id, title, content, post_type, author, price, price_unit, url, crawled_at) in posts: text = f'{title or ""} {content or ""}' codes = extract_codes(text) + total_codes += len(codes) if not codes: skip_count += 1 continue @@ -167,8 +168,7 @@ def main(): print(f' 插入失败 post_id={post_id}, code={code}: {e}') conn.commit() - total = new_count + skip_count - print(f'完成!新增: {new_count} 条, 无冠号跳过: {skip_count} 条, 总处理: {total} 条') + print(f'完成!新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条') cur.execute('SELECT COUNT(*) FROM collections') print(f'collections表当前总量: {cur.fetchone()[0]} 条')