fix: 修复增量提取NOT IN子查询bug - 改用NOT EXISTS避免类型转换问题

scripts/crown_extract_incremental.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-冠字号增量提取脚本 - 只提取新增帖子的藏品数据
+冠字号增量提取脚本 V3 - 修复版
 """
 import sys
 sys.path.insert(0, '/root/coolbot-data')
@@ -111,31 +111,32 @@ def main():
     conn = psycopg2.connect(**DB_CONFIG)
     cur = conn.cursor()
 
-    # 增量：只选还没有在 collections 表中的帖子
+    # 用 NOT EXISTS 替代 NOT IN，避免 TEXT/INTEGER 类型转换问题
     cur.execute("""
-        SELECT id, post_id, title, content, post_type,
-               author_username, price, price_unit, url, crawled_at
-        FROM yichens_posts
-        WHERE category IN ('龙钞', '马钞', '蛇钞', '其他')
-          AND id NOT IN (
-              SELECT DISTINCT CAST(post_id AS INTEGER)
-              FROM collections
-              WHERE post_id IS NOT NULL
+        SELECT p.id, p.post_id, p.title, p.content, p.post_type,
+               p.author_username, p.price, p.price_unit, p.url, p.crawled_at
+        FROM yichens_posts p
+        WHERE p.category IN ('龙钞', '马钞', '蛇钞', '其他')
+          AND NOT EXISTS (
+              SELECT 1 FROM collections c
+              WHERE c.post_id = CAST(p.id AS TEXT)
           )
-        ORDER BY id
-        LIMIT 500
+        ORDER BY p.id
+        LIMIT 1000
     """)
     posts = cur.fetchall()
     print(f'待处理新帖子: {len(posts)} 条')
 
     new_count = 0
     skip_count = 0
+    total_codes = 0
 
     for (pid, post_id, title, content, post_type,
          author, price, price_unit, url, crawled_at) in posts:
 
         text = f'{title or ""} {content or ""}'
         codes = extract_codes(text)
+        total_codes += len(codes)
         if not codes:
             skip_count += 1
             continue
@@ -167,8 +168,7 @@ def main():
                 print(f'  插入失败 post_id={post_id}, code={code}: {e}')
 
     conn.commit()
-    total = new_count + skip_count
-    print(f'完成！新增: {new_count} 条, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
+    print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条')
 
     cur.execute('SELECT COUNT(*) FROM collections')
     print(f'collections表当前总量: {cur.fetchone()[0]} 条')