From c20a2f50ce7c370b1343d3fbe981d2e9c1a05e9e Mon Sep 17 00:00:00 2001
From: caibotmini <caibotmini@openclaw.ai>
Date: Tue, 7 Apr 2026 19:43:16 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E9=87=8F=E8=84=9A=E6=9C=AC?=
 =?UTF-8?q?=E5=8E=BB=E6=8E=89LIMIT=20-=20=E6=AF=8F=E6=AC=A1=E8=B7=91?=
 =?UTF-8?q?=E5=AE=8C=E6=89=80=E6=9C=89=E5=BE=85=E5=A4=84=E7=90=86=E5=B8=96?=
 =?UTF-8?q?=E5=AD=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/crown_extract_incremental.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/scripts/crown_extract_incremental.py b/scripts/crown_extract_incremental.py
index c00a6bd..5752ec4 100644
--- a/scripts/crown_extract_incremental.py
+++ b/scripts/crown_extract_incremental.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-冠字号增量提取脚本 V3 - 修复版
+冠字号全量提取脚本 - 跑完所有待处理帖子
 """
 import sys
 sys.path.insert(0, '/root/coolbot-data')
@@ -106,12 +106,12 @@ def get_category(title):
     return '其他'
 
 def main():
-    print(f'[{datetime.now()}] 冠字号增量提取开始...')
+    print(f'[{datetime.now()}] 冠字号全量提取开始...')
 
     conn = psycopg2.connect(**DB_CONFIG)
     cur = conn.cursor()
 
-    # 用 NOT EXISTS 替代 NOT IN，避免 TEXT/INTEGER 类型转换问题
+    # 无 LIMIT，全量跑完所有待处理帖子
     cur.execute("""
         SELECT p.id, p.post_id, p.title, p.content, p.post_type,
                p.author_username, p.price, p.price_unit, p.url, p.crawled_at
@@ -122,14 +122,15 @@ def main():
               WHERE c.post_id = CAST(p.id AS TEXT)
           )
         ORDER BY p.id
-        LIMIT 1000
     """)
     posts = cur.fetchall()
-    print(f'待处理新帖子: {len(posts)} 条')
+    total = len(posts)
+    print(f'待处理新帖子: {total} 条')
 
     new_count = 0
     skip_count = 0
     total_codes = 0
+    processed = 0
 
     for (pid, post_id, title, content, post_type,
          author, price, price_unit, url, crawled_at) in posts:
@@ -137,8 +138,12 @@ def main():
         text = f'{title or ""} {content or ""}'
         codes = extract_codes(text)
         total_codes += len(codes)
+        processed += 1
+
         if not codes:
             skip_count += 1
+            if processed % 500 == 0:
+                print(f'  已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
             continue
 
         features = extract_features(text)
@@ -167,8 +172,12 @@ def main():
             except Exception as e:
                 print(f'  插入失败 post_id={post_id}, code={code}: {e}')
 
+        if processed % 500 == 0:
+            print(f'  已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
+            conn.commit()
+
     conn.commit()
-    print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条')
+    print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
 
     cur.execute('SELECT COUNT(*) FROM collections')
     print(f'collections表当前总量: {cur.fetchone()[0]} 条')