From b0bb3372faf38a9a70593c9bdcfb63a9756d617d Mon Sep 17 00:00:00 2001
From: caibotmini <caibotmi@coolbot.com>
Date: Tue, 7 Apr 2026 09:15:04 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20save=5Ffull=5Fpost?=
 =?UTF-8?q?=20SQL=20=E5=8F=82=E6=95=B0=E4=B8=8D=E5=8C=B9=E9=85=8D=E9=97=AE?=
 =?UTF-8?q?=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- save_full_post 使用 psycopg2.sql 模块构建动态 SQL
- 修正 INSERT/ON CONFLICT UPDATE 缺失字段(number_features/reply_count/view_count/author_id)
- 修正 VALUES 占位符数量不匹配(%s vs 实际参数)
- 添加 instance_id 字段追踪爬虫来源
- 更新分类逻辑: 出/售/卖>deal, 收/求/购/要>want, 确认/朋友/投诉>normal
---
 crawlers/crawl_today.py | 103 +++++++++++++++++++++++-----------------
 1 file changed, 59 insertions(+), 44 deletions(-)

diff --git a/crawlers/crawl_today.py b/crawlers/crawl_today.py
index 8f41231..e95cebc 100644
--- a/crawlers/crawl_today.py
+++ b/crawlers/crawl_today.py
@@ -310,65 +310,80 @@ class YichensTodaySpider(PaginationSpider):
         if not post or not post.get("post_id"):
             return False
         
-        sql = """
-            INSERT INTO yichens_posts (
-                post_id, title, content, category, post_type,
-                price, price_unit, special_types, number_features,
-                author_username, author_id, contact,
-                has_lifebuoy, reply_count, view_count,
-                post_time, crawled_at, updated_at, url
-            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW(), %s)
-            ON CONFLICT (post_id) DO UPDATE SET
-                title = EXCLUDED.title,
-                content = EXCLUDED.content,
-                category = EXCLUDED.category,
-                post_type = EXCLUDED.post_type,
-                price = EXCLUDED.price,
-                price_unit = EXCLUDED.price_unit,
-                special_types = EXCLUDED.special_types,
-                author_username = EXCLUDED.author_username,
-                contact = EXCLUDED.contact,
-                has_lifebuoy = EXCLUDED.has_lifebuoy,
-                post_time = EXCLUDED.post_time,
-                updated_at = NOW(),
-                crawled_at = NOW(),
-                url = EXCLUDED.url
-        """
+        import psycopg2.sql as sql
         
         has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
         
+        cols = ['post_id', 'title', 'content', 'category', 'post_type', 'price',
+                'price_unit', 'special_types', 'number_features', 'author_username',
+                'author_id', 'contact', 'has_lifebuoy', 'reply_count', 'view_count',
+                'post_time', 'crawled_at', 'updated_at', 'url', 'instance_id']
+        
+        vals = [
+            sql.Literal(post.get("post_id")),
+            sql.Literal(post.get("title")),
+            sql.Literal(post.get("content")),
+            sql.Literal(post.get("category")),
+            sql.Literal(post.get("post_type")),
+            sql.Literal(post.get("price")),
+            sql.Literal(post.get("price_unit")),
+            sql.Literal(post.get("special_types")),
+            sql.Literal(post.get("number_features")),
+            sql.Literal(post.get("author_username")),
+            sql.Literal(f"user_{post.get('post_id')}"),
+            sql.Literal(post.get("contact")),
+            sql.Literal(has_lifebuoy),
+            sql.Literal(0),
+            sql.Literal(0),
+            sql.Literal(post.get("post_time")),
+            sql.SQL('NOW()'),
+            sql.SQL('NOW()'),
+            sql.Literal(post.get("url")),
+            sql.Literal(self.instance_id),
+        ]
+        
+        update_assigns = [
+            "title = EXCLUDED.title",
+            "content = EXCLUDED.content",
+            "category = EXCLUDED.category",
+            "post_type = EXCLUDED.post_type",
+            "price = EXCLUDED.price",
+            "price_unit = EXCLUDED.price_unit",
+            "special_types = EXCLUDED.special_types",
+            "number_features = EXCLUDED.number_features",
+            "author_username = EXCLUDED.author_username",
+            "author_id = EXCLUDED.author_id",
+            "contact = EXCLUDED.contact",
+            "has_lifebuoy = EXCLUDED.has_lifebuoy",
+            "reply_count = EXCLUDED.reply_count",
+            "view_count = EXCLUDED.view_count",
+            "post_time = EXCLUDED.post_time",
+            "updated_at = NOW()",
+            "crawled_at = NOW()",
+            "url = EXCLUDED.url",
+            "instance_id = EXCLUDED.instance_id",
+        ]
+        
+        query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(
+            sql.SQL(', ').join(sql.Identifier(c) for c in cols),
+            sql.SQL(', ').join(vals),
+            sql.SQL(', ').join(sql.SQL(a) for a in update_assigns),
+        )
+        
         try:
             with get_db() as conn:
                 if not conn:
                     print("数据库连接失败")
                     return False
                 cur = conn.cursor()
-                cur.execute(sql, (
-                    post.get("post_id"),
-                    post.get("title"),
-                    post.get("content"),
-                    post.get("category"),
-                    post.get("post_type"),
-                    post.get("price"),
-                    post.get("price_unit"),
-                    post.get("special_types"),
-                    post.get("number_features"),
-                    post.get("author_username"),
-                    f"user_{post.get('post_id')}",
-                    post.get("contact"),
-                    has_lifebuoy,
-                    0,
-                    0,
-                    post.get("post_time"),
-                    post.get("url"),
-                ))
+                cur.execute(query)
                 conn.commit()
                 cur.close()
                 return True
         except Exception as e:
             print(f"保存失败: {e}")
             return False
-    
+
     def run(self):
         log_id = self._log_start()
         try: