CoolBotDataSys/crawlers/crawl_today.py

"""只采集今天的新帖子 - 优化版"""
import re
import sys
import os
sys.path.insert(0, '/root/coolbot-data')

import requests
from bs4 import BeautifulSoup
from datetime import datetime
from crawlers.base import PaginationSpider
from database import get_db

def extract_post_content(html_bytes):
    """从详情页原始HTML提取帖子正文（GBK编码）。
    定位：找到含 min-height:200px 的 div，向下到 <hr />（签名）为止。
    """
    if isinstance(html_bytes, str):
        page = html_bytes
    else:
        page = html_bytes.decode('gbk', errors='replace')

    idx = page.find('min-height:200px')
    if idx == -1:
        return None

    div_start = page.rfind('<div', 0, idx)
    if div_start == -1:
        return None

    hr_idx = page.find('<hr />', idx)
    discl_idx = page.find('\u514d\u8d23\u5371\u58f0\u660e', idx)

    candidates = [x for x in [hr_idx, discl_idx] if x != -1]
    end = min(candidates) if candidates else (hr_idx if hr_idx != -1 else len(page))

    chunk = page[div_start:end]

    text = re.sub(r'<div[^>]*>', '\n', chunk)
    text = re.sub(r'</div>', '', text)
    text = re.sub(r'<p[^>]*>', '\n', text)
    text = re.sub(r'</p>', '', text)
    text = re.sub(r'<br\s*/?>', '\n', text)
    text = re.sub(r'<[^>]+>', '', text)
    text = text.replace('&nbsp;', ' ').replace('&#160;', ' ')
    text = text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')

    lines = [l.strip() for l in text.split('\n') if l.strip()]
    text = '\n'.join(lines)
    text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text)

    # 截取免责声明及之前的内容
    disclaimer = '免责声明及风险提示：所有交易人员，凡未采用本站中介交易的，被骗后果自负。'
    discl_pos = text.find(disclaimer)
    if discl_pos != -1:
        text = text[:discl_pos + len(disclaimer)]
    
    # 清理尾部网站版权和导航信息
    footer_patterns = [
        'BoardJumpListSelect',
        'Powered By Dvbbs',
        '京ICP备',
        '页面执行时间',
        '发短信',
        '购买论坛点券',
        '我能做什么',
        '我发表的主题',
        '我参与的主题',
        '基本资料修改',
        '用户密码修改',
        '联系资料修改',
        '用户短信服务',
        '编辑好友列表',
        '个人文件管理',
        '通行证设置',
        '今日贴数图例',
        '主题数图例',
        '总帖数图例',
        '在线图例',
        '在线情况',
        '用户组在线图例',
        '文件集浏览',
        '图片集浏览',
        'Flash浏览',
        '音乐集浏览',
        '电影集浏览',
        '贺卡发送',
    ]
    for pat in footer_patterns:
        pos = text.find(pat)
        if pos != -1:
            text = text[:pos]
    
    return text if text else None



import logging

logger = logging.getLogger(__name__)

class YichensTodaySpider(PaginationSpider):
    def __init__(self, instance_id=None):
        super().__init__("一尘网今日采集", "pm001")
        self.instance_id = instance_id or os.environ.get('CRAWLER_INSTANCE_ID', 'unknown')
        self.base_url = "http://www.pm001.net"
        self.board_id = "151"
        self.forum_url = f"{self.base_url}/index.asp?boardid={self.board_id}"
        self.max_pages = 2
        self.encoding = "gbk"
        self.min_delay = 1.5
        self.max_delay = 3.0
    
    def get(self, url, **kwargs):
        try:
            self._random_delay()
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            }
            response = requests.get(url, timeout=30, headers=headers, **kwargs)
            try:
                response.encoding = self.encoding
            except:
                response.encoding = "gb2312"
            return response
        except Exception as e:
            print(f"请求失败: {e}")
            return None
    
    def extract_posts_with_dates(self, html):
        """从列表页提取所有帖子ID和"发表于"时间，返回 [(post_id, date_str), ...]
        使用BeautifulSoup解析listtitle div，比正则更准确
        """
        soup = BeautifulSoup(html, 'html.parser')
        posts = []
        listtitle_divs = soup.find_all('div', class_='listtitle')
        
        for div in listtitle_divs:
            link = div.find('a', href=re.compile(r'dispbbs\.asp\?boardID=151&ID=\d+'))
            if not link:
                continue
            
            href = link.get('href', '')
            id_match = re.search(r'\&ID=(\d+)', href)
            if not id_match:
                continue
            post_id = id_match.group(1)
            
            title_attr = link.get('title', '')
            # title格式: 《标题》\n作者：xxx\n发表于：2026/4/5 9:35:00
            date_match = re.search(r'发表于：(\d{4}/\d{1,2}/\d{1,2})', title_attr)
            if date_match:
                date_str = date_match.group(1)
                posts.append((post_id, date_str))
        
        return posts
    
    def parse_post_detail(self, html, url):
        """解析详情页，提取真实发帖时间"""
        soup = BeautifulSoup(html, "html.parser")
        page_text = soup.get_text()
        
        match = re.search(r"\&ID=(\d+)", url)
        post_id = match.group(1) if match else None
        
        title = None
        title_elem = soup.find("title")
        if title_elem:
            title_text = title_elem.get_text(strip=True)
            if "[" in title_text:
                title = title_text.split("[")[0].strip()
        
        author = None
        m = re.search(r">>>\s*([^\s<]+)", page_text)
        if m:
            author = m.group(1).strip()
        
        created_at = None
        m = re.search(r"(\d{4}/\d{1,2}/\d{1,2}\s+\d{1,2}:\d{2}:\d{2})", page_text)
        if m:
            date_str = m.group(1).replace("/", "-")
            try:
                from datetime import datetime
                dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
                if 2020 <= dt.year <= 2030:
                    created_at = date_str
            except:
                pass
        
        phones = re.findall(r'1[3-9]\d{9,10}', page_text)
        contact = ",".join(dict.fromkeys(phones[:5])) if phones else None
        
        price = None
        price_unit = None
        if title:
            price, price_unit = self.extract_price(title)
        if not price:
            price, price_unit = self.extract_price(page_text)
        
        content = extract_post_content(html)
        
        special_types = []
        if title:
            if "救生圈" in title or "88" in title or "888" in title:
                special_types.append("救生圈")
            if "标十" in title:
                special_types.append("标十")
            if "标百" in title:
                special_types.append("标百")
        
        # post_type 分类逻辑：出售 > 求购 > 其他 > 默认出售
        if title:
            if any(c in title for c in ["出", "售", "卖"]):
                post_type = "deal"
            elif any(c in title for c in ["收", "求", "购", "要"]):
                post_type = "want"
            elif any(c in title for c in ["确认", "朋友", "投诉"]):
                post_type = "normal"
            else:
                post_type = "deal"  # 默认出售
        else:
            post_type = "normal"
        
        # category 分类逻辑（优先级：龙钞 > 马钞 > 蛇钞 > 其他）
        category = "其他"
        if title:
            if any(c in title for c in ["龙", "龙钞", "小龙钞", "钞王"]):
                category = "龙钞"
            elif any(c in title for c in ["马", "马钞"]):
                category = "马钞"
            elif any(c in title for c in ["蛇", "蛇钞"]):
                category = "蛇钞"
        
        # 从 postuserinfo div 的 <b> 标签提取用户名（跳过电话号码）
        if not author:
            userinfos = soup.find_all('div', class_='postuserinfo')
            for ui in userinfos:
                for b in ui.find_all('b'):
                    text = b.get_text(strip=True)
                    if text and not re.match(r'^1\d{10}$', text) and len(text) > 1:
                        author = text
                        break
                if author:
                    break

        return {
            "post_id": post_id,
            "title": title,
            "content": content,
            "author_username": author,
            "price": price,
            "price_unit": price_unit,
            "contact": contact,
            "post_time": created_at,
            "special_types": "|".join(special_types) if special_types else None,
            "category": category,
            "post_type": post_type,
            "url": f"http://www.pm001.net/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1",
        }
    
    def extract_price(self, text):
        if not text:
            return None, None
        patterns = [
            r"(\d+)\s*元\s*(?:一张|一刀|一条|单张)?",
            r"[收售出求兑买]\s*(\d+)\s*元?",
        ]
        for pattern in patterns:
            m = re.search(pattern, text)
            if m:
                try:
                    val = float(m.group(1))
                    if val < 0 or val > 999999999:
                        continue
                    price_unit = "元"
                    if "张" in text:
                        price_unit = "元/张"
                    elif "刀" in text:
                        price_unit = "元/刀"
                    elif "条" in text:
                        price_unit = "元/条"
                    return val, price_unit
                except:
                    pass
        return None, None
    
    def crawl_today(self):
        """全量采集：爬取第1-max_pages页所有帖子，不过滤日期
        用于一次性补全所有帖子的post_time等字段
        """
        today = datetime.now().date()
        today_str = today.strftime("%Y/%m/%d").lstrip('0').replace('/0', '/')
        
        print(f"开始全量采集第1-{self.max_pages}页 (今天: {today_str})")
        
        saved = 0
        checked = 0
        
        for page in range(1, self.max_pages + 1):
            page_url = f"{self.forum_url}&page={page}"
            print(f"\n扫描第 {page} 页...")
            
            response = self.get(page_url)
            if not response:
                continue
            
            posts = self.extract_posts_with_dates(response.text)
            print(f"  该页共 {len(posts)} 条帖子")
            
            for post_id, date_str in posts:
                checked += 1
                
                # 标准化日期
                normalized_date = date_str.lstrip('0').replace('/0', '/')
                
                # 访问详情页
                detail_url = f"{self.base_url}/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1"
                detail_response = self.get(detail_url)
                
                if not detail_response:
                    continue
                
                detail = self.parse_post_detail(detail_response.text, detail_url)
                
                # 验证详情页的真实发帖时间
                post_time_str = detail.get('post_time', '')
                if not post_time_str:
                    continue
                
                try:
                    post_time = datetime.strptime(post_time_str, '%Y-%m-%d %H:%M:%S').date()
                    is_today = (post_time == today)
                    label = f"今日{normalized_date}" if is_today else f"历史{post_time}"
                except:
                    label = "时间异常"
                
                # 保存（ON CONFLICT会更新post_time等字段）
                if self.save_full_post(detail):
                    saved += 1
                    print(f"  [保存] {post_id} - {detail.get('title', '')[:25]}... ({label}中已存{saved}条)")
                else:
                    print(f"  [失败] {post_id}")
            
            print(f"  页码 {page} 完成: 检查{len(posts)}条")
        
        print(f"\n===== 采集完成 =====")
        print(f"总计检查: {checked} 条")
        print(f"成功保存: {saved} 条")
        return saved
    
    def save_full_post(self, post):
        if not post or not post.get("post_id"):
            return False
        
        import psycopg2.sql as sql
        
        has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
        
        cols = ['post_id', 'title', 'content', 'category', 'post_type', 'price',
                'price_unit', 'special_types', 'number_features', 'author_username',
                'author_id', 'contact', 'has_lifebuoy', 'reply_count', 'view_count',
                'post_time', 'crawled_at', 'updated_at', 'url', 'instance_id']
        
        vals = [
            sql.Literal(post.get("post_id")),
            sql.Literal(post.get("title")),
            sql.Literal(post.get("content")),
            sql.Literal(post.get("category")),
            sql.Literal(post.get("post_type")),
            sql.Literal(post.get("price")),
            sql.Literal(post.get("price_unit")),
            sql.Literal(post.get("special_types")),
            sql.Literal(post.get("number_features")),
            sql.Literal(post.get("author_username")),
            sql.Literal(f"user_{post.get('post_id')}"),
            sql.Literal(post.get("contact")),
            sql.Literal(has_lifebuoy),
            sql.Literal(0),
            sql.Literal(0),
            sql.Literal(post.get("post_time")),
            sql.SQL('NOW()'),
            sql.SQL('NOW()'),
            sql.Literal(post.get("url")),
            sql.Literal(self.instance_id),
        ]
        
        update_assigns = [
            "title = EXCLUDED.title",
            "content = EXCLUDED.content",
            "category = EXCLUDED.category",
            "post_type = EXCLUDED.post_type",
            "price = EXCLUDED.price",
            "price_unit = EXCLUDED.price_unit",
            "special_types = EXCLUDED.special_types",
            "number_features = EXCLUDED.number_features",
            "author_username = EXCLUDED.author_username",
            "author_id = EXCLUDED.author_id",
            "contact = EXCLUDED.contact",
            "has_lifebuoy = EXCLUDED.has_lifebuoy",
            "reply_count = EXCLUDED.reply_count",
            "view_count = EXCLUDED.view_count",
            "post_time = EXCLUDED.post_time",
            "updated_at = NOW()",
            "crawled_at = NOW()",
            "url = EXCLUDED.url",
            
        ]
        
        query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(
            sql.SQL(', ').join(sql.Identifier(c) for c in cols),
            sql.SQL(', ').join(vals),
            sql.SQL(', ').join(sql.SQL(a) for a in update_assigns),
        )
        
        try:
            with get_db() as conn:
                if not conn:
                    print("数据库连接失败")
                    return False
                cur = conn.cursor()
                cur.execute(query)
                conn.commit()
                cur.close()
                return True
        except Exception as e:
            print(f"保存失败: {e}")
            return False

    def run(self):
        log_id = self._log_start()
        try:
            result = self.crawl_today()
            self._log_finish(log_id, "success", result)
            return result
        except Exception as e:
            print(f"爬虫异常: {e}")
            self._log_finish(log_id, "failed", 0, str(e))
            return 0
    
    def _log_start(self):
        try:
            with get_db() as conn:
                cur = conn.cursor()
                cur.execute(
                    "INSERT INTO crawl_logs (spider_name, status, started_at, instance_id) VALUES (%s, %s, NOW(), %s) RETURNING id",
                    (self.source, "running", self.instance_id)
                )
                log_id = cur.fetchone()[0]
                conn.commit()
                cur.close()
                return log_id
        except Exception as e:
            print(f"记录日志失败: {e}")
            return None
    
    def _log_finish(self, log_id, status, items_count, error=""):
        if log_id is None:
            return
        try:
            with get_db() as conn:
                cur = conn.cursor()
                cur.execute(
                    "UPDATE crawl_logs SET status=%s, items_count=%s, error_message=%s, finished_at=NOW() WHERE id=%s",
                    (status, items_count, error, log_id)
                )
                conn.commit()
                cur.close()
        except Exception as e:
            print(f"更新日志失败: {e}")

if __name__ == "__main__":
    spider = YichensTodaySpider()
    spider.run()