"""一尘网爬虫 - 一尘网钱币论坛数据采集""" import re from typing import Optional, Dict, List, Any from datetime import datetime, timedelta from bs4 import BeautifulSoup import logging from crawlers.base import BaseSpider, PaginationSpider from database import db logger = logging.getLogger(__name__) class YichensUserSpider(PaginationSpider): """一尘网用户爬虫""" def __init__(self): super().__init__("一尘网用户", "yichens") self.base_url = "https://www.yichens.com/user" self.user_list_url = "https://www.yichens.com/user/list" def parse_user(self, html: str, url: str) -> Optional[Dict]: soup = BeautifulSoup(html, "lxml") user_id = None match = re.search(r"user[_\-]?id[=:\s]*['\"]?(\w+)", url, re.I) if match: user_id = match.group(1) match = re.search(r"/user/([^/]+)", url) username = match.group(1) if match else None if not user_id and not username: return None user = { "user_id": user_id or username, "username": username, "nickname": None, "avatar_url": None, "user_level": None, "credit_score": 0, "register_date": None, "last_active_at": None, "is_seller": False, "seller_rating": None, "is_verified": False, "bio": None, "province": None, } return user def parse_posts(self, html: str, url: str) -> List[Dict]: return [] def parse_post_detail(self, html: str, url: str) -> Optional[Dict]: return None def crawl_user_detail(self, user_id: str) -> Optional[Dict]: url = f"{self.base_url}/{user_id}" response = self.get(url) if not response: return None return self.parse_user(response.text, url) def save_user(self, user: Dict) -> bool: if not user or not user.get("user_id"): return False try: with db.get_cursor() as cursor: cursor.execute(""" INSERT INTO yichens_users (user_id, username, nickname, avatar_url, user_level, credit_score, register_date, last_active_at, is_seller, seller_rating, is_verified, bio, province) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE username = VALUES(username), nickname = VALUES(nickname), crawl_latest_at = NOW() """, ( user.get("user_id"), user.get("username"), user.get("nickname"), user.get("avatar_url"), user.get("user_level"), user.get("credit_score", 0), user.get("register_date"), user.get("last_active_at"), user.get("is_seller", False), user.get("seller_rating"), user.get("is_verified", False), user.get("bio"), user.get("province") )) logger.info(f"用户保存成功: {user.get('username')}") return True except Exception as e: logger.error(f"保存用户失败: {e}") return False def run(self) -> List[Dict]: logger.info("开始采集一尘网用户...") return [] class YichensPostSpider(PaginationSpider): """一尘网帖子爬虫""" def __init__(self): super().__init__("一尘网帖子", "yichens") self.base_url = "https://www.yichens.com" self.forum_url = "https://www.yichens.com/forum" self.max_pages = 5 self.categories = { "longchao": {"name": "龙钞", "url": "/forum/longchao"}, "snake": {"name": "蛇钞", "url": "/forum/snake"}, "horse": {"name": "马钞", "url": "/forum/horse"}, } def parse_posts(self, html: str, url: str) -> List[Dict]: soup = BeautifulSoup(html, "lxml") posts = [] post_items = soup.select(".topic-item, .post-item, .thread-item") for item in post_items: try: post = self._extract_post(item, url) if post: posts.append(post) except Exception as e: logger.warning(f"解析帖子项异常: {e}") return posts def _extract_post(self, item, base_url: str) -> Optional[Dict]: post_id = None for attr in ["data-id", "data-post-id", "id"]: val = item.get(attr) if val: post_id = str(val) break if not post_id: return None title_elem = item.select_one(".title, .thread-title, .subject") title = title_elem.get_text(strip=True) if title_elem else f"无标题_{post_id}" author_elem = item.select_one(".author, .thread-author, .username") author_text = author_elem.get_text(strip=True) if author_elem else "匿名" author_id = None for attr in ["data-author-id", "data-user-id", "data-uid"]: val = item.get(attr) if val: author_id = str(val) break price = None price_unit = None price_elem = item.select_one(".price, .deal-price, .cost") if price_elem: price_text = price_elem.get_text(strip=True) match = re.search(r"[\d.]+", price_text.replace(",", "")) if match: price = float(match.group()) if "条" in price_text: price_unit = "元/条" elif "张" in price_text: price_unit = "元/张" else: price_unit = "元" view_count = 0 reply_count = 0 like_count = 0 view_elem = item.select_one(".views, .view-count") if view_elem: match = re.search(r"[\d]+", view_elem.get_text()) if match: view_count = int(match.group()) reply_elem = item.select_one(".replies, .reply-count") if reply_elem: match = re.search(r"[\d]+", reply_elem.get_text()) if match: reply_count = int(match.group()) like_elem = item.select_one(".likes, .like-count") if like_elem: match = re.search(r"[\d]+", like_elem.get_text()) if match: like_count = int(match.group()) created_at = None time_elem = item.select_one(".time, .created-at, .post-time") if time_elem: created_at = self._parse_datetime(time_elem.get_text(strip=True)) post_type = "normal" class_attr = item.get("class", []) if "deal" in class_attr or "trade" in class_attr: post_type = "deal" is_top = False is_essence = False badge_elems = item.select(".badge, .tag") for badge in badge_elems: text = badge.get_text(strip=True).lower() if "顶" in text or "top" in text: is_top = True if "精" in text or "ess" in text: is_essence = True return { "post_id": post_id, "topic_id": post_id, "title": title, "content": None, "content_html": None, "author_id": author_id or f"user_{author_text}", "author_username": author_text, "category": None, "sub_category": None, "post_type": post_type, "price": price, "price_unit": price_unit, "view_count": view_count, "reply_count": reply_count, "like_count": like_count, "is_top": is_top, "is_essence": is_essence, "is_closed": False, "created_at": created_at, "updated_at": created_at, } def parse_post_detail(self, html: str, url: str) -> Optional[Dict]: soup = BeautifulSoup(html, "lxml") post_id = None match = re.search(r"/thread/(\d+)", url) if match: post_id = match.group(1) title_elem = soup.select_one("h1.title, h1.thread-title, .post-title") title = title_elem.get_text(strip=True) if title_elem else None content_elem = soup.select_one(".post-content, .thread-content, .content") content = content_elem.get_text(strip=True, separator="\n") if content_elem else None return {"post_id": post_id, "topic_id": post_id, "title": title, "content": content} def _parse_datetime(self, time_str: str) -> Optional[str]: if not time_str: return None time_str = time_str.strip() patterns = [ (r"\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"), (r"\d{4}-\d{2}-\d{2}", "%Y-%m-%d"), (r"\d+分钟前", "minutes_ago"), (r"\d+小时前", "hours_ago"), ] for pattern, fmt in patterns: match = re.search(pattern, time_str) if match: if fmt == "minutes_ago": mins = int(re.search(r"\d+", match.group()).group()) dt = datetime.now() - timedelta(minutes=mins) return dt.strftime("%Y-%m-%d %H:%M:%S") elif fmt == "hours_ago": hours = int(re.search(r"\d+", match.group()).group()) dt = datetime.now() - timedelta(hours=hours) return dt.strftime("%Y-%m-%d %H:%M:%S") else: try: dt = datetime.strptime(match.group(), fmt) return dt.strftime("%Y-%m-%d %H:%M:%S") except: pass return None def save_post(self, post: Dict) -> bool: if not post or not post.get("post_id"): return False try: with db.get_cursor() as cursor: cursor.execute(""" INSERT INTO yichens_posts (post_id, topic_id, title, content, content_html, author_id, author_username, category, sub_category, post_type, price, price_unit, view_count, reply_count, like_count, is_top, is_essence, is_closed, created_at, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE title = VALUES(title), content = VALUES(content), view_count = VALUES(view_count), reply_count = VALUES(reply_count), updated_at = VALUES(updated_at), crawled_at = NOW() """, ( post.get("post_id"), post.get("topic_id"), post.get("title"), post.get("content"), post.get("content_html"), post.get("author_id"), post.get("author_username"), post.get("category"), post.get("sub_category"), post.get("post_type", "normal"), post.get("price"), post.get("price_unit"), post.get("view_count", 0), post.get("reply_count", 0), post.get("like_count", 0), post.get("is_top", False), post.get("is_essence", False), post.get("is_closed", False), post.get("created_at"), post.get("updated_at") )) logger.info(f"帖子保存成功: {str(post.get('title'))[:30]}") return True except Exception as e: logger.error(f"保存帖子失败: {e}") return False def crawl_forum(self, category_key: str = "longchao", max_pages: int = 5) -> List[Dict]: if category_key not in self.categories: logger.error(f"未知板块: {category_key}") return [] category = self.categories[category_key] base_url = f"{self.base_url}{category['url']}" logger.info(f"开始爬取板块: {category['name']} ({base_url})") self.max_pages = max_pages all_posts = [] for page in range(1, self.max_pages + 1): page_url = f"{base_url}?page={page}" logger.info(f"爬取第 {page} 页: {page_url}") response = self.get(page_url) if not response: logger.warning(f"第 {page} 页请求失败") continue posts = self.parse_posts(response.text, response.url) if not posts: logger.info(f"第 {page} 页无数据") break for post in posts: self.save_post(post) all_posts.append(post) logger.info(f"第 {page} 页获取 {len(posts)} 条帖子") logger.info(f"板块 {category['name']} 共采集 {len(all_posts)} 条帖子") return all_posts def run(self, category: str = "longchao") -> List[Dict]: log_id = self._log_start() try: posts = self.crawl_forum(category, self.max_pages) self._log_finish(log_id, "success", len(posts)) return posts except Exception as e: logger.error(f"爬虫执行失败: {e}") self._log_finish(log_id, "failed", 0, str(e)) return [] def _log_start(self) -> int: with db.get_cursor() as cursor: cursor.execute("INSERT INTO crawl_logs (source, status, started_at) VALUES (%s, %s, NOW())", (self.source, "running")) return cursor.lastrowid def _log_finish(self, log_id: int, status: str, items_count: int, error: str = ""): with db.get_cursor() as cursor: cursor.execute("UPDATE crawl_logs SET status = %s, items_count = %s, error_message = %s, finished_at = NOW() WHERE id = %s", (status, items_count, error, log_id)) def crawl_yichens(category: str = "longchao") -> List[Dict]: spider = YichensPostSpider() return spider.run(category) if __name__ == "__main__": import sys category = sys.argv[1] if len(sys.argv) > 1 else "longchao" crawl_yichens(category)