335 lines
14 KiB
Python
335 lines
14 KiB
Python
"""一尘网爬虫 - 一尘网钱币论坛数据采集"""
|
|
import re
|
|
from typing import Optional, Dict, List, Any
|
|
from datetime import datetime, timedelta
|
|
from bs4 import BeautifulSoup
|
|
import logging
|
|
|
|
from crawlers.base import BaseSpider, PaginationSpider
|
|
from database import db
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class YichensUserSpider(PaginationSpider):
|
|
"""一尘网用户爬虫"""
|
|
|
|
def __init__(self):
|
|
super().__init__("一尘网用户", "yichens")
|
|
self.base_url = "https://www.yichens.com/user"
|
|
self.user_list_url = "https://www.yichens.com/user/list"
|
|
|
|
def parse_user(self, html: str, url: str) -> Optional[Dict]:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
user_id = None
|
|
match = re.search(r"user[_\-]?id[=:\s]*['\"]?(\w+)", url, re.I)
|
|
if match:
|
|
user_id = match.group(1)
|
|
|
|
match = re.search(r"/user/([^/]+)", url)
|
|
username = match.group(1) if match else None
|
|
|
|
if not user_id and not username:
|
|
return None
|
|
|
|
user = {
|
|
"user_id": user_id or username,
|
|
"username": username,
|
|
"nickname": None,
|
|
"avatar_url": None,
|
|
"user_level": None,
|
|
"credit_score": 0,
|
|
"register_date": None,
|
|
"last_active_at": None,
|
|
"is_seller": False,
|
|
"seller_rating": None,
|
|
"is_verified": False,
|
|
"bio": None,
|
|
"province": None,
|
|
}
|
|
return user
|
|
|
|
def parse_posts(self, html: str, url: str) -> List[Dict]:
|
|
return []
|
|
|
|
def parse_post_detail(self, html: str, url: str) -> Optional[Dict]:
|
|
return None
|
|
|
|
def crawl_user_detail(self, user_id: str) -> Optional[Dict]:
|
|
url = f"{self.base_url}/{user_id}"
|
|
response = self.get(url)
|
|
if not response:
|
|
return None
|
|
return self.parse_user(response.text, url)
|
|
|
|
def save_user(self, user: Dict) -> bool:
|
|
if not user or not user.get("user_id"):
|
|
return False
|
|
try:
|
|
with db.get_cursor() as cursor:
|
|
cursor.execute("""
|
|
INSERT INTO yichens_users (user_id, username, nickname, avatar_url, user_level,
|
|
credit_score, register_date, last_active_at, is_seller,
|
|
seller_rating, is_verified, bio, province)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
ON DUPLICATE KEY UPDATE username = VALUES(username),
|
|
nickname = VALUES(nickname), crawl_latest_at = NOW()
|
|
""", (
|
|
user.get("user_id"), user.get("username"), user.get("nickname"),
|
|
user.get("avatar_url"), user.get("user_level"), user.get("credit_score", 0),
|
|
user.get("register_date"), user.get("last_active_at"), user.get("is_seller", False),
|
|
user.get("seller_rating"), user.get("is_verified", False),
|
|
user.get("bio"), user.get("province")
|
|
))
|
|
logger.info(f"用户保存成功: {user.get('username')}")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"保存用户失败: {e}")
|
|
return False
|
|
|
|
def run(self) -> List[Dict]:
|
|
logger.info("开始采集一尘网用户...")
|
|
return []
|
|
|
|
|
|
class YichensPostSpider(PaginationSpider):
|
|
"""一尘网帖子爬虫"""
|
|
|
|
def __init__(self):
|
|
super().__init__("一尘网帖子", "yichens")
|
|
self.base_url = "https://www.yichens.com"
|
|
self.forum_url = "https://www.yichens.com/forum"
|
|
self.max_pages = 5
|
|
self.categories = {
|
|
"longchao": {"name": "龙钞", "url": "/forum/longchao"},
|
|
"snake": {"name": "蛇钞", "url": "/forum/snake"},
|
|
"horse": {"name": "马钞", "url": "/forum/horse"},
|
|
}
|
|
|
|
def parse_posts(self, html: str, url: str) -> List[Dict]:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
posts = []
|
|
post_items = soup.select(".topic-item, .post-item, .thread-item")
|
|
for item in post_items:
|
|
try:
|
|
post = self._extract_post(item, url)
|
|
if post:
|
|
posts.append(post)
|
|
except Exception as e:
|
|
logger.warning(f"解析帖子项异常: {e}")
|
|
return posts
|
|
|
|
def _extract_post(self, item, base_url: str) -> Optional[Dict]:
|
|
post_id = None
|
|
for attr in ["data-id", "data-post-id", "id"]:
|
|
val = item.get(attr)
|
|
if val:
|
|
post_id = str(val)
|
|
break
|
|
if not post_id:
|
|
return None
|
|
|
|
title_elem = item.select_one(".title, .thread-title, .subject")
|
|
title = title_elem.get_text(strip=True) if title_elem else f"无标题_{post_id}"
|
|
author_elem = item.select_one(".author, .thread-author, .username")
|
|
author_text = author_elem.get_text(strip=True) if author_elem else "匿名"
|
|
author_id = None
|
|
for attr in ["data-author-id", "data-user-id", "data-uid"]:
|
|
val = item.get(attr)
|
|
if val:
|
|
author_id = str(val)
|
|
break
|
|
|
|
price = None
|
|
price_unit = None
|
|
price_elem = item.select_one(".price, .deal-price, .cost")
|
|
if price_elem:
|
|
price_text = price_elem.get_text(strip=True)
|
|
match = re.search(r"[\d.]+", price_text.replace(",", ""))
|
|
if match:
|
|
price = float(match.group())
|
|
if "条" in price_text:
|
|
price_unit = "元/条"
|
|
elif "张" in price_text:
|
|
price_unit = "元/张"
|
|
else:
|
|
price_unit = "元"
|
|
|
|
view_count = 0
|
|
reply_count = 0
|
|
like_count = 0
|
|
view_elem = item.select_one(".views, .view-count")
|
|
if view_elem:
|
|
match = re.search(r"[\d]+", view_elem.get_text())
|
|
if match:
|
|
view_count = int(match.group())
|
|
reply_elem = item.select_one(".replies, .reply-count")
|
|
if reply_elem:
|
|
match = re.search(r"[\d]+", reply_elem.get_text())
|
|
if match:
|
|
reply_count = int(match.group())
|
|
like_elem = item.select_one(".likes, .like-count")
|
|
if like_elem:
|
|
match = re.search(r"[\d]+", like_elem.get_text())
|
|
if match:
|
|
like_count = int(match.group())
|
|
|
|
created_at = None
|
|
time_elem = item.select_one(".time, .created-at, .post-time")
|
|
if time_elem:
|
|
created_at = self._parse_datetime(time_elem.get_text(strip=True))
|
|
|
|
post_type = "normal"
|
|
class_attr = item.get("class", [])
|
|
if "deal" in class_attr or "trade" in class_attr:
|
|
post_type = "deal"
|
|
|
|
is_top = False
|
|
is_essence = False
|
|
badge_elems = item.select(".badge, .tag")
|
|
for badge in badge_elems:
|
|
text = badge.get_text(strip=True).lower()
|
|
if "顶" in text or "top" in text:
|
|
is_top = True
|
|
if "精" in text or "ess" in text:
|
|
is_essence = True
|
|
|
|
return {
|
|
"post_id": post_id, "topic_id": post_id, "title": title,
|
|
"content": None, "content_html": None,
|
|
"author_id": author_id or f"user_{author_text}", "author_username": author_text,
|
|
"category": None, "sub_category": None, "post_type": post_type,
|
|
"price": price, "price_unit": price_unit,
|
|
"view_count": view_count, "reply_count": reply_count, "like_count": like_count,
|
|
"is_top": is_top, "is_essence": is_essence, "is_closed": False,
|
|
"created_at": created_at, "updated_at": created_at,
|
|
}
|
|
|
|
def parse_post_detail(self, html: str, url: str) -> Optional[Dict]:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
post_id = None
|
|
match = re.search(r"/thread/(\d+)", url)
|
|
if match:
|
|
post_id = match.group(1)
|
|
title_elem = soup.select_one("h1.title, h1.thread-title, .post-title")
|
|
title = title_elem.get_text(strip=True) if title_elem else None
|
|
content_elem = soup.select_one(".post-content, .thread-content, .content")
|
|
content = content_elem.get_text(strip=True, separator="\n") if content_elem else None
|
|
return {"post_id": post_id, "topic_id": post_id, "title": title, "content": content}
|
|
|
|
def _parse_datetime(self, time_str: str) -> Optional[str]:
|
|
if not time_str:
|
|
return None
|
|
time_str = time_str.strip()
|
|
patterns = [
|
|
(r"\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"),
|
|
(r"\d{4}-\d{2}-\d{2}", "%Y-%m-%d"),
|
|
(r"\d+分钟前", "minutes_ago"),
|
|
(r"\d+小时前", "hours_ago"),
|
|
]
|
|
for pattern, fmt in patterns:
|
|
match = re.search(pattern, time_str)
|
|
if match:
|
|
if fmt == "minutes_ago":
|
|
mins = int(re.search(r"\d+", match.group()).group())
|
|
dt = datetime.now() - timedelta(minutes=mins)
|
|
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
elif fmt == "hours_ago":
|
|
hours = int(re.search(r"\d+", match.group()).group())
|
|
dt = datetime.now() - timedelta(hours=hours)
|
|
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
else:
|
|
try:
|
|
dt = datetime.strptime(match.group(), fmt)
|
|
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
except:
|
|
pass
|
|
return None
|
|
|
|
def save_post(self, post: Dict) -> bool:
|
|
if not post or not post.get("post_id"):
|
|
return False
|
|
try:
|
|
with db.get_cursor() as cursor:
|
|
cursor.execute("""
|
|
INSERT INTO yichens_posts (post_id, topic_id, title, content, content_html,
|
|
author_id, author_username, category, sub_category, post_type,
|
|
price, price_unit, view_count, reply_count, like_count,
|
|
is_top, is_essence, is_closed, created_at, updated_at)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
ON DUPLICATE KEY UPDATE title = VALUES(title), content = VALUES(content),
|
|
view_count = VALUES(view_count), reply_count = VALUES(reply_count),
|
|
updated_at = VALUES(updated_at), crawled_at = NOW()
|
|
""", (
|
|
post.get("post_id"), post.get("topic_id"), post.get("title"),
|
|
post.get("content"), post.get("content_html"), post.get("author_id"),
|
|
post.get("author_username"), post.get("category"), post.get("sub_category"),
|
|
post.get("post_type", "normal"), post.get("price"), post.get("price_unit"),
|
|
post.get("view_count", 0), post.get("reply_count", 0), post.get("like_count", 0),
|
|
post.get("is_top", False), post.get("is_essence", False), post.get("is_closed", False),
|
|
post.get("created_at"), post.get("updated_at")
|
|
))
|
|
logger.info(f"帖子保存成功: {str(post.get('title'))[:30]}")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"保存帖子失败: {e}")
|
|
return False
|
|
|
|
def crawl_forum(self, category_key: str = "longchao", max_pages: int = 5) -> List[Dict]:
|
|
if category_key not in self.categories:
|
|
logger.error(f"未知板块: {category_key}")
|
|
return []
|
|
category = self.categories[category_key]
|
|
base_url = f"{self.base_url}{category['url']}"
|
|
logger.info(f"开始爬取板块: {category['name']} ({base_url})")
|
|
self.max_pages = max_pages
|
|
all_posts = []
|
|
for page in range(1, self.max_pages + 1):
|
|
page_url = f"{base_url}?page={page}"
|
|
logger.info(f"爬取第 {page} 页: {page_url}")
|
|
response = self.get(page_url)
|
|
if not response:
|
|
logger.warning(f"第 {page} 页请求失败")
|
|
continue
|
|
posts = self.parse_posts(response.text, response.url)
|
|
if not posts:
|
|
logger.info(f"第 {page} 页无数据")
|
|
break
|
|
for post in posts:
|
|
self.save_post(post)
|
|
all_posts.append(post)
|
|
logger.info(f"第 {page} 页获取 {len(posts)} 条帖子")
|
|
logger.info(f"板块 {category['name']} 共采集 {len(all_posts)} 条帖子")
|
|
return all_posts
|
|
|
|
def run(self, category: str = "longchao") -> List[Dict]:
|
|
log_id = self._log_start()
|
|
try:
|
|
posts = self.crawl_forum(category, self.max_pages)
|
|
self._log_finish(log_id, "success", len(posts))
|
|
return posts
|
|
except Exception as e:
|
|
logger.error(f"爬虫执行失败: {e}")
|
|
self._log_finish(log_id, "failed", 0, str(e))
|
|
return []
|
|
|
|
def _log_start(self) -> int:
|
|
with db.get_cursor() as cursor:
|
|
cursor.execute("INSERT INTO crawl_logs (source, status, started_at) VALUES (%s, %s, NOW())", (self.source, "running"))
|
|
return cursor.lastrowid
|
|
|
|
def _log_finish(self, log_id: int, status: str, items_count: int, error: str = ""):
|
|
with db.get_cursor() as cursor:
|
|
cursor.execute("UPDATE crawl_logs SET status = %s, items_count = %s, error_message = %s, finished_at = NOW() WHERE id = %s", (status, items_count, error, log_id))
|
|
|
|
|
|
def crawl_yichens(category: str = "longchao") -> List[Dict]:
|
|
spider = YichensPostSpider()
|
|
return spider.run(category)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
category = sys.argv[1] if len(sys.argv) > 1 else "longchao"
|
|
crawl_yichens(category)
|