fix: crawl_today.py - BeautifulSoup解析 + 去掉日期过滤 + 修复post_id正则
This commit is contained in:
parent
79409317f9
commit
9d418f3d8c
|
|
@ -0,0 +1,350 @@
|
||||||
|
"""只采集今天的新帖子 - 优化版"""
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
sys.path.insert(0, '/root/coolbot-data')
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from datetime import datetime
|
||||||
|
from crawlers.base import PaginationSpider
|
||||||
|
from database import get_db
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class YichensTodaySpider(PaginationSpider):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("一尘网今日采集", "pm001")
|
||||||
|
self.base_url = "http://www.pm001.net"
|
||||||
|
self.board_id = "151"
|
||||||
|
self.forum_url = f"{self.base_url}/index.asp?boardid={self.board_id}"
|
||||||
|
self.max_pages = 2
|
||||||
|
self.encoding = "gbk"
|
||||||
|
self.min_delay = 1.5
|
||||||
|
self.max_delay = 3.0
|
||||||
|
|
||||||
|
def get(self, url, **kwargs):
|
||||||
|
try:
|
||||||
|
self._random_delay()
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
|
}
|
||||||
|
response = requests.get(url, timeout=30, headers=headers, **kwargs)
|
||||||
|
try:
|
||||||
|
response.encoding = self.encoding
|
||||||
|
except:
|
||||||
|
response.encoding = "gb2312"
|
||||||
|
return response
|
||||||
|
except Exception as e:
|
||||||
|
print(f"请求失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def extract_posts_with_dates(self, html):
|
||||||
|
"""从列表页提取所有帖子ID和"发表于"时间，返回 [(post_id, date_str), ...]
|
||||||
|
使用BeautifulSoup解析listtitle div，比正则更准确
|
||||||
|
"""
|
||||||
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
posts = []
|
||||||
|
listtitle_divs = soup.find_all('div', class_='listtitle')
|
||||||
|
|
||||||
|
for div in listtitle_divs:
|
||||||
|
link = div.find('a', href=re.compile(r'dispbbs\.asp\?boardID=151&ID=\d+'))
|
||||||
|
if not link:
|
||||||
|
continue
|
||||||
|
|
||||||
|
href = link.get('href', '')
|
||||||
|
id_match = re.search(r'\&ID=(\d+)', href)
|
||||||
|
if not id_match:
|
||||||
|
continue
|
||||||
|
post_id = id_match.group(1)
|
||||||
|
|
||||||
|
title_attr = link.get('title', '')
|
||||||
|
# title格式: 《标题》\n作者：xxx\n发表于：2026/4/5 9:35:00
|
||||||
|
date_match = re.search(r'发表于：(\d{4}/\d{1,2}/\d{1,2})', title_attr)
|
||||||
|
if date_match:
|
||||||
|
date_str = date_match.group(1)
|
||||||
|
posts.append((post_id, date_str))
|
||||||
|
|
||||||
|
return posts
|
||||||
|
|
||||||
|
def parse_post_detail(self, html, url):
|
||||||
|
"""解析详情页，提取真实发帖时间"""
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
page_text = soup.get_text()
|
||||||
|
|
||||||
|
match = re.search(r"ID=(\d+)", url)
|
||||||
|
post_id = match.group(1) if match else None
|
||||||
|
|
||||||
|
title = None
|
||||||
|
title_elem = soup.find("title")
|
||||||
|
if title_elem:
|
||||||
|
title_text = title_elem.get_text(strip=True)
|
||||||
|
if "[" in title_text:
|
||||||
|
title = title_text.split("[")[0].strip()
|
||||||
|
|
||||||
|
author = None
|
||||||
|
m = re.search(r">>>\s*([^\s<]+)", page_text)
|
||||||
|
if m:
|
||||||
|
author = m.group(1).strip()
|
||||||
|
|
||||||
|
created_at = None
|
||||||
|
m = re.search(r"(\d{4}/\d{1,2}/\d{1,2}\s+\d{1,2}:\d{2}:\d{2})", page_text)
|
||||||
|
if m:
|
||||||
|
created_at = m.group(1).replace("/", "-")
|
||||||
|
|
||||||
|
phones = re.findall(r'1[3-9]\d{9,10}', page_text)
|
||||||
|
contact = ",".join(dict.fromkeys(phones[:5])) if phones else None
|
||||||
|
|
||||||
|
price = None
|
||||||
|
price_unit = None
|
||||||
|
if title:
|
||||||
|
price, price_unit = self.extract_price(title)
|
||||||
|
if not price:
|
||||||
|
price, price_unit = self.extract_price(page_text)
|
||||||
|
|
||||||
|
content = None
|
||||||
|
m = re.search(r"(出售|收购|求购|出|售|收)[^\n]{10,500}", page_text)
|
||||||
|
if m:
|
||||||
|
content = m.group(0)[:1000]
|
||||||
|
|
||||||
|
special_types = []
|
||||||
|
if title:
|
||||||
|
if "救生圈" in title or "88" in title or "888" in title:
|
||||||
|
special_types.append("救生圈")
|
||||||
|
if "标十" in title:
|
||||||
|
special_types.append("标十")
|
||||||
|
if "标百" in title:
|
||||||
|
special_types.append("标百")
|
||||||
|
|
||||||
|
post_type = "normal"
|
||||||
|
if title:
|
||||||
|
if any(c in title for c in ["出", "售", "卖", "兑"]):
|
||||||
|
post_type = "deal"
|
||||||
|
elif any(c in title for c in ["求", "收", "购"]):
|
||||||
|
post_type = "want"
|
||||||
|
|
||||||
|
category = "其他"
|
||||||
|
if title:
|
||||||
|
if "龙钞" in title or "龙纪念" in title:
|
||||||
|
category = "龙钞"
|
||||||
|
elif "蛇钞" in title:
|
||||||
|
category = "蛇钞"
|
||||||
|
elif "马钞" in title:
|
||||||
|
category = "马钞"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"post_id": post_id,
|
||||||
|
"title": title,
|
||||||
|
"content": content,
|
||||||
|
"author_username": author,
|
||||||
|
"price": price,
|
||||||
|
"price_unit": price_unit,
|
||||||
|
"contact": contact,
|
||||||
|
"post_time": created_at,
|
||||||
|
"special_types": "|".join(special_types) if special_types else None,
|
||||||
|
"category": category,
|
||||||
|
"post_type": post_type,
|
||||||
|
"url": f"http://www.pm001.net/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1",
|
||||||
|
}
|
||||||
|
|
||||||
|
def extract_price(self, text):
|
||||||
|
if not text:
|
||||||
|
return None, None
|
||||||
|
patterns = [
|
||||||
|
r"(\d+)\s*元\s*(?:一张|一刀|一条|单张)?",
|
||||||
|
r"[收售出求兑买]\s*(\d+)\s*元?",
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
m = re.search(pattern, text)
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
val = float(m.group(1))
|
||||||
|
if val < 0 or val > 999999999:
|
||||||
|
continue
|
||||||
|
price_unit = "元"
|
||||||
|
if "张" in text:
|
||||||
|
price_unit = "元/张"
|
||||||
|
elif "刀" in text:
|
||||||
|
price_unit = "元/刀"
|
||||||
|
elif "条" in text:
|
||||||
|
price_unit = "元/条"
|
||||||
|
return val, price_unit
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
def crawl_today(self):
|
||||||
|
"""全量采集：爬取第1-max_pages页所有帖子，不过滤日期
|
||||||
|
用于一次性补全所有帖子的post_time等字段
|
||||||
|
"""
|
||||||
|
today = datetime.now().date()
|
||||||
|
today_str = today.strftime("%Y/%m/%d").lstrip('0').replace('/0', '/')
|
||||||
|
|
||||||
|
print(f"开始全量采集第1-{self.max_pages}页 (今天: {today_str})")
|
||||||
|
|
||||||
|
saved = 0
|
||||||
|
checked = 0
|
||||||
|
|
||||||
|
for page in range(1, self.max_pages + 1):
|
||||||
|
page_url = f"{self.forum_url}&page={page}"
|
||||||
|
print(f"\n扫描第 {page} 页...")
|
||||||
|
|
||||||
|
response = self.get(page_url)
|
||||||
|
if not response:
|
||||||
|
continue
|
||||||
|
|
||||||
|
posts = self.extract_posts_with_dates(response.text)
|
||||||
|
print(f" 该页共 {len(posts)} 条帖子")
|
||||||
|
|
||||||
|
for post_id, date_str in posts:
|
||||||
|
checked += 1
|
||||||
|
|
||||||
|
# 标准化日期
|
||||||
|
normalized_date = date_str.lstrip('0').replace('/0', '/')
|
||||||
|
|
||||||
|
# 访问详情页
|
||||||
|
detail_url = f"{self.base_url}/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1"
|
||||||
|
detail_response = self.get(detail_url)
|
||||||
|
|
||||||
|
if not detail_response:
|
||||||
|
continue
|
||||||
|
|
||||||
|
detail = self.parse_post_detail(detail_response.text, detail_url)
|
||||||
|
|
||||||
|
# 验证详情页的真实发帖时间
|
||||||
|
post_time_str = detail.get('post_time', '')
|
||||||
|
if not post_time_str:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
post_time = datetime.strptime(post_time_str, '%Y-%m-%d %H:%M:%S').date()
|
||||||
|
is_today = (post_time == today)
|
||||||
|
label = f"今日{normalized_date}" if is_today else f"历史{post_time}"
|
||||||
|
except:
|
||||||
|
label = "时间异常"
|
||||||
|
|
||||||
|
# 保存（ON CONFLICT会更新post_time等字段）
|
||||||
|
if self.save_full_post(detail):
|
||||||
|
saved += 1
|
||||||
|
print(f" [保存] {post_id} - {detail.get('title', '')[:25]}... ({label}中已存{saved}条)")
|
||||||
|
else:
|
||||||
|
print(f" [失败] {post_id}")
|
||||||
|
|
||||||
|
print(f" 页码 {page} 完成: 检查{len(posts)}条")
|
||||||
|
|
||||||
|
print(f"\n===== 采集完成 =====")
|
||||||
|
print(f"总计检查: {checked} 条")
|
||||||
|
print(f"成功保存: {saved} 条")
|
||||||
|
return saved
|
||||||
|
|
||||||
|
def save_full_post(self, post):
|
||||||
|
if not post or not post.get("post_id"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
sql = """
|
||||||
|
INSERT INTO yichens_posts (
|
||||||
|
post_id, title, content, category, post_type,
|
||||||
|
price, price_unit, special_types, number_features,
|
||||||
|
author_username, author_id, contact,
|
||||||
|
has_lifebuoy, reply_count, view_count,
|
||||||
|
post_time, crawled_at, updated_at, url
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW(), %s)
|
||||||
|
ON CONFLICT (post_id) DO UPDATE SET
|
||||||
|
title = EXCLUDED.title,
|
||||||
|
content = EXCLUDED.content,
|
||||||
|
category = EXCLUDED.category,
|
||||||
|
post_type = EXCLUDED.post_type,
|
||||||
|
price = EXCLUDED.price,
|
||||||
|
price_unit = EXCLUDED.price_unit,
|
||||||
|
special_types = EXCLUDED.special_types,
|
||||||
|
author_username = EXCLUDED.author_username,
|
||||||
|
contact = EXCLUDED.contact,
|
||||||
|
has_lifebuoy = EXCLUDED.has_lifebuoy,
|
||||||
|
post_time = EXCLUDED.post_time,
|
||||||
|
updated_at = NOW(),
|
||||||
|
crawled_at = NOW(),
|
||||||
|
url = EXCLUDED.url
|
||||||
|
"""
|
||||||
|
|
||||||
|
has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with get_db() as conn:
|
||||||
|
if not conn:
|
||||||
|
print("数据库连接失败")
|
||||||
|
return False
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(sql, (
|
||||||
|
post.get("post_id"),
|
||||||
|
post.get("title"),
|
||||||
|
post.get("content"),
|
||||||
|
post.get("category"),
|
||||||
|
post.get("post_type"),
|
||||||
|
post.get("price"),
|
||||||
|
post.get("price_unit"),
|
||||||
|
post.get("special_types"),
|
||||||
|
None,
|
||||||
|
post.get("author_username"),
|
||||||
|
f"user_{post.get('post_id')}",
|
||||||
|
post.get("contact"),
|
||||||
|
has_lifebuoy,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
post.get("post_time"),
|
||||||
|
post.get("url"),
|
||||||
|
))
|
||||||
|
conn.commit()
|
||||||
|
cur.close()
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
print(f"保存失败: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
log_id = self._log_start()
|
||||||
|
try:
|
||||||
|
result = self.crawl_today()
|
||||||
|
self._log_finish(log_id, "success", result)
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
print(f"爬虫异常: {e}")
|
||||||
|
self._log_finish(log_id, "failed", 0, str(e))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def _log_start(self):
|
||||||
|
try:
|
||||||
|
with get_db() as conn:
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(
|
||||||
|
"INSERT INTO crawl_logs (spider_name, status, started_at) VALUES (%s, %s, NOW()) RETURNING id",
|
||||||
|
(self.source, "running")
|
||||||
|
)
|
||||||
|
log_id = cur.fetchone()[0]
|
||||||
|
conn.commit()
|
||||||
|
cur.close()
|
||||||
|
return log_id
|
||||||
|
except Exception as e:
|
||||||
|
print(f"记录日志失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _log_finish(self, log_id, status, items_count, error=""):
|
||||||
|
if log_id is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with get_db() as conn:
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE crawl_logs SET status=%s, items_count=%s, error_message=%s, finished_at=NOW() WHERE id=%s",
|
||||||
|
(status, items_count, error, log_id)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
cur.close()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"更新日志失败: {e}")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
spider = YichensTodaySpider()
|
||||||
|
spider.run()
|
||||||
Loading…
Reference in New Issue