725 lines
25 KiB
Python
725 lines
25 KiB
Python
"""一尘网爬虫 v4 - 完整版
|
||
- 爬取帖子 + 所有楼层用户信息
|
||
- 优化字段识别（category/special_types/number_features/price）
|
||
- 增量去重 + 日志记录 + 防封延迟
|
||
"""
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
import re
|
||
import time
|
||
import yaml
|
||
import random
|
||
import json
|
||
from datetime import datetime
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
def load_config():
|
||
with open('/root/coolbot-data/config.yaml', 'r') as f:
|
||
return yaml.safe_load(f)
|
||
|
||
config = load_config()
|
||
DB_CONFIG = config['database']
|
||
|
||
BASE_URL = 'http://www4.pm001.net'
|
||
BOARD_ID = '151'
|
||
|
||
HEADERS = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
}
|
||
|
||
# ============ 数据库操作 ============
|
||
|
||
def get_db_conn():
|
||
import psycopg2
|
||
return psycopg2.connect(
|
||
host=DB_CONFIG['host'],
|
||
port=DB_CONFIG.get('port', 5432),
|
||
user=DB_CONFIG['user'],
|
||
password=DB_CONFIG['password'],
|
||
database=DB_CONFIG['database']
|
||
)
|
||
|
||
def get_existing_post_ids():
|
||
try:
|
||
conn = get_db_conn()
|
||
cur = conn.cursor()
|
||
cur.execute('SELECT post_id FROM yichens_posts')
|
||
existing = set(row[0] for row in cur.fetchall())
|
||
cur.close()
|
||
conn.close()
|
||
return existing
|
||
except Exception as e:
|
||
print(f'获取已有post_id失败: {e}')
|
||
return set()
|
||
|
||
def log_crawl(spider_name, status, items_count, error=''):
|
||
try:
|
||
conn = get_db_conn()
|
||
cur = conn.cursor()
|
||
cur.execute("""
|
||
INSERT INTO crawl_logs (spider_name, status, items_count, error_message, started_at, finished_at)
|
||
VALUES (%s, %s, %s, %s, %s, %s)
|
||
""", (spider_name, status, items_count, error, datetime.now(), datetime.now()))
|
||
conn.commit()
|
||
cur.close()
|
||
conn.close()
|
||
except Exception as e:
|
||
print(f'写入爬虫日志失败: {e}')
|
||
|
||
def save_post(post):
|
||
try:
|
||
conn = get_db_conn()
|
||
cur = conn.cursor()
|
||
sql = """
|
||
INSERT INTO yichens_posts (
|
||
post_id, title, content, category, post_type, price, price_unit,
|
||
special_types, number_features, author_username, author_id,
|
||
contact, has_lifebuoy, reply_count, view_count,
|
||
post_time, crawled_at, url
|
||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (post_id) DO UPDATE SET
|
||
title = EXCLUDED.title,
|
||
content = EXCLUDED.content,
|
||
author_username = EXCLUDED.author_username,
|
||
contact = EXCLUDED.contact,
|
||
reply_count = EXCLUDED.reply_count,
|
||
view_count = EXCLUDED.view_count,
|
||
price = EXCLUDED.price,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
"""
|
||
cur.execute(sql, (
|
||
post.get('post_id'),
|
||
post.get('title'),
|
||
post.get('content'),
|
||
post.get('category'),
|
||
post.get('post_type'),
|
||
post.get('price'),
|
||
post.get('price_unit', '元'),
|
||
post.get('special_types'),
|
||
post.get('number_features'),
|
||
post.get('author_username'),
|
||
post.get('author_id'),
|
||
post.get('contact'),
|
||
post.get('has_lifebuoy', False),
|
||
post.get('reply_count', 0),
|
||
post.get('view_count', 0),
|
||
post.get('post_time'),
|
||
datetime.now(),
|
||
post.get('url')
|
||
))
|
||
conn.commit()
|
||
cur.close()
|
||
conn.close()
|
||
return True
|
||
except Exception as e:
|
||
print(f' DB error: {e}')
|
||
return False
|
||
|
||
def save_member(member):
|
||
"""保存会员信息到 yichens_members 表"""
|
||
if not member or not member.get('user_id'):
|
||
return False
|
||
try:
|
||
conn = get_db_conn()
|
||
cur = conn.cursor()
|
||
sql = """
|
||
INSERT INTO yichens_members (
|
||
user_id, username, transaction_level, credit_score,
|
||
rating_count, post_count, post_points,
|
||
has_business_license, real_name,
|
||
phone, address, bank_accounts, alipay,
|
||
registration_date, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (user_id) DO UPDATE SET
|
||
username = EXCLUDED.username,
|
||
transaction_level = EXCLUDED.transaction_level,
|
||
credit_score = EXCLUDED.credit_score,
|
||
rating_count = EXCLUDED.rating_count,
|
||
post_count = EXCLUDED.post_count,
|
||
has_business_license = EXCLUDED.has_business_license,
|
||
phone = EXCLUDED.phone,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
"""
|
||
bank_accounts_json = json.dumps(member.get('bank_accounts', []), ensure_ascii=False)
|
||
|
||
cur.execute(sql, (
|
||
member.get('user_id'),
|
||
member.get('username'),
|
||
member.get('transaction_level'),
|
||
member.get('credit_score'),
|
||
member.get('rating_count', 0),
|
||
member.get('post_count', 0),
|
||
member.get('post_points', 0),
|
||
member.get('has_business_license', False),
|
||
member.get('real_name'),
|
||
member.get('phone'),
|
||
member.get('address'),
|
||
bank_accounts_json,
|
||
member.get('alipay'),
|
||
member.get('registration_date'),
|
||
datetime.now()
|
||
))
|
||
conn.commit()
|
||
cur.close()
|
||
conn.close()
|
||
return True
|
||
except Exception as e:
|
||
print(f' Save member error: {e}')
|
||
return False
|
||
|
||
# ============ 字段解析 ============
|
||
|
||
def parse_category(text):
|
||
"""识别品类：龙钞、蛇钞、马钞"
|
||
规则：标题或正文中有 龙钞/小龙钞/马钞/蛇钞，或者单独的 龙/马/蛇 字（作为钱币品种）
|
||
"""
|
||
if not text:
|
||
return '其他'
|
||
text = str(text)
|
||
|
||
# 优先精确匹配
|
||
if any(k in text for k in ['龙钞', '小龙钞']):
|
||
return '龙钞'
|
||
if any(k in text for k in ['蛇钞']):
|
||
return '蛇钞'
|
||
if any(k in text for k in ['马钞', '马年']):
|
||
return '马钞'
|
||
|
||
# 单字匹配（作为品种上下文）
|
||
if '龙' in text:
|
||
exclude = ['龙年版', '龙年纪念', '成龙', '恐龙', '龙凤', '天龙', '卧龙', '龙魂', '龙珠笔', '龙马', '龙马精']
|
||
if not any(e in text for e in exclude):
|
||
return '龙钞'
|
||
if '马' in text:
|
||
exclude = ['马自达', '马德里', '马蹄', '马化腾', '马虎', '马蜂窝', '马马虎虎']
|
||
if not any(e in text for e in exclude):
|
||
return '马钞'
|
||
if '蛇' in text:
|
||
exclude = ['眼镜蛇', '蟒蛇', '蛇毒', '捕蛇', '蛇皮']
|
||
if not any(e in text for e in exclude):
|
||
return '蛇钞'
|
||
|
||
return '其他'
|
||
|
||
def parse_special_types(title):
|
||
"""识别特殊规格：标百（刀/刀货）、标十、单张、捆"
|
||
"""
|
||
if not title:
|
||
return None
|
||
title = str(title)
|
||
specials = []
|
||
|
||
# 标十
|
||
if '标十' in title:
|
||
specials.append('标十')
|
||
|
||
# 标百（包含刀、刀货）
|
||
if '标百' in title:
|
||
specials.append('标百')
|
||
elif any(k in title for k in ['刀', '刀货']):
|
||
specials.append('刀')
|
||
|
||
# 单张/散张
|
||
if any(k in title for k in ['单张', '散张']):
|
||
specials.append('单张')
|
||
|
||
# 捆
|
||
if '捆' in title:
|
||
specials.append('捆')
|
||
|
||
# 百连/千连
|
||
if '千连' in title:
|
||
specials.append('千连')
|
||
elif '百连' in title:
|
||
specials.append('百连')
|
||
|
||
# 救生圈
|
||
if any(k in title for k in ['大救生圈', '救生圈']):
|
||
specials.append('救生圈')
|
||
|
||
return '|'.join(specials) if specials else None
|
||
|
||
def parse_number_features(title):
|
||
"""识别号码特征：圆圆、倒置、如意、朦胧、金马、金山、天马、钻石、永恒、无4、无47、带4、无347、无247、豹子、狮子、老虎、大象"
|
||
"""
|
||
if not title:
|
||
return None
|
||
title = str(title)
|
||
features = []
|
||
|
||
feature_map = {
|
||
'倒置': ['倒置'],
|
||
'如意': ['如意'],
|
||
'朦胧': ['朦胧'],
|
||
'金马': ['金马'],
|
||
'金山': ['金山'],
|
||
'天马': ['天马'],
|
||
'钻石': ['钻石'],
|
||
'永恒': ['永恒'],
|
||
'圆圆': ['圆圆'],
|
||
'无4': ['无4', '无四'],
|
||
'无47': ['无47'],
|
||
'无247': ['无247'],
|
||
'无34': ['无34'],
|
||
'无347': ['无347'],
|
||
'带4': ['带4'],
|
||
'豹子': ['豹子'],
|
||
'狮子': ['狮子'],
|
||
'老虎': ['老虎'],
|
||
'大象': ['大象'],
|
||
'生日': ['生日'],
|
||
'满号': ['满号'],
|
||
'首日': ['首日'],
|
||
}
|
||
|
||
for feature, keywords in feature_map.items():
|
||
if any(k in title for k in keywords):
|
||
features.append(feature)
|
||
|
||
return '|'.join(features) if features else None
|
||
|
||
def parse_price(text):
|
||
"""从文本解析价格（标准格式用正则，非标准用启发式）
|
||
标准：XXX元/组、XXX/组、XXX元/张、XXX/刀
|
||
非标准：标题中的模糊价格需要结合上下文
|
||
"""
|
||
if not text:
|
||
return None, None
|
||
|
||
# 优先从标题精确匹配
|
||
patterns = [
|
||
# 格式：数字+元+单位
|
||
(r'(\d{4,5})\s*元\s*/\s*[组张刀条]', '元/组'),
|
||
(r'(\d{4,5})\s*/\s*[组张刀条]', '元/组'),
|
||
(r'(\d{4,5})\s*元', '元'),
|
||
# 求购格式：收XXX元、收XXX
|
||
(r'收\s*(\d{3,5})\s*元?', '元'),
|
||
# 出售格式：出XXX元、售XXX
|
||
(r'[出售售]\s*(\d{3,5})\s*元?', '元'),
|
||
]
|
||
|
||
for pattern, unit in patterns:
|
||
match = re.search(pattern, text)
|
||
if match:
|
||
try:
|
||
price = float(match.group(1))
|
||
# 合理性校验
|
||
if 10 <= price <= 999999:
|
||
return price, unit
|
||
except:
|
||
pass
|
||
|
||
return None, None
|
||
|
||
def parse_price_from_content(title, content):
|
||
"""从正文提取所有价格，返回最高价格和单位"
|
||
用于多货品帖子，取最高出价/要价
|
||
"""
|
||
if not content:
|
||
return parse_price(title)
|
||
|
||
# 收集所有符合格式的价格
|
||
prices = []
|
||
price_pattern = r'(\d{3,5})\s*元|收\s*(\d{3,5})|(\d{4,5})\s*/\s*[组张刀]'
|
||
|
||
for match in re.finditer(price_pattern, content):
|
||
for group in match.groups():
|
||
if group:
|
||
try:
|
||
p = float(group)
|
||
if 10 <= p <= 999999:
|
||
prices.append(p)
|
||
except:
|
||
pass
|
||
|
||
if prices:
|
||
max_price = max(prices)
|
||
# 确定单位
|
||
unit = '元'
|
||
if '元/组' in content or '/组' in content:
|
||
unit = '元/组'
|
||
elif '元/张' in content or '/张' in content:
|
||
unit = '元/张'
|
||
elif '元/刀' in content or '/刀' in content:
|
||
unit = '元/刀'
|
||
return max_price, unit
|
||
|
||
return parse_price(title)
|
||
|
||
def parse_post_type(title):
|
||
"""识别交易类型：deal(出售)/want(求购)/normal"
|
||
"""
|
||
if not title:
|
||
return 'normal'
|
||
title = str(title)
|
||
|
||
# 出售关键词
|
||
deal_keywords = ['出售', '转让', '卖', '低价', '低出', '快出', '亏出', '吐血', '清仓', '出 ', '批出', '批售', '甩卖', '特价']
|
||
if any(k in title for k in deal_keywords):
|
||
return 'deal'
|
||
|
||
# 求购关键词
|
||
want_keywords = ['求购', '收购', '收 ', '要 ', '收', '求', '要']
|
||
if any(k in title for k in want_keywords):
|
||
return 'want'
|
||
|
||
return 'normal'
|
||
|
||
# ============ 页面爬取 ============
|
||
|
||
def crawl_detail(post_id, url):
|
||
"""爬取帖子详情页，提取：内容、用户信息、回复信息"""
|
||
try:
|
||
resp = requests.get(url, headers=HEADERS, timeout=15)
|
||
resp.encoding = 'gb2312'
|
||
html = resp.text
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
result = {
|
||
'content': None,
|
||
'author_username': None,
|
||
'author_id': None,
|
||
'author_info': {},
|
||
'members': [],
|
||
'reply_count': 0,
|
||
'view_count': 0,
|
||
'post_time': None,
|
||
'contact': None,
|
||
'has_lifebuoy': False
|
||
}
|
||
|
||
# 1. 提取浏览数
|
||
view_match = re.search(r'您是本帖的第\s*<b>(\d+)</b>\s*个阅读者', html)
|
||
if view_match:
|
||
result['view_count'] = int(view_match.group(1))
|
||
|
||
# 2. 提取回复数（数 postbottom1/2 数量）
|
||
postbottom_count = len(soup.find_all('div', class_=lambda x: x and 'postbottom' in str(x)))
|
||
# 减去1因为第一个是主帖
|
||
result['reply_count'] = max(0, postbottom_count - 1)
|
||
|
||
# 3. 解析每个楼层
|
||
page_text = soup.get_text(separator='\n', strip=True)
|
||
|
||
# 找所有 postuserinfo 块
|
||
userinfo_blocks = soup.find_all('div', class_='postuserinfo')
|
||
|
||
for i, userinfo in enumerate(userinfo_blocks):
|
||
user_data = parse_userinfo_block(userinfo)
|
||
if user_data and user_data.get('user_id'):
|
||
result['members'].append(user_data)
|
||
|
||
# 第一条是楼主
|
||
if i == 0:
|
||
result['author_username'] = user_data.get('username')
|
||
result['author_id'] = user_data.get('user_id')
|
||
result['author_info'] = user_data
|
||
|
||
# 4. 提取帖子内容
|
||
post_div = soup.find('div', class_='post')
|
||
if post_div:
|
||
text = post_div.get_text(separator='\n', strip=True)
|
||
result['content'] = text[:8000]
|
||
if '救生圈' in text:
|
||
result['has_lifebuoy'] = True
|
||
|
||
# 5. 提取联系方式
|
||
phones = re.findall(r'1[3-9]\d{9}', page_text)
|
||
if phones:
|
||
result['contact'] = ','.join(dict.fromkeys(phones[:3]))
|
||
|
||
# 6. 提取发帖时间（从第一个postbottom）
|
||
time_div = soup.find('div', class_='postbottom1')
|
||
if time_div:
|
||
time_match = re.search(r'(\d{4})/(\d{1,2})/(\d{1,2})\s+(\d{1,2}):(\d{2})', time_div.get_text())
|
||
if time_match:
|
||
result['post_time'] = f"{time_match.group(1)}-{int(time_match.group(2)):02d}-{int(time_match.group(3)):02d} {time_match.group(4)}:{time_match.group(5)}:00"
|
||
|
||
return result
|
||
except Exception as e:
|
||
print(f' Crawl error: {e}')
|
||
return {}
|
||
|
||
def parse_userinfo_block(userinfo_div):
|
||
"""解析单个用户的 userinfo div，提取完整用户信息"""
|
||
try:
|
||
text = userinfo_div.get_text(separator='\n', strip=True)
|
||
|
||
# user_id - from showyyzz() JS call or j_gbook links
|
||
user_id = None
|
||
userinfo_html = str(userinfo_div)
|
||
js_match = re.search(r'showyyzz\((\d+)\)', userinfo_html)
|
||
if js_match:
|
||
user_id = js_match.group(1)
|
||
else:
|
||
link_match = re.search(r'j_gbook_add\.asp\?id=(\d+)', userinfo_html)
|
||
if link_match:
|
||
user_id = link_match.group(1)
|
||
else:
|
||
dispuser_match = re.search(r'dispuser\.asp\?id=(\d+)', userinfo_html)
|
||
if dispuser_match:
|
||
user_id = dispuser_match.group(1)
|
||
|
||
# username
|
||
username = None
|
||
if user_link:
|
||
username = user_link.get_text(strip=True)
|
||
|
||
# transaction_level
|
||
level_match = re.search(r'交易等级[：:]\s*([^\n]+)', text)
|
||
transaction_level = level_match.group(1).strip() if level_match else None
|
||
|
||
# credit_score
|
||
credit_match = re.search(r'信用积分[：:]\s*(\d+)', text)
|
||
credit_score = int(credit_match.group(1)) if credit_match else None
|
||
|
||
# rating_count
|
||
rating_match = re.search(r'评分次数[：:]\s*(\d+)', text)
|
||
rating_count = int(rating_match.group(1)) if rating_match else 0
|
||
|
||
# post_count
|
||
post_match = re.search(r'发贴次数[：:]\s*(\d+)', text)
|
||
post_count = int(post_match.group(1)) if post_match else 0
|
||
|
||
# post_points
|
||
points_match = re.search(r'发帖积分[：:]\s*(\d+)', text)
|
||
post_points = int(points_match.group(1)) if points_match else 0
|
||
|
||
# registration_date
|
||
reg_match = re.search(r'注册日期[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日', text)
|
||
registration_date = None
|
||
if reg_match:
|
||
try:
|
||
registration_date = f"{reg_match.group(1)}-{int(reg_match.group(2)):02d}-{int(reg_match.group(3)):02d}"
|
||
except:
|
||
pass
|
||
|
||
# business license
|
||
has_business_license = '点击查看' in text or '已认证' in text
|
||
|
||
# real_name (from 认证员注)
|
||
real_name = None
|
||
name_match = re.search(r'姓名[：:]([^\s\n]+)', text)
|
||
if name_match:
|
||
real_name = name_match.group(1)
|
||
|
||
# phone (from userinfo or nearby content)
|
||
phone_match = re.search(r'电话[：:]\s*([^\s\n]+)', text)
|
||
phone = phone_match.group(1).strip() if phone_match else None
|
||
|
||
# address
|
||
addr_match = re.search(r'地址[：:]\s*([^\n]+)', text)
|
||
address = addr_match.group(1).strip() if addr_match else None
|
||
|
||
# bank_accounts
|
||
bank_accounts = []
|
||
bank_types = ['农行', '工行', '建行', '中行', '交行', '招行', '兴业', '民生', '光大', '中信']
|
||
for bank in bank_types:
|
||
if bank in text:
|
||
# 匹配账号和户名
|
||
matches = re.findall(rf'{bank}[：:]\s*(\d+)\s*([^\s\n]{{2,10}})', text)
|
||
for acc, name in matches:
|
||
bank_accounts.append({'bank': bank, 'account': acc.strip(), 'name': name.strip()})
|
||
|
||
# alipay
|
||
alipay = None
|
||
|
||
return {
|
||
'user_id': user_id,
|
||
'username': username,
|
||
'transaction_level': transaction_level,
|
||
'credit_score': credit_score,
|
||
'rating_count': rating_count,
|
||
'post_count': post_count,
|
||
'post_points': post_points,
|
||
'registration_date': registration_date,
|
||
'has_business_license': has_business_license,
|
||
'real_name': real_name,
|
||
'phone': phone,
|
||
'address': address,
|
||
'bank_accounts': bank_accounts,
|
||
'alipay': alipay
|
||
}
|
||
except Exception as e:
|
||
print(f' Parse userinfo error: {e}')
|
||
return {}
|
||
|
||
def crawl_list_page(page):
|
||
"""爬取列表页，提取当天所有帖子（使用BeautifulSoup解析）"""
|
||
time.sleep(random.uniform(0.3, 1.5)) # 防封延迟
|
||
|
||
url = f'{BASE_URL}/index.asp?boardid={BOARD_ID}&page={page}'
|
||
try:
|
||
resp = requests.get(url, headers=HEADERS, timeout=15)
|
||
resp.encoding = 'gb2312'
|
||
html = resp.text
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
posts_data = []
|
||
today = datetime.now().strftime('%Y/%-m/%-d')
|
||
|
||
# 找到所有包含 boardID=151 链接的 <div class="listtitle">
|
||
listtitle_divs = soup.find_all('div', class_='listtitle')
|
||
|
||
for div in listtitle_divs:
|
||
# 在 listtitle div 内找 <a> 标签
|
||
link = div.find('a', href=re.compile(r'dispbbs\.asp\?boardID=151&ID=\d+'))
|
||
if not link:
|
||
continue
|
||
|
||
href = link.get('href', '')
|
||
id_match = re.search(r'ID=(\d+)', href)
|
||
if not id_match:
|
||
continue
|
||
post_id = id_match.group(1)
|
||
|
||
# 从 title 属性提取信息：格式《标题》\n作者：xxx\n发表于：2026/4/5 9:35:00
|
||
title_attr = link.get('title', '')
|
||
if not title_attr:
|
||
continue
|
||
|
||
# 提取日期
|
||
date_match = re.search(r'(\d{4}/\d{1,2}/\d{1,2})', title_attr)
|
||
if not date_match:
|
||
continue
|
||
post_date = date_match.group(1)
|
||
|
||
# 只爬当天
|
||
if post_date != today:
|
||
continue
|
||
|
||
# 提取标题
|
||
title_match = re.search(r'《([^》]+)》', title_attr)
|
||
title = title_match.group(1).strip() if title_match else ''
|
||
|
||
if not title or len(title) < 5:
|
||
continue
|
||
|
||
skip_titles = ['栏目交易规范', '固', '精华', '_TOP', '锁', '查看', '页面加载', '投资资讯', '版块主题', '固顶主题']
|
||
if any(t in title for t in skip_titles):
|
||
continue
|
||
|
||
full_url = f'{BASE_URL}/dispbbs.asp?boardid={BOARD_ID}&id={post_id}&page=1'
|
||
|
||
post = {
|
||
'post_id': post_id,
|
||
'title': title,
|
||
'url': full_url,
|
||
'category': parse_category(title),
|
||
'post_type': parse_post_type(title),
|
||
'special_types': parse_special_types(title),
|
||
'number_features': parse_number_features(title),
|
||
}
|
||
|
||
# 从标题提取价格
|
||
price, unit = parse_price(title)
|
||
post['price'] = price
|
||
post['price_unit'] = unit
|
||
post['has_lifebuoy'] = '救生圈' in title
|
||
|
||
posts_data.append(post)
|
||
|
||
return posts_data
|
||
except Exception as e:
|
||
print(f'Crawl page {page} error: {e}')
|
||
return []
|
||
|
||
def process_post(post):
|
||
"""处理单个帖子：爬详情 + 提取价格 + 保存会员 + 入库"""
|
||
detail = crawl_detail(post['post_id'], post['url'])
|
||
post.update(detail)
|
||
|
||
# 更新作者信息
|
||
if detail.get('author_info'):
|
||
author = detail['author_info']
|
||
post['author_username'] = author.get('username')
|
||
post['author_id'] = author.get('user_id')
|
||
|
||
# 保存所有会员
|
||
for member in detail.get('members', []):
|
||
if member.get('user_id'):
|
||
save_member(member)
|
||
|
||
# 从正文提取价格（如果标题没有）
|
||
if not post.get('price') and detail.get('content'):
|
||
price, unit = parse_price_from_content(post.get('title', ''), detail.get('content', ''))
|
||
if price:
|
||
post['price'] = price
|
||
post['price_unit'] = unit
|
||
|
||
# 合并 content
|
||
if detail.get('content'):
|
||
post['content'] = detail['content']
|
||
|
||
if save_post(post):
|
||
return 1
|
||
return 0
|
||
|
||
def run(max_pages=5, workers=10):
|
||
spider_name = 'yichens_spider_v4'
|
||
start_time = datetime.now()
|
||
|
||
print('=' * 60)
|
||
print(f'一尘网爬虫 v4 [{spider_name}]')
|
||
print(f'时间: {start_time.strftime("%Y-%m-%d %H:%M:%S")}')
|
||
print(f'并发: {workers} 线程')
|
||
print('=' * 60)
|
||
|
||
# 1. 已有post_id（增量去重）
|
||
existing_ids = get_existing_post_ids()
|
||
print(f'数据库已有帖子: {len(existing_ids)} 条')
|
||
|
||
# 2. 爬列表页
|
||
all_posts = []
|
||
for page in range(1, max_pages + 1):
|
||
print(f'扫描第 {page} 页...')
|
||
posts = crawl_list_page(page)
|
||
if not posts:
|
||
if page > 2:
|
||
print(f'第 {page} 页无新数据，停止')
|
||
break
|
||
else:
|
||
all_posts.extend(posts)
|
||
print(f' 第 {page} 页: {len(posts)} 条')
|
||
|
||
# 3. 增量过滤
|
||
new_posts = [p for p in all_posts if p['post_id'] not in existing_ids]
|
||
print(f'\n共找到 {len(all_posts)} 条今日帖子，新增 {len(new_posts)} 条')
|
||
|
||
if not new_posts:
|
||
print('没有新帖子，退出')
|
||
log_crawl(spider_name, 'success', 0, '')
|
||
return 0
|
||
|
||
# 4. 并发处理
|
||
saved = 0
|
||
member_count = 0
|
||
print(f'开始并发爬取 ({workers} 线程)...')
|
||
|
||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||
future_to_post = {executor.submit(process_post, post): post for post in new_posts}
|
||
|
||
for i, future in enumerate(as_completed(future_to_post)):
|
||
post = future_to_post[future]
|
||
try:
|
||
if future.result():
|
||
saved += 1
|
||
# 统计会员数（从 post 对象获取）
|
||
if post.get('author_id'):
|
||
member_count += 1
|
||
if (i + 1) % 20 == 0:
|
||
print(f'进度: {i+1}/{len(new_posts)}, 已保存: {saved}')
|
||
except Exception as e:
|
||
print(f'处理 {post["post_id"]} 异常: {e}')
|
||
|
||
elapsed = (datetime.now() - start_time).total_seconds()
|
||
print(f'\n完成! 新增帖子 {saved} 条, 会员 {member_count} 人, 耗时 {elapsed:.1f}秒')
|
||
|
||
log_crawl(spider_name, 'success', saved, '')
|
||
return saved
|
||
|
||
if __name__ == '__main__':
|
||
run(max_pages=5, workers=10)
|