tag inside userinfo div
username = None
b_tag = userinfo_div.find('b')
if b_tag:
username = b_tag.get_text(strip=True)
# transaction_level
level_match = re.search(r'交易等级[：:]\s*([^\n]+)', text)
transaction_level = level_match.group(1).strip() if level_match else None
# credit_score
credit_match = re.search(r'信用积分[：:]\s*(\d+)', text)
credit_score = int(credit_match.group(1)) if credit_match else None
# rating_count
rating_match = re.search(r'评分次数[：:]\s*(\d+)', text)
rating_count = int(rating_match.group(1)) if rating_match else 0
# post_count
post_match = re.search(r'发贴次数[：:]\s*(\d+)', text)
post_count = int(post_match.group(1)) if post_match else 0
# post_points
points_match = re.search(r'发帖积分[：:]\s*(\d+)', text)
post_points = int(points_match.group(1)) if points_match else 0
# registration_date
reg_match = re.search(r'注册日期[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日', text)
registration_date = None
if reg_match:
try:
registration_date = f"{reg_match.group(1)}-{int(reg_match.group(2)):02d}-{int(reg_match.group(3)):02d}"
except:
pass
# business license
has_business_license = '点击查看' in text or '已认证' in text
# real_name (from 认证员注)
real_name = None
name_match = re.search(r'姓名[：:]([^\s\n]+)', text)
if name_match:
real_name = name_match.group(1)
# phone (from userinfo or nearby content)
phone_match = re.search(r'电话[：:]\s*([^\s\n]+)', text)
phone = phone_match.group(1).strip() if phone_match else None
# address
addr_match = re.search(r'地址[：:]\s*([^\n]+)', text)
address = addr_match.group(1).strip() if addr_match else None
# bank_accounts
bank_accounts = []
bank_types = ['农行', '工行', '建行', '中行', '交行', '招行', '兴业', '民生', '光大', '中信']
for bank in bank_types:
if bank in text:
# 匹配账号和户名
matches = re.findall(rf'{bank}[：:]\s*(\d+)\s*([^\s\n]{{2,10}})', text)
for acc, name in matches:
bank_accounts.append({'bank': bank, 'account': acc.strip(), 'name': name.strip()})
# alipay
alipay = None
return {
'user_id': user_id,
'username': username,
'transaction_level': transaction_level,
'credit_score': credit_score,
'rating_count': rating_count,
'post_count': post_count,
'post_points': post_points,
'registration_date': registration_date,
'has_business_license': has_business_license,
'real_name': real_name,
'phone': phone,
'address': address,
'bank_accounts': bank_accounts,
'alipay': alipay
}
except Exception as e:
print(f' Parse userinfo error: {e}')
return {}
def crawl_list_page(page):
"""爬取列表页，提取当天所有帖子（使用BeautifulSoup解析）"""
time.sleep(random.uniform(0.3, 1.5)) # 防封延迟
url = f'{BASE_URL}/index.asp?boardid={BOARD_ID}&page={page}'
try:
resp = requests.get(url, headers=HEADERS, timeout=15)
resp.encoding = 'gb2312'
html = resp.text
soup = BeautifulSoup(html, 'html.parser')
posts_data = []
today = datetime.now().strftime('%Y/%-m/%-d')
# 找到所有包含 boardID=151 链接的
listtitle_divs = soup.find_all('div', class_='listtitle')
for div in listtitle_divs:
# 在 listtitle div 内找
标签
link = div.find('a', href=re.compile(r'dispbbs\.asp\?boardID=151&ID=\d+'))
if not link:
continue
href = link.get('href', '')
id_match = re.search(r'ID=(\d+)', href)
if not id_match:
continue
post_id = id_match.group(1)
# 从 title 属性提取信息：格式《标题》\n作者：xxx\n发表于：2026/4/5 9:35:00
title_attr = link.get('title', '')
if not title_attr:
continue
# 提取日期
date_match = re.search(r'(\d{4}/\d{1,2}/\d{1,2})', title_attr)
if not date_match:
continue
post_date = date_match.group(1)
# 只爬当天
if post_date != today:
continue
# 提取标题
title_match = re.search(r'《([^》]+)》', title_attr)
title = title_match.group(1).strip() if title_match else ''
if not title or len(title) < 5:
continue
skip_titles = ['栏目交易规范', '固', '精华', '_TOP', '锁', '查看', '页面加载', '投资资讯', '版块主题', '固顶主题']
if any(t in title for t in skip_titles):
continue
full_url = f'{BASE_URL}/dispbbs.asp?boardid={BOARD_ID}&id={post_id}&page=1'
post = {
'post_id': post_id,
'title': title,
'url': full_url,
'category': parse_category(title),
'post_type': parse_post_type(title),
'special_types': parse_special_types(title),
'number_features': parse_number_features(title),
}
# 从标题提取价格
price, unit = parse_price(title)
post['price'] = price
post['price_unit'] = unit
post['has_lifebuoy'] = '救生圈' in title
posts_data.append(post)
return posts_data
except Exception as e:
print(f'Crawl page {page} error: {e}')
return []
def process_post(post):
"""处理单个帖子：爬详情 + 提取价格 + 保存会员 + 入库"""
detail = crawl_detail(post['post_id'], post['url'])
post.update(detail)
# 更新作者信息
if detail.get('author_info'):
author = detail['author_info']
post['author_username'] = author.get('username')
post['author_id'] = author.get('user_id')
# 保存所有会员
for member in detail.get('members', []):
if member.get('user_id'):
save_member(member)
# 从正文提取价格（如果标题没有）
if not post.get('price') and detail.get('content'):
price, unit = parse_price_from_content(post.get('title', ''), detail.get('content', ''))
if price:
post['price'] = price
post['price_unit'] = unit
# 合并 content
if detail.get('content'):
post['content'] = detail['content']
if save_post(post):
return 1
return 0
def run(max_pages=5, workers=10):
spider_name = 'yichens_spider_v4'
start_time = datetime.now()
print('=' * 60)
print(f'一尘网爬虫 v4 [{spider_name}]')
print(f'时间: {start_time.strftime("%Y-%m-%d %H:%M:%S")}')
print(f'并发: {workers} 线程')
print('=' * 60)
# 1. 已有post_id（增量去重）
existing_ids = get_existing_post_ids()
print(f'数据库已有帖子: {len(existing_ids)} 条')
# 2. 爬列表页
all_posts = []
for page in range(1, max_pages + 1):
print(f'扫描第 {page} 页...')
posts = crawl_list_page(page)
if not posts:
if page > 2:
print(f'第 {page} 页无新数据，停止')
break
else:
all_posts.extend(posts)
print(f' 第 {page} 页: {len(posts)} 条')
# 3. 增量过滤
new_posts = [p for p in all_posts if p['post_id'] not in existing_ids]
print(f'\n共找到 {len(all_posts)} 条今日帖子，新增 {len(new_posts)} 条')
if not new_posts:
print('没有新帖子，退出')
log_crawl(spider_name, 'success', 0, '')
return 0
# 4. 并发处理
saved = 0
member_count = 0
print(f'开始并发爬取 ({workers} 线程)...')
with ThreadPoolExecutor(max_workers=workers) as executor:
future_to_post = {executor.submit(process_post, post): post for post in new_posts}
for i, future in enumerate(as_completed(future_to_post)):
post = future_to_post[future]
try:
if future.result():
saved += 1
# 统计会员数（从 post 对象获取）
if post.get('author_id'):
member_count += 1
if (i + 1) % 20 == 0:
print(f'进度: {i+1}/{len(new_posts)}, 已保存: {saved}')
except Exception as e:
print(f'处理 {post["post_id"]} 异常: {e}')
elapsed = (datetime.now() - start_time).total_seconds()
print(f'\n完成! 新增帖子 {saved} 条, 会员 {member_count} 人, 耗时 {elapsed:.1f}秒')
log_crawl(spider_name, 'success', saved, '')
return saved
if __name__ == '__main__':
run(max_pages=5, workers=10)