Compare commits
10 Commits
| Author | SHA1 | Date |
|---|---|---|
caibotmini
|
be0bc603b7 | |
caibotmini
|
c20a2f50ce | |
|
caibotmini
|
25129bfc07 | |
|
caibotmini
|
b9554562de | |
|
caibotmini
|
8a100a9a95 | |
|
caibotmini
|
35d238555f | |
|
caibotmini
|
52ece4f412 | |
|
caibotmini
|
b0bb3372fa | |
|
caibotmini
|
e112b4c1e6 | |
|
caibotmini
|
65162298c5 |
|
|
@ -54,6 +54,42 @@ def extract_post_content(html_bytes):
|
||||||
if discl_pos != -1:
|
if discl_pos != -1:
|
||||||
text = text[:discl_pos + len(disclaimer)]
|
text = text[:discl_pos + len(disclaimer)]
|
||||||
|
|
||||||
|
# 清理尾部网站版权和导航信息
|
||||||
|
footer_patterns = [
|
||||||
|
'BoardJumpListSelect',
|
||||||
|
'Powered By Dvbbs',
|
||||||
|
'京ICP备',
|
||||||
|
'页面执行时间',
|
||||||
|
'发短信',
|
||||||
|
'购买论坛点券',
|
||||||
|
'我能做什么',
|
||||||
|
'我发表的主题',
|
||||||
|
'我参与的主题',
|
||||||
|
'基本资料修改',
|
||||||
|
'用户密码修改',
|
||||||
|
'联系资料修改',
|
||||||
|
'用户短信服务',
|
||||||
|
'编辑好友列表',
|
||||||
|
'个人文件管理',
|
||||||
|
'通行证设置',
|
||||||
|
'今日贴数图例',
|
||||||
|
'主题数图例',
|
||||||
|
'总帖数图例',
|
||||||
|
'在线图例',
|
||||||
|
'在线情况',
|
||||||
|
'用户组在线图例',
|
||||||
|
'文件集浏览',
|
||||||
|
'图片集浏览',
|
||||||
|
'Flash浏览',
|
||||||
|
'音乐集浏览',
|
||||||
|
'电影集浏览',
|
||||||
|
'贺卡发送',
|
||||||
|
]
|
||||||
|
for pat in footer_patterns:
|
||||||
|
pos = text.find(pat)
|
||||||
|
if pos != -1:
|
||||||
|
text = text[:pos]
|
||||||
|
|
||||||
return text if text else None
|
return text if text else None
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -143,7 +179,14 @@ class YichensTodaySpider(PaginationSpider):
|
||||||
created_at = None
|
created_at = None
|
||||||
m = re.search(r"(\d{4}/\d{1,2}/\d{1,2}\s+\d{1,2}:\d{2}:\d{2})", page_text)
|
m = re.search(r"(\d{4}/\d{1,2}/\d{1,2}\s+\d{1,2}:\d{2}:\d{2})", page_text)
|
||||||
if m:
|
if m:
|
||||||
created_at = m.group(1).replace("/", "-")
|
date_str = m.group(1).replace("/", "-")
|
||||||
|
try:
|
||||||
|
from datetime import datetime
|
||||||
|
dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
|
||||||
|
if 2020 <= dt.year <= 2030:
|
||||||
|
created_at = date_str
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
phones = re.findall(r'1[3-9]\d{9,10}', page_text)
|
phones = re.findall(r'1[3-9]\d{9,10}', page_text)
|
||||||
contact = ",".join(dict.fromkeys(phones[:5])) if phones else None
|
contact = ",".join(dict.fromkeys(phones[:5])) if phones else None
|
||||||
|
|
@ -166,21 +209,28 @@ class YichensTodaySpider(PaginationSpider):
|
||||||
if "标百" in title:
|
if "标百" in title:
|
||||||
special_types.append("标百")
|
special_types.append("标百")
|
||||||
|
|
||||||
post_type = "normal"
|
# post_type 分类逻辑：出售 > 求购 > 其他 > 默认出售
|
||||||
if title:
|
if title:
|
||||||
if any(c in title for c in ["出", "售", "卖", "兑"]):
|
if any(c in title for c in ["出", "售", "卖"]):
|
||||||
post_type = "deal"
|
post_type = "deal"
|
||||||
elif any(c in title for c in ["求", "收", "购"]):
|
elif any(c in title for c in ["收", "求", "购", "要"]):
|
||||||
post_type = "want"
|
post_type = "want"
|
||||||
|
elif any(c in title for c in ["确认", "朋友", "投诉"]):
|
||||||
|
post_type = "normal"
|
||||||
|
else:
|
||||||
|
post_type = "deal" # 默认出售
|
||||||
|
else:
|
||||||
|
post_type = "normal"
|
||||||
|
|
||||||
|
# category 分类逻辑（优先级：龙钞 > 马钞 > 蛇钞 > 其他）
|
||||||
category = "其他"
|
category = "其他"
|
||||||
if title:
|
if title:
|
||||||
if "龙钞" in title or "龙纪念" in title:
|
if any(c in title for c in ["龙", "龙钞", "小龙钞", "钞王"]):
|
||||||
category = "龙钞"
|
category = "龙钞"
|
||||||
elif "蛇钞" in title:
|
elif any(c in title for c in ["马", "马钞"]):
|
||||||
category = "蛇钞"
|
|
||||||
elif "马钞" in title:
|
|
||||||
category = "马钞"
|
category = "马钞"
|
||||||
|
elif any(c in title for c in ["蛇", "蛇钞"]):
|
||||||
|
category = "蛇钞"
|
||||||
|
|
||||||
# 从 postuserinfo div 的 <b> 标签提取用户名（跳过电话号码）
|
# 从 postuserinfo div 的 <b> 标签提取用户名（跳过电话号码）
|
||||||
if not author:
|
if not author:
|
||||||
|
|
@ -303,58 +353,73 @@ class YichensTodaySpider(PaginationSpider):
|
||||||
if not post or not post.get("post_id"):
|
if not post or not post.get("post_id"):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
sql = """
|
import psycopg2.sql as sql
|
||||||
INSERT INTO yichens_posts (
|
|
||||||
post_id, title, content, category, post_type,
|
|
||||||
price, price_unit, special_types, number_features,
|
|
||||||
author_username, author_id, contact,
|
|
||||||
has_lifebuoy, reply_count, view_count,
|
|
||||||
post_time, crawled_at, updated_at, url
|
|
||||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW(), %s)
|
|
||||||
ON CONFLICT (post_id) DO UPDATE SET
|
|
||||||
title = EXCLUDED.title,
|
|
||||||
content = EXCLUDED.content,
|
|
||||||
category = EXCLUDED.category,
|
|
||||||
post_type = EXCLUDED.post_type,
|
|
||||||
price = EXCLUDED.price,
|
|
||||||
price_unit = EXCLUDED.price_unit,
|
|
||||||
special_types = EXCLUDED.special_types,
|
|
||||||
author_username = EXCLUDED.author_username,
|
|
||||||
contact = EXCLUDED.contact,
|
|
||||||
has_lifebuoy = EXCLUDED.has_lifebuoy,
|
|
||||||
post_time = EXCLUDED.post_time,
|
|
||||||
updated_at = NOW(),
|
|
||||||
crawled_at = NOW(),
|
|
||||||
url = EXCLUDED.url
|
|
||||||
"""
|
|
||||||
|
|
||||||
has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
|
has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
|
||||||
|
|
||||||
|
cols = ['post_id', 'title', 'content', 'category', 'post_type', 'price',
|
||||||
|
'price_unit', 'special_types', 'number_features', 'author_username',
|
||||||
|
'author_id', 'contact', 'has_lifebuoy', 'reply_count', 'view_count',
|
||||||
|
'post_time', 'crawled_at', 'updated_at', 'url', 'instance_id']
|
||||||
|
|
||||||
|
vals = [
|
||||||
|
sql.Literal(post.get("post_id")),
|
||||||
|
sql.Literal(post.get("title")),
|
||||||
|
sql.Literal(post.get("content")),
|
||||||
|
sql.Literal(post.get("category")),
|
||||||
|
sql.Literal(post.get("post_type")),
|
||||||
|
sql.Literal(post.get("price")),
|
||||||
|
sql.Literal(post.get("price_unit")),
|
||||||
|
sql.Literal(post.get("special_types")),
|
||||||
|
sql.Literal(post.get("number_features")),
|
||||||
|
sql.Literal(post.get("author_username")),
|
||||||
|
sql.Literal(f"user_{post.get('post_id')}"),
|
||||||
|
sql.Literal(post.get("contact")),
|
||||||
|
sql.Literal(has_lifebuoy),
|
||||||
|
sql.Literal(0),
|
||||||
|
sql.Literal(0),
|
||||||
|
sql.Literal(post.get("post_time")),
|
||||||
|
sql.SQL('NOW()'),
|
||||||
|
sql.SQL('NOW()'),
|
||||||
|
sql.Literal(post.get("url")),
|
||||||
|
sql.Literal(self.instance_id),
|
||||||
|
]
|
||||||
|
|
||||||
|
update_assigns = [
|
||||||
|
"title = EXCLUDED.title",
|
||||||
|
"content = EXCLUDED.content",
|
||||||
|
"category = EXCLUDED.category",
|
||||||
|
"post_type = EXCLUDED.post_type",
|
||||||
|
"price = EXCLUDED.price",
|
||||||
|
"price_unit = EXCLUDED.price_unit",
|
||||||
|
"special_types = EXCLUDED.special_types",
|
||||||
|
"number_features = EXCLUDED.number_features",
|
||||||
|
"author_username = EXCLUDED.author_username",
|
||||||
|
"author_id = EXCLUDED.author_id",
|
||||||
|
"contact = EXCLUDED.contact",
|
||||||
|
"has_lifebuoy = EXCLUDED.has_lifebuoy",
|
||||||
|
"reply_count = EXCLUDED.reply_count",
|
||||||
|
"view_count = EXCLUDED.view_count",
|
||||||
|
"post_time = EXCLUDED.post_time",
|
||||||
|
"updated_at = NOW()",
|
||||||
|
"crawled_at = NOW()",
|
||||||
|
"url = EXCLUDED.url",
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
|
query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(
|
||||||
|
sql.SQL(', ').join(sql.Identifier(c) for c in cols),
|
||||||
|
sql.SQL(', ').join(vals),
|
||||||
|
sql.SQL(', ').join(sql.SQL(a) for a in update_assigns),
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with get_db() as conn:
|
with get_db() as conn:
|
||||||
if not conn:
|
if not conn:
|
||||||
print("数据库连接失败")
|
print("数据库连接失败")
|
||||||
return False
|
return False
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
cur.execute(sql, (
|
cur.execute(query)
|
||||||
post.get("post_id"),
|
|
||||||
post.get("title"),
|
|
||||||
post.get("content"),
|
|
||||||
post.get("category"),
|
|
||||||
post.get("post_type"),
|
|
||||||
post.get("price"),
|
|
||||||
post.get("price_unit"),
|
|
||||||
post.get("special_types"),
|
|
||||||
post.get("number_features"),
|
|
||||||
post.get("author_username"),
|
|
||||||
f"user_{post.get('post_id')}",
|
|
||||||
post.get("contact"),
|
|
||||||
has_lifebuoy,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
post.get("post_time"),
|
|
||||||
post.get("url"),
|
|
||||||
))
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
cur.close()
|
cur.close()
|
||||||
return True
|
return True
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,250 @@
|
||||||
|
# 一尘数据系统 - 服务器运维手册
|
||||||
|
|
||||||
|
> 适用服务器：2号 (101.37.160.219)、3号 (42.121.116.25)
|
||||||
|
> 统一密码：`Coolbot123`（2号也用此密码登录）
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、服务器概览
|
||||||
|
|
||||||
|
| 服务器 | IP | 用途 | 爬虫 cron |
|
||||||
|
|--------|-----|------|-----------|
|
||||||
|
| 2号 | 101.37.160.219 | 爬虫 + API | 每 15 分钟 |
|
||||||
|
| 3号 | 42.121.116.25 | 爬虫 + API | 每 25 分钟 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、下载更新版本
|
||||||
|
|
||||||
|
### 方式 A：通过 Git 拉取（推荐）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /root/coolbot-data
|
||||||
|
git pull origin main
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注意：可能需要先设置 Git 凭据
|
||||||
|
> ```bash
|
||||||
|
> git config --global credential.helper store
|
||||||
|
> # 首次需要输入一次用户名密码
|
||||||
|
> # 用户名: caibotmi
|
||||||
|
> # 密码: Caibotmi123
|
||||||
|
> ```
|
||||||
|
|
||||||
|
### 方式 B：手动上传
|
||||||
|
|
||||||
|
如果 Git 有问题，可以让机器人（我）从 1 号服务器打包文件，通过中间服务器转发。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、部署启动服务
|
||||||
|
|
||||||
|
### 3.1 安装 Python 依赖
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /root/coolbot-data
|
||||||
|
source venv/bin/activate
|
||||||
|
pip install -r requirements.txt -q
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.2 重启爬虫服务
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 重启 cron（让新代码生效）
|
||||||
|
crontab /root/coolbot-data/scripts/root.cron
|
||||||
|
|
||||||
|
# 或手动重启 crond
|
||||||
|
systemctl restart crond
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 重启 API 服务
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 查看 API 是否在运行
|
||||||
|
ps aux | grep uvicorn | grep -v grep
|
||||||
|
|
||||||
|
# 如果在运行，kill 掉
|
||||||
|
pkill -f uvicorn
|
||||||
|
|
||||||
|
# 重新启动（后台运行）
|
||||||
|
cd /root/coolbot-data
|
||||||
|
source venv/bin/activate
|
||||||
|
nohup python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8080 > /root/coolbot-data/logs/api.log 2>&1 &
|
||||||
|
echo "API 已重启，PID: $!"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 四、验证服务
|
||||||
|
|
||||||
|
### 4.1 验证爬虫日志
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 查看最近爬虫执行记录
|
||||||
|
tail -20 /root/coolbot-data/logs/cron_s2.log # 2号
|
||||||
|
tail -20 /root/coolbot-data/logs/cron_s3.log # 3号
|
||||||
|
```
|
||||||
|
|
||||||
|
**正常的日志格式：**
|
||||||
|
```
|
||||||
|
开始全量采集第1-2页 (今天: 2026/4/7)
|
||||||
|
扫描第 1 页...
|
||||||
|
该页共 391 条帖子
|
||||||
|
[保存] 36193766 - ，，，13100元求购... (今日2026/4/7中已存2条)
|
||||||
|
===== 采集完成 =====
|
||||||
|
总计检查: 781 条
|
||||||
|
成功保存: 23 条
|
||||||
|
```
|
||||||
|
|
||||||
|
**出现以下情况请处理：**
|
||||||
|
- `保存失败: not all arguments converted` → 代码未更新，需重新拉取
|
||||||
|
- `保存失败: connection` → 数据库连接问题
|
||||||
|
- 一直是 0 条 → 检查网络或目标网站
|
||||||
|
|
||||||
|
### 4.2 验证 API 服务
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 测试 API 是否响应
|
||||||
|
curl -s http://localhost:8080/health | python3 -m json.tool
|
||||||
|
```
|
||||||
|
|
||||||
|
**正常响应：**
|
||||||
|
```json
|
||||||
|
{"status": "ok", "message": "CoolBotDataSys API is running", "version": "1.0.0"}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 验证数据库新帖
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -c "
|
||||||
|
import psycopg2
|
||||||
|
conn = psycopg2.connect(host='pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com', port=5432, database='coolbot_data', user='coolbot', password='Coolbot123')
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute('SELECT COUNT(*), MAX(crawled_at) FROM yichens_posts WHERE DATE(crawled_at) = CURRENT_DATE')
|
||||||
|
r = cur.fetchone()
|
||||||
|
print(f'今日帖子: {r[0]}条 | 最后采集: {r[1]}')
|
||||||
|
conn.close()
|
||||||
|
"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 五、定期检查
|
||||||
|
|
||||||
|
### 5.1 每日检查（建议）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. 检查爬虫是否在运行
|
||||||
|
ps aux | grep crawl_today | grep -v grep
|
||||||
|
|
||||||
|
# 2. 检查 cron 是否生效
|
||||||
|
crontab -l | grep crawl
|
||||||
|
|
||||||
|
# 3. 检查今日数据量（应该 > 0）
|
||||||
|
python3 -c "import psycopg2; conn = psycopg2.connect(host='pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com', port=5432, database='coolbot_data', user='coolbot', password='Coolbot123'); cur = conn.cursor(); cur.execute('SELECT COUNT(*) FROM yichens_posts WHERE DATE(crawled_at) = CURRENT_DATE'); print(cur.fetchone()[0], '条'); conn.close()"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.2 每周检查
|
||||||
|
|
||||||
|
- 检查磁盘空间：`df -h`
|
||||||
|
- 检查日志文件大小：`ls -lh /root/coolbot-data/logs/`
|
||||||
|
- 清理旧日志：`find /root/coolbot-data/logs -name "*.log" -mtime +7 -delete`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 六、问题处理
|
||||||
|
|
||||||
|
### 问题 1：爬虫一直报 "保存失败: not all arguments converted"
|
||||||
|
|
||||||
|
**原因：** 代码未更新到最新版本
|
||||||
|
|
||||||
|
**解决：**
|
||||||
|
```bash
|
||||||
|
cd /root/coolbot-data
|
||||||
|
git pull origin main
|
||||||
|
```
|
||||||
|
|
||||||
|
### 问题 2：爬虫返回 0 条新帖
|
||||||
|
|
||||||
|
**排查步骤：**
|
||||||
|
1. 检查网络：`curl -s -o /dev/null -w "%{http_code}" http://www4.pm001.net`
|
||||||
|
2. 查看详细日志：`tail -50 /root/coolbot-data/logs/cron_s*.log`
|
||||||
|
3. 手动跑一次爬虫看输出
|
||||||
|
|
||||||
|
### 问题 3：API 服务无响应
|
||||||
|
|
||||||
|
**解决：**
|
||||||
|
```bash
|
||||||
|
# 检查端口是否监听
|
||||||
|
netstat -tlnp | grep 8080
|
||||||
|
|
||||||
|
# 重启 API
|
||||||
|
cd /root/coolbot-data
|
||||||
|
pkill -f uvicorn
|
||||||
|
source venv/bin/activate
|
||||||
|
nohup python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8080 > logs/api.log 2>&1 &
|
||||||
|
```
|
||||||
|
|
||||||
|
### 问题 4：数据库连接失败
|
||||||
|
|
||||||
|
**原因：** 阿里云 RDS 网络不通 或 密码错误
|
||||||
|
|
||||||
|
**排查：**
|
||||||
|
```bash
|
||||||
|
# 测试数据库连接
|
||||||
|
psql "postgresql://coolbot:Coolbot123@pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com:5432/coolbot_data" -c "SELECT 1"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 问题 5：SSH 连接被拒
|
||||||
|
|
||||||
|
**原因：** 服务器 SSH 策略或网络问题
|
||||||
|
|
||||||
|
**解决：** 等待几分钟后重试，或联系网络管理员
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 七、快速命令汇总
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# === 查看状态 ===
|
||||||
|
git -C /root/coolbot-data log --oneline -1 # 最新版本
|
||||||
|
ps aux | grep crawl_today | grep -v grep # 爬虫进程
|
||||||
|
ps aux | grep uvicorn | grep -v grep # API 进程
|
||||||
|
tail -5 /root/coolbot-data/logs/cron_s*.log # 最新日志
|
||||||
|
|
||||||
|
# === 更新代码 ===
|
||||||
|
cd /root/coolbot-data && git pull origin main
|
||||||
|
|
||||||
|
# === 重启爬虫 ===
|
||||||
|
pkill -f crawl_today
|
||||||
|
# 等待 cron 自动触发，或手动运行：
|
||||||
|
# cd /root/coolbot-data && source venv/bin/activate && python3 scripts/crawl_cron_s2.sh
|
||||||
|
|
||||||
|
# === 重启 API ===
|
||||||
|
cd /root/coolbot-data && pkill -f uvicorn && source venv/bin/activate && nohup python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8080 > logs/api.log 2>&1 &
|
||||||
|
|
||||||
|
# === 验证服务 ===
|
||||||
|
curl -s http://localhost:8080/health # API 健康检查
|
||||||
|
python3 -c "import psycopg2; c=psycopg2.connect(host='pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com',port=5432,database='coolbot_data',user='coolbot',password='Coolbot123'); print(c.cursor().execute('SELECT 1'),'OK')" # DB 连接
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 八、重要文件路径
|
||||||
|
|
||||||
|
| 路径 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| `/root/coolbot-data/crawlers/crawl_today.py` | 爬虫主程序 |
|
||||||
|
| `/root/coolbot-data/scripts/crawl_cron_s2.sh` | 2号 cron 脚本 |
|
||||||
|
| `/root/coolbot-data/scripts/crawl_cron_s3.sh` | 3号 cron 脚本 |
|
||||||
|
| `/root/coolbot-data/logs/cron_s2.log` | 2号爬虫日志 |
|
||||||
|
| `/root/coolbot-data/logs/cron_s3.log` | 3号爬虫日志 |
|
||||||
|
| `/root/coolbot-data/api/main.py` | API 主程序 |
|
||||||
|
| `/root/coolbot-data/logs/api.log` | API 日志 |
|
||||||
|
| `/root/coolbot-data/.env` | 环境变量（包含数据库密码） |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 九、联系方式
|
||||||
|
|
||||||
|
如遇到无法解决的问题，请联系：**菜鸟Mi（caibotmini）**
|
||||||
|
|
@ -0,0 +1,684 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
||||||
|
<title>一尘数据</title>
|
||||||
|
<style>
|
||||||
|
*{margin:0;padding:0;box-sizing:border-box;-webkit-tap-highlight-color:transparent}
|
||||||
|
body{font-family:-apple-system,BlinkMacSystemFont,PingFang SC,Helvetica,Arial,sans-serif;background:#f2f2f2;font-size:14px;color:#333;min-height:100vh;padding-bottom:60px}
|
||||||
|
a{color:inherit;text-decoration:none}
|
||||||
|
|
||||||
|
/* 顶部 */
|
||||||
|
.header{background:linear-gradient(135deg,#1a1a2e,#16213e);color:#fff;padding:12px 16px;position:sticky;top:0;z-index:50;display:flex;align-items:center;justify-content:space-between}
|
||||||
|
.header h1{font-size:17px;font-weight:600}
|
||||||
|
.hdr-btn{font-size:20px;background:none;border:none;cursor:pointer}
|
||||||
|
|
||||||
|
/* 统计 */
|
||||||
|
.stats{padding:12px;display:grid;grid-template-columns:1fr 1fr;gap:10px}
|
||||||
|
.stat{background:#fff;border-radius:10px;padding:14px;text-align:center;cursor:pointer}
|
||||||
|
.stat:active{background:#f0f0f0}
|
||||||
|
.s-num{font-size:26px;font-weight:700;margin-bottom:4px}
|
||||||
|
.s-num.blue{color:#1976d2}.s-num.green{color:#2e7d32}.s-num.orange{color:#e65100}.s-num.red{color:#c62828}
|
||||||
|
.s-label{font-size:12px;color:#999}
|
||||||
|
.stat.full{grid-column:1/-1;display:flex;justify-content:space-around}
|
||||||
|
|
||||||
|
/* 筛选 */
|
||||||
|
.f-bar{background:#fff;padding:10px 12px;border-bottom:1px solid #eee;overflow-x:auto;white-space:nowrap}
|
||||||
|
.f-row{display:flex;gap:8px}
|
||||||
|
.f-btn{flex-shrink:0;padding:5px 14px;border:1px solid #ddd;border-radius:16px;font-size:13px;background:#fff;color:#666;cursor:pointer}
|
||||||
|
.f-btn:active,.f-btn.on{background:#1a1a2e;color:#fff;border-color:#1a1a2e}
|
||||||
|
|
||||||
|
/* 列表 */
|
||||||
|
.list{padding:10px 12px;padding-bottom:65px}
|
||||||
|
.post{background:#fff;border-radius:10px;padding:12px;margin-bottom:10px;cursor:pointer}
|
||||||
|
.post:active{background:#fafafa}
|
||||||
|
.post-title{font-size:14px;font-weight:500;line-height:1.5;margin-bottom:8px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
|
||||||
|
.post-meta{display:flex;flex-wrap:wrap;gap:5px;align-items:center;font-size:12px}
|
||||||
|
.badge{padding:2px 7px;border-radius:4px;font-weight:500}
|
||||||
|
.b-deal{background:#e8f5e9;color:#2e7d32}.b-want{background:#fff3e0;color:#e65100}.b-normal{background:#e3f2fd;color:#1565c0}.b-lifebuoy{background:#ffebee;color:#c62828}
|
||||||
|
.price{color:#e53935;font-weight:600}
|
||||||
|
.author{color:#1976d2}.tm{color:#999}
|
||||||
|
|
||||||
|
/* 分页 */
|
||||||
|
.pager{display:flex;justify-content:center;align-items:center;gap:12px;padding:14px;background:#fff;border-radius:10px;margin-top:10px}
|
||||||
|
.pg-btn{padding:8px 18px;background:#f5f5f5;border:1px solid #ddd;border-radius:6px;font-size:14px;cursor:pointer}
|
||||||
|
.pg-btn:disabled{opacity:.4;cursor:not-allowed}
|
||||||
|
.pg-info{font-size:13px;color:#999}
|
||||||
|
/* 统计页 */
|
||||||
|
.stats-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}
|
||||||
|
.st-card{background:#fff;border-radius:10px;padding:16px;text-align:center;cursor:pointer}
|
||||||
|
.st-card:active{background:#f0f0f0}
|
||||||
|
.st-num{font-size:32px;font-weight:700;color:#e65100;line-height:1.2}
|
||||||
|
.st-card.st-deal .st-num{color:#2e7d32}
|
||||||
|
.st-label{font-size:14px;color:#333;margin:6px 0 4px;font-weight:500}
|
||||||
|
.st-price{font-size:12px;color:#999}
|
||||||
|
|
||||||
|
/* 空/加载 */
|
||||||
|
.empty,.loading{text-align:center;padding:50px;color:#999}
|
||||||
|
.empty::before{content:'📭';font-size:40px;display:block;margin-bottom:10px}
|
||||||
|
|
||||||
|
/* 底部导航 */
|
||||||
|
.tab{position:fixed;bottom:0;left:0;right:0;background:#fff;border-top:1px solid #eee;display:flex;z-index:50;padding-bottom:env(safe-area-inset-bottom,0)}
|
||||||
|
.tab-i{flex:1;text-align:center;padding:10px 0;font-size:11px;color:#999;cursor:pointer}
|
||||||
|
.tab-i.on{color:#1a1a2e}
|
||||||
|
.tab-i span{display:block;font-size:22px;margin-bottom:2px}
|
||||||
|
|
||||||
|
/* 详情页 */
|
||||||
|
.detail{position:fixed;inset:0;background:#fff;z-index:100;overflow-y:auto}
|
||||||
|
.detail-hdr{position:sticky;top:0;background:#fff;padding:12px 16px;border-bottom:1px solid #eee;display:flex;align-items:center;justify-content:space-between;z-index:5}
|
||||||
|
.back{background:none;border:none;font-size:16px;color:#1976d2;cursor:pointer}
|
||||||
|
.detail-link{font-size:13px;color:#1976d2}
|
||||||
|
.detail-body{padding:16px}
|
||||||
|
.detail-title{font-size:16px;font-weight:600;line-height:1.5;margin-bottom:14px}
|
||||||
|
.sec{margin-bottom:20px}
|
||||||
|
.sec-title{font-size:13px;font-weight:600;margin-bottom:10px;padding-bottom:6px;border-bottom:2px solid #1a1a2e}
|
||||||
|
.info-g{ display:grid;grid-template-columns:1fr 1fr;gap:10px}
|
||||||
|
.info-i{display:flex;flex-direction:column;gap:2px}
|
||||||
|
.info-l{font-size:11px;color:#999}
|
||||||
|
.info-v{font-size:14px;color:#333}
|
||||||
|
.info-v.prime{font-size:22px;color:#e53935;font-weight:700}
|
||||||
|
.content{background:#f8f9fa;padding:12px;border-radius:8px;white-space:pre-wrap;font-size:14px;line-height:1.7;word-break:break-all;max-height:300px;overflow-y:auto}
|
||||||
|
.tag-l{display:flex;flex-wrap:wrap;gap:6px}
|
||||||
|
.tag{padding:3px 10px;background:#e3f2fd;color:#1565c0;border-radius:12px;font-size:12px}
|
||||||
|
|
||||||
|
/* 品类 */
|
||||||
|
.cat-list{background:#fff;border-radius:10px;padding:12px;margin:0 12px 80px}
|
||||||
|
.cat-row{display:flex;justify-content:space-between;padding:8px 0;border-bottom:1px solid #f0f0f0;font-size:13px}
|
||||||
|
.cat-row:last-child{border:none}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<div id="app"></div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const API = '/api/v1'
|
||||||
|
let S = {
|
||||||
|
v: 'home', // home | list | detail
|
||||||
|
stats: {},
|
||||||
|
posts: [],
|
||||||
|
cur: null,
|
||||||
|
loading: false,
|
||||||
|
pg: 1,
|
||||||
|
ps: 300,
|
||||||
|
total: 0,
|
||||||
|
ft: { type: '', cat: '', lifebuoy: '' }
|
||||||
|
}
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const app = document.getElementById('app')
|
||||||
|
if (S.v === 'home') {
|
||||||
|
const s = S.stats
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="header">
|
||||||
|
<h1>📊 一尘数据分析</h1>
|
||||||
|
<button class="hdr-btn" onclick="loadStats()">🔄</button>
|
||||||
|
</div>
|
||||||
|
<div class="stats">
|
||||||
|
<div class="stat" onclick="goList('deal')">
|
||||||
|
<div class="s-num green">${s.deal_count||0}</div>
|
||||||
|
<div class="s-label">今日出售</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat" onclick="goList('want')">
|
||||||
|
<div class="s-num orange">${s.want_count||0}</div>
|
||||||
|
<div class="s-label">今日求购</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat" onclick="goList('')">
|
||||||
|
<div class="s-num blue">${s.today_count||0}</div>
|
||||||
|
<div class="s-label">今日新增</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat" onclick="goList('lifebuoy')">
|
||||||
|
<div class="s-num red">${s.lifebuoy_count||0}</div>
|
||||||
|
<div class="s-label">大救生圈</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat full">
|
||||||
|
<div>
|
||||||
|
<div class="s-num blue" style="font-size:20px">${s.total_posts||0}</div>
|
||||||
|
<div class="s-label">总帖子</div>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div style="font-size:22px">🟢</div>
|
||||||
|
<div class="s-label">在线</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div style="padding:0 12px 20px">
|
||||||
|
<div class="sec-title">📋 品类分布</div>
|
||||||
|
<div class="cat-list">
|
||||||
|
<div style="margin-bottom:12px">
|
||||||
|
<div style="font-size:12px;color:#1976d2;margin-bottom:8px">📅 当日新增</div>
|
||||||
|
${((S.catReport&&S.catReport.today&&S.catReport.today.categories)||[]).map(c=>`
|
||||||
|
<div class="cat-row" onclick="goList('','today','${c.category}')">
|
||||||
|
<span>${c.category}</span>
|
||||||
|
<span style="color:#1976d2;font-weight:600">${c.count}条</span>
|
||||||
|
</div>
|
||||||
|
`).join('')}
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div style="font-size:12px;color:#e65100;margin-bottom:8px">⏰ 近1小时新增</div>
|
||||||
|
${((S.catReport&&S.catReport.last_1h&&S.catReport.last_1h.categories)||[]).map(c=>`
|
||||||
|
<div class="cat-row" onclick="goList('','1h','${c.category}')">
|
||||||
|
<span>${c.category}</span>
|
||||||
|
<span style="color:#e65100;font-weight:600">${c.count}条</span>
|
||||||
|
</div>
|
||||||
|
`).join('')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="tab">
|
||||||
|
<div class="tab-i on" onclick="S.v='home';render()"><span>📊</span>首页</div>
|
||||||
|
<div class="tab-i" onclick="goList('')"><span>📋</span>帖子</div>
|
||||||
|
<div class="tab-i" onclick="goCollections()"><span>💎</span>藏品</div>
|
||||||
|
<div class="tab-i" onclick="goUsers()"><span>👤</span>用户</div>
|
||||||
|
<div class="tab-i" onclick="S.v='stats';loadStats()"><span>📈</span>统计</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if (S.v === 'collections') {
|
||||||
|
const cols = S.collections || []
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="header"><h1>💎 藏品列表</h1><button class="hdr-btn" onclick="loadCollections()">🔄</button></div>
|
||||||
|
<div class="filter-bar">
|
||||||
|
<select id="col-cat" onchange="S.colCat=this.value;render()">
|
||||||
|
<option value="">全部品类</option>
|
||||||
|
<option value="龙钞">🐉 龙钞</option>
|
||||||
|
<option value="马钞">🐴 马钞</option>
|
||||||
|
<option value="蛇钞">🐍 蛇钞</option>
|
||||||
|
<option value="其他">📦 其他</option>
|
||||||
|
</select>
|
||||||
|
<input type="text" id="col-search" placeholder="搜索名称/冠号" onkeyup="S.colSearch=this.value;render()" style="flex:1;padding:8px;border:1px solid #ddd;border-radius:6px">
|
||||||
|
</div>
|
||||||
|
<div class="list">
|
||||||
|
${cols.filter(c => (!S.colCat || c.category === S.colCat) && (!S.colSearch || (c.name&&c.name.includes(S.colSearch)) || (c.crown_code&&c.crown_code.includes(S.colSearch)))).map(c => `
|
||||||
|
<div class="post-item" onclick="goCollectionDetail('${c.id}')">
|
||||||
|
<div class="post-title">${c.name || c.crown_code || '无名'}</div>
|
||||||
|
<div class="post-meta">
|
||||||
|
<span class="${c.category==='龙钞'?'green':c.category==='马钞'?'orange':c.category==='蛇钞'?'purple':'gray'}">${c.category||'其他'}</span>
|
||||||
|
<span>🏷️ ${c.series||'-'}</span>
|
||||||
|
<span>📅 ${c.issue_year||'-'}</span>
|
||||||
|
<span>🖥️ ${c.instance_id?c.instance_id.replace('server_','S'):'-'}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`).join('')}
|
||||||
|
${cols.length===0?'<div style="padding:40px;text-align:center;color:#999">暂无藏品，点击刷新</div>':''}
|
||||||
|
</div>
|
||||||
|
<div class="tab">
|
||||||
|
<div class="tab-i" onclick="S.v='home';render()"><span>📊</span>首页</div>
|
||||||
|
<div class="tab-i" onclick="goList('')"><span>📋</span>帖子</div>
|
||||||
|
<div class="tab-i on" onclick="goCollections()"><span>💎</span>藏品</div>
|
||||||
|
<div class="tab-i" onclick="goUsers()"><span>👤</span>用户</div>
|
||||||
|
<div class="tab-i" onclick="S.v='stats';loadStats()"><span>📈</span>统计</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if (S.v === 'users') {
|
||||||
|
const users = S.users || []
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="header"><h1>👤 用户列表</h1><button class="hdr-btn" onclick="loadUsers()">🔄</button></div>
|
||||||
|
<div class="filter-bar">
|
||||||
|
<input type="text" id="user-search" placeholder="搜索用户名" onkeyup="S.userSearch=this.value;render()" style="flex:1;padding:8px;border:1px solid #ddd;border-radius:6px">
|
||||||
|
</div>
|
||||||
|
<div class="list">
|
||||||
|
${users.filter(u => !S.userSearch || u.username.includes(S.userSearch)).map(u => `
|
||||||
|
<div class="post-item">
|
||||||
|
<div class="post-title">${u.username}</div>
|
||||||
|
<div class="post-meta">
|
||||||
|
<span>${u.is_seller?'🏪 商家':'👤 个人'}</span>
|
||||||
|
<span>📝 ${u.post_count||0}帖</span>
|
||||||
|
<span>⭐ ${u.credit_score||0}分</span>
|
||||||
|
<span>📅 ${u.registration_date?u.registration_date.substring(0,10):'-'}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`).join('')}
|
||||||
|
${users.length===0?'<div style="padding:40px;text-align:center;color:#999">暂无用户，点击刷新</div>':''}
|
||||||
|
</div>
|
||||||
|
<div class="tab">
|
||||||
|
<div class="tab-i" onclick="S.v='home';render()"><span>📊</span>首页</div>
|
||||||
|
<div class="tab-i" onclick="goList('')"><span>📋</span>帖子</div>
|
||||||
|
<div class="tab-i" onclick="goCollections()"><span>💎</span>藏品</div>
|
||||||
|
<div class="tab-i on" onclick="goUsers()"><span>👤</span>用户</div>
|
||||||
|
<div class="tab-i" onclick="S.v='stats';loadStats()"><span>📈</span>统计</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if (S.v === 'stats') {
|
||||||
|
const st = S.statsData || {}
|
||||||
|
const want = st.want || {}
|
||||||
|
const deal = st.deal || {}
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="header"><h1>📈 统计分析</h1><button class="hdr-btn" onclick="loadStats()">🔄</button></div>
|
||||||
|
<div style="padding:12px">
|
||||||
|
<div style="font-size:13px;font-weight:600;color:#e65100;margin-bottom:10px">🔍 求购龙钞类</div>
|
||||||
|
<div class="stats-grid">
|
||||||
|
<div class="st-card" onclick="goStatPosts('want_dai4_biao10')">
|
||||||
|
<div class="st-num">${want.dai4_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">带4标十</div>
|
||||||
|
<div class="st-price">均 ${want.dai4_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card" onclick="goStatPosts('want_wu47_biao10')">
|
||||||
|
<div class="st-num">${want.wu47_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">无47标十</div>
|
||||||
|
<div class="st-price">均 ${want.wu47_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card" onclick="goStatPosts('want_wu247_biao10')">
|
||||||
|
<div class="st-num">${want.wu247_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">无247/天马标十</div>
|
||||||
|
<div class="st-price">均 ${want.wu247_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card" onclick="goStatPosts('want_wu247_biaobai')">
|
||||||
|
<div class="st-num">${want.wu247_biaobai?.count||0}</div>
|
||||||
|
<div class="st-label">无247/天马标百/刀</div>
|
||||||
|
<div class="st-price">均 ${want.wu247_biaobai?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card" onclick="goStatPosts('want_wu47_biaobai')">
|
||||||
|
<div class="st-num">${want.wu47_biaobai?.count||0}</div>
|
||||||
|
<div class="st-label">无47标百/刀</div>
|
||||||
|
<div class="st-price">均 ${want.wu47_biaobai?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div style="font-size:13px;font-weight:600;color:#2e7d32;margin:16px 0 10px">🔍 出售龙钞类</div>
|
||||||
|
<div class="stats-grid">
|
||||||
|
<div class="st-card st-deal" onclick="goStatPosts('deal_dai4_biao10')">
|
||||||
|
<div class="st-num">${deal.dai4_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">带4标十</div>
|
||||||
|
<div class="st-price">均 ${deal.dai4_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card st-deal" onclick="goStatPosts('deal_wu47_biao10')">
|
||||||
|
<div class="st-num">${deal.wu47_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">无47标十</div>
|
||||||
|
<div class="st-price">均 ${deal.wu47_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card st-deal" onclick="goStatPosts('deal_wu247_biao10')">
|
||||||
|
<div class="st-num">${deal.wu247_biao10?.count||0}</div>
|
||||||
|
<div class="st-label">无247/天马标十</div>
|
||||||
|
<div class="st-price">均 ${deal.wu247_biao10?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card st-deal" onclick="goStatPosts('deal_wu247_biaobai')">
|
||||||
|
<div class="st-num">${deal.wu247_biaobai?.count||0}</div>
|
||||||
|
<div class="st-label">无247/天马标百/刀</div>
|
||||||
|
<div class="st-price">均 ${deal.wu247_biaobai?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
<div class="st-card st-deal" onclick="goStatPosts('deal_wu47_biaobai')">
|
||||||
|
<div class="st-num">${deal.wu47_biaobai?.count||0}</div>
|
||||||
|
<div class="st-label">无47标百/刀</div>
|
||||||
|
<div class="st-price">均 ${deal.wu47_biaobai?.avg||0}元</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="tab">
|
||||||
|
<div class="tab-i" onclick="S.v='home';render()"><span>📊</span>首页</div>
|
||||||
|
<div class="tab-i" onclick="goList('')"><span>📋</span>帖子</div>
|
||||||
|
<div class="tab-i" onclick="goCollections()"><span>💎</span>藏品</div>
|
||||||
|
<div class="tab-i" onclick="goUsers()"><span>👤</span>用户</div>
|
||||||
|
<div class="tab-i on" onclick="S.v='stats';loadStats()"><span>📈</span>统计</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if (S.v === 'list') {
|
||||||
|
const ft = S.ft
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="header"><h1>📋 帖子列表</h1></div>
|
||||||
|
<div class="f-bar">
|
||||||
|
<div class="f-row">
|
||||||
|
<button class="f-btn${ft.type===''?' on':''}" onclick="setF('type','')">全部</button>
|
||||||
|
<button class="f-btn${ft.type==='deal'?' on':''}" onclick="setF('type','deal')">出售</button>
|
||||||
|
<button class="f-btn${ft.type==='want'?' on':''}" onclick="setF('type','want')">求购</button>
|
||||||
|
<button class="f-btn${ft.type==='other'?' on':''}" onclick="setF('type','other')">其他</button>
|
||||||
|
</div>
|
||||||
|
<div class="f-row" style="margin-top:8px">
|
||||||
|
<button class="f-btn${ft.cat===''?' on':''}" onclick="setF('cat','')">全品类</button>
|
||||||
|
<button class="f-btn${ft.cat==='龙钞'?' on':''}" onclick="setF('cat','龙钞')">龙钞</button>
|
||||||
|
<button class="f-btn${ft.cat==='蛇钞'?' on':''}" onclick="setF('cat','蛇钞')">蛇钞</button>
|
||||||
|
<button class="f-btn${ft.cat==='马钞'?' on':''}" onclick="setF('cat','马钞')">马钞</button>
|
||||||
|
<button class="f-btn${ft.cat==='其他'?' on':''}" onclick="setF('cat','其他')">其他</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="list">
|
||||||
|
${S.loading?'<div class="loading">加载中...</div>':''}
|
||||||
|
${!S.loading&&S.posts.length===0?'<div class="empty">暂无数据</div>':''}
|
||||||
|
${!S.loading&&S.posts.length>0?S.posts.map(p=>`
|
||||||
|
<div class="post" onclick="showDetail(${p.id})">
|
||||||
|
<div class="post-title">${p.title||'无标题'}</div>
|
||||||
|
<div class="post-meta">
|
||||||
|
<span class="badge b-${p.post_type||'normal'}">${p.post_type==='deal'?'出售':p.post_type==='want'?'求购':'普通'}</span>
|
||||||
|
${p.has_lifebuoy?'<span class="badge b-lifebuoy">🔥大救生圈</span>':''}
|
||||||
|
${p.price?'<span class="price">'+p.price+'元</span>':''}
|
||||||
|
${p.category?'<span class="tm">'+p.category+'</span>':''}
|
||||||
|
${p.author_username?'<span class="author">'+p.author_username+'</span>':''}
|
||||||
|
<span class="tm">${fmtTime(p.post_time)}</span>
|
||||||
|
${p.instance_id?'<span class="tm">🖥️'+p.instance_id.replace('server_','S')+'</span>':''}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`).join(''):''}
|
||||||
|
${!S.loading&&S.posts.length>0?`
|
||||||
|
<div class="pager">
|
||||||
|
<button class="pg-btn"${S.pg<=1?' disabled':''} onclick="prevP()">上一页</button>
|
||||||
|
<span class="pg-info">${S.pg} / ${Math.max(1,Math.ceil(S.total/S.ps))} (共${S.total}条)</span>
|
||||||
|
<button class="pg-btn"${S.pg>=Math.ceil(S.total/S.ps)?' disabled':''} onclick="nextP()">下一页</button>
|
||||||
|
</div>
|
||||||
|
`:''}
|
||||||
|
</div>
|
||||||
|
<div class="tab">
|
||||||
|
<div class="tab-i" onclick="S.v='home';render()"><span>📊</span>首页</div>
|
||||||
|
<div class="tab-i on" onclick="goList('')"><span>📋</span>帖子</div>
|
||||||
|
<div class="tab-i" onclick="goCollections()"><span>💎</span>藏品</div>
|
||||||
|
<div class="tab-i" onclick="goUsers()"><span>👤</span>用户</div>
|
||||||
|
<div class="tab-i" onclick="S.v='stats';loadStats()"><span>📈</span>统计</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if (S.v === 'collection-detail' && S.curCollection) {
|
||||||
|
const c = S.curCollection
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="detail">
|
||||||
|
<div class="detail-hdr">
|
||||||
|
<button class="back" onclick="S.v='collections';render()">← 返回藏品</button>
|
||||||
|
<a class="detail-link" href="${c.post_url||'#'}" target="_blank">原文链接</a>
|
||||||
|
</div>
|
||||||
|
<div class="detail-body">
|
||||||
|
<div class="detail-title">${c.name || c.crown_code || '无名称'}</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">🏷️ 藏品信息</div>
|
||||||
|
<div class="info-g">
|
||||||
|
<div class="info-i"><span class="info-l">品类</span><span class="${c.category==='龙钞'?'green':c.category==='马钞'?'orange':c.category==='蛇钞'?'purple':'gray'}">${c.category||'其他'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">系列</span><span class="info-v">${c.series||'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">发行年份</span><span class="info-v">${c.issue_year||'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">冠号</span><span class="info-v" style="font-size:16px;font-weight:600;color:#1976d2">${c.crown_code||'-'}</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">📝 描述</div>
|
||||||
|
<div class="content">${c.description||'暂无描述'}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">🔗 关联信息</div>
|
||||||
|
<div class="info-g">
|
||||||
|
<div class="info-i"><span class="info-l">帖子ID</span><span class="info-v">${c.post_id||'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">藏品ID</span><span class="info-v">${c.id||'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">创建时间</span><span class="info-v">${c.created_at?new Date(c.created_at).toLocaleString('zh-CN'):'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">更新时间</span><span class="info-v">${c.updated_at?new Date(c.updated_at).toLocaleString('zh-CN'):'-'}</span></div>
|
||||||
|
<div class="info-i"><span class="info-l">采集服务器</span><span class="info-v">${c.instance_id||'-'}</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="margin-top:20px">
|
||||||
|
<button onclick="goCollectionPost('${c.post_id}')" style="width:100%;padding:12px;background:#1976d2;color:#fff;border:none;border-radius:8px;font-size:15px;cursor:pointer">查看关联帖子 →</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
if (S.v === 'detail' && S.cur) {
|
||||||
|
const p = S.cur
|
||||||
|
app.innerHTML = `
|
||||||
|
<div class="detail">
|
||||||
|
<div class="detail-hdr">
|
||||||
|
<button class="back" onclick="S.v='list';render()">← 返回</button>
|
||||||
|
<a class="detail-link" href="${p.url||'#'}" target="_blank">原文链接</a>
|
||||||
|
</div>
|
||||||
|
<div class="detail-body">
|
||||||
|
<div class="detail-title">${p.title||'无标题'}</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">💰 价格信息</div>
|
||||||
|
<div class="info-g">
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">价格</span>
|
||||||
|
<span class="info-v prime">${p.price?p.price+'元':'-'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">品类</span>
|
||||||
|
<span class="info-v">${p.category||'-'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">包装</span>
|
||||||
|
<span class="info-v">${p.special_types||'-'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">号码</span>
|
||||||
|
<span class="info-v">${p.number_features||'-'}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
${p.special_types?`
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">📦 包装分类</div>
|
||||||
|
<div class="tag-l">
|
||||||
|
${p.special_types.split(/[,\/|]/).map(s=>`<span class="tag">${s.trim()}</span>`).join('')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`:''}
|
||||||
|
|
||||||
|
${p.number_features?`
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">🔢 号码分类</div>
|
||||||
|
<div class="tag-l">
|
||||||
|
${p.number_features.split(/[,\/|]/).map(s=>`<span class="tag">${s.trim()}</span>`).join('')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`:''}
|
||||||
|
|
||||||
|
${p.special_tags?`
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">🏷️ 特殊标识</div>
|
||||||
|
<div class="tag-l">
|
||||||
|
${p.special_tags.split(/[,\/|]/).map(s=>`<span class="tag">${s.trim()}</span>`).join('')}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`:''}
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">👤 作者信息</div>
|
||||||
|
<div class="info-g">
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">用户名</span>
|
||||||
|
<span class="info-v" style="color:#1976d2">${p.author_username||'-'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">联系方式</span>
|
||||||
|
<span class="info-v">${p.contact||'-'}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">📊 帖子状态</div>
|
||||||
|
<div class="info-g">
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">类型</span>
|
||||||
|
<span class="badge b-${p.post_type||'normal'}">${p.post_type==='deal'?'出售':p.post_type==='want'?'求购':'普通'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">大救生圈</span>
|
||||||
|
<span class="info-v">${p.has_lifebuoy?'🔥 是':'否'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">发帖时间</span>
|
||||||
|
<span class="info-v">${p.post_time?new Date(p.post_time).toLocaleString('zh-CN'):'-'}</span>
|
||||||
|
</div>
|
||||||
|
<div class="info-i">
|
||||||
|
<span class="info-l">回复数</span>
|
||||||
|
<span class="info-v">${p.reply_count||0}</span>
|
||||||
|
</div>
|
||||||
|
${p.instance_id?'<div class="info-i"><span class="info-l">来源</span><span class="info-v">🖥️ '+p.instance_id.replace('server_','服务器')+'</span></div>':''}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="sec">
|
||||||
|
<div class="sec-title">📝 帖子内容</div>
|
||||||
|
<div class="content">${p.content||'暂无内容'}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function goCollections() {
|
||||||
|
console.log('goCollections clicked')
|
||||||
|
S.v = 'collections'
|
||||||
|
loadCollections()
|
||||||
|
}
|
||||||
|
function goUsers() {
|
||||||
|
console.log('goUsers clicked')
|
||||||
|
S.v = 'users'
|
||||||
|
loadUsers()
|
||||||
|
}
|
||||||
|
async function loadCollections() {
|
||||||
|
console.log('loadCollections called, current S.v=', S.v)
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/v1/collections?page=1&page_size=100')
|
||||||
|
const json = await res.json()
|
||||||
|
console.log('collections API response code:', json.code, 'items:', (json.data&&json.data.items||[]).length)
|
||||||
|
if (json.code === 0) {
|
||||||
|
S.collections = json.data.items || []
|
||||||
|
render()
|
||||||
|
}
|
||||||
|
} catch(e) { console.error('loadCollections error:', e.message) }
|
||||||
|
}
|
||||||
|
async function loadUsers() {
|
||||||
|
console.log('loadUsers called, current S.v=', S.v)
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/v1/users?page=1&page_size=100')
|
||||||
|
const json = await res.json()
|
||||||
|
console.log('users API response code:', json.code, 'items:', (json.data&&json.data.items||[]).length)
|
||||||
|
if (json.code === 0) {
|
||||||
|
S.users = json.data.items || []
|
||||||
|
render()
|
||||||
|
}
|
||||||
|
} catch(e) { console.error('loadUsers error:', e.message) }
|
||||||
|
}
|
||||||
|
function showCollectionDetail(c) {
|
||||||
|
alert(['名称: '+(c.name||'-'), '品类: '+(c.category||'-'), '系列: '+(c.series||'-'),
|
||||||
|
'年份: '+(c.issue_year||'-'), '冠号: '+(c.crown_code||'-'), '描述: '+(c.description||'-')].join('\n'))
|
||||||
|
}
|
||||||
|
|
||||||
|
async function goCollectionDetail(collectionId) {
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/v1/collections/'+collectionId)
|
||||||
|
const json = await res.json()
|
||||||
|
if (json.code === 0) {
|
||||||
|
S.curCollection = json.data
|
||||||
|
S.v = 'collection-detail'
|
||||||
|
render()
|
||||||
|
} else {
|
||||||
|
alert('藏品不存在')
|
||||||
|
}
|
||||||
|
} catch(e) { alert('加载失败: '+e.message) }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function goCollectionPost(postId) {
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/v1/yichens/posts/'+encodeURIComponent(postId))
|
||||||
|
const json = await res.json()
|
||||||
|
if (json.code === 0) {
|
||||||
|
S.cur = json.data
|
||||||
|
S.v = 'detail'
|
||||||
|
render()
|
||||||
|
} else {
|
||||||
|
alert('帖子不存在或已删除')
|
||||||
|
}
|
||||||
|
} catch(e) { alert('加载失败: '+e.message) }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadStats() {
|
||||||
|
try {
|
||||||
|
const r = await fetch(API+'/yichens/statistics')
|
||||||
|
const d = await r.json()
|
||||||
|
if (d.code === 0) { S.stats = d.data }
|
||||||
|
const r2 = await fetch(API+'/yichens/report/categories')
|
||||||
|
const d2 = await r2.json()
|
||||||
|
if (d2.code === 0) { S.catReport = d2.data }
|
||||||
|
const r3 = await fetch(API+'/yichens/report/stats')
|
||||||
|
const d3 = await r3.json()
|
||||||
|
if (d3.code === 0) { S.statsData = d3.data }
|
||||||
|
render()
|
||||||
|
} catch(e) { console.error(e) }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadPosts() {
|
||||||
|
S.loading = true; render()
|
||||||
|
try {
|
||||||
|
let url = API+'/yichens/posts?page='+S.pg+'&page_size='+S.ps
|
||||||
|
if (S.ft.type) url += '&post_type='+S.ft.type
|
||||||
|
if (S.ft.cat) url += '&category='+S.ft.cat
|
||||||
|
if (S.ft.lifebuoy === 'yes') url += '&has_lifebuoy=true'
|
||||||
|
if (S.ft.timeRange) url += '&time_range='+S.ft.timeRange
|
||||||
|
if (S.ft.statType) { S.loading = false; loadStatPosts(); return }
|
||||||
|
const r = await fetch(url)
|
||||||
|
const d = await r.json()
|
||||||
|
if (d.code === 0) { S.posts = d.data.items||[]; S.total = d.data.total||0 }
|
||||||
|
} catch(e) { console.error(e) }
|
||||||
|
S.loading = false; render()
|
||||||
|
}
|
||||||
|
|
||||||
|
function goList(type, timeRange, cat) {
|
||||||
|
S.ft = { type: '', cat: '', lifebuoy: '', timeRange: '' }
|
||||||
|
if (type === 'lifebuoy') S.ft.lifebuoy = 'yes'
|
||||||
|
else if (type) S.ft.type = type
|
||||||
|
if (timeRange) S.ft.timeRange = timeRange
|
||||||
|
if (cat) S.ft.cat = cat
|
||||||
|
S.pg = 1; S.v = 'list'; loadPosts()
|
||||||
|
S.collections = []; S.users = []; S.colCat = ''; S.colSearch = ''; S.userSearch = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function goStatPosts(statType) {
|
||||||
|
S.ft = { type: '', cat: '', lifebuoy: '', timeRange: '', statType: statType }
|
||||||
|
S.pg = 1; S.v = 'list'; loadStatPosts()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadStatPosts() {
|
||||||
|
if (!S.ft.statType) return
|
||||||
|
S.loading = true; render()
|
||||||
|
try {
|
||||||
|
let url = API+'/yichens/report/posts?stat_type='+S.ft.statType+'&page='+S.pg+'&page_size='+S.ps
|
||||||
|
const r = await fetch(url)
|
||||||
|
const d = await r.json()
|
||||||
|
if (d.code === 0) { S.posts = d.data.items||[]; S.total = d.data.total||0 }
|
||||||
|
} catch(e) { console.error(e) }
|
||||||
|
S.loading = false; render()
|
||||||
|
}
|
||||||
|
|
||||||
|
function setF(key, val) {
|
||||||
|
if (key === 'type') { S.ft.type = val; S.ft.timeRange = '' }
|
||||||
|
if (key === 'cat') { S.ft.cat = val; S.ft.timeRange = '' }
|
||||||
|
S.pg = 1; loadPosts()
|
||||||
|
}
|
||||||
|
|
||||||
|
function showDetail(id) {
|
||||||
|
S.cur = S.posts.find(p => p.id === id)
|
||||||
|
S.v = 'detail'; render()
|
||||||
|
}
|
||||||
|
|
||||||
|
function prevP() { if (S.pg > 1) { S.pg--; loadPosts(); window.scrollTo(0,0) } }
|
||||||
|
function nextP() { S.pg++; loadPosts(); window.scrollTo(0,0) }
|
||||||
|
|
||||||
|
function fmtTime(ts) {
|
||||||
|
if (!ts) return '-'
|
||||||
|
const d = new Date(ts), n = new Date()
|
||||||
|
const diff = n - d
|
||||||
|
if (diff < 60000) return '刚刚'
|
||||||
|
if (diff < 3600000) return Math.floor(diff/60000)+'分钟前'
|
||||||
|
if (diff < 86400000) return Math.floor(diff/3600000)+'小时前'
|
||||||
|
if (diff < 604800000) return Math.floor(diff/86400000)+'天前'
|
||||||
|
return d.toLocaleDateString('zh-CN')
|
||||||
|
}
|
||||||
|
|
||||||
|
// init
|
||||||
|
loadStats()
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,147 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
冠字号提取脚本 - 最终版
|
||||||
|
规则：只提取J0开头的冠字号（J0+8位或J0+9位）
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
sys.path.insert(0, '/root/coolbot-data')
|
||||||
|
|
||||||
|
import re
|
||||||
|
import psycopg2
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
DB_CONFIG = {
|
||||||
|
'host': 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com',
|
||||||
|
'port': 5432,
|
||||||
|
'database': 'coolbot_data',
|
||||||
|
'user': 'coolbot',
|
||||||
|
'password': 'Coolbot123'
|
||||||
|
}
|
||||||
|
|
||||||
|
FEATURE_KEYWORDS = [
|
||||||
|
'无47', '无34', '无347', '无4', '无3', '无2',
|
||||||
|
'标十', '标百', '标九', '标准十', '标准百',
|
||||||
|
'首日', '生日', '金马', '银马',
|
||||||
|
'金钩', '倒置', '满号', '圆圆',
|
||||||
|
'豹子', '顺子', '恐龙', '天龙',
|
||||||
|
'PMG', '爱藏', '尾号',
|
||||||
|
'大象', '麒麟', '老虎', '狮子',
|
||||||
|
'金马王', '天马', '龙马精神',
|
||||||
|
'连号', '散号', '一刀', '龙凤', '熊猫',
|
||||||
|
]
|
||||||
|
|
||||||
|
def extract_codes(text):
|
||||||
|
"""只提取J0开头的冠字号：J0+8位或J0+9位"""
|
||||||
|
if not text:
|
||||||
|
return set()
|
||||||
|
codes = set()
|
||||||
|
for p in [r'J0\d{8}', r'J0\d{9}']:
|
||||||
|
for c in re.findall(p, text):
|
||||||
|
# J0+8位: 总长10, J0+9位: 总长11
|
||||||
|
if len(c) in (10, 11):
|
||||||
|
codes.add(c)
|
||||||
|
return codes
|
||||||
|
|
||||||
|
def extract_features(text):
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
feat = [kw for kw in FEATURE_KEYWORDS if kw in text]
|
||||||
|
return '|'.join(feat) if feat else None
|
||||||
|
|
||||||
|
def extract_price(text):
|
||||||
|
if not text:
|
||||||
|
return None, '元'
|
||||||
|
prices = []
|
||||||
|
for p in [r'(\d+(?:\.\d+)?)\s*元', r'(\d+(?:\.\d+)?)\s*/\s*[张件组百]']:
|
||||||
|
for x in re.findall(p, text):
|
||||||
|
try:
|
||||||
|
v = float(x)
|
||||||
|
if 1 <= v < 100000:
|
||||||
|
prices.append(v)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return (min(prices), '元') if prices else (None, '元')
|
||||||
|
|
||||||
|
def get_category(title):
|
||||||
|
if not title:
|
||||||
|
return '其他'
|
||||||
|
for kw in ['龙', '龙钞', '小龙钞', '钞王']:
|
||||||
|
if kw in title:
|
||||||
|
return '龙钞'
|
||||||
|
for kw in ['马', '马钞']:
|
||||||
|
if kw in title:
|
||||||
|
return '马钞'
|
||||||
|
for kw in ['蛇', '蛇钞']:
|
||||||
|
if kw in title:
|
||||||
|
return '蛇钞'
|
||||||
|
return '其他'
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print(f'[{datetime.now()}] 冠字号提取开始（仅J0冠字号）...')
|
||||||
|
|
||||||
|
conn = psycopg2.connect(**DB_CONFIG)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
# 清空旧数据
|
||||||
|
cur.execute('DELETE FROM collections WHERE crown_code IS NOT NULL')
|
||||||
|
conn.commit()
|
||||||
|
print(f'清空旧数据完成')
|
||||||
|
|
||||||
|
BATCH = 100
|
||||||
|
offset = 0
|
||||||
|
total_new = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
cur.execute(f'''
|
||||||
|
SELECT id, post_id, title, content, post_type,
|
||||||
|
author_username, price, price_unit, url, crawled_at
|
||||||
|
FROM yichens_posts
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT {BATCH} OFFSET {offset}
|
||||||
|
''')
|
||||||
|
posts = cur.fetchall()
|
||||||
|
if not posts:
|
||||||
|
break
|
||||||
|
|
||||||
|
for (pid, post_id, title, content, post_type,
|
||||||
|
author, price, price_unit, url, crawled_at) in posts:
|
||||||
|
|
||||||
|
text = f'{title or ""} {content or ""}'
|
||||||
|
codes = extract_codes(text)
|
||||||
|
if not codes:
|
||||||
|
offset += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
features = extract_features(text)
|
||||||
|
price_val, _ = extract_price(text)
|
||||||
|
price_val = price_val if price_val else price
|
||||||
|
category = get_category(title)
|
||||||
|
|
||||||
|
for code in codes:
|
||||||
|
try:
|
||||||
|
cur.execute('''
|
||||||
|
INSERT INTO collections (
|
||||||
|
name, crown_code, category, post_id, post_url,
|
||||||
|
author, price, price_unit,
|
||||||
|
number_feature, post_title, post_type,
|
||||||
|
post_crawled_at, created_at, updated_at
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,NOW(),NOW())
|
||||||
|
''', (
|
||||||
|
code, code, category, str(post_id), url,
|
||||||
|
author, price_val, price_unit or '元',
|
||||||
|
features, title, post_type,
|
||||||
|
crawled_at
|
||||||
|
))
|
||||||
|
total_new += 1
|
||||||
|
except Exception as e:
|
||||||
|
pass
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
offset += len(posts)
|
||||||
|
print(f' 已处理 {offset} 条，新增 {total_new} 条')
|
||||||
|
|
||||||
|
print(f'\n完成！总计新增: {total_new} 条')
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,187 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
冠字号全量提取脚本 - 跑完所有待处理帖子
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
sys.path.insert(0, '/root/coolbot-data')
|
||||||
|
|
||||||
|
import re, psycopg2
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
DB_CONFIG = {
|
||||||
|
'host': 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com',
|
||||||
|
'port': 5432,
|
||||||
|
'database': 'coolbot_data',
|
||||||
|
'user': 'coolbot',
|
||||||
|
'password': 'Coolbot123'
|
||||||
|
}
|
||||||
|
|
||||||
|
FEATURE_KEYWORDS = [
|
||||||
|
'无47', '无34', '无347', '无4', '无3', '无2',
|
||||||
|
'标十', '标百', '标九', '标准十', '标准百',
|
||||||
|
'首日', '生日', '金马', '银马',
|
||||||
|
'金钩', '倒置', '满号', '圆圆',
|
||||||
|
'豹子', '顺子', '恐龙', '天龙',
|
||||||
|
'PMG', '爱藏', '尾号',
|
||||||
|
'大象', '麒麟', '老虎', '狮子',
|
||||||
|
'金马王', '天马', '龙马精神',
|
||||||
|
'连号', '散号', '一刀', '龙凤', '熊猫',
|
||||||
|
]
|
||||||
|
|
||||||
|
def is_date_code(code):
|
||||||
|
if len(code) != 8:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
year = int(code[:4])
|
||||||
|
month = int(code[4:6])
|
||||||
|
day = int(code[6:8])
|
||||||
|
if 2000 <= year <= 2030 and 1 <= month <= 12 and 1 <= day <= 31:
|
||||||
|
return True
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_phone(code):
|
||||||
|
return len(code) == 11 and code.startswith('1')
|
||||||
|
|
||||||
|
def is_valid_crown(code):
|
||||||
|
if not code:
|
||||||
|
return False
|
||||||
|
if code.startswith('J0'):
|
||||||
|
return len(code) in (10, 11)
|
||||||
|
if len(code) not in (8, 9):
|
||||||
|
return False
|
||||||
|
if is_phone(code):
|
||||||
|
return False
|
||||||
|
if len(code) == 8 and is_date_code(code):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def extract_codes(text):
|
||||||
|
if not text:
|
||||||
|
return set()
|
||||||
|
codes = set()
|
||||||
|
for p in [r'J0\d{8}', r'J0\d{9}']:
|
||||||
|
codes.update(re.findall(p, text))
|
||||||
|
for c in re.findall(r'\b\d{8}\b', text):
|
||||||
|
if is_valid_crown(c):
|
||||||
|
codes.add(c)
|
||||||
|
for c in re.findall(r'\b\d{9}\b', text):
|
||||||
|
if is_valid_crown(c):
|
||||||
|
codes.add(c)
|
||||||
|
return codes
|
||||||
|
|
||||||
|
def extract_features(text):
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
feat = [kw for kw in FEATURE_KEYWORDS if kw in text]
|
||||||
|
return '|'.join(feat) if feat else None
|
||||||
|
|
||||||
|
def extract_price(text):
|
||||||
|
if not text:
|
||||||
|
return None, '元'
|
||||||
|
prices = []
|
||||||
|
for p in [r'(\d+(?:\.\d+)?)\s*元', r'(\d+(?:\.\d+)?)\s*/\s*[张件组百]']:
|
||||||
|
for x in re.findall(p, text):
|
||||||
|
try:
|
||||||
|
v = float(x)
|
||||||
|
if 1 <= v < 100000:
|
||||||
|
prices.append(v)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
return (min(prices), '元') if prices else (None, '元')
|
||||||
|
|
||||||
|
def get_category(title):
|
||||||
|
if not title:
|
||||||
|
return '其他'
|
||||||
|
for kw in ['龙', '龙钞', '小龙钞', '钞王']:
|
||||||
|
if kw in title:
|
||||||
|
return '龙钞'
|
||||||
|
for kw in ['马', '马钞']:
|
||||||
|
if kw in title:
|
||||||
|
return '马钞'
|
||||||
|
for kw in ['蛇', '蛇钞']:
|
||||||
|
if kw in title:
|
||||||
|
return '蛇钞'
|
||||||
|
return '其他'
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print(f'[{datetime.now()}] 冠字号全量提取开始...')
|
||||||
|
|
||||||
|
conn = psycopg2.connect(**DB_CONFIG)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
# 无 LIMIT，全量跑完所有待处理帖子
|
||||||
|
cur.execute("""
|
||||||
|
SELECT p.id, p.post_id, p.title, p.content, p.post_type,
|
||||||
|
p.author_username, p.price, p.price_unit, p.url, p.crawled_at
|
||||||
|
FROM yichens_posts p
|
||||||
|
WHERE p.category IN ('龙钞', '马钞', '蛇钞', '其他')
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM collections c
|
||||||
|
WHERE c.post_id = CAST(p.id AS TEXT)
|
||||||
|
)
|
||||||
|
ORDER BY p.id
|
||||||
|
""")
|
||||||
|
posts = cur.fetchall()
|
||||||
|
total = len(posts)
|
||||||
|
print(f'待处理新帖子: {total} 条')
|
||||||
|
|
||||||
|
new_count = 0
|
||||||
|
skip_count = 0
|
||||||
|
total_codes = 0
|
||||||
|
processed = 0
|
||||||
|
|
||||||
|
for (pid, post_id, title, content, post_type,
|
||||||
|
author, price, price_unit, url, crawled_at) in posts:
|
||||||
|
|
||||||
|
text = f'{title or ""} {content or ""}'
|
||||||
|
codes = extract_codes(text)
|
||||||
|
total_codes += len(codes)
|
||||||
|
processed += 1
|
||||||
|
|
||||||
|
if not codes:
|
||||||
|
skip_count += 1
|
||||||
|
if processed % 500 == 0:
|
||||||
|
print(f' 已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
|
||||||
|
continue
|
||||||
|
|
||||||
|
features = extract_features(text)
|
||||||
|
price_val, _ = extract_price(text)
|
||||||
|
price_val = price_val if price_val else price
|
||||||
|
category = get_category(title)
|
||||||
|
|
||||||
|
for code in codes:
|
||||||
|
try:
|
||||||
|
cur.execute("""
|
||||||
|
INSERT INTO collections (
|
||||||
|
name, crown_code, category, post_id, post_url,
|
||||||
|
author, price, price_unit,
|
||||||
|
number_feature, post_title, post_type,
|
||||||
|
post_crawled_at, created_at, updated_at
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,NOW(),NOW())
|
||||||
|
ON CONFLICT (post_id, crown_code) DO NOTHING
|
||||||
|
""", (
|
||||||
|
code, code, category, str(post_id), url,
|
||||||
|
author, price_val, price_unit or '元',
|
||||||
|
features, title, post_type,
|
||||||
|
crawled_at
|
||||||
|
))
|
||||||
|
if cur.rowcount > 0:
|
||||||
|
new_count += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f' 插入失败 post_id={post_id}, code={code}: {e}')
|
||||||
|
|
||||||
|
if processed % 500 == 0:
|
||||||
|
print(f' 已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
|
||||||
|
|
||||||
|
cur.execute('SELECT COUNT(*) FROM collections')
|
||||||
|
print(f'collections表当前总量: {cur.fetchone()[0]} 条')
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Loading…
Reference in New Issue