feat: v0.0.2 - 帖子正文爬取优化
- extract_post_content 新增免责声明截断逻辑：只保留到"免责声明及风险提示： 所有交易人员，凡未采用本站中介交易的，被骗后果自负。"为止 - 爬虫脚本 crawl_cron_s1/s2/s3 支持 instance_id 日志记录 - crawler_monitor.py 监控脚本：检查三台服务器爬虫运行状态 - 修复 crawl_today 日志写入 crawl_logs 表的问题
This commit is contained in:
caibotmini
parent
07b2e5c7b5
commit
1c0f74ccb6
|
|
@ -47,6 +47,13 @@ def extract_post_content(html_bytes):
|
|||
lines = [l.strip() for l in text.split('\n') if l.strip()]
|
||||
text = '\n'.join(lines)
|
||||
text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text)
|
||||
|
||||
# 截取免责声明及之前的内容
|
||||
disclaimer = '免责声明及风险提示： 所有交易人员，凡未采用本站中介交易的，被骗后果自负。'
|
||||
discl_pos = text.find(disclaimer)
|
||||
if discl_pos != -1:
|
||||
text = text[:discl_pos + len(disclaimer)]
|
||||
|
||||
return text if text else None
|
||||
|
||||
|
||||
|
|
@ -56,8 +63,9 @@ import logging
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
class YichensTodaySpider(PaginationSpider):
|
||||
def __init__(self):
|
||||
def __init__(self, instance_id=None):
|
||||
super().__init__("一尘网今日采集", "pm001")
|
||||
self.instance_id = instance_id or os.environ.get('CRAWLER_INSTANCE_ID', 'unknown')
|
||||
self.base_url = "http://www.pm001.net"
|
||||
self.board_id = "151"
|
||||
self.forum_url = f"{self.base_url}/index.asp?boardid={self.board_id}"
|
||||
|
|
@ -370,8 +378,8 @@ class YichensTodaySpider(PaginationSpider):
|
|||
with get_db() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"INSERT INTO crawl_logs (spider_name, status, started_at) VALUES (%s, %s, NOW()) RETURNING id",
|
||||
(self.source, "running")
|
||||
"INSERT INTO crawl_logs (spider_name, status, started_at, instance_id) VALUES (%s, %s, NOW(), %s) RETURNING id",
|
||||
(self.source, "running", self.instance_id)
|
||||
)
|
||||
log_id = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
cd /root/coolbot-data
|
||||
source venv/bin/activate
|
||||
export CRAWLER_INSTANCE_ID=server_1
|
||||
python3 -u -c "
|
||||
import sys, os
|
||||
sys.path.insert(0, '/root/coolbot-data')
|
||||
os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'
|
||||
os.environ['DB_PORT'] = '5432'
|
||||
os.environ['DB_USER'] = 'coolbot'
|
||||
os.environ['DB_PASSWORD'] = 'Coolbot123'
|
||||
os.environ['DB_NAME'] = 'coolbot_data'
|
||||
os.environ['CRAWLER_INSTANCE_ID'] = 'server_1'
|
||||
from crawlers.crawl_today import YichensTodaySpider
|
||||
spider = YichensTodaySpider()
|
||||
spider.max_pages = 2
|
||||
spider.min_delay = 0.2
|
||||
spider.max_delay = 0.5
|
||||
log_id = spider._log_start()
|
||||
try:
|
||||
result = spider.crawl_today()
|
||||
spider._log_finish(log_id, 'success', result)
|
||||
print('[server_1] 采集完成: ' + str(result) + ' 条', flush=True)
|
||||
except Exception as e:
|
||||
spider._log_finish(log_id, 'failed', 0, str(e))
|
||||
raise
|
||||
"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/bash
|
||||
cd /root/coolbot-data
|
||||
source venv/bin/activate
|
||||
export CRAWLER_INSTANCE_ID=server_2
|
||||
python3 -u -c "
|
||||
import sys, os
|
||||
sys.path.insert(0, '/root/coolbot-data')
|
||||
os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'
|
||||
os.environ['DB_PORT'] = '5432'
|
||||
os.environ['DB_USER'] = 'coolbot'
|
||||
os.environ['DB_PASSWORD'] = 'Coolbot123'
|
||||
os.environ['DB_NAME'] = 'coolbot_data'
|
||||
os.environ['CRAWLER_INSTANCE_ID'] = 'server_2'
|
||||
from crawlers.crawl_today import YichensTodaySpider
|
||||
spider = YichensTodaySpider()
|
||||
spider.max_pages = 2
|
||||
spider.min_delay = 0.2
|
||||
spider.max_delay = 0.5
|
||||
result = spider.crawl_today()
|
||||
print('[server_2] 采集完成: ' + str(result) + ' 条', flush=True)
|
||||
"
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
#!/bin/bash
|
||||
cd /root/coolbot-data
|
||||
export PYTHONPATH=/usr/lib64/python3.6/site-packages:$PYTHONPATH
|
||||
export CRAWLER_INSTANCE_ID=server_3
|
||||
python3 -u -c "
|
||||
import sys
|
||||
sys.path.insert(0, '/usr/lib64/python3.6/site-packages')
|
||||
sys.path.insert(0, '/root/coolbot-data')
|
||||
import os
|
||||
os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'
|
||||
os.environ['DB_PORT'] = '5432'
|
||||
os.environ['DB_USER'] = 'coolbot'
|
||||
os.environ['DB_PASSWORD'] = 'Coolbot123'
|
||||
os.environ['DB_NAME'] = 'coolbot_data'
|
||||
os.environ['CRAWLER_INSTANCE_ID'] = 'server_3'
|
||||
from crawlers.crawl_today import YichensTodaySpider
|
||||
spider = YichensTodaySpider()
|
||||
spider.max_pages = 2
|
||||
spider.min_delay = 0.2
|
||||
spider.max_delay = 0.5
|
||||
log_id = spider._log_start()
|
||||
try:
|
||||
result = spider.crawl_today()
|
||||
spider._log_finish(log_id, 'success', result)
|
||||
print('[server_3] 采集完成: ' + str(result) + '条', flush=True)
|
||||
except Exception as e:
|
||||
spider._log_finish(log_id, 'failed', 0, str(e))
|
||||
raise
|
||||
"
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawler 健康监控 - 检查所有服务器爬虫是否正常运行
|
||||
每10分钟运行一次，检查最近30分钟内各实例是否有记录
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, '/root/coolbot-data')
|
||||
|
||||
import psycopg2
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# ========== 配置 ==========
|
||||
# 各服务器实例ID（对应crawl_logs.instance_id）
|
||||
INSTANCES = {
|
||||
'server_1': '1号服务器-每00/30分',
|
||||
'server_2': '2号服务器-每10/40分',
|
||||
'server_3': '3号服务器-每20/50分',
|
||||
}
|
||||
|
||||
DB_CONFIG = {
|
||||
'host': os.environ.get('DB_HOST', 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'),
|
||||
'port': int(os.environ.get('DB_PORT', 5432)),
|
||||
'database': os.environ.get('DB_NAME', 'coolbot_data'),
|
||||
'user': os.environ.get('DB_USER', 'coolbot'),
|
||||
'password': os.environ.get('DB_PASSWORD', 'Coolbot123'),
|
||||
}
|
||||
|
||||
# 飞书 Webhook
|
||||
FEISHU_WEBHOOK = os.environ.get('FEISHU_WEBHOOK_URL', '')
|
||||
FEISHU_SECRET = os.environ.get('FEISHU_APP_SECRET', '')
|
||||
|
||||
# 检查时间窗口（分钟）
|
||||
WINDOW_MINUTES = 35 # 宽松5分钟容错
|
||||
|
||||
# ==========================
|
||||
|
||||
def get_db():
|
||||
return psycopg2.connect(**DB_CONFIG)
|
||||
|
||||
def check_instance_status(conn, instance_id):
|
||||
"""检查某实例最近是否在窗口内运行过"""
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT COUNT(*), MAX(finished_at)
|
||||
FROM crawl_logs
|
||||
WHERE instance_id = %s
|
||||
AND finished_at >= NOW() - INTERVAL '%s minutes'
|
||||
AND status = 'success'
|
||||
""", (instance_id, WINDOW_MINUTES))
|
||||
r = cur.fetchone()
|
||||
cur.close()
|
||||
return {'count': r[0], 'last_run': r[1]}
|
||||
|
||||
def send_feishu_alert(message):
|
||||
"""发送飞书告警"""
|
||||
if not FEISHU_WEBHOOK:
|
||||
print("⚠️ 未配置飞书Webhook，跳过告警")
|
||||
return
|
||||
|
||||
now = datetime.now().strftime('%H:%M')
|
||||
card = {
|
||||
"msg_type": "interactive",
|
||||
"card": {
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": f"⚠️ 爬虫监控告警 {now}"},
|
||||
"template": "red"
|
||||
},
|
||||
"elements": [
|
||||
{"tag": "div", "text": {"tag": "lark_md", "content": message}}
|
||||
]
|
||||
}
|
||||
}
|
||||
try:
|
||||
r = requests.post(FEISHU_WEBHOOK, json=card, timeout=10)
|
||||
print(f"飞书告警响应: {r.status_code}")
|
||||
except Exception as e:
|
||||
print(f"飞书发送失败: {e}")
|
||||
|
||||
def send_feishu_healthy():
|
||||
"""发送健康状态（仅关键时段发送，避免骚扰）"""
|
||||
hour = datetime.now().hour
|
||||
if hour not in [8, 9, 17, 18, 19]: # 只在工作时段发送
|
||||
return
|
||||
|
||||
if not FEISHU_WEBHOOK:
|
||||
return
|
||||
|
||||
card = {
|
||||
"msg_type": "interactive",
|
||||
"card": {
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": f"✅ 爬虫监控正常 {datetime.now().strftime('%H:%M')}"},
|
||||
"template": "green"
|
||||
},
|
||||
"elements": [
|
||||
{"tag": "div", "text": {"tag": "lark_md", "content": "所有服务器爬虫运行正常"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
try:
|
||||
requests.post(FEISHU_WEBHOOK, json=card, timeout=10)
|
||||
except:
|
||||
pass
|
||||
|
||||
def main():
|
||||
print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 爬虫健康检查...")
|
||||
|
||||
conn = get_db()
|
||||
|
||||
all_ok = True
|
||||
details = []
|
||||
|
||||
for instance_id, name in INSTANCES.items():
|
||||
status = check_instance_status(conn, instance_id)
|
||||
last_run_str = status['last_run'].strftime('%H:%M') if status['last_run'] else '从未运行'
|
||||
|
||||
if status['count'] > 0:
|
||||
details.append(f"✅ {name}: 最近 {last_run_str} (共{status['count']}次)")
|
||||
else:
|
||||
details.append(f"❌ {name}: ⚠️ 超过{WINDOW_MINUTES}分钟无记录！")
|
||||
all_ok = False
|
||||
|
||||
conn.close()
|
||||
|
||||
print('\n'.join(details))
|
||||
|
||||
if all_ok:
|
||||
print("✅ 所有服务器正常")
|
||||
send_feishu_healthy()
|
||||
else:
|
||||
msg = "**爬虫告警：部分服务器超过30分钟无爬取记录**\n\n" + '\n'.join(details)
|
||||
print(f"\n❌ 告警: {msg}")
|
||||
send_feishu_alert(msg)
|
||||
|
||||
return 0 if all_ok else 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue