diff --git a/crawlers/crawl_today.py b/crawlers/crawl_today.py index 92d0de3..90bc784 100644 --- a/crawlers/crawl_today.py +++ b/crawlers/crawl_today.py @@ -47,6 +47,13 @@ def extract_post_content(html_bytes): lines = [l.strip() for l in text.split('\n') if l.strip()] text = '\n'.join(lines) text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text) + + # 截取免责声明及之前的内容 + disclaimer = '免责声明及风险提示： 所有交易人员，凡未采用本站中介交易的，被骗后果自负。' + discl_pos = text.find(disclaimer) + if discl_pos != -1: + text = text[:discl_pos + len(disclaimer)] + return text if text else None @@ -56,8 +63,9 @@ import logging logger = logging.getLogger(__name__) class YichensTodaySpider(PaginationSpider): - def __init__(self): + def __init__(self, instance_id=None): super().__init__("一尘网今日采集", "pm001") + self.instance_id = instance_id or os.environ.get('CRAWLER_INSTANCE_ID', 'unknown') self.base_url = "http://www.pm001.net" self.board_id = "151" self.forum_url = f"{self.base_url}/index.asp?boardid={self.board_id}" @@ -370,8 +378,8 @@ class YichensTodaySpider(PaginationSpider): with get_db() as conn: cur = conn.cursor() cur.execute( - "INSERT INTO crawl_logs (spider_name, status, started_at) VALUES (%s, %s, NOW()) RETURNING id", - (self.source, "running") + "INSERT INTO crawl_logs (spider_name, status, started_at, instance_id) VALUES (%s, %s, NOW(), %s) RETURNING id", + (self.source, "running", self.instance_id) ) log_id = cur.fetchone()[0] conn.commit() diff --git a/scripts/crawl_cron_s1.sh b/scripts/crawl_cron_s1.sh new file mode 100755 index 0000000..7edc1d7 --- /dev/null +++ b/scripts/crawl_cron_s1.sh @@ -0,0 +1,27 @@ +#!/bin/bash +cd /root/coolbot-data +source venv/bin/activate +export CRAWLER_INSTANCE_ID=server_1 +python3 -u -c " +import sys, os +sys.path.insert(0, '/root/coolbot-data') +os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com' +os.environ['DB_PORT'] = '5432' +os.environ['DB_USER'] = 'coolbot' +os.environ['DB_PASSWORD'] = 'Coolbot123' +os.environ['DB_NAME'] = 'coolbot_data' +os.environ['CRAWLER_INSTANCE_ID'] = 'server_1' +from crawlers.crawl_today import YichensTodaySpider +spider = YichensTodaySpider() +spider.max_pages = 2 +spider.min_delay = 0.2 +spider.max_delay = 0.5 +log_id = spider._log_start() +try: + result = spider.crawl_today() + spider._log_finish(log_id, 'success', result) + print('[server_1] 采集完成: ' + str(result) + ' 条', flush=True) +except Exception as e: + spider._log_finish(log_id, 'failed', 0, str(e)) + raise +" diff --git a/scripts/crawl_cron_s2.sh b/scripts/crawl_cron_s2.sh new file mode 100755 index 0000000..c9729f5 --- /dev/null +++ b/scripts/crawl_cron_s2.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cd /root/coolbot-data +source venv/bin/activate +export CRAWLER_INSTANCE_ID=server_2 +python3 -u -c " +import sys, os +sys.path.insert(0, '/root/coolbot-data') +os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com' +os.environ['DB_PORT'] = '5432' +os.environ['DB_USER'] = 'coolbot' +os.environ['DB_PASSWORD'] = 'Coolbot123' +os.environ['DB_NAME'] = 'coolbot_data' +os.environ['CRAWLER_INSTANCE_ID'] = 'server_2' +from crawlers.crawl_today import YichensTodaySpider +spider = YichensTodaySpider() +spider.max_pages = 2 +spider.min_delay = 0.2 +spider.max_delay = 0.5 +result = spider.crawl_today() +print('[server_2] 采集完成: ' + str(result) + ' 条', flush=True) +" diff --git a/scripts/crawl_cron_s3.sh b/scripts/crawl_cron_s3.sh new file mode 100644 index 0000000..ad3bf68 --- /dev/null +++ b/scripts/crawl_cron_s3.sh @@ -0,0 +1,29 @@ +#!/bin/bash +cd /root/coolbot-data +export PYTHONPATH=/usr/lib64/python3.6/site-packages:$PYTHONPATH +export CRAWLER_INSTANCE_ID=server_3 +python3 -u -c " +import sys +sys.path.insert(0, '/usr/lib64/python3.6/site-packages') +sys.path.insert(0, '/root/coolbot-data') +import os +os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com' +os.environ['DB_PORT'] = '5432' +os.environ['DB_USER'] = 'coolbot' +os.environ['DB_PASSWORD'] = 'Coolbot123' +os.environ['DB_NAME'] = 'coolbot_data' +os.environ['CRAWLER_INSTANCE_ID'] = 'server_3' +from crawlers.crawl_today import YichensTodaySpider +spider = YichensTodaySpider() +spider.max_pages = 2 +spider.min_delay = 0.2 +spider.max_delay = 0.5 +log_id = spider._log_start() +try: + result = spider.crawl_today() + spider._log_finish(log_id, 'success', result) + print('[server_3] 采集完成: ' + str(result) + '条', flush=True) +except Exception as e: + spider._log_finish(log_id, 'failed', 0, str(e)) + raise +" diff --git a/scripts/crawler_monitor.py b/scripts/crawler_monitor.py new file mode 100755 index 0000000..d2cb754 --- /dev/null +++ b/scripts/crawler_monitor.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +Crawler 健康监控 - 检查所有服务器爬虫是否正常运行 +每10分钟运行一次，检查最近30分钟内各实例是否有记录 +""" +import sys +import os +sys.path.insert(0, '/root/coolbot-data') + +import psycopg2 +import requests +from datetime import datetime, timedelta + +# ========== 配置 ========== +# 各服务器实例ID（对应crawl_logs.instance_id） +INSTANCES = { + 'server_1': '1号服务器-每00/30分', + 'server_2': '2号服务器-每10/40分', + 'server_3': '3号服务器-每20/50分', +} + +DB_CONFIG = { + 'host': os.environ.get('DB_HOST', 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'), + 'port': int(os.environ.get('DB_PORT', 5432)), + 'database': os.environ.get('DB_NAME', 'coolbot_data'), + 'user': os.environ.get('DB_USER', 'coolbot'), + 'password': os.environ.get('DB_PASSWORD', 'Coolbot123'), +} + +# 飞书 Webhook +FEISHU_WEBHOOK = os.environ.get('FEISHU_WEBHOOK_URL', '') +FEISHU_SECRET = os.environ.get('FEISHU_APP_SECRET', '') + +# 检查时间窗口（分钟） +WINDOW_MINUTES = 35 # 宽松5分钟容错 + +# ========================== + +def get_db(): + return psycopg2.connect(**DB_CONFIG) + +def check_instance_status(conn, instance_id): + """检查某实例最近是否在窗口内运行过""" + cur = conn.cursor() + cur.execute(""" + SELECT COUNT(*), MAX(finished_at) + FROM crawl_logs + WHERE instance_id = %s + AND finished_at >= NOW() - INTERVAL '%s minutes' + AND status = 'success' + """, (instance_id, WINDOW_MINUTES)) + r = cur.fetchone() + cur.close() + return {'count': r[0], 'last_run': r[1]} + +def send_feishu_alert(message): + """发送飞书告警""" + if not FEISHU_WEBHOOK: + print("⚠️ 未配置飞书Webhook，跳过告警") + return + + now = datetime.now().strftime('%H:%M') + card = { + "msg_type": "interactive", + "card": { + "header": { + "title": {"tag": "plain_text", "content": f"⚠️ 爬虫监控告警 {now}"}, + "template": "red" + }, + "elements": [ + {"tag": "div", "text": {"tag": "lark_md", "content": message}} + ] + } + } + try: + r = requests.post(FEISHU_WEBHOOK, json=card, timeout=10) + print(f"飞书告警响应: {r.status_code}") + except Exception as e: + print(f"飞书发送失败: {e}") + +def send_feishu_healthy(): + """发送健康状态（仅关键时段发送，避免骚扰）""" + hour = datetime.now().hour + if hour not in [8, 9, 17, 18, 19]: # 只在工作时段发送 + return + + if not FEISHU_WEBHOOK: + return + + card = { + "msg_type": "interactive", + "card": { + "header": { + "title": {"tag": "plain_text", "content": f"✅ 爬虫监控正常 {datetime.now().strftime('%H:%M')}"}, + "template": "green" + }, + "elements": [ + {"tag": "div", "text": {"tag": "lark_md", "content": "所有服务器爬虫运行正常"}} + ] + } + } + try: + requests.post(FEISHU_WEBHOOK, json=card, timeout=10) + except: + pass + +def main(): + print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 爬虫健康检查...") + + conn = get_db() + + all_ok = True + details = [] + + for instance_id, name in INSTANCES.items(): + status = check_instance_status(conn, instance_id) + last_run_str = status['last_run'].strftime('%H:%M') if status['last_run'] else '从未运行' + + if status['count'] > 0: + details.append(f"✅ {name}: 最近 {last_run_str} (共{status['count']}次)") + else: + details.append(f"❌ {name}: ⚠️ 超过{WINDOW_MINUTES}分钟无记录！") + all_ok = False + + conn.close() + + print('\n'.join(details)) + + if all_ok: + print("✅ 所有服务器正常") + send_feishu_healthy() + else: + msg = "**爬虫告警：部分服务器超过30分钟无爬取记录**\n\n" + '\n'.join(details) + print(f"\n❌ 告警: {msg}") + send_feishu_alert(msg) + + return 0 if all_ok else 1 + +if __name__ == '__main__': + sys.exit(main())