""" 定时任务调度器 - 修复版 集成 APScheduler 实现爬虫自动化 """ import logging from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.interval import IntervalTrigger logger = logging.getLogger(__name__) # 全局调度器实例 scheduler = BlockingScheduler(timezone="Asia/Shanghai") def crawl_today_job(): """每日采集任务""" from crawlers.crawl_today import YichensTodaySpider logger.info("[定时任务] 开始执行一尘网今日采集") try: spider = YichensTodaySpider() result = spider.run() logger.info(f"[定时任务] 采集完成,新增 {result} 条") except Exception as e: logger.error(f"[定时任务] 采集失败: {e}") def crawl_incremental_job(): """增量采集任务(每小时)""" from crawlers.crawl_today import YichensTodaySpider logger.info("[定时任务] 开始执行增量采集") try: spider = YichensTodaySpider() spider.max_pages = 2 # 增量只扫2页 result = spider.crawl_today() logger.info(f"[定时任务] 增量采集完成,新增 {result} 条") except Exception as e: logger.error(f"[定时任务] 增量采集失败: {e}") def init_scheduler(): """初始化定时任务""" # 每天早上 8 点采集 scheduler.add_job( crawl_today_job, CronTrigger(hour=8, minute=0, timezone="Asia/Shanghai"), id='yichens_daily_crawl', name='一尘网每日采集', replace_existing=True ) # 每小时增量采集(如果需要) scheduler.add_job( crawl_incremental_job, IntervalTrigger(hours=1, timezone="Asia/Shanghai"), id='yichens_incremental', name='一尘网增量采集', replace_existing=True ) logger.info("定时任务已注册: 每日8点采集 + 每小时增量") return scheduler def start_scheduler(): """启动调度器""" init_scheduler() logger.info("调度器启动") try: scheduler.start() except (KeyboardInterrupt, SystemExit): logger.info("调度器停止") scheduler.shutdown() if __name__ == "__main__": logging.basicConfig(level=logging.INFO) start_scheduler()