20 lines
505 B
Bash
20 lines
505 B
Bash
|
#!/bin/bash
|
||
|
|
# 每日一尘网爬虫任务
|
||
|
|
DATE=$(date +%Y-%m-%d)
|
||
|
|
LOG_FILE="/root/coolbot-data/logs/crawl_${DATE}.log"
|
||
|
|
|
||
|
|
echo "[$(date)] 开始每日爬虫任务..." >> $LOG_FILE
|
||
|
|
|
||
|
|
cd /root/coolbot-data
|
||
|
|
source venv/bin/activate
|
||
|
|
export PYTHONPATH=/root/coolbot-data:$PYTHONPATH
|
||
|
|
|
||
|
|
python3 -c "
|
||
|
|
from crawlers.yichens_spider import YichensSpider
|
||
|
|
spider = YichensSpider()
|
||
|
|
posts = spider.run(max_pages=5)
|
||
|
|
print(f'采集帖子数: {len(posts)}')
|
||
|
|
" >> $LOG_FILE 2>&1
|
||
|
|
|
||
|
|
echo "[$(date)] 爬虫任务完成" >> $LOG_FILE
|