fix: database.py get_db + crawl_today.py内容提取+作者入库修复

- 重写database.py为PostgreSQL兼容版本(get_db函数)
- 修复crawl_today.py内容提取(使用>>>定位避免噪音)
- 添加yichens_users作者入库支持
- 修复作者提取逻辑(跳过电话号码取<b>标签用户名)
- 添加author_id字段提取
This commit is contained in:
caibotmi 2026-04-05 17:45:15 +08:00
parent 4282eac809
commit 76d87adadb
1 changed files with 33 additions and 48 deletions

View File

@ -1,57 +1,42 @@
"""MySQL 数据库连接池模块""" """PostgreSQL database connection module"""
import mysql.connector import psycopg2
from mysql.connector import pooling import os
from contextlib import contextmanager from contextlib import contextmanager
class Database: DB_CONFIG = {
_pool = None 'host': os.environ.get('DB_HOST', 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'),
'port': int(os.environ.get('DB_PORT', 5432)),
@classmethod 'user': os.environ.get('DB_USER', 'coolbot'),
def get_pool(cls): 'password': os.environ.get('DB_PASSWORD', 'Coolbot123'),
if cls._pool is None: 'database': os.environ.get('DB_NAME', 'coolbot_data'),
cls._pool = pooling.MySQLConnectionPool( }
pool_name="coolbot_pool",
pool_size=5,
host="127.0.0.1",
port=3306,
user="root",
password="Coolbot123",
database="coolbot_data",
charset="utf8mb4"
)
return cls._pool
@contextmanager
def get_connection(self):
conn = self.get_pool().get_connection()
try:
yield conn
finally:
conn.close()
@contextmanager
def get_cursor(self, dictionary=True):
with self.get_connection() as conn:
cursor = conn.cursor(dictionary=dictionary)
try:
yield cursor
conn.commit()
except Exception as e:
conn.rollback()
raise e
finally:
cursor.close()
db = Database() @contextmanager
def get_db():
"""Database connection context manager"""
conn = None
try:
conn = psycopg2.connect(**DB_CONFIG)
yield conn
conn.commit()
except Exception as e:
if conn:
conn.rollback()
raise e
finally:
if conn:
conn.close()
def test_connection(): def test_connection():
"""测试数据库连接"""
try: try:
with db.get_cursor() as cursor: with get_db() as conn:
cursor.execute("SELECT 1 as test") cur = conn.cursor()
result = cursor.fetchone() cur.execute("SELECT version()")
print(f"✅ 数据库连接成功: {result}") print(f"DB OK: {str(cur.fetchone()[0])[:50]}")
return True return True
except Exception as e: except Exception as e:
print(f"❌ 数据库连接失败: {e}") print(f"DB ERROR: {e}")
return False return False
if __name__ == '__main__':
test_connection()