""" 缓存服务 - SQLite 数据库操作 """ import json import logging from datetime import datetime, timedelta from typing import Dict, List, Optional from sqlalchemy.orm import Session from app.models import MarketData, ScheduledTask, SymbolTimestamp from app.config import CACHE_TTL_SECONDS from app.storage_manager import get_storage_manager logger = logging.getLogger(__name__) # ===== 市场数据缓存 ===== def is_cache_valid( db: Session, symbol: str, data_type: str, period: str, ttl_seconds: int = CACHE_TTL_SECONDS, ) -> bool: """检查指定品种+周期的缓存是否在有效期内""" record = db.query(MarketData).filter_by( symbol=symbol, data_type=data_type, period=period, ).first() if not record: return False age = (datetime.now() - record.fetched_at).total_seconds() return age < ttl_seconds def check_cache_status( db: Session, symbol: str, data_type: str, periods: List[str], ttl_seconds: int = CACHE_TTL_SECONDS, ) -> dict: """ 检查一组周期的缓存状态。 Returns: { "all_valid": bool, # 所有周期都有有效缓存 "valid_periods": [...], "missing_periods": [...], } """ valid = [] missing = [] for p in periods: if is_cache_valid(db, symbol, data_type, p, ttl_seconds): valid.append(p) else: missing.append(p) return { "all_valid": len(missing) == 0, "valid_periods": valid, "missing_periods": missing, } def _save_to_sqlite(db: Session, symbol: str, data: Dict) -> MarketData: """ 将采集结果写入 SQLite 缓存，并同步更新合约时间戳。 Args: db: 数据库会话 symbol: 品种代码 data: 采集脚本返回的完整数据 Returns: 保存的 MarketData 记录 """ now = datetime.now() # 按 period 拆分存储（每个周期一条记录） for period, candles in data.get("timeframes", {}).items(): record = db.query(MarketData).filter_by( symbol=symbol, data_type=data.get("type", "futures"), period=period, ).first() candles_json = json.dumps(candles, ensure_ascii=False) if record: record.candles_json = candles_json record.current_price = data.get("current_price") record.fetched_at = now record.candle_count = len(candles) else: record = MarketData( symbol=symbol, data_type=data.get("type", "futures"), period=period, candles_json=candles_json, current_price=data.get("current_price"), fetched_at=now, candle_count=len(candles), ) db.add(record) # 更新合约时间戳 _update_symbol_timestamp_sqlite(db, symbol, data.get("type", "futures"), now) db.commit() logger.info(f"缓存已更新: {symbol}, {len(data.get('timeframes', {}))} 个周期") # 返回最新的一条作为代表 return db.query(MarketData).filter_by( symbol=symbol, data_type=data.get("type", "futures"), ).order_by(MarketData.fetched_at.desc()).first() def _update_symbol_timestamp_sqlite(db: Session, symbol: str, data_type: str, refresh_time: datetime) -> None: """更新或创建合约时间戳记录（SQLite 兜底）。""" timestamp_record = db.query(SymbolTimestamp).filter_by( symbol=symbol, data_type=data_type ).first() if timestamp_record: timestamp_record.last_refresh_at = refresh_time timestamp_record.refresh_count += 1 else: timestamp_record = SymbolTimestamp( symbol=symbol, data_type=data_type, last_refresh_at=refresh_time, refresh_count=1 ) db.add(timestamp_record) def _get_symbol_timestamp_sqlite(db: Session, symbol: str, data_type: str = "futures") -> Optional[datetime]: """获取合约最后刷新时间（SQLite 兜底）。""" record = db.query(SymbolTimestamp).filter_by( symbol=symbol, data_type=data_type ).first() return record.last_refresh_at if record else None def _get_from_sqlite( db: Session, symbol: str, data_type: str = "futures", periods: Optional[List[str]] = None, end_time: Optional[datetime] = None, max_candles: int = 100, ) -> Optional[Dict]: """ 从 SQLite 缓存中获取完整的多周期数据。 Args: db: 数据库会话 symbol: 品种代码 data_type: 数据类型 periods: 周期列表 end_time: 结束时间(可选),默认为当前时间 max_candles: 每个周期最大K线数量,默认100 Returns: 与采集脚本相同格式的数据，或 None """ query = db.query(MarketData).filter_by(symbol=symbol, data_type=data_type) if periods: query = query.filter(MarketData.period.in_(periods)) records = query.all() if not records: return None # 检查缓存是否过期 now = datetime.now() newest = max(r.fetched_at for r in records) is_fresh = (now - newest).total_seconds() < CACHE_TTL_SECONDS # 如果未指定结束时间,默认为当前时间 filter_end_time = end_time if end_time else now # 确保filter_end_time是naive datetime(无时区) if filter_end_time.tzinfo is not None: filter_end_time = filter_end_time.replace(tzinfo=None) timeframes = {} current_price = None for r in records: candles = json.loads(r.candles_json) # 过滤结束时间之前的K线数据 filtered_candles = [] for candle in candles: candle_time = candle.get('datetime') or candle.get('time') if candle_time: # 解析K线时间 if isinstance(candle_time, str): try: candle_dt = datetime.fromisoformat(candle_time.replace('Z', '+00:00')) # 转换为naive datetime进行比较 if candle_dt.tzinfo is not None: candle_dt = candle_dt.replace(tzinfo=None) except: filtered_candles.append(candle) continue else: candle_dt = candle_time # 如果是aware datetime,转换为naive if candle_dt.tzinfo is not None: candle_dt = candle_dt.replace(tzinfo=None) # 只保留结束时间之前的数据 if candle_dt <= filter_end_time: filtered_candles.append(candle) else: filtered_candles.append(candle) # 限制K线数量,超过max_candles则取最新的max_candles条 if len(filtered_candles) > max_candles: filtered_candles = filtered_candles[-max_candles:] timeframes[r.period] = filtered_candles if current_price is None: current_price = r.current_price return { "symbol": symbol, "type": data_type, "current_price": current_price, "timestamp": newest.isoformat(), "timeframes": timeframes, "is_fresh": is_fresh, "fetched_at": newest.isoformat(), } def save_market_data(db: Session, symbol: str, data: Dict) -> Optional[MarketData]: """ 保存采集结果到缓存，优先写入 StorageManager（Redis/MySQL），失败时降级到 SQLite。 Args: symbol: 品种代码 data: 采集脚本返回的完整数据 Returns: SQLite 写入时返回 MarketData 记录；StorageManager 成功时返回 None """ storage = get_storage_manager() data_type = data.get("type", "futures") now = datetime.now() if storage.is_available(): try: if storage.save_market_data(symbol, data): storage.save_symbol_timestamp(symbol, data_type, now) return None else: raise Exception("save_market_data returned False") except Exception as e: logger.warning(f"StorageManager 写入失败，降级到 SQLite: {e}") return _save_to_sqlite(db, symbol, data) def update_symbol_timestamp(db: Session, symbol: str, data_type: str, refresh_time: datetime) -> None: """更新或创建合约时间戳记录，优先使用 StorageManager。""" storage = get_storage_manager() if storage.is_available(): try: storage.save_symbol_timestamp(symbol, data_type, refresh_time) return except Exception as e: logger.warning(f"StorageManager 时间戳写入失败，降级到 SQLite: {e}") _update_symbol_timestamp_sqlite(db, symbol, data_type, refresh_time) db.commit() def get_symbol_timestamp(db: Session, symbol: str, data_type: str = "futures") -> Optional[datetime]: """获取合约最后刷新时间，优先使用 StorageManager。""" storage = get_storage_manager() if storage.is_available(): try: result = storage.get_symbol_timestamp(symbol, data_type) if result: return datetime.fromisoformat(result["last_refresh_at"]) except Exception as e: logger.warning(f"StorageManager 时间戳读取失败，降级到 SQLite: {e}") return _get_symbol_timestamp_sqlite(db, symbol, data_type) def needs_refresh(db: Session, symbol: str, data_type: str = "futures", threshold_seconds: int = 300) -> bool: """ 检查合约是否需要刷新（数据是否超过阈值时间） Args: db: 数据库会话 symbol: 品种代码 data_type: 数据类型 threshold_seconds: 阈值时间（秒），默认300秒（5分钟） Returns: True 表示需要刷新，False 表示数据仍然新鲜 """ last_refresh = get_symbol_timestamp(db, symbol, data_type) if last_refresh is None: return True # 从未刷新过，需要刷新 age = (datetime.now() - last_refresh).total_seconds() return age > threshold_seconds def get_latest_cached( db: Session, symbol: str, data_type: str = "futures", period: Optional[str] = None, ) -> List[MarketData]: """获取最新缓存数据""" query = db.query(MarketData).filter_by(symbol=symbol, data_type=data_type) if period: query = query.filter_by(period=period) return query.order_by(MarketData.fetched_at.desc()).all() def _merge_storage_market_data( symbol: str, data_type: str, periods: Optional[List[str]], storage_results: Dict[str, Optional[Dict]], ) -> Optional[Dict]: """将 StorageManager 按周期返回的数据合并为 get_cached_data 标准格式。""" timeframes = {} current_price = None newest = None for period, result in storage_results.items(): if not result: continue candles = result.get("candles", []) timeframes[period] = candles if current_price is None: current_price = result.get("current_price") timestamp_raw = result.get("timestamp") if timestamp_raw: try: timestamp_dt = datetime.fromisoformat(timestamp_raw) if newest is None or timestamp_dt > newest: newest = timestamp_dt except Exception: pass if not timeframes: return None if newest is None: newest = datetime.now() now = datetime.now() is_fresh = (now - newest).total_seconds() < CACHE_TTL_SECONDS return { "symbol": symbol, "type": data_type, "current_price": current_price, "timestamp": newest.isoformat(), "timeframes": timeframes, "is_fresh": is_fresh, "fetched_at": newest.isoformat(), } def get_cached_data( db: Session, symbol: str, data_type: str = "futures", periods: Optional[List[str]] = None, end_time: Optional[datetime] = None, max_candles: int = 100, ) -> Optional[Dict]: """ 从缓存中获取完整的多周期数据，优先使用 StorageManager，失败时降级到 SQLite。 Args: db: 数据库会话 symbol: 品种代码 data_type: 数据类型 periods: 周期列表 end_time: 结束时间(可选),默认为当前时间 max_candles: 每个周期最大K线数量,默认100 Returns: 与采集脚本相同格式的数据，或 None """ storage = get_storage_manager() if storage.is_available(): try: storage_results = {} query_periods = periods if periods else [] for period in query_periods: storage_results[period] = storage.get_market_data(symbol, data_type, period) merged = _merge_storage_market_data(symbol, data_type, periods, storage_results) if merged: return merged except Exception as e: logger.warning(f"StorageManager 读取失败，降级到 SQLite: {e}") return _get_from_sqlite(db, symbol, data_type, periods, end_time, max_candles) # ===== 定时任务管理 ===== def create_task( db: Session, symbol: str, data_type: str, periods: str, interval_seconds: int, task_type: str = "interval", run_time: Optional[str] = None, ) -> ScheduledTask: """创建定时任务配置""" existing = db.query(ScheduledTask).filter_by( symbol=symbol, data_type=data_type ).first() if existing: existing.periods = periods existing.interval_seconds = interval_seconds existing.task_type = task_type existing.run_time = run_time existing.enabled = True existing.updated_at = datetime.now() db.commit() db.refresh(existing) return existing task = ScheduledTask( symbol=symbol, data_type=data_type, periods=periods, interval_seconds=interval_seconds, task_type=task_type, run_time=run_time, enabled=True, ) db.add(task) db.commit() db.refresh(task) return task def list_tasks(db: Session) -> List[ScheduledTask]: """列出所有任务""" return db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all() def get_task(db: Session, task_id: int) -> Optional[ScheduledTask]: """获取单个任务""" return db.query(ScheduledTask).filter_by(id=task_id).first() def disable_task(db: Session, task_id: int) -> Optional[ScheduledTask]: """禁用任务""" task = db.query(ScheduledTask).filter_by(id=task_id).first() if task: task.enabled = False task.updated_at = datetime.now() db.commit() db.refresh(task) return task def enable_task(db: Session, task_id: int) -> Optional[ScheduledTask]: """启用任务""" task = db.query(ScheduledTask).filter_by(id=task_id).first() if task: task.enabled = True task.updated_at = datetime.now() db.commit() db.refresh(task) return task def delete_task(db: Session, task_id: int) -> bool: """删除任务""" task = db.query(ScheduledTask).filter_by(id=task_id).first() if task: db.delete(task) db.commit() return True return False def update_task_status( db: Session, task_id: int, status: str ) -> None: """更新任务执行状态""" task = db.query(ScheduledTask).filter_by(id=task_id).first() if task: task.last_run = datetime.now() task.last_status = status db.commit()