commit
4eaee5c594
@ -0,0 +1 @@
|
|||||||
|
# 数据缓冲平台
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
|||||||
|
# api
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,36 @@
|
|||||||
|
"""
|
||||||
|
数据缓冲平台 - 配置
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# 项目根目录
|
||||||
|
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||||
|
|
||||||
|
# 数据库路径
|
||||||
|
DB_PATH = Path(os.getenv(
|
||||||
|
"BUFFER_DB_PATH",
|
||||||
|
str(Path(__file__).resolve().parent.parent / "data" / "buffer.db")
|
||||||
|
))
|
||||||
|
|
||||||
|
# 原始采集脚本路径
|
||||||
|
COLLECTOR_SCRIPT = os.getenv(
|
||||||
|
"COLLECTOR_SCRIPT",
|
||||||
|
str(BASE_DIR / "market_data_colector_platform" / "futures_data_collector.py")
|
||||||
|
)
|
||||||
|
|
||||||
|
# FastAPI 服务配置
|
||||||
|
HOST = os.getenv("BUFFER_HOST", "0.0.0.0")
|
||||||
|
PORT = int(os.getenv("BUFFER_PORT", "8600"))
|
||||||
|
|
||||||
|
# 数据缓存
|
||||||
|
CACHE_TTL_SECONDS = int(os.getenv("CACHE_TTL", "300")) # 默认5分钟过期
|
||||||
|
|
||||||
|
# 并发采集
|
||||||
|
MAX_WORKERS = int(os.getenv("MAX_WORKERS", "2"))
|
||||||
|
|
||||||
|
# 日志
|
||||||
|
LOG_LEVEL = os.getenv("BUFFER_LOG_LEVEL", "INFO")
|
||||||
|
|
||||||
|
# 调度器
|
||||||
|
SCHEDULER_MAX_INSTANCES = 1 # 同一任务不允许重叠执行
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
"""
|
||||||
|
数据缓冲平台 - 数据库连接
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker, declarative_base
|
||||||
|
from app.config import DB_PATH
|
||||||
|
|
||||||
|
# 确保数据目录存在
|
||||||
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
engine = create_engine(
|
||||||
|
f"sqlite:///{DB_PATH}",
|
||||||
|
connect_args={"check_same_thread": False},
|
||||||
|
pool_pre_ping=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
Base = declarative_base()
|
||||||
|
|
||||||
|
|
||||||
|
def get_db():
|
||||||
|
"""获取数据库会话"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
yield db
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
@ -0,0 +1,108 @@
|
|||||||
|
"""
|
||||||
|
数据缓冲平台 - FastAPI 主入口
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
from fastapi.responses import FileResponse
|
||||||
|
|
||||||
|
from app.database import engine, Base
|
||||||
|
from app.config import HOST, PORT, LOG_LEVEL
|
||||||
|
from app.api import data, tasks, config
|
||||||
|
from app.services.scheduler import start_scheduler, stop_scheduler
|
||||||
|
|
||||||
|
# 配置日志
|
||||||
|
logging.basicConfig(
|
||||||
|
level=getattr(logging, LOG_LEVEL.upper(), logging.INFO),
|
||||||
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||||
|
)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
"""应用生命周期管理"""
|
||||||
|
# 启动时:建表 + 启动调度器
|
||||||
|
logger.info("创建数据库表...")
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
logger.info("启动定时调度器...")
|
||||||
|
start_scheduler()
|
||||||
|
|
||||||
|
# 恢复已启用的任务
|
||||||
|
from app.database import SessionLocal
|
||||||
|
from app.services.cache import list_tasks
|
||||||
|
from app.services.scheduler import add_job
|
||||||
|
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
enabled_tasks = [t for t in list_tasks(db) if t.enabled]
|
||||||
|
for t in enabled_tasks:
|
||||||
|
add_job(t.id, t.interval_seconds)
|
||||||
|
logger.info(f"恢复定时任务: {t.symbol} (每 {t.interval_seconds}s)")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
logger.info(f"数据缓冲平台已启动 http://{HOST}:{PORT}")
|
||||||
|
|
||||||
|
yield
|
||||||
|
|
||||||
|
# 关闭时
|
||||||
|
logger.info("停止调度器...")
|
||||||
|
stop_scheduler()
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
title="数据缓冲平台",
|
||||||
|
description="期货/股票行情数据缓存与定时采集平台",
|
||||||
|
version="1.0.0",
|
||||||
|
lifespan=lifespan,
|
||||||
|
)
|
||||||
|
|
||||||
|
# CORS
|
||||||
|
app.add_middleware(
|
||||||
|
CORSMiddleware,
|
||||||
|
allow_origins=["*"],
|
||||||
|
allow_credentials=True,
|
||||||
|
allow_methods=["*"],
|
||||||
|
allow_headers=["*"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 静态文件服务
|
||||||
|
STATIC_DIR = Path(__file__).resolve().parent / "static"
|
||||||
|
STATIC_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/ui")
|
||||||
|
def ui_page():
|
||||||
|
"""品种配置管理页面"""
|
||||||
|
return FileResponse(str(STATIC_DIR / "index.html"))
|
||||||
|
|
||||||
|
# 注册路由
|
||||||
|
app.include_router(data.router, prefix="/api/v1")
|
||||||
|
app.include_router(tasks.router, prefix="/api/v1")
|
||||||
|
app.include_router(config.router, prefix="/api/v1")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/health")
|
||||||
|
def health():
|
||||||
|
return {"status": "ok", "service": "market-data-buffer"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/")
|
||||||
|
def root():
|
||||||
|
return {
|
||||||
|
"message": "数据缓冲平台 API",
|
||||||
|
"docs": "/docs",
|
||||||
|
"health": "/api/v1/health",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import uvicorn
|
||||||
|
uvicorn.run("app.main:app", host=HOST, port=PORT, reload=True)
|
||||||
@ -0,0 +1,52 @@
|
|||||||
|
"""
|
||||||
|
数据缓冲平台 - 数据模型 (SQLAlchemy ORM)
|
||||||
|
"""
|
||||||
|
from datetime import datetime
|
||||||
|
from sqlalchemy import Column, String, Integer, Float, Text, DateTime, Boolean, Index, UniqueConstraint
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class MarketData(Base):
|
||||||
|
"""缓存的市场数据表"""
|
||||||
|
__tablename__ = "market_data"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
symbol = Column(String(32), nullable=False, index=True, comment="品种合约代码")
|
||||||
|
data_type = Column(String(16), nullable=False, default="futures", comment="数据类型: futures/stock")
|
||||||
|
period = Column(String(16), nullable=False, index=True, comment="周期: 5min/15min/30min/60min/daily")
|
||||||
|
# K线数据以 JSON 字符串形式存储
|
||||||
|
candles_json = Column(Text, nullable=False, comment="K线数据JSON")
|
||||||
|
current_price = Column(Float, nullable=True, comment="当前价格")
|
||||||
|
fetched_at = Column(DateTime, nullable=False, default=datetime.now, index=True, comment="获取时间")
|
||||||
|
candle_count = Column(Integer, default=0, comment="K线数量")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("symbol", "data_type", "period", name="uq_symbol_period"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<MarketData {self.symbol} {self.period} candles={self.candle_count}>"
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduledTask(Base):
|
||||||
|
"""定时任务配置表"""
|
||||||
|
__tablename__ = "scheduled_tasks"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
symbol = Column(String(32), nullable=False, comment="品种合约代码")
|
||||||
|
data_type = Column(String(16), nullable=False, default="futures", comment="数据类型")
|
||||||
|
periods = Column(String(256), nullable=False, comment="周期列表(逗号分隔), 如 5min,15min,60min")
|
||||||
|
interval_seconds = Column(Integer, nullable=False, default=300, comment="轮询间隔(秒)")
|
||||||
|
enabled = Column(Boolean, nullable=False, default=True, comment="是否启用")
|
||||||
|
job_id = Column(String(64), nullable=True, unique=True, comment="APScheduler job_id")
|
||||||
|
last_run = Column(DateTime, nullable=True, comment="最后执行时间")
|
||||||
|
last_status = Column(String(16), nullable=True, comment="最后状态: success/failed")
|
||||||
|
created_at = Column(DateTime, nullable=False, default=datetime.now)
|
||||||
|
updated_at = Column(DateTime, nullable=False, default=datetime.now, onupdate=datetime.now)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("symbol", "data_type", name="uq_task_symbol"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<Task {self.symbol} every {self.interval_seconds}s enabled={self.enabled}>"
|
||||||
@ -0,0 +1,102 @@
|
|||||||
|
"""Pydantic 数据校验模型"""
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import List, Optional
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
# ===== 采集请求 =====
|
||||||
|
|
||||||
|
class BatchFetchRequest(BaseModel):
|
||||||
|
"""批量获取数据请求"""
|
||||||
|
symbols: List[str] = Field(..., description="品种合约列表,如 ['SN2504', 'AG2506']")
|
||||||
|
data_type: str = Field(default="futures", description="数据类型: futures / stock")
|
||||||
|
periods: List[str] = Field(
|
||||||
|
default=["5min", "15min", "30min", "60min", "daily"],
|
||||||
|
description="周期列表: 5min / 15min / 30min / 60min / daily"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ===== 数据响应 =====
|
||||||
|
|
||||||
|
class CandleItem(BaseModel):
|
||||||
|
"""单根K线"""
|
||||||
|
datetime: str
|
||||||
|
open: float
|
||||||
|
high: float
|
||||||
|
low: float
|
||||||
|
close: float
|
||||||
|
volume: float
|
||||||
|
|
||||||
|
|
||||||
|
class TimeframeData(BaseModel):
|
||||||
|
"""一个周期的数据"""
|
||||||
|
period: str
|
||||||
|
candles: List[CandleItem]
|
||||||
|
candle_count: int
|
||||||
|
fetched_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class SymbolDataResponse(BaseModel):
|
||||||
|
"""单个品种的数据响应"""
|
||||||
|
symbol: str
|
||||||
|
data_type: str
|
||||||
|
current_price: Optional[float] = None
|
||||||
|
timeframes: List[TimeframeData]
|
||||||
|
source: str = "cache|live"
|
||||||
|
|
||||||
|
|
||||||
|
class BatchFetchResponse(BaseModel):
|
||||||
|
"""批量获取响应"""
|
||||||
|
success: List[str] = Field(default_factory=list, description="成功的品种")
|
||||||
|
failed: List[str] = Field(default_factory=list, description="失败的品种")
|
||||||
|
details: dict = Field(default_factory=dict, description="每个品种的详细数据")
|
||||||
|
|
||||||
|
|
||||||
|
class LatestDataResponse(BaseModel):
|
||||||
|
"""获取最新数据响应"""
|
||||||
|
symbol: str
|
||||||
|
data_type: str
|
||||||
|
period: str
|
||||||
|
candles: List[CandleItem]
|
||||||
|
candle_count: int
|
||||||
|
current_price: Optional[float] = None
|
||||||
|
fetched_at: str
|
||||||
|
is_fresh: bool = Field(description="数据是否在缓存有效期内")
|
||||||
|
|
||||||
|
|
||||||
|
# ===== 定时任务 =====
|
||||||
|
|
||||||
|
class CreateTaskRequest(BaseModel):
|
||||||
|
"""创建定时任务请求"""
|
||||||
|
symbol: str = Field(..., description="品种合约代码")
|
||||||
|
data_type: str = Field(default="futures", description="数据类型")
|
||||||
|
periods: List[str] = Field(
|
||||||
|
default=["5min", "15min", "30min", "60min", "daily"],
|
||||||
|
description="需要定时获取的周期"
|
||||||
|
)
|
||||||
|
interval_seconds: int = Field(
|
||||||
|
default=300,
|
||||||
|
ge=30,
|
||||||
|
le=86400,
|
||||||
|
description="轮询间隔(秒),范围 30~86400"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskInfo(BaseModel):
|
||||||
|
"""任务信息"""
|
||||||
|
id: int
|
||||||
|
symbol: str
|
||||||
|
data_type: str
|
||||||
|
periods: List[str]
|
||||||
|
interval_seconds: int
|
||||||
|
enabled: bool
|
||||||
|
running: bool = Field(description="当前是否正在运行")
|
||||||
|
last_run: Optional[str] = None
|
||||||
|
last_status: Optional[str] = None
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class TaskListResponse(BaseModel):
|
||||||
|
tasks: List[TaskInfo]
|
||||||
|
total: int
|
||||||
@ -0,0 +1 @@
|
|||||||
|
# services
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,82 @@
|
|||||||
|
"""
|
||||||
|
数据采集服务 - 包装原始采集脚本
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 获取原始采集脚本路径 (buffer_platform/app/services -> buffer_platform -> parent = market_data_colector_platform)
|
||||||
|
SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
|
||||||
|
if SCRIPT_DIR not in sys.path:
|
||||||
|
sys.path.insert(0, SCRIPT_DIR)
|
||||||
|
logger.info(f"已添加采集脚本路径到sys.path: {SCRIPT_DIR}")
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_symbol_data(
|
||||||
|
symbol: str,
|
||||||
|
data_type: str = "futures",
|
||||||
|
periods: Optional[List[str]] = None,
|
||||||
|
max_workers: int = 2,
|
||||||
|
) -> Dict:
|
||||||
|
"""
|
||||||
|
获取单个品种的多周期数据。
|
||||||
|
|
||||||
|
返回格式:
|
||||||
|
{
|
||||||
|
"symbol": "SN2504",
|
||||||
|
"type": "futures",
|
||||||
|
"current_price": 12345.0,
|
||||||
|
"timestamp": "2025-01-15T10:30:00+08:00",
|
||||||
|
"timeframes": {
|
||||||
|
"5min": [{"datetime": ..., "open": ..., ...}, ...],
|
||||||
|
...
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from futures_data_collector import collect_futures_data, collect_stock_data
|
||||||
|
|
||||||
|
if data_type == "stock":
|
||||||
|
result = collect_stock_data(symbol)
|
||||||
|
else:
|
||||||
|
result = collect_futures_data(symbol)
|
||||||
|
|
||||||
|
# 如果指定了周期,只保留需要的
|
||||||
|
if periods:
|
||||||
|
filtered = {}
|
||||||
|
for p in periods:
|
||||||
|
if p in result.get("timeframes", {}):
|
||||||
|
filtered[p] = result["timeframes"][p]
|
||||||
|
result["timeframes"] = filtered
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"采集 {symbol} 数据失败: {e}")
|
||||||
|
return {
|
||||||
|
"symbol": symbol,
|
||||||
|
"type": data_type,
|
||||||
|
"current_price": None,
|
||||||
|
"timestamp": datetime.now().isoformat(),
|
||||||
|
"timeframes": {},
|
||||||
|
"error": str(e),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_batch(
|
||||||
|
symbols: List[str],
|
||||||
|
data_type: str = "futures",
|
||||||
|
periods: Optional[List[str]] = None,
|
||||||
|
max_workers: int = 2,
|
||||||
|
) -> Dict[str, Dict]:
|
||||||
|
"""批量获取多个品种数据(串行,避免过度并发)"""
|
||||||
|
results = {}
|
||||||
|
for sym in symbols:
|
||||||
|
logger.info(f"开始采集 {sym} ...")
|
||||||
|
results[sym] = fetch_symbol_data(sym, data_type, periods, max_workers)
|
||||||
|
return results
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"futures": {
|
||||||
|
"原油": "SC2606",
|
||||||
|
"燃油": "FU2606",
|
||||||
|
"低硫燃油": "LU2607",
|
||||||
|
"沪银": "AG2606",
|
||||||
|
"沪金": "AU2606",
|
||||||
|
"沪铜": "CU2606",
|
||||||
|
"沪镍": "NI2606",
|
||||||
|
"沪锡": "SN2606",
|
||||||
|
"沪铝": "AL2606",
|
||||||
|
"沪锌": "PB2606",
|
||||||
|
"氧化铝": "AO2609",
|
||||||
|
"工业硅": "SI2609",
|
||||||
|
"多晶硅": "PS2606",
|
||||||
|
"碳酸锂": "LC2609",
|
||||||
|
"纯碱": "SA2609",
|
||||||
|
"烧碱": "SH2607",
|
||||||
|
"玻璃": "FG2609",
|
||||||
|
"橡胶": "RU2609",
|
||||||
|
"合成橡胶": "BR2606",
|
||||||
|
"20号胶": "NR2607",
|
||||||
|
"螺纹钢": "RB2610",
|
||||||
|
"铁矿石": "I2609",
|
||||||
|
"焦煤": "JM2606",
|
||||||
|
"焦炭": "J2606",
|
||||||
|
"PTA": "TA2609",
|
||||||
|
"棕榈油": "P2609",
|
||||||
|
"豆粕": "M2609",
|
||||||
|
"白糖": "SR2609",
|
||||||
|
"棉花": "CF2609",
|
||||||
|
"甲醇": "MA2609",
|
||||||
|
"尿素": "UR2609",
|
||||||
|
"中证1000": "IM2606"
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
@ -0,0 +1,11 @@
|
|||||||
|
fastapi>=0.110.0
|
||||||
|
uvicorn>=0.29.0
|
||||||
|
sqlalchemy>=2.0.0
|
||||||
|
aiosqlite>=0.20.0
|
||||||
|
pydantic>=2.0.0
|
||||||
|
apscheduler>=3.10.0
|
||||||
|
akshare>=1.14.0
|
||||||
|
pandas>=2.0.0
|
||||||
|
tenacity>=8.2.0
|
||||||
|
requests>=2.31.0
|
||||||
|
httpx>=0.27.0
|
||||||
Loading…
Reference in new issue