commit
4eaee5c594
@ -0,0 +1 @@
|
||||
# 数据缓冲平台
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
||||
# api
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,36 @@
|
||||
"""
|
||||
数据缓冲平台 - 配置
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# 项目根目录
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
# 数据库路径
|
||||
DB_PATH = Path(os.getenv(
|
||||
"BUFFER_DB_PATH",
|
||||
str(Path(__file__).resolve().parent.parent / "data" / "buffer.db")
|
||||
))
|
||||
|
||||
# 原始采集脚本路径
|
||||
COLLECTOR_SCRIPT = os.getenv(
|
||||
"COLLECTOR_SCRIPT",
|
||||
str(BASE_DIR / "market_data_colector_platform" / "futures_data_collector.py")
|
||||
)
|
||||
|
||||
# FastAPI 服务配置
|
||||
HOST = os.getenv("BUFFER_HOST", "0.0.0.0")
|
||||
PORT = int(os.getenv("BUFFER_PORT", "8600"))
|
||||
|
||||
# 数据缓存
|
||||
CACHE_TTL_SECONDS = int(os.getenv("CACHE_TTL", "300")) # 默认5分钟过期
|
||||
|
||||
# 并发采集
|
||||
MAX_WORKERS = int(os.getenv("MAX_WORKERS", "2"))
|
||||
|
||||
# 日志
|
||||
LOG_LEVEL = os.getenv("BUFFER_LOG_LEVEL", "INFO")
|
||||
|
||||
# 调度器
|
||||
SCHEDULER_MAX_INSTANCES = 1 # 同一任务不允许重叠执行
|
||||
@ -0,0 +1,28 @@
|
||||
"""
|
||||
数据缓冲平台 - 数据库连接
|
||||
"""
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, declarative_base
|
||||
from app.config import DB_PATH
|
||||
|
||||
# 确保数据目录存在
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{DB_PATH}",
|
||||
connect_args={"check_same_thread": False},
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
def get_db():
|
||||
"""获取数据库会话"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@ -0,0 +1,108 @@
|
||||
"""
|
||||
数据缓冲平台 - FastAPI 主入口
|
||||
"""
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from pathlib import Path
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.database import engine, Base
|
||||
from app.config import HOST, PORT, LOG_LEVEL
|
||||
from app.api import data, tasks, config
|
||||
from app.services.scheduler import start_scheduler, stop_scheduler
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用生命周期管理"""
|
||||
# 启动时:建表 + 启动调度器
|
||||
logger.info("创建数据库表...")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
logger.info("启动定时调度器...")
|
||||
start_scheduler()
|
||||
|
||||
# 恢复已启用的任务
|
||||
from app.database import SessionLocal
|
||||
from app.services.cache import list_tasks
|
||||
from app.services.scheduler import add_job
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
enabled_tasks = [t for t in list_tasks(db) if t.enabled]
|
||||
for t in enabled_tasks:
|
||||
add_job(t.id, t.interval_seconds)
|
||||
logger.info(f"恢复定时任务: {t.symbol} (每 {t.interval_seconds}s)")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
logger.info(f"数据缓冲平台已启动 http://{HOST}:{PORT}")
|
||||
|
||||
yield
|
||||
|
||||
# 关闭时
|
||||
logger.info("停止调度器...")
|
||||
stop_scheduler()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="数据缓冲平台",
|
||||
description="期货/股票行情数据缓存与定时采集平台",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# 静态文件服务
|
||||
STATIC_DIR = Path(__file__).resolve().parent / "static"
|
||||
STATIC_DIR.mkdir(parents=True, exist_ok=True)
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
|
||||
|
||||
@app.get("/ui")
|
||||
def ui_page():
|
||||
"""品种配置管理页面"""
|
||||
return FileResponse(str(STATIC_DIR / "index.html"))
|
||||
|
||||
# 注册路由
|
||||
app.include_router(data.router, prefix="/api/v1")
|
||||
app.include_router(tasks.router, prefix="/api/v1")
|
||||
app.include_router(config.router, prefix="/api/v1")
|
||||
|
||||
|
||||
@app.get("/api/v1/health")
|
||||
def health():
|
||||
return {"status": "ok", "service": "market-data-buffer"}
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def root():
|
||||
return {
|
||||
"message": "数据缓冲平台 API",
|
||||
"docs": "/docs",
|
||||
"health": "/api/v1/health",
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host=HOST, port=PORT, reload=True)
|
||||
@ -0,0 +1,52 @@
|
||||
"""
|
||||
数据缓冲平台 - 数据模型 (SQLAlchemy ORM)
|
||||
"""
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, String, Integer, Float, Text, DateTime, Boolean, Index, UniqueConstraint
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class MarketData(Base):
|
||||
"""缓存的市场数据表"""
|
||||
__tablename__ = "market_data"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
symbol = Column(String(32), nullable=False, index=True, comment="品种合约代码")
|
||||
data_type = Column(String(16), nullable=False, default="futures", comment="数据类型: futures/stock")
|
||||
period = Column(String(16), nullable=False, index=True, comment="周期: 5min/15min/30min/60min/daily")
|
||||
# K线数据以 JSON 字符串形式存储
|
||||
candles_json = Column(Text, nullable=False, comment="K线数据JSON")
|
||||
current_price = Column(Float, nullable=True, comment="当前价格")
|
||||
fetched_at = Column(DateTime, nullable=False, default=datetime.now, index=True, comment="获取时间")
|
||||
candle_count = Column(Integer, default=0, comment="K线数量")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("symbol", "data_type", "period", name="uq_symbol_period"),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<MarketData {self.symbol} {self.period} candles={self.candle_count}>"
|
||||
|
||||
|
||||
class ScheduledTask(Base):
|
||||
"""定时任务配置表"""
|
||||
__tablename__ = "scheduled_tasks"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
symbol = Column(String(32), nullable=False, comment="品种合约代码")
|
||||
data_type = Column(String(16), nullable=False, default="futures", comment="数据类型")
|
||||
periods = Column(String(256), nullable=False, comment="周期列表(逗号分隔), 如 5min,15min,60min")
|
||||
interval_seconds = Column(Integer, nullable=False, default=300, comment="轮询间隔(秒)")
|
||||
enabled = Column(Boolean, nullable=False, default=True, comment="是否启用")
|
||||
job_id = Column(String(64), nullable=True, unique=True, comment="APScheduler job_id")
|
||||
last_run = Column(DateTime, nullable=True, comment="最后执行时间")
|
||||
last_status = Column(String(16), nullable=True, comment="最后状态: success/failed")
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.now)
|
||||
updated_at = Column(DateTime, nullable=False, default=datetime.now, onupdate=datetime.now)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("symbol", "data_type", name="uq_task_symbol"),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Task {self.symbol} every {self.interval_seconds}s enabled={self.enabled}>"
|
||||
@ -0,0 +1,102 @@
|
||||
"""Pydantic 数据校验模型"""
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ===== 采集请求 =====
|
||||
|
||||
class BatchFetchRequest(BaseModel):
|
||||
"""批量获取数据请求"""
|
||||
symbols: List[str] = Field(..., description="品种合约列表,如 ['SN2504', 'AG2506']")
|
||||
data_type: str = Field(default="futures", description="数据类型: futures / stock")
|
||||
periods: List[str] = Field(
|
||||
default=["5min", "15min", "30min", "60min", "daily"],
|
||||
description="周期列表: 5min / 15min / 30min / 60min / daily"
|
||||
)
|
||||
|
||||
|
||||
# ===== 数据响应 =====
|
||||
|
||||
class CandleItem(BaseModel):
|
||||
"""单根K线"""
|
||||
datetime: str
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float
|
||||
|
||||
|
||||
class TimeframeData(BaseModel):
|
||||
"""一个周期的数据"""
|
||||
period: str
|
||||
candles: List[CandleItem]
|
||||
candle_count: int
|
||||
fetched_at: str
|
||||
|
||||
|
||||
class SymbolDataResponse(BaseModel):
|
||||
"""单个品种的数据响应"""
|
||||
symbol: str
|
||||
data_type: str
|
||||
current_price: Optional[float] = None
|
||||
timeframes: List[TimeframeData]
|
||||
source: str = "cache|live"
|
||||
|
||||
|
||||
class BatchFetchResponse(BaseModel):
|
||||
"""批量获取响应"""
|
||||
success: List[str] = Field(default_factory=list, description="成功的品种")
|
||||
failed: List[str] = Field(default_factory=list, description="失败的品种")
|
||||
details: dict = Field(default_factory=dict, description="每个品种的详细数据")
|
||||
|
||||
|
||||
class LatestDataResponse(BaseModel):
|
||||
"""获取最新数据响应"""
|
||||
symbol: str
|
||||
data_type: str
|
||||
period: str
|
||||
candles: List[CandleItem]
|
||||
candle_count: int
|
||||
current_price: Optional[float] = None
|
||||
fetched_at: str
|
||||
is_fresh: bool = Field(description="数据是否在缓存有效期内")
|
||||
|
||||
|
||||
# ===== 定时任务 =====
|
||||
|
||||
class CreateTaskRequest(BaseModel):
|
||||
"""创建定时任务请求"""
|
||||
symbol: str = Field(..., description="品种合约代码")
|
||||
data_type: str = Field(default="futures", description="数据类型")
|
||||
periods: List[str] = Field(
|
||||
default=["5min", "15min", "30min", "60min", "daily"],
|
||||
description="需要定时获取的周期"
|
||||
)
|
||||
interval_seconds: int = Field(
|
||||
default=300,
|
||||
ge=30,
|
||||
le=86400,
|
||||
description="轮询间隔(秒),范围 30~86400"
|
||||
)
|
||||
|
||||
|
||||
class TaskInfo(BaseModel):
|
||||
"""任务信息"""
|
||||
id: int
|
||||
symbol: str
|
||||
data_type: str
|
||||
periods: List[str]
|
||||
interval_seconds: int
|
||||
enabled: bool
|
||||
running: bool = Field(description="当前是否正在运行")
|
||||
last_run: Optional[str] = None
|
||||
last_status: Optional[str] = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class TaskListResponse(BaseModel):
|
||||
tasks: List[TaskInfo]
|
||||
total: int
|
||||
@ -0,0 +1 @@
|
||||
# services
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,82 @@
|
||||
"""
|
||||
数据采集服务 - 包装原始采集脚本
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 获取原始采集脚本路径 (buffer_platform/app/services -> buffer_platform -> parent = market_data_colector_platform)
|
||||
SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
|
||||
if SCRIPT_DIR not in sys.path:
|
||||
sys.path.insert(0, SCRIPT_DIR)
|
||||
logger.info(f"已添加采集脚本路径到sys.path: {SCRIPT_DIR}")
|
||||
|
||||
|
||||
def fetch_symbol_data(
|
||||
symbol: str,
|
||||
data_type: str = "futures",
|
||||
periods: Optional[List[str]] = None,
|
||||
max_workers: int = 2,
|
||||
) -> Dict:
|
||||
"""
|
||||
获取单个品种的多周期数据。
|
||||
|
||||
返回格式:
|
||||
{
|
||||
"symbol": "SN2504",
|
||||
"type": "futures",
|
||||
"current_price": 12345.0,
|
||||
"timestamp": "2025-01-15T10:30:00+08:00",
|
||||
"timeframes": {
|
||||
"5min": [{"datetime": ..., "open": ..., ...}, ...],
|
||||
...
|
||||
}
|
||||
}
|
||||
"""
|
||||
try:
|
||||
from futures_data_collector import collect_futures_data, collect_stock_data
|
||||
|
||||
if data_type == "stock":
|
||||
result = collect_stock_data(symbol)
|
||||
else:
|
||||
result = collect_futures_data(symbol)
|
||||
|
||||
# 如果指定了周期,只保留需要的
|
||||
if periods:
|
||||
filtered = {}
|
||||
for p in periods:
|
||||
if p in result.get("timeframes", {}):
|
||||
filtered[p] = result["timeframes"][p]
|
||||
result["timeframes"] = filtered
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"采集 {symbol} 数据失败: {e}")
|
||||
return {
|
||||
"symbol": symbol,
|
||||
"type": data_type,
|
||||
"current_price": None,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"timeframes": {},
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
def fetch_batch(
|
||||
symbols: List[str],
|
||||
data_type: str = "futures",
|
||||
periods: Optional[List[str]] = None,
|
||||
max_workers: int = 2,
|
||||
) -> Dict[str, Dict]:
|
||||
"""批量获取多个品种数据(串行,避免过度并发)"""
|
||||
results = {}
|
||||
for sym in symbols:
|
||||
logger.info(f"开始采集 {sym} ...")
|
||||
results[sym] = fetch_symbol_data(sym, data_type, periods, max_workers)
|
||||
return results
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,36 @@
|
||||
{
|
||||
"futures": {
|
||||
"原油": "SC2606",
|
||||
"燃油": "FU2606",
|
||||
"低硫燃油": "LU2607",
|
||||
"沪银": "AG2606",
|
||||
"沪金": "AU2606",
|
||||
"沪铜": "CU2606",
|
||||
"沪镍": "NI2606",
|
||||
"沪锡": "SN2606",
|
||||
"沪铝": "AL2606",
|
||||
"沪锌": "PB2606",
|
||||
"氧化铝": "AO2609",
|
||||
"工业硅": "SI2609",
|
||||
"多晶硅": "PS2606",
|
||||
"碳酸锂": "LC2609",
|
||||
"纯碱": "SA2609",
|
||||
"烧碱": "SH2607",
|
||||
"玻璃": "FG2609",
|
||||
"橡胶": "RU2609",
|
||||
"合成橡胶": "BR2606",
|
||||
"20号胶": "NR2607",
|
||||
"螺纹钢": "RB2610",
|
||||
"铁矿石": "I2609",
|
||||
"焦煤": "JM2606",
|
||||
"焦炭": "J2606",
|
||||
"PTA": "TA2609",
|
||||
"棕榈油": "P2609",
|
||||
"豆粕": "M2609",
|
||||
"白糖": "SR2609",
|
||||
"棉花": "CF2609",
|
||||
"甲醇": "MA2609",
|
||||
"尿素": "UR2609",
|
||||
"中证1000": "IM2606"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -0,0 +1,11 @@
|
||||
fastapi>=0.110.0
|
||||
uvicorn>=0.29.0
|
||||
sqlalchemy>=2.0.0
|
||||
aiosqlite>=0.20.0
|
||||
pydantic>=2.0.0
|
||||
apscheduler>=3.10.0
|
||||
akshare>=1.14.0
|
||||
pandas>=2.0.0
|
||||
tenacity>=8.2.0
|
||||
requests>=2.31.0
|
||||
httpx>=0.27.0
|
||||
Loading…
Reference in new issue