You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

569 lines
21 KiB

"""
数据库迁移脚本 v2.2
创建股票/期货 K 线表结构 (TimescaleDB)
"""
import logging
from datetime import datetime
from sqlalchemy import text, create_engine
from sqlalchemy.orm import sessionmaker
from app.config import settings
logger = logging.getLogger(__name__)
# TimescaleDB 连接
engine = create_engine(settings.TIMESCALE_DB_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# ==================== 股票 K 线表 ====================
STOCK_KLINE_TABLES = {
"stock_klines_1m": """
CREATE TABLE IF NOT EXISTS stock_klines_1m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
is_limit_up BOOLEAN DEFAULT FALSE,
is_limit_down BOOLEAN DEFAULT FALSE,
total_market_cap NUMERIC(20, 2),
float_market_cap NUMERIC(20, 2),
inst_holding_ratio NUMERIC(5, 2),
trading_days INTEGER,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_5m": """
CREATE TABLE IF NOT EXISTS stock_klines_5m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_15m": """
CREATE TABLE IF NOT EXISTS stock_klines_15m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_30m": """
CREATE TABLE IF NOT EXISTS stock_klines_30m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_1h": """
CREATE TABLE IF NOT EXISTS stock_klines_1h (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_1d": """
CREATE TABLE IF NOT EXISTS stock_klines_1d (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
is_limit_up BOOLEAN DEFAULT FALSE,
is_limit_down BOOLEAN DEFAULT FALSE,
total_market_cap NUMERIC(20, 2),
float_market_cap NUMERIC(20, 2),
inst_holding_ratio NUMERIC(5, 2),
trading_days INTEGER,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_1w": """
CREATE TABLE IF NOT EXISTS stock_klines_1w (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"stock_klines_1month": """
CREATE TABLE IF NOT EXISTS stock_klines_1month (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
amount NUMERIC(20, 4) NOT NULL,
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
}
# ==================== 期货 K 线表 ====================
FUTURES_KLINE_TABLES = {
"futures_klines_1m": """
CREATE TABLE IF NOT EXISTS futures_klines_1m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_5m": """
CREATE TABLE IF NOT EXISTS futures_klines_5m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_15m": """
CREATE TABLE IF NOT EXISTS futures_klines_15m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_30m": """
CREATE TABLE IF NOT EXISTS futures_klines_30m (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_1h": """
CREATE TABLE IF NOT EXISTS futures_klines_1h (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_1d": """
CREATE TABLE IF NOT EXISTS futures_klines_1d (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_1w": """
CREATE TABLE IF NOT EXISTS futures_klines_1w (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
"futures_klines_1month": """
CREATE TABLE IF NOT EXISTS futures_klines_1month (
id BIGSERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ts TIMESTAMPTZ NOT NULL,
open NUMERIC(18, 4) NOT NULL,
high NUMERIC(18, 4) NOT NULL,
low NUMERIC(18, 4) NOT NULL,
close NUMERIC(18, 4) NOT NULL,
volume BIGINT NOT NULL,
open_interest BIGINT NOT NULL,
settlement_price NUMERIC(18, 4),
trade_date DATE NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
}
# ==================== 辅助表 ====================
AUXILIARY_TABLES = {
# 股票基本信息表
"stock_symbols": """
CREATE TABLE IF NOT EXISTS stock_symbols (
id SERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL UNIQUE,
symbol_name VARCHAR(50) NOT NULL,
exchange VARCHAR(10) NOT NULL,
industry VARCHAR(50),
list_date DATE,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
# 股票复权因子表
"stock_adjust_factors": """
CREATE TABLE IF NOT EXISTS stock_adjust_factors (
id SERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL,
ex_date DATE NOT NULL,
adjust_factor NUMERIC(10, 4) NOT NULL,
dividend_ratio NUMERIC(10, 4) DEFAULT 0,
split_ratio NUMERIC(10, 4) DEFAULT 1,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(symbol_id, ex_date)
)
""",
# 期货品种信息表
"futures_products": """
CREATE TABLE IF NOT EXISTS futures_products (
id SERIAL PRIMARY KEY,
product_code VARCHAR(10) NOT NULL UNIQUE,
product_name VARCHAR(50) NOT NULL,
exchange VARCHAR(10) NOT NULL,
product_type VARCHAR(20),
multiplier NUMERIC(10, 2) NOT NULL,
min_price_tick NUMERIC(10, 4) NOT NULL,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
# 期货合约信息表
"futures_contracts": """
CREATE TABLE IF NOT EXISTS futures_contracts (
id SERIAL PRIMARY KEY,
symbol_id VARCHAR(20) NOT NULL UNIQUE,
product_code VARCHAR(10) NOT NULL,
contract_month VARCHAR(6) NOT NULL,
exchange VARCHAR(10) NOT NULL,
list_date DATE,
last_trade_date DATE,
delivery_date DATE,
is_main BOOLEAN DEFAULT FALSE,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
""",
# 交易日历表
"trading_calendar": """
CREATE TABLE IF NOT EXISTS trading_calendar (
id SERIAL PRIMARY KEY,
exchange VARCHAR(10) NOT NULL,
trade_date DATE NOT NULL,
is_trading_day BOOLEAN NOT NULL,
pre_trade_date DATE,
next_trade_date DATE,
UNIQUE(exchange, trade_date)
)
""",
# API Key 存储表(认证增强)
"api_keys": """
CREATE TABLE IF NOT EXISTS api_keys (
id SERIAL PRIMARY KEY,
api_key VARCHAR(64) NOT NULL UNIQUE,
user_id INTEGER,
name VARCHAR(100),
permissions VARCHAR(100),
expires_at TIMESTAMPTZ,
is_active BOOLEAN DEFAULT TRUE,
last_used_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW()
)
""",
}
# ==================== 索引定义 ====================
INDEX_DEFINITIONS = """
-- 股票 K 线索引
CREATE INDEX IF NOT EXISTS idx_stock_klines_1m_symbol_ts ON stock_klines_1m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_5m_symbol_ts ON stock_klines_5m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_15m_symbol_ts ON stock_klines_15m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_30m_symbol_ts ON stock_klines_30m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_1h_symbol_ts ON stock_klines_1h(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_1d_symbol_ts ON stock_klines_1d(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_1w_symbol_ts ON stock_klines_1w(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_stock_klines_1month_symbol_ts ON stock_klines_1month(symbol_id, ts DESC);
-- 期货 K 线索引
CREATE INDEX IF NOT EXISTS idx_futures_klines_1m_symbol_ts ON futures_klines_1m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_5m_symbol_ts ON futures_klines_5m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_15m_symbol_ts ON futures_klines_15m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_30m_symbol_ts ON futures_klines_30m(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_1h_symbol_ts ON futures_klines_1h(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_1d_symbol_ts ON futures_klines_1d(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_1w_symbol_ts ON futures_klines_1w(symbol_id, ts DESC);
CREATE INDEX IF NOT EXISTS idx_futures_klines_1month_symbol_ts ON futures_klines_1month(symbol_id, ts DESC);
-- 辅助表索引
CREATE INDEX IF NOT EXISTS idx_stock_symbols_exchange ON stock_symbols(exchange);
CREATE INDEX IF NOT EXISTS idx_stock_adjust_factors_symbol_date ON stock_adjust_factors(symbol_id, ex_date DESC);
CREATE INDEX IF NOT EXISTS idx_futures_products_exchange ON futures_products(exchange);
CREATE INDEX IF NOT EXISTS idx_futures_contracts_product ON futures_contracts(product_code);
CREATE INDEX IF NOT EXISTS idx_futures_contracts_main ON futures_contracts(product_code, is_main) WHERE is_main = TRUE;
CREATE INDEX IF NOT EXISTS idx_trading_calendar_date ON trading_calendar(exchange, trade_date);
"""
# ==================== TimescaleDB Hypertable 转换 ====================
HYPERTABLE_DEFINITIONS = """
-- 股票 K 线超表转换
SELECT create_hypertable('stock_klines_1m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_5m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_15m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_30m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_1h', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_1d', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_1w', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('stock_klines_1month', 'ts', if_not_exists => TRUE);
-- 期货 K 线超表转换
SELECT create_hypertable('futures_klines_1m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_5m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_15m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_30m', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_1h', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_1d', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_1w', 'ts', if_not_exists => TRUE);
SELECT create_hypertable('futures_klines_1month', 'ts', if_not_exists => TRUE);
"""
async def run_migration_v2_2():
"""
执行 v2.2 数据库迁移
创建所有必要的表结构索引和 TimescaleDB 超表
"""
logger.info("=" * 60)
logger.info("开始 v2.2 数据库迁移...")
logger.info("=" * 60)
db = SessionLocal()
try:
# 1. 创建股票 K 线表
logger.info("\n[1/5] 创建股票 K 线表...")
for table_name, create_sql in STOCK_KLINE_TABLES.items():
logger.info(f" 创建表: {table_name}")
db.execute(text(create_sql))
db.commit()
logger.info(f" ✅ 完成 {len(STOCK_KLINE_TABLES)} 个股票 K 线表")
# 2. 创建期货 K 线表
logger.info("\n[2/5] 创建期货 K 线表...")
for table_name, create_sql in FUTURES_KLINE_TABLES.items():
logger.info(f" 创建表: {table_name}")
db.execute(text(create_sql))
db.commit()
logger.info(f" ✅ 完成 {len(FUTURES_KLINE_TABLES)} 个期货 K 线表")
# 3. 创建辅助表
logger.info("\n[3/5] 创建辅助表...")
for table_name, create_sql in AUXILIARY_TABLES.items():
logger.info(f" 创建表: {table_name}")
db.execute(text(create_sql))
db.commit()
logger.info(f" ✅ 完成 {len(AUXILIARY_TABLES)} 个辅助表")
# 4. 创建索引
logger.info("\n[4/5] 创建索引...")
db.execute(text(INDEX_DEFINITIONS))
db.commit()
logger.info(" ✅ 完成所有索引创建")
# 5. 转换为 TimescaleDB 超表
logger.info("\n[5/5] 转换 TimescaleDB 超表...")
db.execute(text(HYPERTABLE_DEFINITIONS))
db.commit()
logger.info(" ✅ 完成超表转换")
logger.info("\n" + "=" * 60)
logger.info("✅ v2.2 数据库迁移完成!")
logger.info("=" * 60)
# 返回迁移统计
return {
"version": "2.2",
"status": "success",
"tables_created": len(STOCK_KLINE_TABLES) + len(FUTURES_KLINE_TABLES) + len(AUXILIARY_TABLES),
"stock_kline_tables": list(STOCK_KLINE_TABLES.keys()),
"futures_kline_tables": list(FUTURES_KLINE_TABLES.keys()),
"auxiliary_tables": list(AUXILIARY_TABLES.keys()),
"migration_time": datetime.now().isoformat()
}
except Exception as e:
db.rollback()
logger.error(f"❌ 数据库迁移失败: {e}")
raise e
finally:
db.close()
def check_migration_status():
"""
检查迁移状态
Returns:
dict: 各表是否存在的状态
"""
db = SessionLocal()
status = {}
try:
all_tables = (
list(STOCK_KLINE_TABLES.keys()) +
list(FUTURES_KLINE_TABLES.keys()) +
list(AUXILIARY_TABLES.keys())
)
for table_name in all_tables:
result = db.execute(text(f"""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = '{table_name}'
)
""")).first()
status[table_name] = result[0] if result else False
return status
finally:
db.close()
# 命令行执行入口
if __name__ == "__main__":
import asyncio
asyncio.run(run_migration_v2_2())