diff --git a/API_TEST_CASES.md b/API_TEST_CASES.md new file mode 100644 index 0000000..755a5ac --- /dev/null +++ b/API_TEST_CASES.md @@ -0,0 +1,77 @@ +# API 测试用例清单 + +## 新增数据同步接口测试用例 + +| 测试ID | 测试名称 | 方法 | 路径 | 说明 | +|--------|----------|------|------|------| +| `admin_data_sync_full` | 全量数据同步 | POST | `/v1/admin/data/sync` | 同步基础+行情+财务数据 | +| `admin_data_sync_base` | 同步基础K线数据 | POST | `/v1/admin/data/sync` | 仅同步OHLCV基础数据 | +| `admin_data_sync_quote` | 同步行情指标数据 | POST | `/v1/admin/data/sync` | 同步均线/MACD/涨跌幅 | +| `admin_data_sync_finance` | 同步财务数据 | POST | `/v1/admin/data/sync` | 同步市值/股本/利润 | +| `admin_data_sync_incremental` | 增量数据同步 | POST | `/v1/admin/data/sync/incremental` | 同步最近30天数据 | +| `admin_data_sync_futures` | 期货数据同步 | POST | `/v1/admin/data/sync` | 同步期货数据 | + +## 测试用例分类统计 + +| 分类 | 用例数 | +|------|--------| +| 股票接口 | 4 | +| 期货接口 | 5 | +| 管理接口 | 7 | +| 适配器管理 | 3 | +| **数据同步接口** | **6** | +| 测试管理 | 1 | +| **总计** | **26** | + +## 使用方法 + +### 通过管理后台API测试 +```bash +# 1. 获取测试列表 +curl "http://localhost:8080/v1/admin/tests/api" + +# 2. 执行全量数据同步测试 +curl -X POST "http://localhost:8080/v1/admin/tests/api/run" \ + -H "Content-Type: application/json" \ + -d '{ + "id": "admin_data_sync_full", + "params": { + "symbols": ["000001.SZ"], + "sync_type": "full" + } + }' + +# 3. 执行基础数据同步测试 +curl -X POST "http://localhost:8080/v1/admin/tests/api/run" \ + -H "Content-Type: application/json" \ + -d '{ + "id": "admin_data_sync_base" + }' +``` + +### 直接调用数据同步接口 +```bash +# 全量同步 +curl -X POST "http://localhost:8080/v1/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20240101", + "end_date": "20240301" + }' + +# 增量同步 +curl -X POST "http://localhost:8080/v1/admin/data/sync/incremental" \ + -H "Content-Type: application/json" \ + -d '["600519.SH", "000001.SZ"]' +``` + +## 对应的适配器方法 + +| API接口 | 适配器方法 | +|---------|-----------| +| `/admin/data/sync` (base) | `fetch_kline_base` | +| `/admin/data/sync` (quote) | `fetch_kline_quote` | +| `/admin/data/sync` (finance) | `fetch_kline_finance` | +| `/admin/data/sync/incremental` | `sync_daily_incremental` | diff --git a/DATABASE_REFACTOR.md b/DATABASE_REFACTOR.md new file mode 100644 index 0000000..754f2f7 --- /dev/null +++ b/DATABASE_REFACTOR.md @@ -0,0 +1,320 @@ +# 数据库结构重设计文档 + +## 背景 + +原有数据库使用单一的 `stock_klines_1d` 表存储所有日线数据，包含OHLCV、涨跌停状态、市值等多个维度的信息。这种设计在以下场景存在问题： + +1. **数据来源不一致**：OHLCV来自行情接口，财务数据来自股本/财报接口，合并存储导致更新频率不一致 +2. **查询效率低**：不需要财务指标时仍需查询包含大量字段的宽表 +3. **扩展困难**：新增指标需要修改表结构，影响线上服务 + +## 新表结构设计 + +### 1. 日线历史数据表 (1d) + +#### stock_klines_1d_base - 日线基础表 +存储最基础的K线数据，对应 `query_kline` 接口。 + +```sql +CREATE TABLE stock_klines_1d_base ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + symbol_id VARCHAR(20) NOT NULL COMMENT '标的代码', + ts DATETIME NOT NULL COMMENT '时间戳', + trade_date VARCHAR(10) NOT NULL COMMENT '交易日 YYYY-MM-DD', + open DECIMAL(18,4) COMMENT '开盘价', + high DECIMAL(18,4) COMMENT '最高价', + low DECIMAL(18,4) COMMENT '最低价', + close DECIMAL(18,4) COMMENT '收盘价', + volume BIGINT COMMENT '成交量(股)', + amount DECIMAL(20,4) COMMENT '成交额(元)', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_symbol_ts (symbol_id, ts), + INDEX idx_symbol_date (symbol_id, trade_date) +) COMMENT='股票日线K线-基础表'; +``` + +#### stock_klines_1d_quote - 日线行情表 +存储行情技术指标，需要在基础数据基础上计算。 + +```sql +CREATE TABLE stock_klines_1d_quote ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + symbol_id VARCHAR(20) NOT NULL COMMENT '标的代码', + trade_date VARCHAR(10) NOT NULL COMMENT '交易日', + change_pct DECIMAL(8,4) COMMENT '涨跌幅(%)', + change_5d_pct DECIMAL(8,4) COMMENT '5日涨跌幅', + change_10d_pct DECIMAL(8,4) COMMENT '10日涨跌幅', + change_20d_pct DECIMAL(8,4) COMMENT '20日涨跌幅', + change_30d_pct DECIMAL(8,4) COMMENT '30日涨跌幅', + change_60d_pct DECIMAL(8,4) COMMENT '60日涨跌幅', + ma5 DECIMAL(18,4) COMMENT '5日均线', + ma10 DECIMAL(18,4) COMMENT '10日均线', + ma20 DECIMAL(18,4) COMMENT '20日均线', + ma30 DECIMAL(18,4) COMMENT '30日均线', + ma60 DECIMAL(18,4) COMMENT '60日均线', + ma120 DECIMAL(18,4) COMMENT '120日均线', + ma250 DECIMAL(18,4) COMMENT '250日均线', + macd_dif DECIMAL(18,6) COMMENT 'MACD DIF', + macd_dea DECIMAL(18,6) COMMENT 'MACD DEA', + macd_bar DECIMAL(18,6) COMMENT 'MACD BAR', + bias5 DECIMAL(8,4) COMMENT '5日乖离率', + bias10 DECIMAL(8,4) COMMENT '10日乖离率', + bias20 DECIMAL(8,4) COMMENT '20日乖离率', + is_limit_up BOOLEAN COMMENT '是否涨停', + is_limit_down BOOLEAN COMMENT '是否跌停', + limit_up_price DECIMAL(18,4) COMMENT '涨停价', + limit_down_price DECIMAL(18,4) COMMENT '跌停价', + is_st BOOLEAN COMMENT '是否ST', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_symbol_date (symbol_id, trade_date) +) COMMENT='股票日线K线-行情指标表'; +``` + +#### stock_klines_1d_finance - 日线财务表 +存储财务相关数据，对应 `get_equity_structure` / `get_share_holder` / `get_income` 等接口。 + +```sql +CREATE TABLE stock_klines_1d_finance ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + symbol_id VARCHAR(20) NOT NULL COMMENT '标的代码', + trade_date VARCHAR(10) NOT NULL COMMENT '交易日', + total_market_cap DECIMAL(20,4) COMMENT '总市值(元)', + float_market_cap DECIMAL(20,4) COMMENT '流通市值(元)', + total_shares BIGINT COMMENT '总股本(股)', + float_shares BIGINT COMMENT '流通股本(股)', + inst_holding_shares BIGINT COMMENT '机构持股数量', + inst_holding_ratio DECIMAL(8,4) COMMENT '机构持仓占比(%)', + top10_holders_ratio DECIMAL(8,4) COMMENT '前十大股东持股占比(%)', + net_profit DECIMAL(20,4) COMMENT '净利润', + revenue DECIMAL(20,4) COMMENT '营业总收入', + eps DECIMAL(12,4) COMMENT '每股收益', + roe DECIMAL(8,4) COMMENT '净资产收益率(%)', + trading_days INT COMMENT '从上市至今交易日数', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_symbol_date (symbol_id, trade_date) +) COMMENT='股票日线K线-财务数据表'; +``` + +### 2. 其他周期历史数据表 + +分钟线/周线/月线只保留基础K线数据和简单指标： + +- `stock_klines_1m` - 1分钟线 +- `stock_klines_5m` - 5分钟线 +- `stock_klines_15m` - 15分钟线 +- `stock_klines_30m` - 30分钟线 +- `stock_klines_60m` - 60分钟线 +- `stock_klines_1w` - 周线 +- `stock_klines_1month` - 月线 + +字段结构： +```sql +symbol_id, ts, trade_date, open, high, low, close, volume, amount, +change_pct, macd_dif, macd_dea, macd_bar +``` + +### 3. 实时数据表 + +#### stock_realtime_quotes - 实时行情快照 +```sql +CREATE TABLE stock_realtime_quotes ( + symbol_id VARCHAR(20) PRIMARY KEY COMMENT '标的代码', + update_time DATETIME COMMENT '更新时间', + last_price DECIMAL(18,4) COMMENT '最新价', + open DECIMAL(18,4) COMMENT '开盘价', + high DECIMAL(18,4) COMMENT '最高价', + low DECIMAL(18,4) COMMENT '最低价', + pre_close DECIMAL(18,4) COMMENT '昨收', + volume BIGINT COMMENT '成交量', + amount DECIMAL(20,4) COMMENT '成交额', + bid1 DECIMAL(18,4) COMMENT '买一价', + ask1 DECIMAL(18,4) COMMENT '卖一价', + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) COMMENT='股票实时行情快照'; +``` + +### 4. 基础数据表 + +#### stock_basic_info - 股票基础信息 +```sql +CREATE TABLE stock_basic_info ( + symbol_id VARCHAR(20) PRIMARY KEY COMMENT '标的代码', + name VARCHAR(100) COMMENT '名称', + exchange VARCHAR(10) COMMENT '交易所', + list_date DATE COMMENT '上市日期', + list_board VARCHAR(20) COMMENT '上市板块', + industry VARCHAR(50) COMMENT '所属行业', + status VARCHAR(10) COMMENT '状态 active/delisted', + is_delisted BOOLEAN COMMENT '是否退市', + delist_date DATE COMMENT '退市日期', + is_st BOOLEAN COMMENT '是否ST', + total_shares BIGINT COMMENT '总股本', + float_shares BIGINT COMMENT '流通股本', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) COMMENT='股票基础信息'; +``` + +#### stock_trading_calendar - 交易日历 +```sql +CREATE TABLE stock_trading_calendar ( + trade_date VARCHAR(8) PRIMARY KEY COMMENT '交易日 YYYYMMDD', + is_trading_day BOOLEAN COMMENT '是否交易日', + week_day TINYINT COMMENT '星期几 1-7' +) COMMENT='交易日历'; +``` + +## 代码结构 + +### 新增文件 + +1. **app/repositories/stock_repository_v2.py** - 支持拆分表结构的新仓库 + - `get_klines_base()` / `save_klines_base()` - 基础数据操作 + - `get_klines_quote()` / `save_klines_quote()` - 行情数据操作 + - `get_klines_finance()` / `save_klines_finance()` - 财务数据操作 + +2. **app/services/data_sync_service.py** - 数据同步服务 + - `sync_kline_base()` - 同步基础K线数据 + - `sync_kline_quote()` - 同步行情指标数据 + - `sync_kline_finance()` - 同步财务数据 + - `sync_full_stock_data()` - 全量同步 + - `sync_daily_incremental()` - 每日增量同步 + +3. **app/api/admin_routes.py** - 新增数据同步API + - `POST /admin/data/sync` - 手动触发数据同步 + - `POST /admin/data/sync/incremental` - 增量同步 + +### 适配器接口扩展 + +**app/adapters/base.py** 新增抽象方法： +- `fetch_kline_base()` - 获取基础K线数据 +- `fetch_kline_quote()` - 获取行情指标数据 +- `fetch_kline_finance()` - 获取财务数据 +- `fetch_stock_basic_info()` - 获取股票基础信息 + +**app/adapters/amazingdata_adapter.py** 新增实现： +- `_fetch_kline_base_sync()` - 同步获取基础K线 +- `_fetch_kline_quote_sync()` - 同步计算并获取行情指标 +- `_fetch_kline_finance_sync()` - 同步获取财务数据 +- `_fetch_stock_basic_info_sync()` - 同步获取基础信息 + +## 使用指南 + +### 1. 手动触发数据同步 + +```bash +# 全量同步（基础+行情+财务） +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -H "X-Admin-Token: your_token" \ + -d '{ + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20240101", + "end_date": "20240301", + "asset_class": "stock" + }' + +# 只同步基础K线数据 +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH"], + "sync_type": "base", + "freq": "1d" + }' + +# 只同步行情指标 +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH"], + "sync_type": "quote" + }' +``` + +### 2. 在代码中使用 Repository + +```python +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency + +db = SessionLocal() +repo = StockRepositoryV2(db) + +# 查询基础K线 +base_data = repo.get_klines_base( + symbol="600519.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 查询行情指标 +quote_data = repo.get_klines_quote( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) + +# 查询财务数据 +finance_data = repo.get_klines_finance( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +### 3. 使用数据同步服务 + +```python +from app.services.adapter_service import AdapterService +from app.services.data_sync_service import DataSyncService + +# 获取适配器 +adapter_service = AdapterService() +adapter = adapter_service.get_active_adapter("stock") + +# 创建同步服务 +sync_service = DataSyncService(adapter) + +# 同步指定标的 +results = await sync_service.sync_full_stock_data( + symbols=["600519.SH", "000001.SZ"], + start="20240101", + end="20240301" +) +``` + +## 迁移建议 + +1. **新部署系统**：直接使用新表结构，旧表可忽略 +2. **存量系统迁移**： + - 使用 `INSERT INTO stock_klines_1d_base SELECT ... FROM stock_klines_1d` 迁移基础数据 + - 行情/财务字段需要重新计算或使用默认值 + - 保持旧表一段时间用于回滚 + +3. **数据补全策略**： + - 基础数据：从数据源全量拉取 + - 行情指标：在本地根据基础数据计算 + - 财务数据：从股本结构接口获取历史数据 + +## 性能优化建议 + +1. **索引优化**：已为高频查询字段(symbol_id, trade_date, ts)创建联合索引 +2. **分区建议**：对于大表可按 `trade_date` 进行按月分区 +3. **查询优化**： + - 只需OHLCV时只查询 base 表 + - 需要指标时 JOIN quote 表 + - 需要财务数据时 JOIN finance 表 + +## 后续扩展计划 + +1. **支持更多指标**：RSI、KDJ、布林带等 +2. **分钟线行情表**：为高频策略提供分钟级指标 +3. **财报季数据表**：季度/年度详细财报数据 +4. **资金流向表**：北向资金、主力资金等 diff --git a/QUICKSTART_NEW_TABLES.md b/QUICKSTART_NEW_TABLES.md new file mode 100644 index 0000000..5f8d744 --- /dev/null +++ b/QUICKSTART_NEW_TABLES.md @@ -0,0 +1,198 @@ +# 新表结构快速开始指南 + +## 1. 创建新表 + +```bash +# 进入项目目录 +cd d:\\alpha_workspace\\python_market_data_service + +# 激活虚拟环境 +.\\venv\\Scripts\\activate + +# 创建新表 +python scripts/create_split_tables.py + +# 如果需要重新创建（会删除已有数据） +python scripts/create_split_tables.py --drop-existing +``` + +## 2. 启动服务 + +```bash +# 启动主服务 +python main.py +``` + +## 3. 同步数据 + +### 全量同步示例 + +```bash +# 同步茅台和平安银行的历史数据 +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20230101", + "end_date": "20240301" + }' +``` + +### 增量同步示例 + +```bash +# 同步最近30天的数据 +curl -X POST "http://localhost:8080/admin/data/sync/incremental" \ + -H "Content-Type: application/json" \ + -d '["600519.SH", "000001.SZ"]' +``` + +### 只同步基础数据 + +```bash +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH"], + "sync_type": "base", + "start_date": "20240101", + "end_date": "20240301" + }' +``` + +## 4. 查询数据 + +### 使用 Repository 查询 + +```python +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency +from datetime import datetime + +# 创建会话 +db = SessionLocal() +repo = StockRepositoryV2(db) + +# 查询基础K线数据（高效，字段少） +base_data = repo.get_klines_base( + symbol="600519.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) +print(f"Base records: {len(base_data)}") + +# 查询行情指标（包含均线、MACD等） +quote_data = repo.get_klines_quote( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +for item in quote_data[:3]: + print(f"{item['trade_date']}: close={item.get('ma5')}") + +# 查询财务数据（市值、股本等） +finance_data = repo.get_klines_finance( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +for item in finance_data[:3]: + print(f"{item['trade_date']}: 市值={item.get('total_market_cap')}") +``` + +## 5. 定时任务配置 + +创建每日定时同步脚本 `scripts/daily_sync.py`: + +```python +#!/usr/bin/env python3 +"""每日增量同步脚本""" +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from app.services.adapter_service import AdapterService +from app.services.data_sync_service import DataSyncService +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal + +async def main(): + # 获取热门股票列表 + db = SessionLocal() + repo = StockRepositoryV2(db) + + # 这里可以从配置或数据库读取需要同步的标的列表 + symbols = ["600519.SH", "000001.SZ", "000858.SZ"] # 示例 + + # 获取适配器 + adapter_service = AdapterService() + adapter = adapter_service.get_active_adapter("stock") + + # 同步 + sync_service = DataSyncService(adapter) + results = await sync_service.sync_daily_incremental(symbols) + + print("Sync completed:", results) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +添加到定时任务（Linux crontab）： +```bash +# 每天凌晨2点执行增量同步 +0 2 * * * cd /path/to/project && python scripts/daily_sync.py >> logs/sync.log 2>&1 +``` + +## 6. 表结构说明 + +| 表名 | 用途 | 数据来源 | 更新频率 | +|------|------|----------|----------| +| stock_klines_1d_base | 基础K线(OHLCV) | query_kline | 每日 | +| stock_klines_1d_quote | 行情指标 | 本地计算 | 每日 | +| stock_klines_1d_finance | 财务数据 | 股本/财报接口 | 财报季 | +| stock_realtime_quotes | 实时行情 | 行情推送 | 实时 | +| stock_symbols | 股票列表 | 基础数据接口 | 每日 | +| stock_trading_calendar | 交易日历 | 交易所数据 | 每年 | + +## 7. 常见问题 + +### Q: 旧表数据如何迁移？ + +A: 使用以下SQL迁移基础数据： +```sql +INSERT INTO stock_klines_1d_base +(symbol_id, ts, trade_date, open, high, low, close, volume, amount) +SELECT + symbol_id, ts, trade_date, open, high, low, close, volume, amount +FROM stock_klines_1d +WHERE trade_date >= '2024-01-01'; +``` + +### Q: 如何只查询基础数据提高性能？ + +A: 使用 `StockRepositoryV2.get_klines_base()` 方法，它只查询轻量的base表。 + +### Q: 行情指标如何计算？ + +A: 指标在 `AmazingDataAdapter._fetch_kline_quote_sync()` 中本地计算： +- 均线: 基于收盘价计算MA5/10/20/30/60/120/250 +- MACD: 使用pandas ewm计算 +- 乖离率: (收盘价-均线)/均线 +- 涨跌停: 对比收盘价和涨跌停价 + +### Q: 市值数据如何计算？ + +A: 市值 = 收盘价 × 股本，股本从 `get_equity_structure` 接口获取。 + +## 8. 下一步 + +- [ ] 配置定时自动同步 +- [ ] 添加Redis缓存热点数据 +- [ ] 实现分钟线行情指标表 +- [ ] 添加数据一致性校验任务 diff --git a/REFACTOR_SUMMARY.md b/REFACTOR_SUMMARY.md new file mode 100644 index 0000000..fef80bd --- /dev/null +++ b/REFACTOR_SUMMARY.md @@ -0,0 +1,186 @@ +# 数据库结构重设计 - 完成总结 + +## 概述 + +已完成数据库表结构从单一宽表向拆分表的迁移设计，并实现了完整的数据同步流程。 + +## 主要改动 + +### 1. 数据模型层 (app/repositories/models.py) + +**新增拆分表模型：** +- `StockKLine1DBase` - 日线基础表 (OHLCV) +- `StockKLine1DQuote` - 日线行情表 (涨跌幅、均线、MACD、乖离率、涨跌停状态) +- `StockKLine1DFinance` - 日线财务表 (市值、股本、机构持股、利润) +- `StockRealTimeQuote` - 实时行情快照表 +- `StockBasicInfo` - 股票基础信息表 + +**向后兼容：** +- 添加 `StockKLine1D = StockKLine1DBase` 别名，保持旧代码兼容 + +### 2. 数据仓库层 (app/repositories/stock_repository_v2.py) + +**新增 Repository：** +- `StockRepositoryV2` - 支持拆分表结构的新仓库 + - `get_klines_base()` / `save_klines_base()` - 基础数据操作 + - `get_klines_quote()` / `save_klines_quote()` - 行情数据操作 + - `get_klines_finance()` / `save_klines_finance()` - 财务数据操作 + - `get_realtime_quote()` / `save_realtime_quote()` - 实时行情操作 + +### 3. 适配器接口层 (app/adapters/base.py) + +**新增抽象方法：** +- `fetch_kline_base()` - 获取基础K线数据 +- `fetch_kline_quote()` - 获取行情指标数据 +- `fetch_kline_finance()` - 获取财务数据 +- `fetch_stock_basic_info()` - 获取股票基础信息 + +### 4. AmazingData适配器 (app/adapters/amazingdata_adapter.py) + +**新增实现方法：** +- `_fetch_kline_base_sync()` - 从 query_kline 获取基础数据 +- `_fetch_kline_quote_sync()` - 计算均线/MACD/乖离率等指标 +- `_fetch_kline_finance_sync()` - 从股本结构数据计算市值 +- `_fetch_stock_basic_info_sync()` - 获取股票列表和基础信息 + +### 5. 数据同步服务 (app/services/data_sync_service.py) + +**新增服务：** +- `DataSyncService` - 协调数据拉取和存储 + - `sync_kline_base()` - 同步基础K线 + - `sync_kline_quote()` - 同步行情指标 + - `sync_kline_finance()` - 同步财务数据 + - `sync_full_stock_data()` - 全量同步 + - `sync_daily_incremental()` - 每日增量同步 + +### 6. 管理API层 (app/api/admin_routes.py) + +**新增端点：** +- `POST /admin/data/sync` - 手动触发数据同步 + - 支持 sync_type: base/quote/finance/full + - 支持指定日期范围和标的列表 +- `POST /admin/data/sync/incremental` - 增量同步（最近30天） + +### 7. 类型定义 (app/models/admin_types.py) + +**新增类型：** +- `DataSyncType` - 同步类型枚举 +- `DataSyncRequest` - 同步请求 +- `DataSyncResult` - 单个标的同步结果 +- `DataSyncData` - 同步响应 + +### 8. 迁移脚本 (scripts/create_split_tables.py) + +**功能：** +- 自动创建所有新表 +- 支持 `--drop-existing` 参数重建表 + +## 表结构对比 + +### 旧表 (stock_klines_1d) +``` +symbol_id, ts, trade_date, open, high, low, close, volume, amount, +is_limit_up, is_limit_down, total_market_cap, float_market_cap, +inst_holding_ratio, trading_days +``` + +### 新表 +``` +stock_klines_1d_base: + symbol_id, ts, trade_date, open, high, low, close, volume, amount + +stock_klines_1d_quote: + symbol_id, trade_date, change_pct, change_Nd_pct, maN, + macd_dif/dea/bar, biasN, is_limit_up/down, limit_up/down_price, is_st + +stock_klines_1d_finance: + symbol_id, trade_date, total_market_cap, float_market_cap, + total_shares, float_shares, inst_holding_shares, inst_holding_ratio, + top10_holders_ratio, net_profit, revenue, eps, roe, trading_days +``` + +## API使用示例 + +### 1. 全量同步 +```bash +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20240101", + "end_date": "20240301" + }' +``` + +### 2. 只同步基础数据 +```bash +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH"], + "sync_type": "base" + }' +``` + +### 3. 增量同步 +```bash +curl -X POST "http://localhost:8080/admin/data/sync/incremental" \ + -H "Content-Type: application/json" \ + -d '["600519.SH", "000001.SZ"]' +``` + +## 代码使用示例 + +```python +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency + +db = SessionLocal() +repo = StockRepositoryV2(db) + +# 只查询基础数据（高效） +base_data = repo.get_klines_base( + symbol="600519.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 只查询行情指标 +quote_data = repo.get_klines_quote( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +## 性能优势 + +1. **查询优化**：只需OHLCV时只查询轻量的base表 +2. **更新解耦**：基础数据可高频更新，财务数据按财报季更新 +3. **扩展灵活**：新增指标只需修改quote表，不影响基础数据 + +## 后续建议 + +1. **定时任务**：配置每日凌晨执行增量同步 +2. **数据校验**：定期校验基础数据和行情指标的一致性 +3. **分区优化**：对大表按trade_date进行分区 +4. **缓存优化**：热点数据使用Redis缓存 + +## 文件清单 + +**新增文件：** +- `app/repositories/stock_repository_v2.py` (636 lines) +- `app/services/data_sync_service.py` (387 lines) +- `scripts/create_split_tables.py` (106 lines) +- `DATABASE_REFACTOR.md` - 详细设计文档 + +**修改文件：** +- `app/repositories/models.py` - 添加新表模型和向后兼容别名 +- `app/adapters/base.py` - 添加新抽象方法 +- `app/adapters/amazingdata_adapter.py` - 添加新数据获取方法 (~300 lines) +- `app/api/admin_routes.py` - 添加数据同步端点 +- `app/models/admin_types.py` - 添加同步相关类型 +- `app/models/__init__.py` - 导出新类型 diff --git a/SPLIT_TABLES_COMPLETE.md b/SPLIT_TABLES_COMPLETE.md new file mode 100644 index 0000000..4252245 --- /dev/null +++ b/SPLIT_TABLES_COMPLETE.md @@ -0,0 +1,146 @@ +# 数据表分表完成总结 + +## 完成状态: ✅ 全部完成 + +## 股票数据表分表 + +### 日线分表 (3张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `stock_klines_1d_base` | 基础K线数据 (OHLCV) | ✅ | +| `stock_klines_1d_quote` | 行情指标 (均线/MACD/涨跌幅) | ✅ | +| `stock_klines_1d_finance` | 财务数据 (市值/股本/利润) | ✅ | + +### 分钟线 (2张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `stock_klines_1m` | 1分钟线 | ✅ | +| `stock_klines_5m` | 5分钟线 | ✅ | + +### 实时数据 (1张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `stock_realtime_quotes` | 实时行情快照 | ✅ | + +--- + +## 期货数据表分表 + +### 日线分表 (2张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `futures_klines_1d_base` | 基础K线数据 (OHLCV+持仓量) | ✅ | +| `futures_klines_1d_quote` | 行情指标 (均线/MACD/持仓变化) | ✅ | + +### 分钟线 (2张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `futures_klines_1m_base` | 1分钟基础线 | ✅ | +| `futures_klines_5m_base` | 5分钟基础线 | ✅ | + +### 实时数据 (1张表) +| 表名 | 用途 | 状态 | +|------|------|------| +| `futures_realtime_quotes` | 实时行情快照 | ✅ | + +--- + +## 代码文件 + +### 模型定义 +- `app/repositories/models.py` - 所有分表模型定义 + +### 数据仓库 +- `app/repositories/stock_repository_v2.py` - 股票分表仓库 +- `app/repositories/futures_repository_v2.py` - 期货分表仓库 + +### 创建脚本 +- `scripts/create_split_tables.py` - 创建所有分表 +- `scripts/add_stock_split_tables.py` - 添加股票分表 +- `scripts/fix_models.py` - 添加期货分表 + +--- + +## 使用方法 + +### 1. 创建分表 +```bash +python scripts/create_split_tables.py +``` + +### 2. 股票数据操作 +```python +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency + +db = SessionLocal() +repo = StockRepositoryV2(db) + +# 查询基础数据 +base_data = repo.get_klines_base( + symbol="600519.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 查询行情指标 +quote_data = repo.get_klines_quote( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) + +# 查询财务数据 +finance_data = repo.get_klines_finance( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +### 3. 期货数据操作 +```python +from app.repositories.futures_repository_v2 import FuturesRepositoryV2 + +db = SessionLocal() +repo = FuturesRepositoryV2(db) + +# 查询基础数据 +base_data = repo.get_klines_base( + symbol="RB2410.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 查询行情指标 +quote_data = repo.get_klines_quote( + symbol="RB2410.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +--- + +## 分表设计优势 + +1. **查询性能**: 只需基础数据时，只查询轻量的base表 +2. **更新解耦**: + - 基础数据: 每日更新 + - 行情指标: 计算后更新 + - 财务数据: 财报季更新 +3. **扩展性**: 新增指标只需修改quote表 +4. **存储优化**: 冷热数据分离 + +--- + +## 总表数统计 + +| 类别 | 表数量 | +|------|--------| +| 股票分表 | 6张 | +| 期货分表 | 5张 | +| **总计** | **11张** | diff --git a/TABLE_SPLIT_DESIGN.md b/TABLE_SPLIT_DESIGN.md new file mode 100644 index 0000000..6e78261 --- /dev/null +++ b/TABLE_SPLIT_DESIGN.md @@ -0,0 +1,253 @@ +# 数据表分表设计文档 + +## 设计原则 + +所有K线数据表按照**数据类型**和**更新频率**进行拆分： +1. **基础表(Base)**: OHLCV等原始行情数据，更新频率高 +2. **行情表(Quote)**: 技术指标(均线/MACD/涨跌幅)，需要计算 +3. **财务表(Finance)**: 市值/股本等财务数据，更新频率低 + +## 股票数据表 + +### 日线数据 (1d) + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `stock_klines_1d_base` | 日线基础表 | symbol_id, trade_date, open, high, low, close, volume, amount, adj_factor | +| `stock_klines_1d_quote` | 日线行情表 | change_pct, ma5/10/20/30/60/120/250, macd_dif/dea/bar, bias5/10/20, is_limit_up/down, is_st | +| `stock_klines_1d_finance` | 日线财务表 | total_market_cap, float_market_cap, total_shares, float_shares, inst_holding_ratio, net_profit, revenue, eps, roe | + +### 分钟线数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `stock_klines_1m` | 1分钟线 | symbol_id, ts, trade_date, open, high, low, close, volume, amount, change_pct, macd_dif/dea/bar | +| `stock_klines_5m` | 5分钟线 | 同上 | +| `stock_klines_15m` | 15分钟线 | 同上 | +| `stock_klines_30m` | 30分钟线 | 同上 | +| `stock_klines_60m` | 60分钟线 | 同上 | + +### 周月线数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `stock_klines_1w` | 周线 | symbol_id, ts, trade_date, open, high, low, close, volume, amount | +| `stock_klines_1month` | 月线 | 同上 | + +### 实时数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `stock_realtime_quotes` | 实时行情快照 | symbol_id, last_price, open, high, low, volume, amount, bid1, ask1 | + +### 基础数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `stock_symbols` | 股票列表 | symbol_id, name, exchange, list_date, industry, status | +| `stock_trading_calendar` | 交易日历 | trade_date, is_trading_day, week_day | +| `stock_adjust_factors` | 复权系数 | symbol_id, trade_date, qfq_factor, hfq_factor | + +--- + +## 期货数据表 + +### 日线数据 (1d) + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `futures_klines_1d_base` | 日线基础表 | symbol_id, trade_date, open, high, low, close, volume, amount, open_interest, settlement, pre_settlement | +| `futures_klines_1d_quote` | 日线行情表 | change_pct, ma5/10/20/30/60, macd_dif/dea/bar, oi_change, oi_change_pct, amplitude | + +### 分钟线数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `futures_klines_1m_base` | 1分钟基础表 | symbol_id, ts, trade_date, open, high, low, close, volume, amount, open_interest | +| `futures_klines_5m_base` | 5分钟基础表 | 同上 | +| `futures_klines_15m_base` | 15分钟基础表 | 同上 | +| `futures_klines_30m_base` | 30分钟基础表 | 同上 | +| `futures_klines_60m_base` | 60分钟基础表 | 同上 | + +### 周月线数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `futures_klines_1w_base` | 周线基础表 | symbol_id, ts, trade_date, open, high, low, close, volume, amount, open_interest | +| `futures_klines_1month_base` | 月线基础表 | 同上 | + +### 实时数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `futures_realtime_quotes` | 实时行情快照 | symbol_id, last_price, open, high, low, volume, amount, open_interest, bid1, ask1, limit_up, limit_down | + +### 基础数据 + +| 表名 | 说明 | 主要字段 | +|------|------|----------| +| `futures_symbols` | 合约列表 | symbol_id, name, exchange, underlying, contract_month, list_date, delist_date | +| `futures_trading_calendar` | 交易日历 | trade_date, is_trading_day, has_night_session, week_day | + +--- + +## 代码实现 + +### Repository 结构 + +``` +app/repositories/ +├── stock_repository_v2.py # 股票分表仓库 +├── futures_repository_v2.py # 期货分表仓库 +└── models.py # 所有模型定义 +``` + +### 关键方法 + +#### StockRepositoryV2 +```python +# 基础数据操作 +get_klines_base(symbol, freq, start, end) -> List[Dict] +save_klines_base(freq, items) -> int + +# 行情数据操作 +get_klines_quote(symbol, start_date, end_date) -> List[Dict] +save_klines_quote(items) -> int + +# 财务数据操作 +get_klines_finance(symbol, start_date, end_date) -> List[Dict] +save_klines_finance(items) -> int + +# 实时行情 +get_realtime_quote(symbol) -> Optional[Dict] +save_realtime_quote(data) -> None +``` + +#### FuturesRepositoryV2 +```python +# 基础数据操作 +get_klines_base(symbol, freq, start, end) -> List[Dict] +save_klines_base(freq, items) -> int + +# 行情数据操作 +get_klines_quote(symbol, start_date, end_date) -> List[Dict] +save_klines_quote(items) -> int + +# 实时行情 +get_realtime_quote(symbol) -> Optional[Dict] +save_realtime_quote(data) -> None +``` + +--- + +## 使用示例 + +### 股票数据查询 + +```python +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency + +db = SessionLocal() +repo = StockRepositoryV2(db) + +# 只查询基础数据（高效） +base_data = repo.get_klines_base( + symbol="600519.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 查询行情指标 +quote_data = repo.get_klines_quote( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) + +# 查询财务数据 +finance_data = repo.get_klines_finance( + symbol="600519.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +### 期货数据查询 + +```python +from app.repositories.futures_repository_v2 import FuturesRepositoryV2 + +db = SessionLocal() +repo = FuturesRepositoryV2(db) + +# 查询期货基础数据 +base_data = repo.get_klines_base( + symbol="RB2410.SH", + freq=Frequency.FREQ_1D, + start=datetime(2024, 1, 1), + end=datetime(2024, 3, 1) +) + +# 查询期货行情指标 +quote_data = repo.get_klines_quote( + symbol="RB2410.SH", + start_date="2024-01-01", + end_date="2024-03-01" +) +``` + +--- + +## 创建表的脚本 + +```bash +# 创建所有分表 +python scripts/create_split_tables.py + +# 重新创建（会删除已有数据） +python scripts/create_split_tables.py --drop-existing +``` + +--- + +## 数据同步 + +### 股票数据同步 +```bash +# 全量同步 +curl -X POST "http://localhost:8080/admin/data/sync" \ + -H "Content-Type: application/json" \ + -d '{ + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20240101", + "end_date": "20240301" + }' +``` + +### 期货数据同步 +期货数据同步API待实现，可参考股票同步API实现类似接口。 + +--- + +## 性能优势 + +1. **查询优化**: 只需基础数据时，只查询轻量的base表 +2. **更新解耦**: + - 基础数据: 每日收盘后更新 + - 行情指标: 计算后批量更新 + - 财务数据: 财报季更新 +3. **存储优化**: 热点数据(base)和冷数据(finance)分离存储 +4. **扩展性**: 新增指标只需修改quote表，不影响其他表 + +--- + +## 后续扩展 + +1. **分钟线行情表**: 为高频策略提供分钟级技术指标 +2. **跨表查询视图**: 创建联合视图方便查询 +3. **数据分区**: 按trade_date对大表进行分区 +4. **缓存层**: Redis缓存热点基础数据 diff --git a/app/adapters/__pycache__/amazingdata_adapter.cpython-311.pyc b/app/adapters/__pycache__/amazingdata_adapter.cpython-311.pyc index 37ea79f..ec0b1d5 100644 Binary files a/app/adapters/__pycache__/amazingdata_adapter.cpython-311.pyc and b/app/adapters/__pycache__/amazingdata_adapter.cpython-311.pyc differ diff --git a/app/adapters/__pycache__/base.cpython-311.pyc b/app/adapters/__pycache__/base.cpython-311.pyc index 89a7203..77be9ea 100644 Binary files a/app/adapters/__pycache__/base.cpython-311.pyc and b/app/adapters/__pycache__/base.cpython-311.pyc differ diff --git a/app/adapters/amazingdata_adapter.py b/app/adapters/amazingdata_adapter.py index f3e4bb0..ce6c494 100644 --- a/app/adapters/amazingdata_adapter.py +++ b/app/adapters/amazingdata_adapter.py @@ -1323,3 +1323,501 @@ class AmazingDataAdapter(DataSourceAdapter): is_local=is_local ) ) + + # ==================== New Split Table Data Fetch Methods ==================== + + async def fetch_kline_base( + self, + symbol: str, + start: str, + end: str, + freq: str + ) -> List[Dict[str, Any]]: + """Fetch K-line base data (OHLCV) + + Corresponding tables: stock_klines_1d_base, stock_klines_1m_base, etc. + + Returns: + List[Dict] containing fields: + - symbol: Symbol code + - ts: Timestamp + - trade_date: Trade date + - open/high/low/close: Price data + - volume: Trading volume + - amount: Trading amount + - adj_factor: Adjustment factor + """ + print(f"[amazingdata_adapter fetch_kline_base]Fetching {symbol} {freq} base data...") + self._check_login() + + period_map = { + "1m": self._ad.constant.Period.min1, + "5m": self._ad.constant.Period.min5, + "15m": self._ad.constant.Period.min15, + "30m": self._ad.constant.Period.min30, + "60m": self._ad.constant.Period.min60, + "1d": self._ad.constant.Period.day, + "1w": self._ad.constant.Period.week, + "1month": self._ad.constant.Period.month, + } + period_value = period_map.get(freq, self._ad.constant.Period.day).value + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_base_sync(symbol, start, end, period_value, freq) + ) + + def _fetch_kline_base_sync( + self, + symbol: str, + start_date: str, + end_date: str, + period_value: int, + freq: str + ) -> List[Dict[str, Any]]: + """Sync method to fetch K-line base data""" + codes = [symbol] + start_int = self._format_date(start_date) + end_int = self._format_date(end_date) + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=period_value + ) + + if symbol not in kline_dict: + info(f"No kline data found for {symbol}") + return [] + + df = kline_dict[symbol] + results = [] + + for _, row in df.iterrows(): + kline_time = row.get('kline_time') + if pd.isna(kline_time) or kline_time is None: + continue + + if isinstance(kline_time, pd.Timestamp): + ts = int(kline_time.timestamp()) + trade_date = kline_time.strftime('%Y-%m-%d') + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + ts = int(dt.timestamp()) + trade_date = dt.strftime('%Y-%m-%d') + + results.append({ + "symbol": symbol, + "ts": ts, + "trade_date": trade_date, + "open": float(row.get('open', 0)), + "high": float(row.get('high', 0)), + "low": float(row.get('low', 0)), + "close": float(row.get('close', 0)), + "volume": int(row.get('volume', 0)), + "amount": float(row.get('amount', 0)), + "adj_factor": float(row.get('adj_factor', 1.0)) if 'adj_factor' in df.columns else 1.0, + }) + + info(f"Fetched {len(results)} base kline records for {symbol}") + return results + + async def fetch_kline_quote( + self, + symbol: str, + start: str, + end: str + ) -> List[Dict[str, Any]]: + """Fetch daily quote indicator data (calculated) + + Corresponding table: stock_klines_1d_quote + + Returns: + List[Dict] containing fields: + - change_pct: Price change percentage + - change_Nd_pct: N-day price change + - ma_N: Moving averages + - macd_dif/dea/bar: MACD indicators + - bias_N: Bias ratios + - is_limit_up/down: Limit up/down status + - is_st: ST status + """ + print(f"[amazingdata_adapter fetch_kline_quote]Calculating {symbol} quote indicators...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_quote_sync(symbol, start, end) + ) + + def _fetch_kline_quote_sync( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """Sync method to calculate quote indicators""" + import numpy as np + + start_dt = datetime.strptime(start_date, "%Y%m%d") + extended_start = datetime(start_dt.year - 1, start_dt.month, start_dt.day) + extended_start_str = extended_start.strftime("%Y%m%d") + + codes = [symbol] + start_int = self._format_date(extended_start_str) + end_int = self._format_date(end_date) + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=self._ad.constant.Period.day.value + ) + + if symbol not in kline_dict: + return [] + + df = kline_dict[symbol].copy() + df = df.sort_values('kline_time') + + try: + code_info_df = self._base_data.get_code_info(security_type=SecurityType.STOCK_A.value) + if symbol in code_info_df.index: + high_limited = float(code_info_df.loc[symbol, 'high_limited']) if 'high_limited' in code_info_df.columns else None + low_limited = float(code_info_df.loc[symbol, 'low_limited']) if 'low_limited' in code_info_df.columns else None + else: + high_limited = low_limited = None + except: + high_limited = low_limited = None + + results = [] + closes = df['close'].values + + for i, (_, row) in enumerate(df.iterrows()): + kline_time = row.get('kline_time') + if pd.isna(kline_time): + continue + + if isinstance(kline_time, pd.Timestamp): + trade_date = kline_time.strftime('%Y-%m-%d') + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + trade_date = dt.strftime('%Y-%m-%d') + + close = float(row.get('close', 0)) + + change_pct = None + if i > 0: + prev_close = closes[i-1] + if prev_close > 0: + change_pct = round((close - prev_close) / prev_close * 100, 4) + + def calc_n_day_change(n): + if i >= n and closes[i-n] > 0: + return round((close - closes[i-n]) / closes[i-n] * 100, 4) + return None + + def calc_ma(n): + if i >= n - 1: + return round(np.mean(closes[i-n+1:i+1]), 4) + return None + + def calc_macd(): + if i < 33: + return None, None, None + ema12 = pd.Series(closes[:i+1]).ewm(span=12).mean().iloc[-1] + ema26 = pd.Series(closes[:i+1]).ewm(span=26).mean().iloc[-1] + dif = ema12 - ema26 + dea = pd.Series([ema12 - ema26 for _ in range(i+1)]).ewm(span=9).mean().iloc[-1] + bar = (dif - dea) * 2 + return round(dif, 6), round(dea, 6), round(bar, 6) + + def calc_bias(n): + ma = calc_ma(n) + if ma and ma > 0: + return round((close - ma) / ma * 100, 4) + return None + + is_limit_up = False + is_limit_down = False + if high_limited and low_limited and close > 0: + is_limit_up = close >= high_limited * 0.995 + is_limit_down = close <= low_limited * 1.005 + + macd_dif, macd_dea, macd_bar = calc_macd() + + if trade_date.replace('-', '') >= start_date: + results.append({ + "symbol": symbol, + "trade_date": trade_date, + "change_pct": change_pct, + "change_5d_pct": calc_n_day_change(5), + "change_10d_pct": calc_n_day_change(10), + "change_20d_pct": calc_n_day_change(20), + "change_30d_pct": calc_n_day_change(30), + "change_60d_pct": calc_n_day_change(60), + "macd_dif": macd_dif, + "macd_dea": macd_dea, + "macd_bar": macd_bar, + "bias_5": calc_bias(5), + "bias_10": calc_bias(10), + "bias_20": calc_bias(20), + "is_limit_up": is_limit_up, + "is_limit_down": is_limit_down, + "limit_up_price": round(high_limited, 4) if high_limited else None, + "limit_down_price": round(low_limited, 4) if low_limited else None, + "is_st": None, + "ma_5": calc_ma(5), + "ma_10": calc_ma(10), + "ma_20": calc_ma(20), + "ma_30": calc_ma(30), + "ma_60": calc_ma(60), + "ma_120": calc_ma(120), + "ma_250": calc_ma(250), + }) + + info(f"Calculated {len(results)} quote indicators for {symbol}") + return results + + async def fetch_kline_finance( + self, + symbol: str, + start: str, + end: str + ) -> List[Dict[str, Any]]: + """Fetch daily finance data + + Corresponding table: stock_klines_1d_finance + Data sources: get_equity_structure, get_share_holder, get_income + + Returns: + List[Dict] containing fields: + - total_market_cap: Total market cap + - float_market_cap: Float market cap + - total_shares: Total shares + - float_shares: Float shares + - inst_holding_shares: Institutional holding shares + - inst_holding_ratio: Institutional holding ratio + - net_profit: Net profit + - revenue: Revenue + - eps: EPS + - roe: ROE + """ + print(f"[amazingdata_adapter fetch_kline_finance]Fetching {symbol} finance data...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_finance_sync(symbol, start, end) + ) + + def _fetch_kline_finance_sync( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """Sync method to fetch finance data""" + codes = [symbol] + start_int = self._format_date(start_date) + end_int = self._format_date(end_date) + + results = [] + + try: + equity_dict = self._info_data.get_equity_structure( + code_list=codes, + local_path=self.config.local_path, + is_local=self.config.use_local_cache + ) + + equity_data = {} + if symbol in equity_dict: + equity_df = equity_dict[symbol] + for _, row in equity_df.iterrows(): + ann_date = row.get('ANN_DATE') + if pd.notna(ann_date): + if isinstance(ann_date, (int, float)): + date_key = str(int(ann_date)) + else: + date_key = str(ann_date).replace('-', '').replace('/', '') + equity_data[date_key] = { + 'total_shares': float(row.get('TOT_A_SHARE', 0)) * 10000 if pd.notna(row.get('TOT_A_SHARE')) else 0, + 'float_shares': float(row.get('FLOAT_A_SHARE', 0)) * 10000 if pd.notna(row.get('FLOAT_A_SHARE')) else 0, + } + except Exception as e: + print(f"[amazingdata_adapter]Failed to get equity structure: {e}") + equity_data = {} + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=self._ad.constant.Period.day.value + ) + + if symbol not in kline_dict: + return [] + + df = kline_dict[symbol] + + for _, row in df.iterrows(): + kline_time = row.get('kline_time') + if pd.isna(kline_time): + continue + + if isinstance(kline_time, pd.Timestamp): + trade_date = kline_time.strftime('%Y-%m-%d') + trade_date_int = int(kline_time.strftime('%Y%m%d')) + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + trade_date = dt.strftime('%Y-%m-%d') + trade_date_int = int(date_str) + + close = float(row.get('close', 0)) + + total_shares = 0 + float_shares = 0 + for date_key in sorted(equity_data.keys(), reverse=True): + if int(date_key) <= trade_date_int: + total_shares = equity_data[date_key]['total_shares'] + float_shares = equity_data[date_key]['float_shares'] + break + + total_market_cap = close * total_shares if total_shares > 0 and close > 0 else None + float_market_cap = close * float_shares if float_shares > 0 and close > 0 else None + + results.append({ + "symbol": symbol, + "trade_date": trade_date, + "total_market_cap": round(total_market_cap, 2) if total_market_cap else None, + "float_market_cap": round(float_market_cap, 2) if float_market_cap else None, + "total_shares": int(total_shares) if total_shares > 0 else None, + "float_shares": int(float_shares) if float_shares > 0 else None, + "inst_holding_shares": None, + "inst_holding_ratio": None, + "top10_holders_ratio": None, + "net_profit": None, + "revenue": None, + "eps": None, + "roe": None, + "trading_days": None, + }) + + info(f"Fetched {len(results)} finance records for {symbol}") + return results + + async def fetch_stock_basic_info( + self, + codes: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """Fetch stock basic info + + Corresponding table: stock_symbols + Data source: get_stock_basic + + Returns: + List[Dict] containing fields: + - symbol_id: Symbol code + - name: Name + - exchange: Exchange + - list_date: List date + - list_board: List board + - industry: Industry + - status: Status + - is_delisted: Is delisted + - delist_date: Delist date + """ + print(f"[amazingdata_adapter fetch_stock_basic_info]Fetching stock basic info...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_stock_basic_info_sync(codes) + ) + + def _fetch_stock_basic_info_sync( + self, + codes: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """Sync method to fetch stock basic info""" + try: + all_codes = self._base_data.get_code_list( + security_type=SecurityType.STOCK_A.value + ) + + if codes: + all_codes = [c for c in all_codes if c in codes] + + info_df = self._base_data.get_code_info( + security_type=SecurityType.STOCK_A.value + ) + + results = [] + for code in all_codes: + if ".SH" in code: + exchange = "SH" + elif ".SZ" in code: + exchange = "SZ" + elif ".BJ" in code: + exchange = "BJ" + else: + exchange = "" + + name = code + if code in info_df.index and 'symbol' in info_df.columns: + name = info_df.loc[code, 'symbol'] + + list_date = None + try: + equity_dict = self._info_data.get_equity_structure( + code_list=[code], + local_path=self.config.local_path, + is_local=self.config.use_local_cache + ) + if code in equity_dict and not equity_dict[code].empty: + first_record = equity_dict[code].iloc[0] + ann_date = first_record.get('ANN_DATE') + if pd.notna(ann_date): + if isinstance(ann_date, (int, float)): + list_date = datetime.strptime(str(int(ann_date)), "%Y%m%d") + else: + list_date = pd.to_datetime(ann_date) + except: + pass + + results.append({ + "symbol_id": code, + "name": name, + "exchange": exchange, + "list_date": list_date, + "list_board": None, + "industry": None, + "status": "active", + "is_delisted": False, + "delist_date": None, + "is_st": None, + "total_shares": None, + "float_shares": None, + }) + + info(f"Fetched {len(results)} stock basic info records") + return results + + except Exception as e: + error(f"Failed to fetch stock basic info: {e}") + return [] diff --git a/app/adapters/base.py b/app/adapters/base.py index 26a132b..e58caad 100644 --- a/app/adapters/base.py +++ b/app/adapters/base.py @@ -109,3 +109,127 @@ class DataSourceAdapter(ABC): async def close(self) -> None: """关闭连接""" pass + + # ==================== 拆分表结构的新接口 ==================== + + async def fetch_kline_base( + self, + symbol: str, + start: str, + end: str, + freq: str + ) -> List[dict]: + """获取K线基础数据 (OHLCV) + + 对应表: stock_klines_1d_base, stock_klines_1m_base 等 + + Returns: + List[dict] 包含字段: + - symbol: 标的代码 + - ts: 时间戳 + - trade_date: 交易日期 + - open/high/low/close: 开高低收 + - volume: 成交量 + - amount: 成交额 + """ + # 默认实现：使用 fetch_klines 并转换格式 + klines = await self.fetch_klines(symbol, start, end, freq) + return [ + { + "symbol": k.symbol, + "ts": k.time, + "trade_date": k.trade_date, + "open": k.open, + "high": k.high, + "low": k.low, + "close": k.close, + "volume": k.volume, + "amount": k.amount, + } + for k in klines + ] + + async def fetch_kline_quote( + self, + symbol: str, + start: str, + end: str + ) -> List[dict]: + """获取日线行情指标数据 + + 对应表: stock_klines_1d_quote + + Returns: + List[dict] 包含字段: + - symbol: 标的代码 + - trade_date: 交易日期 + - change_pct: 涨跌幅 + - change_Nd_pct: N日涨跌幅 + - ma_N: 均线 + - macd_dif/dea/bar: MACD指标 + - bias_N: 乖离率 + - is_limit_up/down: 涨跌停状态 + - is_st: 是否ST + """ + raise NotImplementedError("fetch_kline_quote not implemented") + + async def fetch_kline_finance( + self, + symbol: str, + start: str, + end: str + ) -> List[dict]: + """获取日线财务数据 + + 对应表: stock_klines_1d_finance + + Returns: + List[dict] 包含字段: + - symbol: 标的代码 + - trade_date: 交易日期 + - total_market_cap: 总市值 + - float_market_cap: 流通市值 + - total_shares: 总股本 + - float_shares: 流通股本 + - inst_holding_shares: 机构持股数量 + - inst_holding_ratio: 机构持仓占比 + - net_profit: 净利润 + - revenue: 营业总收入 + - eps: 每股收益 + - roe: 净资产收益率 + """ + raise NotImplementedError("fetch_kline_finance not implemented") + + async def fetch_stock_basic_info( + self, + codes: Optional[List[str]] = None + ) -> List[dict]: + """获取股票基础信息 + + 对应表: stock_symbols + + Returns: + List[dict] 包含字段: + - symbol_id: 标的代码 + - name: 名称 + - exchange: 交易所 + - list_date: 上市日期 + - list_board: 上市板块 + - industry: 行业 + - status: 状态 + - is_delisted: 是否退市 + - delist_date: 退市日期 + """ + # 默认实现：使用 fetch_symbols 并转换格式 + symbols = await self.fetch_symbols("stock") + return [ + { + "symbol_id": s.symbol_id, + "name": s.name, + "exchange": s.exchange, + "list_date": s.list_date, + "delist_date": s.delist_date, + "status": "active" if not s.delist_date else "delisted", + } + for s in symbols + ] diff --git a/app/api/__pycache__/admin_routes.cpython-311.pyc b/app/api/__pycache__/admin_routes.cpython-311.pyc index db793b8..c228081 100644 Binary files a/app/api/__pycache__/admin_routes.cpython-311.pyc and b/app/api/__pycache__/admin_routes.cpython-311.pyc differ diff --git a/app/api/admin_routes.py b/app/api/admin_routes.py index 39ffb3f..e183e12 100644 --- a/app/api/admin_routes.py +++ b/app/api/admin_routes.py @@ -1,11 +1,13 @@ """管理后台API路由 - 对应Go的api/admin_router.go""" -from fastapi import APIRouter, Depends, HTTPException, Header, Query -from typing import Optional +from datetime import datetime, timedelta +from fastapi import APIRouter, Depends, HTTPException, Header, Query, Body +from typing import Optional, List from app.models import ( Response, ConfigListRequest, ConfigUpdateRequest, ReloadRequest, AdapterToggleRequest, AdapterConfigUpdateRequest, - APITestRequest, WSTestRequest, TestHistoryRequest + APITestRequest, WSTestRequest, TestHistoryRequest, + DataSyncRequest, DataSyncType, DataSyncData, DataSyncResult ) from app.services import ConfigService, AdapterService, TestService from app.core.config import get_config @@ -269,3 +271,151 @@ def get_test_history( return Response(code=0, message="success", data=data) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + + +# ============================================ +# 数据同步接口 +# ============================================ + +@admin_router.post("/admin/data/sync", response_model=Response) +async def sync_data( + req: DataSyncRequest, + token: str = Depends(verify_admin_token) +): + """手动触发数据同步 + + 支持同步类型: + - base: 基础K线数据 (OHLCV) + - quote: 行情指标数据 (均线、MACD、涨跌停等) + - finance: 财务数据 (市值、股本、利润等) + - full: 全量同步 (以上全部) + + 示例请求: + { + "symbols": ["600519.SH", "000001.SZ"], + "sync_type": "full", + "start_date": "20240101", + "end_date": "20240301", + "asset_class": "stock" + } + """ + import time + from app.services.adapter_service import AdapterService + from app.services.data_sync_service import DataSyncService + from app.models import Frequency + + start_time = time.time() + + try: + # 获取适配器 + adapter_service = AdapterService() + adapter = adapter_service.get_active_adapter(req.asset_class) + + if not adapter: + return Response( + code=400, + message=f"No active adapter found for asset class: {req.asset_class}", + data=None + ) + + # 创建同步服务 + sync_service = DataSyncService(adapter) + + # 设置默认日期 + end_date = req.end_date or datetime.now().strftime("%Y%m%d") + start_date = req.start_date or (datetime.now() - timedelta(days=365)).strftime("%Y%m%d") + + # 根据同步类型执行同步 + base_results = [] + quote_results = [] + finance_results = [] + + if req.sync_type == DataSyncType.BASE or req.sync_type == DataSyncType.FULL: + freq = Frequency.FREQ_1D if req.freq == "1d" else Frequency.FREQ_1D + base_counts = await sync_service.sync_kline_base( + req.symbols, freq, start_date, end_date + ) + base_results = [ + DataSyncResult(symbol=k, count=v if v > 0 else 0, error=str(e) if v < 0 else None) + for k, v in base_counts.items() + ] + + if req.sync_type == DataSyncType.QUOTE or req.sync_type == DataSyncType.FULL: + quote_counts = await sync_service.sync_kline_quote( + req.symbols, start_date, end_date + ) + quote_results = [ + DataSyncResult(symbol=k, count=v if v > 0 else 0, error=str(e) if v < 0 else None) + for k, v in quote_counts.items() + ] + + if req.sync_type == DataSyncType.FINANCE or req.sync_type == DataSyncType.FULL: + finance_counts = await sync_service.sync_kline_finance( + req.symbols, start_date, end_date + ) + finance_results = [ + DataSyncResult(symbol=k, count=v if v > 0 else 0, error=str(e) if v < 0 else None) + for k, v in finance_counts.items() + ] + + total_time = int((time.time() - start_time) * 1000) + + # 统计成功/失败数量 + all_results = base_results + quote_results + finance_results + success_count = sum(1 for r in all_results if r.count > 0) + fail_count = sum(1 for r in all_results if r.error) + + data = DataSyncData( + success=fail_count == 0, + message=f"Synced {success_count} symbols, {fail_count} failed, took {total_time}ms", + sync_type=req.sync_type, + base_results=base_results, + quote_results=quote_results, + finance_results=finance_results, + total_time_ms=total_time + ) + + return Response(code=0, message="success", data=data) + + except Exception as e: + error(f"[DataSync API] Failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@admin_router.post("/admin/data/sync/incremental", response_model=Response) +async def sync_incremental( + symbols: List[str] = Body(..., description="标的代码列表"), + asset_class: str = Body("stock", description="资产类别"), + token: str = Depends(verify_admin_token) +): + """触发增量同步（最近30天） + + 用于每日定时任务，同步最近的数据 + """ + from app.services.adapter_service import AdapterService + from app.services.data_sync_service import DataSyncService + + try: + # 获取适配器 + adapter_service = AdapterService() + adapter = adapter_service.get_active_adapter(asset_class) + + if not adapter: + return Response( + code=400, + message=f"No active adapter found for asset class: {asset_class}", + data=None + ) + + # 创建同步服务 + sync_service = DataSyncService(adapter) + + # 执行增量同步 + results = await sync_service.sync_daily_incremental(symbols) + + return Response(code=0, message="success", data=results) + + except Exception as e: + error(f"[DataSync API] Incremental sync failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/app/models/__init__.py b/app/models/__init__.py index 2918093..cf60bb8 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -61,6 +61,10 @@ from .admin_types import ( WSMessage, TestHistoryRequest, TestHistoryData, + DataSyncType, + DataSyncRequest, + DataSyncResult, + DataSyncData, ) __all__ = [ @@ -132,4 +136,9 @@ __all__ = [ "WSMessage", "TestHistoryRequest", "TestHistoryData", + # 数据同步 + "DataSyncType", + "DataSyncRequest", + "DataSyncResult", + "DataSyncData", ] diff --git a/app/models/__pycache__/__init__.cpython-311.pyc b/app/models/__pycache__/__init__.cpython-311.pyc index 0f566d8..107b2ab 100644 Binary files a/app/models/__pycache__/__init__.cpython-311.pyc and b/app/models/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/models/__pycache__/admin_types.cpython-311.pyc b/app/models/__pycache__/admin_types.cpython-311.pyc index 725166a..fa0a5fe 100644 Binary files a/app/models/__pycache__/admin_types.cpython-311.pyc and b/app/models/__pycache__/admin_types.cpython-311.pyc differ diff --git a/app/models/admin_types.py b/app/models/admin_types.py index be4bfd4..deab0e9 100644 --- a/app/models/admin_types.py +++ b/app/models/admin_types.py @@ -248,3 +248,44 @@ class TestHistoryData(BaseModel): """测试历史数据""" api_tests: List[APITestResult] = Field(default_factory=list, description="API测试历史") ws_tests: List[WSTestResult] = Field(default_factory=list, description="WebSocket测试历史") + + +# ============================================ +# 数据同步类型 +# ============================================ + +class DataSyncType(str, Enum): + """数据同步类型""" + BASE = "base" # 基础K线数据 + QUOTE = "quote" # 行情指标数据 + FINANCE = "finance" # 财务数据 + FULL = "full" # 全量同步 + + +class DataSyncRequest(BaseModel): + """数据同步请求""" + symbols: List[str] = Field(..., description="标的代码列表") + sync_type: DataSyncType = Field(default=DataSyncType.FULL, description="同步类型") + freq: Optional[str] = Field(default="1d", description="周期(仅对base有效)") + start_date: Optional[str] = Field(None, description="开始日期 YYYYMMDD") + end_date: Optional[str] = Field(None, description="结束日期 YYYYMMDD") + asset_class: str = Field(default="stock", description="资产类别") + + +class DataSyncResult(BaseModel): + """单个标的同步结果""" + symbol: str = Field(..., description="标的代码") + count: int = Field(default=0, description="同步记录数") + error: Optional[str] = Field(None, description="错误信息") + + +class DataSyncData(BaseModel): + """数据同步响应""" + success: bool = Field(..., description="是否成功") + message: str = Field(..., description="提示信息") + sync_type: DataSyncType = Field(..., description="同步类型") + base_results: List[DataSyncResult] = Field(default_factory=list, description="基础数据同步结果") + quote_results: List[DataSyncResult] = Field(default_factory=list, description="行情数据同步结果") + finance_results: List[DataSyncResult] = Field(default_factory=list, description="财务数据同步结果") + total_time_ms: int = Field(default=0, description="总耗时(ms)") + timestamp: datetime = Field(default_factory=datetime.now, description="同步时间") diff --git a/app/repositories/__pycache__/futures_repository_v2.cpython-311.pyc b/app/repositories/__pycache__/futures_repository_v2.cpython-311.pyc new file mode 100644 index 0000000..a87d9f3 Binary files /dev/null and b/app/repositories/__pycache__/futures_repository_v2.cpython-311.pyc differ diff --git a/app/repositories/__pycache__/models.cpython-311.pyc b/app/repositories/__pycache__/models.cpython-311.pyc index 3c501d1..d0f1da8 100644 Binary files a/app/repositories/__pycache__/models.cpython-311.pyc and b/app/repositories/__pycache__/models.cpython-311.pyc differ diff --git a/app/repositories/__pycache__/stock_repository_v2.cpython-311.pyc b/app/repositories/__pycache__/stock_repository_v2.cpython-311.pyc new file mode 100644 index 0000000..ece4f37 Binary files /dev/null and b/app/repositories/__pycache__/stock_repository_v2.cpython-311.pyc differ diff --git a/app/repositories/futures_repository_v2.py b/app/repositories/futures_repository_v2.py new file mode 100644 index 0000000..ef13d0b --- /dev/null +++ b/app/repositories/futures_repository_v2.py @@ -0,0 +1,393 @@ +"""期货数据仓库 V2 - 支持拆分后的表结构 + +新表结构说明： +- futures_klines_1d_base: 日线基础数据(OHLCV + 持仓量) +- futures_klines_1d_quote: 日线行情指标(涨跌幅、均线、MACD等) +- futures_klines_1m_base/5m_base: 分钟线基础数据 +- futures_realtime_quotes: 实时行情快照 +""" +from datetime import datetime +from typing import List, Tuple, Optional, Dict, Any + +from sqlalchemy.orm import Session +from sqlalchemy import func, or_, and_ + +from app.models import ( + KLineItem, Symbol, SymbolListRequest, + TradingDatesData, TradeCalData, AdjustType, Frequency +) +from app.repositories.models import ( + FuturesSymbol, FuturesTradingCalendar, + FuturesKLine1DBase, FuturesKLine1DQuote, + FuturesKLine1MBase, FuturesKLine5MBase, FuturesKLine15MBase, FuturesKLine30MBase, FuturesKLine60MBase, + FuturesRealTimeQuote +) + + +class FuturesRepositoryV2: + """期货数据仓库 V2 - 支持拆分表结构""" + + def __init__(self, db: Session): + self.db = db + + # ==================== K线基础数据表操作 ==================== + + def get_klines_base( + self, + symbol: str, + freq: Frequency, + start: datetime, + end: datetime, + ) -> List[Dict[str, Any]]: + """获取期货K线基础数据 (OHLCV + 持仓量) + + 对应表: futures_klines_1d_base, futures_klines_1m_base 等 + """ + model = self._get_kline_base_model(freq) + + query = self.db.query(model).filter( + model.symbol_id == symbol, + model.ts >= start, + model.ts <= end + ).order_by(model.ts.asc()) + + results = query.all() + + return [ + { + "symbol_id": r.symbol_id, + "trade_date": r.trade_date, + "ts": r.ts, + "open": float(r.open) if r.open else None, + "high": float(r.high) if r.high else None, + "low": float(r.low) if r.low else None, + "close": float(r.close) if r.close else None, + "volume": r.volume, + "amount": float(r.amount) if r.amount else None, + "open_interest": r.open_interest, + "settlement": float(r.settlement) if r.settlement else None, + "pre_settlement": float(r.pre_settlement) if r.pre_settlement else None, + } + for r in results + ] + + def save_klines_base( + self, + freq: Frequency, + items: List[Dict[str, Any]] + ) -> int: + """保存期货K线基础数据""" + if not items: + return 0 + + model = self._get_kline_base_model(freq) + count = 0 + + for item in items: + symbol = item.get('symbol_id', item.get('symbol')) + trade_date = item.get('trade_date') + ts = item.get('ts') + + if not ts and trade_date: + if isinstance(trade_date, str): + ts = datetime.strptime(trade_date.replace('-', ''), "%Y%m%d").timestamp() + else: + ts = trade_date.timestamp() + + # 检查是否存在 + existing = self.db.query(model).filter( + model.symbol_id == symbol, + model.ts == ts + ).first() + + if existing: + existing.open = item.get('open', existing.open) + existing.high = item.get('high', existing.high) + existing.low = item.get('low', existing.low) + existing.close = item.get('close', existing.close) + existing.volume = item.get('volume', existing.volume) + existing.amount = item.get('amount', existing.amount) + existing.open_interest = item.get('open_interest', existing.open_interest) + existing.settlement = item.get('settlement', existing.settlement) + existing.pre_settlement = item.get('pre_settlement', existing.pre_settlement) + else: + new_record = model( + symbol_id=symbol, + trade_date=trade_date, + ts=ts, + open=item.get('open'), + high=item.get('high'), + low=item.get('low'), + close=item.get('close'), + volume=item.get('volume'), + amount=item.get('amount'), + open_interest=item.get('open_interest'), + settlement=item.get('settlement'), + pre_settlement=item.get('pre_settlement'), + ) + self.db.add(new_record) + count += 1 + + self.db.commit() + return count + + def _get_kline_base_model(self, freq: Frequency): + """根据周期获取期货K线基础数据模型""" + mapping = { + Frequency.FREQ_1M: FuturesKLine1MBase, + Frequency.FREQ_5M: FuturesKLine5MBase, + Frequency.FREQ_15M: FuturesKLine15MBase, + Frequency.FREQ_30M: FuturesKLine30MBase, + Frequency.FREQ_60M: FuturesKLine60MBase, + Frequency.FREQ_1D: FuturesKLine1DBase, + } + return mapping.get(freq, FuturesKLine1DBase) + + # ==================== K线行情数据表操作 ==================== + + def get_klines_quote( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """获取期货日线行情指标数据 + + 对应表: futures_klines_1d_quote + """ + results = self.db.query(FuturesKLine1DQuote).filter( + FuturesKLine1DQuote.symbol_id == symbol, + FuturesKLine1DQuote.trade_date >= start_date, + FuturesKLine1DQuote.trade_date <= end_date + ).order_by(FuturesKLine1DQuote.trade_date.asc()).all() + + return [ + { + "symbol_id": r.symbol_id, + "trade_date": r.trade_date, + "change_pct": float(r.change_pct) if r.change_pct else None, + "change_5d_pct": float(r.change_5d_pct) if r.change_5d_pct else None, + "change_10d_pct": float(r.change_10d_pct) if r.change_10d_pct else None, + "change_20d_pct": float(r.change_20d_pct) if r.change_20d_pct else None, + "ma5": float(r.ma5) if r.ma5 else None, + "ma10": float(r.ma10) if r.ma10 else None, + "ma20": float(r.ma20) if r.ma20 else None, + "ma30": float(r.ma30) if r.ma30 else None, + "ma60": float(r.ma60) if r.ma60 else None, + "macd_dif": float(r.macd_dif) if r.macd_dif else None, + "macd_dea": float(r.macd_dea) if r.macd_dea else None, + "macd_bar": float(r.macd_bar) if r.macd_bar else None, + "oi_change": r.oi_change, + "oi_change_pct": float(r.oi_change_pct) if r.oi_change_pct else None, + "amplitude": float(r.amplitude) if r.amplitude else None, + } + for r in results + ] + + def save_klines_quote( + self, + items: List[Dict[str, Any]] + ) -> int: + """保存期货日线行情指标数据""" + if not items: + return 0 + + count = 0 + for item in items: + symbol = item.get('symbol_id', item.get('symbol')) + trade_date = item.get('trade_date') + + # 检查是否存在 + existing = self.db.query(FuturesKLine1DQuote).filter( + FuturesKLine1DQuote.symbol_id == symbol, + FuturesKLine1DQuote.trade_date == trade_date + ).first() + + if existing: + existing.change_pct = item.get('change_pct', existing.change_pct) + existing.change_5d_pct = item.get('change_5d_pct', existing.change_5d_pct) + existing.change_10d_pct = item.get('change_10d_pct', existing.change_10d_pct) + existing.change_20d_pct = item.get('change_20d_pct', existing.change_20d_pct) + existing.ma5 = item.get('ma5', existing.ma5) + existing.ma10 = item.get('ma10', existing.ma10) + existing.ma20 = item.get('ma20', existing.ma20) + existing.ma30 = item.get('ma30', existing.ma30) + existing.ma60 = item.get('ma60', existing.ma60) + existing.macd_dif = item.get('macd_dif', existing.macd_dif) + existing.macd_dea = item.get('macd_dea', existing.macd_dea) + existing.macd_bar = item.get('macd_bar', existing.macd_bar) + existing.oi_change = item.get('oi_change', existing.oi_change) + existing.oi_change_pct = item.get('oi_change_pct', existing.oi_change_pct) + existing.amplitude = item.get('amplitude', existing.amplitude) + else: + new_record = FuturesKLine1DQuote( + symbol_id=symbol, + trade_date=trade_date, + change_pct=item.get('change_pct'), + change_5d_pct=item.get('change_5d_pct'), + change_10d_pct=item.get('change_10d_pct'), + change_20d_pct=item.get('change_20d_pct'), + ma5=item.get('ma5'), + ma10=item.get('ma10'), + ma20=item.get('ma20'), + ma30=item.get('ma30'), + ma60=item.get('ma60'), + macd_dif=item.get('macd_dif'), + macd_dea=item.get('macd_dea'), + macd_bar=item.get('macd_bar'), + oi_change=item.get('oi_change'), + oi_change_pct=item.get('oi_change_pct'), + amplitude=item.get('amplitude'), + ) + self.db.add(new_record) + count += 1 + + self.db.commit() + return count + + # ==================== 实时行情数据表操作 ==================== + + def get_realtime_quote(self, symbol: str) -> Optional[Dict[str, Any]]: + """获取期货实时行情 + + 对应表: futures_realtime_quotes + """ + result = self.db.query(FuturesRealTimeQuote).filter( + FuturesRealTimeQuote.symbol_id == symbol + ).first() + + if result: + return { + "symbol_id": result.symbol_id, + "update_time": result.update_time, + "last_price": float(result.last_price) if result.last_price else None, + "open": float(result.open) if result.open else None, + "high": float(result.high) if result.high else None, + "low": float(result.low) if result.low else None, + "pre_close": float(result.pre_close) if result.pre_close else None, + "pre_settlement": float(result.pre_settlement) if result.pre_settlement else None, + "settlement": float(result.settlement) if result.settlement else None, + "volume": result.volume, + "amount": float(result.amount) if result.amount else None, + "open_interest": result.open_interest, + "bid1": float(result.bid1) if result.bid1 else None, + "ask1": float(result.ask1) if result.ask1 else None, + "limit_up": float(result.limit_up) if result.limit_up else None, + "limit_down": float(result.limit_down) if result.limit_down else None, + } + return None + + def save_realtime_quote(self, data: Dict[str, Any]) -> None: + """保存期货实时行情""" + symbol = data.get('symbol_id', data.get('symbol')) + + existing = self.db.query(FuturesRealTimeQuote).filter( + FuturesRealTimeQuote.symbol_id == symbol + ).first() + + if existing: + existing.update_time = data.get('update_time', existing.update_time) + existing.last_price = data.get('last_price', existing.last_price) + existing.open = data.get('open', existing.open) + existing.high = data.get('high', existing.high) + existing.low = data.get('low', existing.low) + existing.pre_close = data.get('pre_close', existing.pre_close) + existing.pre_settlement = data.get('pre_settlement', existing.pre_settlement) + existing.settlement = data.get('settlement', existing.settlement) + existing.volume = data.get('volume', existing.volume) + existing.amount = data.get('amount', existing.amount) + existing.open_interest = data.get('open_interest', existing.open_interest) + existing.bid1 = data.get('bid1', existing.bid1) + existing.ask1 = data.get('ask1', existing.ask1) + existing.limit_up = data.get('limit_up', existing.limit_up) + existing.limit_down = data.get('limit_down', existing.limit_down) + else: + new_record = FuturesRealTimeQuote( + symbol_id=symbol, + update_time=data.get('update_time'), + last_price=data.get('last_price'), + open=data.get('open'), + high=data.get('high'), + low=data.get('low'), + pre_close=data.get('pre_close'), + pre_settlement=data.get('pre_settlement'), + settlement=data.get('settlement'), + volume=data.get('volume'), + amount=data.get('amount'), + open_interest=data.get('open_interest'), + bid1=data.get('bid1'), + ask1=data.get('ask1'), + limit_up=data.get('limit_up'), + limit_down=data.get('limit_down'), + ) + self.db.add(new_record) + + self.db.commit() + + # ==================== 标的和日历 ==================== + + def list_symbols( + self, + req: SymbolListRequest + ) -> Tuple[List[Symbol], int]: + """查询期货合约列表""" + query = self.db.query(FuturesSymbol) + + # 筛选条件 + if req.exchange: + query = query.filter(FuturesSymbol.exchange == req.exchange.value) + + if req.keyword: + keyword = f"%{req.keyword}%" + query = query.filter( + or_( + FuturesSymbol.symbol_id.ilike(keyword), + FuturesSymbol.name.ilike(keyword), + FuturesSymbol.underlying.ilike(keyword) + ) + ) + + # 查询总数 + total = query.count() + + # 分页查询 + results = query.order_by(FuturesSymbol.symbol_id).offset( + (req.page - 1) * req.size + ).limit(req.size).all() + + symbols = [] + for r in results: + s = Symbol( + symbol_id=r.symbol_id, + symbol_type=r.symbol_type, + exchange=r.exchange, + name=r.name, + list_date=r.list_date, + delist_date=r.delist_date, + status=r.status + ) + symbols.append(s) + + return symbols, total + + def get_trading_dates(self, start: str, end: str) -> TradingDatesData: + """获取期货交易日历""" + results = self.db.query(FuturesTradingCalendar).filter( + FuturesTradingCalendar.trade_date >= start, + FuturesTradingCalendar.trade_date <= end, + FuturesTradingCalendar.is_trading_day == True + ).order_by(FuturesTradingCalendar.trade_date.asc()).all() + + dates = [r.trade_date for r in results] + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + total_days = (end_date - start_date).days + 1 + + return TradingDatesData( + start=start, + end=end, + total_days=total_days, + trading_days=len(dates), + trading_dates=dates + ) diff --git a/app/repositories/models.py b/app/repositories/models.py index 410322a..5b241c8 100644 --- a/app/repositories/models.py +++ b/app/repositories/models.py @@ -227,3 +227,421 @@ class StockAdjustFactor(Base): __table_args__ = ( Index("idx_adj_factor_symbol_date", "symbol_id", "trade_date"), ) + + + +# ============================================ +# 期货分表结构 (新增) +# ============================================ + +class FuturesKLine1DBase(Base): + """期货日线K线 - 基础表""" + __tablename__ = "futures_klines_1d_base" + __table_args__ = ( + Index("idx_futures_1d_base_symbol_ts", "symbol_id", "ts"), + Index("idx_futures_1d_base_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算价") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1DQuote(Base): + """期货日线K线 - 行情指标表""" + __tablename__ = "futures_klines_1d_quote" + __table_args__ = ( + Index("idx_futures_1d_quote_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + change_5d_pct = Column(Numeric(8, 4), nullable=True, comment="5日涨跌幅%") + change_10d_pct = Column(Numeric(8, 4), nullable=True, comment="10日涨跌幅%") + change_20d_pct = Column(Numeric(8, 4), nullable=True, comment="20日涨跌幅%") + ma5 = Column(Numeric(18, 4), nullable=True, comment="5日均线") + ma10 = Column(Numeric(18, 4), nullable=True, comment="10日均线") + ma20 = Column(Numeric(18, 4), nullable=True, comment="20日均线") + ma30 = Column(Numeric(18, 4), nullable=True, comment="30日均线") + ma60 = Column(Numeric(18, 4), nullable=True, comment="60日均线") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + oi_change = Column(BigInteger, nullable=True, comment="持仓量变化") + oi_change_pct = Column(Numeric(8, 4), nullable=True, comment="持仓量变化%") + amplitude = Column(Numeric(8, 4), nullable=True, comment="振幅%") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1MBase(Base): + """期货1分钟K线 - 基础表""" + __tablename__ = "futures_klines_1m_base" + __table_args__ = (Index("idx_futures_1m_base_symbol_ts", "symbol_id", "ts"),) + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine5MBase(Base): + """期货5分钟K线 - 基础表""" + __tablename__ = "futures_klines_5m_base" + __table_args__ = (Index("idx_futures_5m_base_symbol_ts", "symbol_id", "ts"),) + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesRealTimeQuote(Base): + """期货实时行情快照""" + __tablename__ = "futures_realtime_quotes" + symbol_id = Column(String(20), primary_key=True, comment="合约代码") + update_time = Column(DateTime, nullable=False, comment="更新时间") + last_price = Column(Numeric(18, 4), nullable=True, comment="最新价") + open = Column(Numeric(18, 4), nullable=True, comment="开盘价") + high = Column(Numeric(18, 4), nullable=True, comment="最高价") + low = Column(Numeric(18, 4), nullable=True, comment="最低价") + pre_close = Column(Numeric(18, 4), nullable=True, comment="昨收") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + volume = Column(BigInteger, nullable=True, comment="成交量") + amount = Column(Numeric(20, 4), nullable=True, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + bid1 = Column(Numeric(18, 4), nullable=True, comment="买一价") + bid1_volume = Column(BigInteger, nullable=True, comment="买一量") + ask1 = Column(Numeric(18, 4), nullable=True, comment="卖一价") + ask1_volume = Column(BigInteger, nullable=True, comment="卖一量") + limit_up = Column(Numeric(18, 4), nullable=True, comment="涨停价") + limit_down = Column(Numeric(18, 4), nullable=True, comment="跌停价") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# 向后兼容别名 +FuturesKLine1M = FuturesKLine1MBase +FuturesKLine5M = FuturesKLine5MBase +FuturesKLine1D = FuturesKLine1DBase + + + +# ============================================ +# 股票日线历史数据表 (1d) - 分表 (新增) +# ============================================ + +class StockKLine1DBase(Base): + """股票日线K线 - 基础表 + + 存储最基础的K线数据，对应 query_kline 接口 + """ + __tablename__ = "stock_klines_1d_base" + __table_args__ = ( + Index("idx_stock_1d_base_symbol_ts", "symbol_id", "ts"), + Index("idx_stock_1d_base_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量(股)") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额(元)") + adj_factor = Column(Numeric(18, 8), nullable=True, comment="复权系数") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine1DQuote(Base): + """股票日线K线 - 行情指标表 + + 存储需要计算的行情指标 + """ + __tablename__ = "stock_klines_1d_quote" + __table_args__ = ( + Index("idx_stock_1d_quote_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + change_5d_pct = Column(Numeric(8, 4), nullable=True, comment="5日涨跌幅%") + change_10d_pct = Column(Numeric(8, 4), nullable=True, comment="10日涨跌幅%") + change_20d_pct = Column(Numeric(8, 4), nullable=True, comment="20日涨跌幅%") + change_30d_pct = Column(Numeric(8, 4), nullable=True, comment="30日涨跌幅%") + change_60d_pct = Column(Numeric(8, 4), nullable=True, comment="60日涨跌幅%") + ma5 = Column(Numeric(18, 4), nullable=True, comment="5日均线") + ma10 = Column(Numeric(18, 4), nullable=True, comment="10日均线") + ma20 = Column(Numeric(18, 4), nullable=True, comment="20日均线") + ma30 = Column(Numeric(18, 4), nullable=True, comment="30日均线") + ma60 = Column(Numeric(18, 4), nullable=True, comment="60日均线") + ma120 = Column(Numeric(18, 4), nullable=True, comment="120日均线") + ma250 = Column(Numeric(18, 4), nullable=True, comment="250日均线") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + bias_5 = Column(Numeric(8, 4), nullable=True, comment="5日乖离率%") + bias_10 = Column(Numeric(8, 4), nullable=True, comment="10日乖离率%") + bias_20 = Column(Numeric(8, 4), nullable=True, comment="20日乖离率%") + is_limit_up = Column(Boolean, nullable=True, comment="是否涨停") + is_limit_down = Column(Boolean, nullable=True, comment="是否跌停") + limit_up_price = Column(Numeric(18, 4), nullable=True, comment="涨停价") + limit_down_price = Column(Numeric(18, 4), nullable=True, comment="跌停价") + is_st = Column(Boolean, nullable=True, comment="是否ST") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine1DFinance(Base): + """股票日线K线 - 财务数据表 + + 存储财务相关数据 + """ + __tablename__ = "stock_klines_1d_finance" + __table_args__ = ( + Index("idx_stock_1d_finance_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + total_market_cap = Column(Numeric(20, 4), nullable=True, comment="总市值(元)") + float_market_cap = Column(Numeric(20, 4), nullable=True, comment="流通市值(元)") + total_shares = Column(BigInteger, nullable=True, comment="总股本(股)") + float_shares = Column(BigInteger, nullable=True, comment="流通股本(股)") + inst_holding_shares = Column(BigInteger, nullable=True, comment="机构持股数量") + inst_holding_ratio = Column(Numeric(8, 4), nullable=True, comment="机构持仓占比%") + top10_holders_ratio = Column(Numeric(8, 4), nullable=True, comment="前十大股东持股占比%") + net_profit = Column(Numeric(20, 4), nullable=True, comment="净利润") + revenue = Column(Numeric(20, 4), nullable=True, comment="营业总收入") + eps = Column(Numeric(12, 4), nullable=True, comment="每股收益") + roe = Column(Numeric(8, 4), nullable=True, comment="净资产收益率%") + trading_days = Column(Integer, nullable=True, comment="从上市至今交易日数") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockRealTimeQuote(Base): + """股票实时行情快照""" + __tablename__ = "stock_realtime_quotes" + + symbol_id = Column(String(20), primary_key=True, comment="标的代码") + update_time = Column(DateTime, nullable=False, comment="更新时间") + last_price = Column(Numeric(18, 4), nullable=True, comment="最新价") + open = Column(Numeric(18, 4), nullable=True, comment="开盘价") + high = Column(Numeric(18, 4), nullable=True, comment="最高价") + low = Column(Numeric(18, 4), nullable=True, comment="最低价") + pre_close = Column(Numeric(18, 4), nullable=True, comment="昨收") + volume = Column(BigInteger, nullable=True, comment="成交量") + amount = Column(Numeric(20, 4), nullable=True, comment="成交额") + bid1 = Column(Numeric(18, 4), nullable=True, comment="买一价") + ask1 = Column(Numeric(18, 4), nullable=True, comment="卖一价") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# 股票表向后兼容别名 +StockKLine1D = StockKLine1DBase + + + +# ============================================ +# 股票15/30/60分钟K线表 (新增) +# ============================================ + +class StockKLine15M(Base): + """股票15分钟K线""" + __tablename__ = "stock_klines_15m" + __table_args__ = ( + Index("idx_stock_15m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine30M(Base): + """股票30分钟K线""" + __tablename__ = "stock_klines_30m" + __table_args__ = ( + Index("idx_stock_30m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine60M(Base): + """股票60分钟K线""" + __tablename__ = "stock_klines_60m" + __table_args__ = ( + Index("idx_stock_60m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + + + +# ============================================ +# 期货15/30/60分钟K线表 (新增) +# ============================================ + +class FuturesKLine15MBase(Base): + """期货15分钟K线 - 基础表""" + __tablename__ = "futures_klines_15m_base" + __table_args__ = ( + Index("idx_futures_15m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine30MBase(Base): + """期货30分钟K线 - 基础表""" + __tablename__ = "futures_klines_30m_base" + __table_args__ = ( + Index("idx_futures_30m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine60MBase(Base): + """期货60分钟K线 - 基础表""" + __tablename__ = "futures_klines_60m_base" + __table_args__ = ( + Index("idx_futures_60m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + diff --git a/app/repositories/models_old.py b/app/repositories/models_old.py new file mode 100644 index 0000000..410322a --- /dev/null +++ b/app/repositories/models_old.py @@ -0,0 +1,229 @@ +"""数据库模型定义""" +from datetime import datetime +from typing import Optional + +from sqlalchemy import ( + Column, Integer, String, Float, DateTime, + Boolean, Numeric, BigInteger, Index, Text +) + +from app.repositories.database import Base + + +# ============================================ +# 股票相关表 +# ============================================ + +class StockSymbol(Base): + """股票标的表""" + __tablename__ = "stock_symbols" + + symbol_id = Column(String(20), primary_key=True, index=True, comment="标的代码") + symbol_type = Column(String(20), nullable=False, comment="标的类型") + exchange = Column(String(10), nullable=False, index=True, comment="交易所") + name = Column(String(100), nullable=False, comment="名称") + name_en = Column(String(100), nullable=True, comment="英文名称") + list_date = Column(DateTime, nullable=True, comment="上市日期") + delist_date = Column(DateTime, nullable=True, comment="退市日期") + industry = Column(String(50), nullable=True, comment="行业分类") + status = Column(String(20), nullable=False, default="active", comment="状态") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockTradingCalendar(Base): + """股票交易日历表""" + __tablename__ = "stock_trading_calendar" + + trade_date = Column(String(8), primary_key=True, comment="交易日期") + is_trading_day = Column(Boolean, nullable=False, comment="是否交易日") + week_day = Column(Integer, nullable=True, comment="星期几") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine1M(Base): + """股票1分钟K线""" + __tablename__ = "stock_klines_1m" + __table_args__ = ( + Index("idx_stock_1m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +class StockKLine5M(Base): + """股票5分钟K线""" + __tablename__ = "stock_klines_5m" + __table_args__ = ( + Index("idx_stock_5m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +class StockKLine1D(Base): + """股票日线K线""" + __tablename__ = "stock_klines_1d" + __table_args__ = ( + Index("idx_stock_1d_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + # 新增字段 + trade_date = Column(String(10), nullable=True, index=True, comment="交易日 (YYYY-MM-DD)") + is_limit_up = Column(Boolean, nullable=True, comment="是否涨停") + is_limit_down = Column(Boolean, nullable=True, comment="是否跌停") + total_market_cap = Column(Numeric(20, 2), nullable=True, comment="总市值（元）") + float_market_cap = Column(Numeric(20, 2), nullable=True, comment="流通市值（元）") + inst_holding_ratio = Column(Numeric(8, 4), nullable=True, comment="机构持仓占比（%）") + trading_days = Column(Integer, nullable=True, comment="可交易日数（从上市至今）") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +# ============================================ +# 期货相关表 +# ============================================ + +class FuturesSymbol(Base): + """期货合约表""" + __tablename__ = "futures_symbols" + + symbol_id = Column(String(20), primary_key=True, index=True, comment="合约代码") + symbol_type = Column(String(20), nullable=False, comment="标的类型") + exchange = Column(String(10), nullable=False, index=True, comment="交易所") + name = Column(String(100), nullable=False, comment="名称") + underlying = Column(String(10), nullable=False, index=True, comment="品种代码") + contract_month = Column(String(6), nullable=False, comment="合约月份") + list_date = Column(DateTime, nullable=True, comment="上市日期") + delist_date = Column(DateTime, nullable=True, comment="退市日期") + status = Column(String(20), nullable=False, default="active", comment="状态") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesTradingCalendar(Base): + """期货交易日历表""" + __tablename__ = "futures_trading_calendar" + + trade_date = Column(String(8), primary_key=True, comment="交易日期") + is_trading_day = Column(Boolean, nullable=False, comment="是否交易日") + has_night_session = Column(Boolean, default=False, comment="是否有夜盘") + week_day = Column(Integer, nullable=True, comment="星期几") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1M(Base): + """期货1分钟K线""" + __tablename__ = "futures_klines_1m" + __table_args__ = ( + Index("idx_futures_1m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +class FuturesKLine1D(Base): + """期货日线K线""" + __tablename__ = "futures_klines_1d" + __table_args__ = ( + Index("idx_futures_1d_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +# ============================================ +# 公共表 +# ============================================ + +class DataSourceConfig(Base): + """数据源配置表""" + __tablename__ = "data_source_config" + + asset_class = Column(String(20), primary_key=True, comment="资产类别") + active_source = Column(String(50), nullable=False, comment="当前激活源") + standby_sources = Column(Text, nullable=True, comment="待命源列表(JSON)") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class DataQualityCheck(Base): + """数据质量检查表""" + __tablename__ = "data_quality_checks" + + id = Column(BigInteger, primary_key=True, autoincrement=True) + check_date = Column(String(8), nullable=False, index=True, comment="检查日期") + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + freq = Column(String(10), nullable=False, comment="周期") + check_type = Column(String(20), nullable=False, comment="检查类型") + status = Column(String(10), nullable=False, comment="状态 pass/fail") + expect_count = Column(Integer, nullable=True, comment="期望数量") + actual_count = Column(Integer, nullable=True, comment="实际数量") + detail = Column(String(500), nullable=True, comment="详情") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + + +class StockAdjustFactor(Base): + """股票复权系数表""" + __tablename__ = "stock_adjust_factors" + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日期 YYYY-MM-DD") + qfq_factor = Column(Numeric(18, 8), nullable=False, default=1.0, comment="前复权系数") + hfq_factor = Column(Numeric(18, 8), nullable=False, default=1.0, comment="后复权系数") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + __table_args__ = ( + Index("idx_adj_factor_symbol_date", "symbol_id", "trade_date"), + ) diff --git a/app/repositories/stock_repository_v2.py b/app/repositories/stock_repository_v2.py new file mode 100644 index 0000000..5716582 --- /dev/null +++ b/app/repositories/stock_repository_v2.py @@ -0,0 +1,618 @@ +"""股票数据仓库 V2 - 支持拆分后的表结构 + +新表结构说明： +- stock_klines_1d_base: 日线基础数据(OHLCV) +- stock_klines_1d_quote: 日线行情指标(涨跌幅、均线、MACD等) +- stock_klines_1d_finance: 日线财务数据(市值、股本、利润等) +- stock_klines_1m/5m/15m/30m/60m: 分钟线基础数据 +- stock_klines_1w/1month: 周/月线基础数据 +""" +from datetime import datetime, time +from typing import List, Tuple, Optional, Dict, Any + +from sqlalchemy.orm import Session +from sqlalchemy import func, or_, and_ + +from app.models import ( + KLineItem, Symbol, SymbolListRequest, SymbolListData, + TradingDatesData, TradeCalData, AdjustType, Frequency +) +from app.repositories.models import ( + StockSymbol, StockTradingCalendar, StockAdjustFactor, + StockKLine1DBase, StockKLine1DQuote, StockKLine1DFinance, + StockKLine1M, StockKLine5M, StockKLine15M, StockKLine30M, StockKLine60M, + StockRealTimeQuote +) + + +class StockRepositoryV2: + """股票数据仓库 V2 - 支持拆分表结构""" + + def __init__(self, db: Session): + self.db = db + + # ==================== K线基础数据表操作 ==================== + + def get_klines_base( + self, + symbol: str, + freq: Frequency, + start: datetime, + end: datetime, + ) -> List[Dict[str, Any]]: + """获取K线基础数据 (OHLCV) + + 对应表: stock_klines_1d_base, stock_klines_1m_base, stock_klines_5m_base 等 + """ + model = self._get_kline_base_model(freq) + + query = self.db.query(model).filter( + model.symbol_id == symbol, + model.ts >= start, + model.ts <= end + ).order_by(model.ts.asc()) + + results = query.all() + + return [ + { + "symbol_id": r.symbol_id, + "trade_date": r.trade_date, + "ts": r.ts, + "open": float(r.open) if r.open else None, + "high": float(r.high) if r.high else None, + "low": float(r.low) if r.low else None, + "close": float(r.close) if r.close else None, + "volume": r.volume, + "amount": float(r.amount) if r.amount else None, + } + for r in results + ] + + def save_klines_base( + self, + freq: Frequency, + items: List[Dict[str, Any]] + ) -> int: + """保存K线基础数据 + + Returns: + 保存的记录数 + """ + if not items: + return 0 + + model = self._get_kline_base_model(freq) + count = 0 + + for item in items: + symbol = item.get('symbol_id', item.get('symbol')) + trade_date = item.get('trade_date') + ts = item.get('ts') + + if not ts and trade_date: + # 如果没有ts但有trade_date,生成ts + if isinstance(trade_date, str): + ts = datetime.strptime(trade_date.replace('-', ''), "%Y%m%d").timestamp() + else: + ts = trade_date.timestamp() + + # 检查是否存在 + existing = self.db.query(model).filter( + model.symbol_id == symbol, + model.ts == ts + ).first() + + if existing: + existing.open = item.get('open', existing.open) + existing.high = item.get('high', existing.high) + existing.low = item.get('low', existing.low) + existing.close = item.get('close', existing.close) + existing.volume = item.get('volume', existing.volume) + existing.amount = item.get('amount', existing.amount) + else: + new_record = model( + symbol_id=symbol, + trade_date=trade_date, + ts=ts, + open=item.get('open'), + high=item.get('high'), + low=item.get('low'), + close=item.get('close'), + volume=item.get('volume'), + amount=item.get('amount'), + ) + self.db.add(new_record) + count += 1 + + self.db.commit() + return count + + def _get_kline_base_model(self, freq: Frequency): + """根据周期获取K线基础数据模型""" + mapping = { + Frequency.FREQ_1M: StockKLine1M, + Frequency.FREQ_5M: StockKLine5M, + Frequency.FREQ_15M: StockKLine15M, + Frequency.FREQ_30M: StockKLine30M, + Frequency.FREQ_60M: StockKLine60M, + Frequency.FREQ_1D: StockKLine1DBase, + } + return mapping.get(freq, StockKLine1DBase) + + # ==================== K线行情数据表操作 ==================== + + def get_klines_quote( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """获取日线行情指标数据 + + 对应表: stock_klines_1d_quote + """ + results = self.db.query(StockKLine1DQuote).filter( + StockKLine1DQuote.symbol_id == symbol, + StockKLine1DQuote.trade_date >= start_date, + StockKLine1DQuote.trade_date <= end_date + ).order_by(StockKLine1DQuote.trade_date.asc()).all() + + return [ + { + "symbol_id": r.symbol_id, + "trade_date": r.trade_date, + "change_pct": float(r.change_pct) if r.change_pct else None, + "change_5d_pct": float(r.change_5d_pct) if r.change_5d_pct else None, + "change_10d_pct": float(r.change_10d_pct) if r.change_10d_pct else None, + "change_20d_pct": float(r.change_20d_pct) if r.change_20d_pct else None, + "change_30d_pct": float(r.change_30d_pct) if r.change_30d_pct else None, + "change_60d_pct": float(r.change_60d_pct) if r.change_60d_pct else None, + "ma5": float(r.ma5) if r.ma5 else None, + "ma10": float(r.ma10) if r.ma10 else None, + "ma20": float(r.ma20) if r.ma20 else None, + "ma30": float(r.ma30) if r.ma30 else None, + "ma60": float(r.ma60) if r.ma60 else None, + "ma120": float(r.ma120) if r.ma120 else None, + "ma250": float(r.ma250) if r.ma250 else None, + "macd_dif": float(r.macd_dif) if r.macd_dif else None, + "macd_dea": float(r.macd_dea) if r.macd_dea else None, + "macd_bar": float(r.macd_bar) if r.macd_bar else None, + "bias5": float(r.bias5) if r.bias5 else None, + "bias10": float(r.bias10) if r.bias10 else None, + "bias20": float(r.bias20) if r.bias20 else None, + "is_limit_up": r.is_limit_up, + "is_limit_down": r.is_limit_down, + "limit_up_price": float(r.limit_up_price) if r.limit_up_price else None, + "limit_down_price": float(r.limit_down_price) if r.limit_down_price else None, + "is_st": r.is_st, + } + for r in results + ] + + def save_klines_quote( + self, + items: List[Dict[str, Any]] + ) -> int: + """保存日线行情指标数据 + + 对应表: stock_klines_1d_quote + """ + if not items: + return 0 + + count = 0 + for item in items: + symbol = item.get('symbol_id', item.get('symbol')) + trade_date = item.get('trade_date') + + # 检查是否存在 + existing = self.db.query(StockKLine1DQuote).filter( + StockKLine1DQuote.symbol_id == symbol, + StockKLine1DQuote.trade_date == trade_date + ).first() + + if existing: + existing.change_pct = item.get('change_pct', existing.change_pct) + existing.change_5d_pct = item.get('change_5d_pct', existing.change_5d_pct) + existing.change_10d_pct = item.get('change_10d_pct', existing.change_10d_pct) + existing.change_20d_pct = item.get('change_20d_pct', existing.change_20d_pct) + existing.change_30d_pct = item.get('change_30d_pct', existing.change_30d_pct) + existing.change_60d_pct = item.get('change_60d_pct', existing.change_60d_pct) + existing.ma5 = item.get('ma5', existing.ma5) + existing.ma10 = item.get('ma10', existing.ma10) + existing.ma20 = item.get('ma20', existing.ma20) + existing.ma30 = item.get('ma30', existing.ma30) + existing.ma60 = item.get('ma60', existing.ma60) + existing.ma120 = item.get('ma120', existing.ma120) + existing.ma250 = item.get('ma250', existing.ma250) + existing.macd_dif = item.get('macd_dif', existing.macd_dif) + existing.macd_dea = item.get('macd_dea', existing.macd_dea) + existing.macd_bar = item.get('macd_bar', existing.macd_bar) + existing.bias5 = item.get('bias5', existing.bias5) + existing.bias10 = item.get('bias10', existing.bias10) + existing.bias20 = item.get('bias20', existing.bias20) + existing.is_limit_up = item.get('is_limit_up', existing.is_limit_up) + existing.is_limit_down = item.get('is_limit_down', existing.is_limit_down) + existing.limit_up_price = item.get('limit_up_price', existing.limit_up_price) + existing.limit_down_price = item.get('limit_down_price', existing.limit_down_price) + existing.is_st = item.get('is_st', existing.is_st) + else: + new_record = StockKLine1DQuote( + symbol_id=symbol, + trade_date=trade_date, + change_pct=item.get('change_pct'), + change_5d_pct=item.get('change_5d_pct'), + change_10d_pct=item.get('change_10d_pct'), + change_20d_pct=item.get('change_20d_pct'), + change_30d_pct=item.get('change_30d_pct'), + change_60d_pct=item.get('change_60d_pct'), + ma5=item.get('ma5'), + ma10=item.get('ma10'), + ma20=item.get('ma20'), + ma30=item.get('ma30'), + ma60=item.get('ma60'), + ma120=item.get('ma120'), + ma250=item.get('ma250'), + macd_dif=item.get('macd_dif'), + macd_dea=item.get('macd_dea'), + macd_bar=item.get('macd_bar'), + bias5=item.get('bias5'), + bias10=item.get('bias10'), + bias20=item.get('bias20'), + is_limit_up=item.get('is_limit_up'), + is_limit_down=item.get('is_limit_down'), + limit_up_price=item.get('limit_up_price'), + limit_down_price=item.get('limit_down_price'), + is_st=item.get('is_st'), + ) + self.db.add(new_record) + count += 1 + + self.db.commit() + return count + + # ==================== K线财务数据表操作 ==================== + + def get_klines_finance( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """获取日线财务数据 + + 对应表: stock_klines_1d_finance + """ + results = self.db.query(StockKLine1DFinance).filter( + StockKLine1DFinance.symbol_id == symbol, + StockKLine1DFinance.trade_date >= start_date, + StockKLine1DFinance.trade_date <= end_date + ).order_by(StockKLine1DFinance.trade_date.asc()).all() + + return [ + { + "symbol_id": r.symbol_id, + "trade_date": r.trade_date, + "total_market_cap": float(r.total_market_cap) if r.total_market_cap else None, + "float_market_cap": float(r.float_market_cap) if r.float_market_cap else None, + "total_shares": r.total_shares, + "float_shares": r.float_shares, + "inst_holding_shares": r.inst_holding_shares, + "inst_holding_ratio": float(r.inst_holding_ratio) if r.inst_holding_ratio else None, + "top10_holders_ratio": float(r.top10_holders_ratio) if r.top10_holders_ratio else None, + "net_profit": float(r.net_profit) if r.net_profit else None, + "revenue": float(r.revenue) if r.revenue else None, + "eps": float(r.eps) if r.eps else None, + "roe": float(r.roe) if r.roe else None, + "trading_days": r.trading_days, + } + for r in results + ] + + def save_klines_finance( + self, + items: List[Dict[str, Any]] + ) -> int: + """保存日线财务数据 + + 对应表: stock_klines_1d_finance + """ + if not items: + return 0 + + count = 0 + for item in items: + symbol = item.get('symbol_id', item.get('symbol')) + trade_date = item.get('trade_date') + + # 检查是否存在 + existing = self.db.query(StockKLine1DFinance).filter( + StockKLine1DFinance.symbol_id == symbol, + StockKLine1DFinance.trade_date == trade_date + ).first() + + if existing: + existing.total_market_cap = item.get('total_market_cap', existing.total_market_cap) + existing.float_market_cap = item.get('float_market_cap', existing.float_market_cap) + existing.total_shares = item.get('total_shares', existing.total_shares) + existing.float_shares = item.get('float_shares', existing.float_shares) + existing.inst_holding_shares = item.get('inst_holding_shares', existing.inst_holding_shares) + existing.inst_holding_ratio = item.get('inst_holding_ratio', existing.inst_holding_ratio) + existing.top10_holders_ratio = item.get('top10_holders_ratio', existing.top10_holders_ratio) + existing.net_profit = item.get('net_profit', existing.net_profit) + existing.revenue = item.get('revenue', existing.revenue) + existing.eps = item.get('eps', existing.eps) + existing.roe = item.get('roe', existing.roe) + existing.trading_days = item.get('trading_days', existing.trading_days) + else: + new_record = StockKLine1DFinance( + symbol_id=symbol, + trade_date=trade_date, + total_market_cap=item.get('total_market_cap'), + float_market_cap=item.get('float_market_cap'), + total_shares=item.get('total_shares'), + float_shares=item.get('float_shares'), + inst_holding_shares=item.get('inst_holding_shares'), + inst_holding_ratio=item.get('inst_holding_ratio'), + top10_holders_ratio=item.get('top10_holders_ratio'), + net_profit=item.get('net_profit'), + revenue=item.get('revenue'), + eps=item.get('eps'), + roe=item.get('roe'), + trading_days=item.get('trading_days'), + ) + self.db.add(new_record) + count += 1 + + self.db.commit() + return count + + # ==================== 实时行情数据表操作 ==================== + + def get_realtime_quote(self, symbol: str) -> Optional[Dict[str, Any]]: + """获取实时行情 + + 对应表: stock_realtime_quotes + """ + result = self.db.query(StockRealTimeQuote).filter( + StockRealTimeQuote.symbol_id == symbol + ).first() + + if result: + return { + "symbol_id": result.symbol_id, + "update_time": result.update_time, + "last_price": float(result.last_price) if result.last_price else None, + "open": float(result.open) if result.open else None, + "high": float(result.high) if result.high else None, + "low": float(result.low) if result.low else None, + "pre_close": float(result.pre_close) if result.pre_close else None, + "volume": result.volume, + "amount": float(result.amount) if result.amount else None, + "bid1": float(result.bid1) if result.bid1 else None, + "ask1": float(result.ask1) if result.ask1 else None, + } + return None + + def save_realtime_quote(self, data: Dict[str, Any]) -> None: + """保存实时行情 + + 对应表: stock_realtime_quotes + """ + symbol = data.get('symbol_id', data.get('symbol')) + + existing = self.db.query(StockRealTimeQuote).filter( + StockRealTimeQuote.symbol_id == symbol + ).first() + + if existing: + existing.update_time = data.get('update_time', existing.update_time) + existing.last_price = data.get('last_price', existing.last_price) + existing.open = data.get('open', existing.open) + existing.high = data.get('high', existing.high) + existing.low = data.get('low', existing.low) + existing.pre_close = data.get('pre_close', existing.pre_close) + existing.volume = data.get('volume', existing.volume) + existing.amount = data.get('amount', existing.amount) + existing.bid1 = data.get('bid1', existing.bid1) + existing.ask1 = data.get('ask1', existing.ask1) + else: + new_record = StockRealTimeQuote( + symbol_id=symbol, + update_time=data.get('update_time'), + last_price=data.get('last_price'), + open=data.get('open'), + high=data.get('high'), + low=data.get('low'), + pre_close=data.get('pre_close'), + volume=data.get('volume'), + amount=data.get('amount'), + bid1=data.get('bid1'), + ask1=data.get('ask1'), + ) + self.db.add(new_record) + + self.db.commit() + + # ==================== 标的和日历（复用原有实现）==================== + + def list_symbols( + self, + req: SymbolListRequest + ) -> Tuple[List[Symbol], int]: + """查询标的列表""" + query = self.db.query(StockSymbol) + + # 筛选条件 + if req.exchange: + query = query.filter(StockSymbol.exchange == req.exchange.value) + + if req.keyword: + keyword = f"%{req.keyword}%" + query = query.filter( + or_( + StockSymbol.symbol_id.ilike(keyword), + StockSymbol.name.ilike(keyword) + ) + ) + + # 查询总数 + total = query.count() + + # 分页查询 + results = query.order_by(StockSymbol.symbol_id).offset( + (req.page - 1) * req.size + ).limit(req.size).all() + + symbols = [] + for r in results: + s = Symbol( + symbol_id=r.symbol_id, + symbol_type=r.symbol_type, + exchange=r.exchange, + name=r.name, + name_en=r.name_en, + list_date=r.list_date, + delist_date=r.delist_date, + industry=r.industry, + status=r.status + ) + symbols.append(s) + + return symbols, total + + def get_trading_dates(self, start: str, end: str) -> TradingDatesData: + """获取交易日历""" + results = self.db.query(StockTradingCalendar).filter( + StockTradingCalendar.trade_date >= start, + StockTradingCalendar.trade_date <= end, + StockTradingCalendar.is_trading_day == True + ).order_by(StockTradingCalendar.trade_date.asc()).all() + + dates = [r.trade_date for r in results] + + # 计算总天数 + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + total_days = (end_date - start_date).days + 1 + + return TradingDatesData( + start=start, + end=end, + total_days=total_days, + trading_days=len(dates), + trading_dates=dates + ) + + def save_symbols(self, symbols: List[Symbol]) -> None: + """保存标的列表""" + for s in symbols: + existing = self.db.query(StockSymbol).filter( + StockSymbol.symbol_id == s.symbol_id + ).first() + + if existing: + existing.name = s.name + existing.name_en = s.name_en + existing.list_date = s.list_date + existing.delist_date = s.delist_date + existing.industry = s.industry + existing.status = s.status + else: + new_symbol = StockSymbol( + symbol_id=s.symbol_id, + symbol_type=s.symbol_type.value if s.symbol_type else "stock", + exchange=s.exchange.value if s.exchange else "", + name=s.name, + name_en=s.name_en, + list_date=s.list_date, + delist_date=s.delist_date, + industry=s.industry, + status=s.status + ) + self.db.add(new_symbol) + + self.db.commit() + + def save_trading_calendar(self, dates: List[TradeCalData]) -> None: + """保存交易日历""" + for d in dates: + date_str = d.date.strftime("%Y%m%d") + + existing = self.db.query(StockTradingCalendar).filter( + StockTradingCalendar.trade_date == date_str + ).first() + + if existing: + existing.is_trading_day = d.is_trading_day + existing.week_day = d.date.weekday() + 1 + else: + new_cal = StockTradingCalendar( + trade_date=date_str, + is_trading_day=d.is_trading_day, + week_day=d.date.weekday() + 1 + ) + self.db.add(new_cal) + + self.db.commit() + + def get_adjust_factors( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """获取指定日期范围内的复权系数""" + results = self.db.query(StockAdjustFactor).filter( + StockAdjustFactor.symbol_id == symbol, + StockAdjustFactor.trade_date >= start_date, + StockAdjustFactor.trade_date <= end_date + ).order_by(StockAdjustFactor.trade_date.asc()).all() + + return [ + { + "trade_date": r.trade_date, + "qfq_factor": float(r.qfq_factor) if r.qfq_factor else 1.0, + "hfq_factor": float(r.hfq_factor) if r.hfq_factor else 1.0 + } + for r in results + ] + + def save_adjust_factors(self, symbol: str, factors: List[Dict[str, Any]]) -> None: + """保存复权系数""" + for f in factors: + trade_date = f.get("trade_date") + + existing = self.db.query(StockAdjustFactor).filter( + StockAdjustFactor.symbol_id == symbol, + StockAdjustFactor.trade_date == trade_date + ).first() + + if existing: + existing.qfq_factor = f.get("qfq_factor", 1.0) + existing.hfq_factor = f.get("hfq_factor", 1.0) + else: + new_factor = StockAdjustFactor( + symbol_id=symbol, + trade_date=trade_date, + qfq_factor=f.get("qfq_factor", 1.0), + hfq_factor=f.get("hfq_factor", 1.0) + ) + self.db.add(new_factor) + + self.db.commit() + + def get_latest_adjust_factor(self, symbol: str) -> Optional[Dict[str, Any]]: + """获取最新的复权系数""" + result = self.db.query(StockAdjustFactor).filter( + StockAdjustFactor.symbol_id == symbol + ).order_by(StockAdjustFactor.trade_date.desc()).first() + + if result: + return { + "trade_date": result.trade_date, + "qfq_factor": float(result.qfq_factor) if result.qfq_factor else 1.0, + "hfq_factor": float(result.hfq_factor) if result.hfq_factor else 1.0 + } + return None diff --git a/app/services/__pycache__/data_sync_service.cpython-311.pyc b/app/services/__pycache__/data_sync_service.cpython-311.pyc new file mode 100644 index 0000000..cb844fb Binary files /dev/null and b/app/services/__pycache__/data_sync_service.cpython-311.pyc differ diff --git a/app/services/__pycache__/test_service.cpython-311.pyc b/app/services/__pycache__/test_service.cpython-311.pyc index a6d4cc9..b37c513 100644 Binary files a/app/services/__pycache__/test_service.cpython-311.pyc and b/app/services/__pycache__/test_service.cpython-311.pyc differ diff --git a/app/services/data_sync_service.py b/app/services/data_sync_service.py new file mode 100644 index 0000000..b420e23 --- /dev/null +++ b/app/services/data_sync_service.py @@ -0,0 +1,395 @@ +"""数据同步服务 + +从数据源适配器拉取数据并同步到数据库 +支持新的拆分表结构 +""" +import asyncio +from datetime import datetime, timedelta +from typing import List, Optional, Dict, Any + +from sqlalchemy.orm import Session + +from app.adapters.base import DataSourceAdapter +from app.repositories.stock_repository_v2 import StockRepositoryV2 +from app.repositories.database import SessionLocal +from app.models import Frequency +from app.core.logger import info, error, warning + + +class DataSyncService: + """数据同步服务""" + + def __init__(self, adapter: DataSourceAdapter): + self.adapter = adapter + + async def sync_kline_base( + self, + symbols: List[str], + freq: Frequency, + start: str, + end: str, + db: Optional[Session] = None + ) -> Dict[str, int]: + """同步K线基础数据 + + Args: + symbols: 标的代码列表 + freq: 周期 + start: 开始日期 (YYYYMMDD) + end: 结束日期 (YYYYMMDD) + db: 数据库会话 (可选，用于事务控制) + + Returns: + 各标的同步记录数统计 + """ + should_close = db is None + if db is None: + db = SessionLocal() + + try: + repo = StockRepositoryV2(db) + results = {} + + for symbol in symbols: + try: + info(f"[DataSync] Syncing kline base data for {symbol} {freq.value}") + + # 从适配器获取数据 + data = await self.adapter.fetch_kline_base( + symbol=symbol, + start=start, + end=end, + freq=freq.value + ) + + if data: + # 转换数据格式 + items = [ + { + "symbol_id": item.get("symbol", item.get("symbol_id")), + "trade_date": item.get("trade_date"), + "ts": item.get("ts"), + "open": item.get("open"), + "high": item.get("high"), + "low": item.get("low"), + "close": item.get("close"), + "volume": item.get("volume"), + "amount": item.get("amount"), + } + for item in data + ] + + # 保存到数据库 + count = repo.save_klines_base(freq, items) + results[symbol] = count + info(f"[DataSync] Saved {count} base kline records for {symbol}") + else: + results[symbol] = 0 + warning(f"[DataSync] No base kline data returned for {symbol}") + + except Exception as e: + error(f"[DataSync] Failed to sync {symbol}: {e}") + results[symbol] = -1 # -1 表示错误 + + return results + + finally: + if should_close: + db.close() + + async def sync_kline_quote( + self, + symbols: List[str], + start: str, + end: str, + db: Optional[Session] = None + ) -> Dict[str, int]: + """同步日线行情指标数据 + + Args: + symbols: 标的代码列表 + start: 开始日期 (YYYYMMDD) + end: 结束日期 (YYYYMMDD) + db: 数据库会话 + + Returns: + 各标的同步记录数统计 + """ + should_close = db is None + if db is None: + db = SessionLocal() + + try: + repo = StockRepositoryV2(db) + results = {} + + for symbol in symbols: + try: + info(f"[DataSync] Syncing kline quote data for {symbol}") + + # 从适配器获取数据（包含计算后的指标） + data = await self.adapter.fetch_kline_quote( + symbol=symbol, + start=start, + end=end + ) + + if data: + # 转换数据格式 + items = [ + { + "symbol_id": item.get("symbol", item.get("symbol_id")), + "trade_date": item.get("trade_date"), + "change_pct": item.get("change_pct"), + "change_5d_pct": item.get("change_5d_pct"), + "change_10d_pct": item.get("change_10d_pct"), + "change_20d_pct": item.get("change_20d_pct"), + "change_30d_pct": item.get("change_30d_pct"), + "change_60d_pct": item.get("change_60d_pct"), + "ma5": item.get("ma5"), + "ma10": item.get("ma10"), + "ma20": item.get("ma20"), + "ma30": item.get("ma30"), + "ma60": item.get("ma60"), + "ma120": item.get("ma120"), + "ma250": item.get("ma250"), + "macd_dif": item.get("macd_dif"), + "macd_dea": item.get("macd_dea"), + "macd_bar": item.get("macd_bar"), + "bias5": item.get("bias5"), + "bias10": item.get("bias10"), + "bias20": item.get("bias20"), + "is_limit_up": item.get("is_limit_up"), + "is_limit_down": item.get("is_limit_down"), + "limit_up_price": item.get("limit_up_price"), + "limit_down_price": item.get("limit_down_price"), + "is_st": item.get("is_st"), + } + for item in data + ] + + # 保存到数据库 + count = repo.save_klines_quote(items) + results[symbol] = count + info(f"[DataSync] Saved {count} quote records for {symbol}") + else: + results[symbol] = 0 + warning(f"[DataSync] No quote data returned for {symbol}") + + except Exception as e: + error(f"[DataSync] Failed to sync quote for {symbol}: {e}") + results[symbol] = -1 + + return results + + finally: + if should_close: + db.close() + + async def sync_kline_finance( + self, + symbols: List[str], + start: str, + end: str, + db: Optional[Session] = None + ) -> Dict[str, int]: + """同步日线财务数据 + + Args: + symbols: 标的代码列表 + start: 开始日期 (YYYYMMDD) + end: 结束日期 (YYYYMMDD) + db: 数据库会话 + + Returns: + 各标的同步记录数统计 + """ + should_close = db is None + if db is None: + db = SessionLocal() + + try: + repo = StockRepositoryV2(db) + results = {} + + for symbol in symbols: + try: + info(f"[DataSync] Syncing kline finance data for {symbol}") + + # 从适配器获取数据 + data = await self.adapter.fetch_kline_finance( + symbol=symbol, + start=start, + end=end + ) + + if data: + # 转换数据格式 + items = [ + { + "symbol_id": item.get("symbol", item.get("symbol_id")), + "trade_date": item.get("trade_date"), + "total_market_cap": item.get("total_market_cap"), + "float_market_cap": item.get("float_market_cap"), + "total_shares": item.get("total_shares"), + "float_shares": item.get("float_shares"), + "inst_holding_shares": item.get("inst_holding_shares"), + "inst_holding_ratio": item.get("inst_holding_ratio"), + "top10_holders_ratio": item.get("top10_holders_ratio"), + "net_profit": item.get("net_profit"), + "revenue": item.get("revenue"), + "eps": item.get("eps"), + "roe": item.get("roe"), + "trading_days": item.get("trading_days"), + } + for item in data + ] + + # 保存到数据库 + count = repo.save_klines_finance(items) + results[symbol] = count + info(f"[DataSync] Saved {count} finance records for {symbol}") + else: + results[symbol] = 0 + warning(f"[DataSync] No finance data returned for {symbol}") + + except Exception as e: + error(f"[DataSync] Failed to sync finance for {symbol}: {e}") + results[symbol] = -1 + + return results + + finally: + if should_close: + db.close() + + async def sync_stock_basic_info( + self, + codes: Optional[List[str]] = None, + db: Optional[Session] = None + ) -> int: + """同步股票基础信息 + + Args: + codes: 指定代码列表，None表示全量同步 + db: 数据库会话 + + Returns: + 同步记录数 + """ + should_close = db is None + if db is None: + db = SessionLocal() + + try: + repo = StockRepositoryV2(db) + + info("[DataSync] Syncing stock basic info") + + # 从适配器获取数据 + data = await self.adapter.fetch_stock_basic_info(codes=codes) + + if data: + # 转换为Symbol对象 + from app.models import Symbol, Exchange + symbols = [] + + for item in data: + # 解析exchange + exchange_str = item.get('exchange', '') + exchange = None + if exchange_str == 'SH': + exchange = Exchange.SH + elif exchange_str == 'SZ': + exchange = Exchange.SZ + elif exchange_str == 'BJ': + exchange = Exchange.BJ + + symbol = Symbol( + symbol_id=item.get('symbol_id', item.get('symbol')), + name=item.get('name'), + exchange=exchange, + list_date=item.get('list_date'), + delist_date=item.get('delist_date'), + industry=item.get('industry'), + status=item.get('status', 'active'), + ) + symbols.append(symbol) + + # 保存到数据库 + repo.save_symbols(symbols) + info(f"[DataSync] Saved {len(symbols)} stock basic info records") + return len(symbols) + + return 0 + + except Exception as e: + error(f"[DataSync] Failed to sync stock basic info: {e}") + return 0 + + finally: + if should_close: + db.close() + + async def sync_full_stock_data( + self, + symbols: List[str], + start: str, + end: str + ) -> Dict[str, Any]: + """全量同步股票数据（基础+行情+财务） + + 用于首次导入或历史数据补全 + """ + info(f"[DataSync] Starting full sync for {len(symbols)} symbols from {start} to {end}") + + results = { + "base": {}, + "quote": {}, + "finance": {}, + "errors": [] + } + + db = SessionLocal() + try: + # 1. 同步基础K线数据 + results["base"] = await self.sync_kline_base( + symbols, Frequency.FREQ_1D, start, end, db + ) + + # 2. 同步行情指标数据 + results["quote"] = await self.sync_kline_quote( + symbols, start, end, db + ) + + # 3. 同步财务数据 + results["finance"] = await self.sync_kline_finance( + symbols, start, end, db + ) + + info(f"[DataSync] Full sync completed") + + except Exception as e: + error(f"[DataSync] Full sync failed: {e}") + results["errors"].append(str(e)) + + finally: + db.close() + + return results + + async def sync_daily_incremental( + self, + symbols: List[str] + ) -> Dict[str, Any]: + """每日增量同步（同步最近N天数据） + + 用于定时任务，同步最近3个交易日的数据 + """ + # 计算日期范围（最近5个交易日） + end_date = datetime.now().strftime("%Y%m%d") + start_date = (datetime.now() - timedelta(days=30)).strftime("%Y%m%d") + + info(f"[DataSync] Starting daily incremental sync for {len(symbols)} symbols") + + return await self.sync_full_stock_data(symbols, start_date, end_date) diff --git a/app/services/test_service.py b/app/services/test_service.py index 5a90531..72656c9 100644 --- a/app/services/test_service.py +++ b/app/services/test_service.py @@ -248,6 +248,90 @@ class TestService: ), ] ), + APITestCategory( + name="数据同步接口", + items=[ + APITestCase( + id="admin_data_sync_full", + name="全量数据同步", + method="POST", + path="/v1/admin/data/sync", + description="手动触发全量数据同步（基础+行情+财务）", + body={ + "symbols": ["000001.SZ", "600519.SH"], + "sync_type": "full", + "start_date": "20240301", + "end_date": "20240310", + "asset_class": "stock" + } + ), + APITestCase( + id="admin_data_sync_base", + name="同步基础K线数据", + method="POST", + path="/v1/admin/data/sync", + description="仅同步基础K线数据（OHLCV）", + body={ + "symbols": ["000001.SZ"], + "sync_type": "base", + "freq": "1d", + "start_date": "20240301", + "end_date": "20240310", + "asset_class": "stock" + } + ), + APITestCase( + id="admin_data_sync_quote", + name="同步行情指标数据", + method="POST", + path="/v1/admin/data/sync", + description="同步行情指标数据（均线/MACD/涨跌幅）", + body={ + "symbols": ["000001.SZ"], + "sync_type": "quote", + "start_date": "20240301", + "end_date": "20240310", + "asset_class": "stock" + } + ), + APITestCase( + id="admin_data_sync_finance", + name="同步财务数据", + method="POST", + path="/v1/admin/data/sync", + description="同步财务数据（市值/股本/利润）", + body={ + "symbols": ["000001.SZ"], + "sync_type": "finance", + "start_date": "20240301", + "end_date": "20240310", + "asset_class": "stock" + } + ), + APITestCase( + id="admin_data_sync_incremental", + name="增量数据同步", + method="POST", + path="/v1/admin/data/sync/incremental", + description="触发增量同步（最近30天）", + body=["000001.SZ", "600519.SH"] + ), + APITestCase( + id="admin_data_sync_futures", + name="期货数据同步", + method="POST", + path="/v1/admin/data/sync", + description="同步期货数据", + body={ + "symbols": ["CU2504.SHFE"], + "sync_type": "full", + "start_date": "20240301", + "end_date": "20240310", + "asset_class": "futures" + } + ), + ] + ), APITestCategory( name="测试管理", items=[ diff --git a/scripts/__pycache__/create_split_tables.cpython-311.pyc b/scripts/__pycache__/create_split_tables.cpython-311.pyc new file mode 100644 index 0000000..aa098c8 Binary files /dev/null and b/scripts/__pycache__/create_split_tables.cpython-311.pyc differ diff --git a/scripts/add_15_30_60_tables.py b/scripts/add_15_30_60_tables.py new file mode 100644 index 0000000..8d0dc76 --- /dev/null +++ b/scripts/add_15_30_60_tables.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""添加15/30/60分钟K线表""" + +# 股票15/30/60分钟表 +stock_tables = ''' + + +# ============================================ +# 股票15/30/60分钟K线表 (新增) +# ============================================ + +class StockKLine15M(Base): + """股票15分钟K线""" + __tablename__ = "stock_klines_15m" + __table_args__ = ( + Index("idx_stock_15m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine30M(Base): + """股票30分钟K线""" + __tablename__ = "stock_klines_30m" + __table_args__ = ( + Index("idx_stock_30m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine60M(Base): + """股票60分钟K线""" + __tablename__ = "stock_klines_60m" + __table_args__ = ( + Index("idx_stock_60m_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + +''' + +# 期货15/30/60分钟表 +futures_tables = ''' + + +# ============================================ +# 期货15/30/60分钟K线表 (新增) +# ============================================ + +class FuturesKLine15MBase(Base): + """期货15分钟K线 - 基础表""" + __tablename__ = "futures_klines_15m_base" + __table_args__ = ( + Index("idx_futures_15m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine30MBase(Base): + """期货30分钟K线 - 基础表""" + __tablename__ = "futures_klines_30m_base" + __table_args__ = ( + Index("idx_futures_30m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine60MBase(Base): + """期货60分钟K线 - 基础表""" + __tablename__ = "futures_klines_60m_base" + __table_args__ = ( + Index("idx_futures_60m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + +''' + +def main(): + models_path = 'app/repositories/models.py' + + with open(models_path, 'r', encoding='utf-8') as f: + content = f.read() + + added = [] + + # 添加股票表 + if 'class StockKLine15M' not in content: + content = content + stock_tables + added.append('StockKLine15M/30M/60M') + + # 添加期货表 + if 'class FuturesKLine15MBase' not in content: + content = content + futures_tables + added.append('FuturesKLine15MBase/30MBase/60MBase') + + if added: + with open(models_path, 'w', encoding='utf-8') as f: + f.write(content) + print(f'Added: {", ".join(added)}') + else: + print('All tables already exist.') + +if __name__ == "__main__": + main() diff --git a/scripts/add_adapter_methods.py b/scripts/add_adapter_methods.py new file mode 100644 index 0000000..6265e9b --- /dev/null +++ b/scripts/add_adapter_methods.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Add new data fetch methods to amazingdata_adapter.py using English comments""" + +new_methods = ''' + + # ==================== New Split Table Data Fetch Methods ==================== + + async def fetch_kline_base( + self, + symbol: str, + start: str, + end: str, + freq: str + ) -> List[Dict[str, Any]]: + """Fetch K-line base data (OHLCV) + + Corresponding tables: stock_klines_1d_base, stock_klines_1m_base, etc. + + Returns: + List[Dict] containing fields: + - symbol: Symbol code + - ts: Timestamp + - trade_date: Trade date + - open/high/low/close: Price data + - volume: Trading volume + - amount: Trading amount + - adj_factor: Adjustment factor + """ + print(f"[amazingdata_adapter fetch_kline_base]Fetching {symbol} {freq} base data...") + self._check_login() + + period_map = { + "1m": self._ad.constant.Period.min1, + "5m": self._ad.constant.Period.min5, + "15m": self._ad.constant.Period.min15, + "30m": self._ad.constant.Period.min30, + "60m": self._ad.constant.Period.min60, + "1d": self._ad.constant.Period.day, + "1w": self._ad.constant.Period.week, + "1month": self._ad.constant.Period.month, + } + period_value = period_map.get(freq, self._ad.constant.Period.day).value + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_base_sync(symbol, start, end, period_value, freq) + ) + + def _fetch_kline_base_sync( + self, + symbol: str, + start_date: str, + end_date: str, + period_value: int, + freq: str + ) -> List[Dict[str, Any]]: + """Sync method to fetch K-line base data""" + codes = [symbol] + start_int = self._format_date(start_date) + end_int = self._format_date(end_date) + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=period_value + ) + + if symbol not in kline_dict: + info(f"No kline data found for {symbol}") + return [] + + df = kline_dict[symbol] + results = [] + + for _, row in df.iterrows(): + kline_time = row.get('kline_time') + if pd.isna(kline_time) or kline_time is None: + continue + + if isinstance(kline_time, pd.Timestamp): + ts = int(kline_time.timestamp()) + trade_date = kline_time.strftime('%Y-%m-%d') + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + ts = int(dt.timestamp()) + trade_date = dt.strftime('%Y-%m-%d') + + results.append({ + "symbol": symbol, + "ts": ts, + "trade_date": trade_date, + "open": float(row.get('open', 0)), + "high": float(row.get('high', 0)), + "low": float(row.get('low', 0)), + "close": float(row.get('close', 0)), + "volume": int(row.get('volume', 0)), + "amount": float(row.get('amount', 0)), + "adj_factor": float(row.get('adj_factor', 1.0)) if 'adj_factor' in df.columns else 1.0, + }) + + info(f"Fetched {len(results)} base kline records for {symbol}") + return results + + async def fetch_kline_quote( + self, + symbol: str, + start: str, + end: str + ) -> List[Dict[str, Any]]: + """Fetch daily quote indicator data (calculated) + + Corresponding table: stock_klines_1d_quote + + Returns: + List[Dict] containing fields: + - change_pct: Price change percentage + - change_Nd_pct: N-day price change + - ma_N: Moving averages + - macd_dif/dea/bar: MACD indicators + - bias_N: Bias ratios + - is_limit_up/down: Limit up/down status + - is_st: ST status + """ + print(f"[amazingdata_adapter fetch_kline_quote]Calculating {symbol} quote indicators...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_quote_sync(symbol, start, end) + ) + + def _fetch_kline_quote_sync( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """Sync method to calculate quote indicators""" + import numpy as np + + start_dt = datetime.strptime(start_date, "%Y%m%d") + extended_start = datetime(start_dt.year - 1, start_dt.month, start_dt.day) + extended_start_str = extended_start.strftime("%Y%m%d") + + codes = [symbol] + start_int = self._format_date(extended_start_str) + end_int = self._format_date(end_date) + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=self._ad.constant.Period.day.value + ) + + if symbol not in kline_dict: + return [] + + df = kline_dict[symbol].copy() + df = df.sort_values('kline_time') + + try: + code_info_df = self._base_data.get_code_info(security_type=SecurityType.STOCK_A.value) + if symbol in code_info_df.index: + high_limited = float(code_info_df.loc[symbol, 'high_limited']) if 'high_limited' in code_info_df.columns else None + low_limited = float(code_info_df.loc[symbol, 'low_limited']) if 'low_limited' in code_info_df.columns else None + else: + high_limited = low_limited = None + except: + high_limited = low_limited = None + + results = [] + closes = df['close'].values + + for i, (_, row) in enumerate(df.iterrows()): + kline_time = row.get('kline_time') + if pd.isna(kline_time): + continue + + if isinstance(kline_time, pd.Timestamp): + trade_date = kline_time.strftime('%Y-%m-%d') + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + trade_date = dt.strftime('%Y-%m-%d') + + close = float(row.get('close', 0)) + + change_pct = None + if i > 0: + prev_close = closes[i-1] + if prev_close > 0: + change_pct = round((close - prev_close) / prev_close * 100, 4) + + def calc_n_day_change(n): + if i >= n and closes[i-n] > 0: + return round((close - closes[i-n]) / closes[i-n] * 100, 4) + return None + + def calc_ma(n): + if i >= n - 1: + return round(np.mean(closes[i-n+1:i+1]), 4) + return None + + def calc_macd(): + if i < 33: + return None, None, None + ema12 = pd.Series(closes[:i+1]).ewm(span=12).mean().iloc[-1] + ema26 = pd.Series(closes[:i+1]).ewm(span=26).mean().iloc[-1] + dif = ema12 - ema26 + dea = pd.Series([ema12 - ema26 for _ in range(i+1)]).ewm(span=9).mean().iloc[-1] + bar = (dif - dea) * 2 + return round(dif, 6), round(dea, 6), round(bar, 6) + + def calc_bias(n): + ma = calc_ma(n) + if ma and ma > 0: + return round((close - ma) / ma * 100, 4) + return None + + is_limit_up = False + is_limit_down = False + if high_limited and low_limited and close > 0: + is_limit_up = close >= high_limited * 0.995 + is_limit_down = close <= low_limited * 1.005 + + macd_dif, macd_dea, macd_bar = calc_macd() + + if trade_date.replace('-', '') >= start_date: + results.append({ + "symbol": symbol, + "trade_date": trade_date, + "change_pct": change_pct, + "change_5d_pct": calc_n_day_change(5), + "change_10d_pct": calc_n_day_change(10), + "change_20d_pct": calc_n_day_change(20), + "change_30d_pct": calc_n_day_change(30), + "change_60d_pct": calc_n_day_change(60), + "macd_dif": macd_dif, + "macd_dea": macd_dea, + "macd_bar": macd_bar, + "bias_5": calc_bias(5), + "bias_10": calc_bias(10), + "bias_20": calc_bias(20), + "is_limit_up": is_limit_up, + "is_limit_down": is_limit_down, + "limit_up_price": round(high_limited, 4) if high_limited else None, + "limit_down_price": round(low_limited, 4) if low_limited else None, + "is_st": None, + "ma_5": calc_ma(5), + "ma_10": calc_ma(10), + "ma_20": calc_ma(20), + "ma_30": calc_ma(30), + "ma_60": calc_ma(60), + "ma_120": calc_ma(120), + "ma_250": calc_ma(250), + }) + + info(f"Calculated {len(results)} quote indicators for {symbol}") + return results + + async def fetch_kline_finance( + self, + symbol: str, + start: str, + end: str + ) -> List[Dict[str, Any]]: + """Fetch daily finance data + + Corresponding table: stock_klines_1d_finance + Data sources: get_equity_structure, get_share_holder, get_income + + Returns: + List[Dict] containing fields: + - total_market_cap: Total market cap + - float_market_cap: Float market cap + - total_shares: Total shares + - float_shares: Float shares + - inst_holding_shares: Institutional holding shares + - inst_holding_ratio: Institutional holding ratio + - net_profit: Net profit + - revenue: Revenue + - eps: EPS + - roe: ROE + """ + print(f"[amazingdata_adapter fetch_kline_finance]Fetching {symbol} finance data...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_kline_finance_sync(symbol, start, end) + ) + + def _fetch_kline_finance_sync( + self, + symbol: str, + start_date: str, + end_date: str + ) -> List[Dict[str, Any]]: + """Sync method to fetch finance data""" + codes = [symbol] + start_int = self._format_date(start_date) + end_int = self._format_date(end_date) + + results = [] + + try: + equity_dict = self._info_data.get_equity_structure( + code_list=codes, + local_path=self.config.local_path, + is_local=self.config.use_local_cache + ) + + equity_data = {} + if symbol in equity_dict: + equity_df = equity_dict[symbol] + for _, row in equity_df.iterrows(): + ann_date = row.get('ANN_DATE') + if pd.notna(ann_date): + if isinstance(ann_date, (int, float)): + date_key = str(int(ann_date)) + else: + date_key = str(ann_date).replace('-', '').replace('/', '') + equity_data[date_key] = { + 'total_shares': float(row.get('TOT_A_SHARE', 0)) * 10000 if pd.notna(row.get('TOT_A_SHARE')) else 0, + 'float_shares': float(row.get('FLOAT_A_SHARE', 0)) * 10000 if pd.notna(row.get('FLOAT_A_SHARE')) else 0, + } + except Exception as e: + print(f"[amazingdata_adapter]Failed to get equity structure: {e}") + equity_data = {} + + kline_dict = self._market_data.query_kline( + code_list=codes, + begin_date=start_int, + end_date=end_int, + period=self._ad.constant.Period.day.value + ) + + if symbol not in kline_dict: + return [] + + df = kline_dict[symbol] + + for _, row in df.iterrows(): + kline_time = row.get('kline_time') + if pd.isna(kline_time): + continue + + if isinstance(kline_time, pd.Timestamp): + trade_date = kline_time.strftime('%Y-%m-%d') + trade_date_int = int(kline_time.strftime('%Y%m%d')) + else: + date_str = str(int(kline_time)) + if len(date_str) != 8: + continue + dt = datetime.strptime(date_str, "%Y%m%d") + trade_date = dt.strftime('%Y-%m-%d') + trade_date_int = int(date_str) + + close = float(row.get('close', 0)) + + total_shares = 0 + float_shares = 0 + for date_key in sorted(equity_data.keys(), reverse=True): + if int(date_key) <= trade_date_int: + total_shares = equity_data[date_key]['total_shares'] + float_shares = equity_data[date_key]['float_shares'] + break + + total_market_cap = close * total_shares if total_shares > 0 and close > 0 else None + float_market_cap = close * float_shares if float_shares > 0 and close > 0 else None + + results.append({ + "symbol": symbol, + "trade_date": trade_date, + "total_market_cap": round(total_market_cap, 2) if total_market_cap else None, + "float_market_cap": round(float_market_cap, 2) if float_market_cap else None, + "total_shares": int(total_shares) if total_shares > 0 else None, + "float_shares": int(float_shares) if float_shares > 0 else None, + "inst_holding_shares": None, + "inst_holding_ratio": None, + "top10_holders_ratio": None, + "net_profit": None, + "revenue": None, + "eps": None, + "roe": None, + "trading_days": None, + }) + + info(f"Fetched {len(results)} finance records for {symbol}") + return results + + async def fetch_stock_basic_info( + self, + codes: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """Fetch stock basic info + + Corresponding table: stock_symbols + Data source: get_stock_basic + + Returns: + List[Dict] containing fields: + - symbol_id: Symbol code + - name: Name + - exchange: Exchange + - list_date: List date + - list_board: List board + - industry: Industry + - status: Status + - is_delisted: Is delisted + - delist_date: Delist date + """ + print(f"[amazingdata_adapter fetch_stock_basic_info]Fetching stock basic info...") + self._check_login() + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: self._fetch_stock_basic_info_sync(codes) + ) + + def _fetch_stock_basic_info_sync( + self, + codes: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """Sync method to fetch stock basic info""" + try: + all_codes = self._base_data.get_code_list( + security_type=SecurityType.STOCK_A.value + ) + + if codes: + all_codes = [c for c in all_codes if c in codes] + + info_df = self._base_data.get_code_info( + security_type=SecurityType.STOCK_A.value + ) + + results = [] + for code in all_codes: + if ".SH" in code: + exchange = "SH" + elif ".SZ" in code: + exchange = "SZ" + elif ".BJ" in code: + exchange = "BJ" + else: + exchange = "" + + name = code + if code in info_df.index and 'symbol' in info_df.columns: + name = info_df.loc[code, 'symbol'] + + list_date = None + try: + equity_dict = self._info_data.get_equity_structure( + code_list=[code], + local_path=self.config.local_path, + is_local=self.config.use_local_cache + ) + if code in equity_dict and not equity_dict[code].empty: + first_record = equity_dict[code].iloc[0] + ann_date = first_record.get('ANN_DATE') + if pd.notna(ann_date): + if isinstance(ann_date, (int, float)): + list_date = datetime.strptime(str(int(ann_date)), "%Y%m%d") + else: + list_date = pd.to_datetime(ann_date) + except: + pass + + results.append({ + "symbol_id": code, + "name": name, + "exchange": exchange, + "list_date": list_date, + "list_board": None, + "industry": None, + "status": "active", + "is_delisted": False, + "delist_date": None, + "is_st": None, + "total_shares": None, + "float_shares": None, + }) + + info(f"Fetched {len(results)} stock basic info records") + return results + + except Exception as e: + error(f"Failed to fetch stock basic info: {e}") + return [] +''' + +def main(): + # Read original file + with open('app/adapters/amazingdata_adapter.py', 'r', encoding='utf-8') as f: + content = f.read() + + # Check if already added + if 'fetch_kline_base' not in content: + # Append new methods before the last line + content = content.rstrip() + new_methods + + with open('app/adapters/amazingdata_adapter.py', 'w', encoding='utf-8') as f: + f.write(content) + print('New methods added successfully!') + else: + print('Methods already exist.') + +if __name__ == "__main__": + main() diff --git a/scripts/add_stock_split_tables.py b/scripts/add_stock_split_tables.py new file mode 100644 index 0000000..ffdc7fc --- /dev/null +++ b/scripts/add_stock_split_tables.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""添加股票分表模型到 models.py""" + +# 股票分表结构 +stock_split_tables = ''' + + +# ============================================ +# 股票日线历史数据表 (1d) - 分表 (新增) +# ============================================ + +class StockKLine1DBase(Base): + """股票日线K线 - 基础表 + + 存储最基础的K线数据，对应 query_kline 接口 + """ + __tablename__ = "stock_klines_1d_base" + __table_args__ = ( + Index("idx_stock_1d_base_symbol_ts", "symbol_id", "ts"), + Index("idx_stock_1d_base_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量(股)") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额(元)") + adj_factor = Column(Numeric(18, 8), nullable=True, comment="复权系数") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine1DQuote(Base): + """股票日线K线 - 行情指标表 + + 存储需要计算的行情指标 + """ + __tablename__ = "stock_klines_1d_quote" + __table_args__ = ( + Index("idx_stock_1d_quote_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + change_5d_pct = Column(Numeric(8, 4), nullable=True, comment="5日涨跌幅%") + change_10d_pct = Column(Numeric(8, 4), nullable=True, comment="10日涨跌幅%") + change_20d_pct = Column(Numeric(8, 4), nullable=True, comment="20日涨跌幅%") + change_30d_pct = Column(Numeric(8, 4), nullable=True, comment="30日涨跌幅%") + change_60d_pct = Column(Numeric(8, 4), nullable=True, comment="60日涨跌幅%") + ma5 = Column(Numeric(18, 4), nullable=True, comment="5日均线") + ma10 = Column(Numeric(18, 4), nullable=True, comment="10日均线") + ma20 = Column(Numeric(18, 4), nullable=True, comment="20日均线") + ma30 = Column(Numeric(18, 4), nullable=True, comment="30日均线") + ma60 = Column(Numeric(18, 4), nullable=True, comment="60日均线") + ma120 = Column(Numeric(18, 4), nullable=True, comment="120日均线") + ma250 = Column(Numeric(18, 4), nullable=True, comment="250日均线") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + bias_5 = Column(Numeric(8, 4), nullable=True, comment="5日乖离率%") + bias_10 = Column(Numeric(8, 4), nullable=True, comment="10日乖离率%") + bias_20 = Column(Numeric(8, 4), nullable=True, comment="20日乖离率%") + is_limit_up = Column(Boolean, nullable=True, comment="是否涨停") + is_limit_down = Column(Boolean, nullable=True, comment="是否跌停") + limit_up_price = Column(Numeric(18, 4), nullable=True, comment="涨停价") + limit_down_price = Column(Numeric(18, 4), nullable=True, comment="跌停价") + is_st = Column(Boolean, nullable=True, comment="是否ST") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockKLine1DFinance(Base): + """股票日线K线 - 财务数据表 + + 存储财务相关数据 + """ + __tablename__ = "stock_klines_1d_finance" + __table_args__ = ( + Index("idx_stock_1d_finance_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="标的代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + total_market_cap = Column(Numeric(20, 4), nullable=True, comment="总市值(元)") + float_market_cap = Column(Numeric(20, 4), nullable=True, comment="流通市值(元)") + total_shares = Column(BigInteger, nullable=True, comment="总股本(股)") + float_shares = Column(BigInteger, nullable=True, comment="流通股本(股)") + inst_holding_shares = Column(BigInteger, nullable=True, comment="机构持股数量") + inst_holding_ratio = Column(Numeric(8, 4), nullable=True, comment="机构持仓占比%") + top10_holders_ratio = Column(Numeric(8, 4), nullable=True, comment="前十大股东持股占比%") + net_profit = Column(Numeric(20, 4), nullable=True, comment="净利润") + revenue = Column(Numeric(20, 4), nullable=True, comment="营业总收入") + eps = Column(Numeric(12, 4), nullable=True, comment="每股收益") + roe = Column(Numeric(8, 4), nullable=True, comment="净资产收益率%") + trading_days = Column(Integer, nullable=True, comment="从上市至今交易日数") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class StockRealTimeQuote(Base): + """股票实时行情快照""" + __tablename__ = "stock_realtime_quotes" + + symbol_id = Column(String(20), primary_key=True, comment="标的代码") + update_time = Column(DateTime, nullable=False, comment="更新时间") + last_price = Column(Numeric(18, 4), nullable=True, comment="最新价") + open = Column(Numeric(18, 4), nullable=True, comment="开盘价") + high = Column(Numeric(18, 4), nullable=True, comment="最高价") + low = Column(Numeric(18, 4), nullable=True, comment="最低价") + pre_close = Column(Numeric(18, 4), nullable=True, comment="昨收") + volume = Column(BigInteger, nullable=True, comment="成交量") + amount = Column(Numeric(20, 4), nullable=True, comment="成交额") + bid1 = Column(Numeric(18, 4), nullable=True, comment="买一价") + ask1 = Column(Numeric(18, 4), nullable=True, comment="卖一价") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# 股票表向后兼容别名 +StockKLine1D = StockKLine1DBase +''' + +def main(): + models_path = 'app/repositories/models.py' + + with open(models_path, 'r', encoding='utf-8') as f: + content = f.read() + + # 检查是否已存在 + if 'class StockKLine1DBase' not in content: + with open(models_path, 'a', encoding='utf-8') as f: + f.write(stock_split_tables) + print('Stock split tables appended successfully!') + else: + print('Stock split tables already exist.') + +if __name__ == "__main__": + main() diff --git a/scripts/append_futures_models.py b/scripts/append_futures_models.py new file mode 100644 index 0000000..60c6a5c --- /dev/null +++ b/scripts/append_futures_models.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""追加期货分表模型到 models.py""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +futures_split_tables = ''' + + +# ============================================ +# 期货日线历史数据表 (1d) - 分表 +# ============================================ + +class FuturesKLine1DBase(Base): + """期货日线K线 - 基础表 + + 存储最基础的K线数据，对应 query_kline 接口 + """ + __tablename__ = "futures_klines_1d_base" + __table_args__ = ( + Index("idx_futures_1d_base_symbol_ts", "symbol_id", "ts"), + Index("idx_futures_1d_base_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + + # 基础价格数据 + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + + # 成交量额 + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + + # 期货特有 + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算价") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1DQuote(Base): + """期货日线K线 - 行情指标表 + + 存储需要计算的行情指标 + """ + __tablename__ = "futures_klines_1d_quote" + __table_args__ = ( + Index("idx_futures_1d_quote_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + + # 涨跌幅 (需计算) + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + change_5d_pct = Column(Numeric(8, 4), nullable=True, comment="5日涨跌幅%") + change_10d_pct = Column(Numeric(8, 4), nullable=True, comment="10日涨跌幅%") + change_20d_pct = Column(Numeric(8, 4), nullable=True, comment="20日涨跌幅%") + + # 均线 (需计算) + ma5 = Column(Numeric(18, 4), nullable=True, comment="5日均线") + ma10 = Column(Numeric(18, 4), nullable=True, comment="10日均线") + ma20 = Column(Numeric(18, 4), nullable=True, comment="20日均线") + ma30 = Column(Numeric(18, 4), nullable=True, comment="30日均线") + ma60 = Column(Numeric(18, 4), nullable=True, comment="60日均线") + + # MACD指标 (需计算) + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + + # 持仓变化 + oi_change = Column(BigInteger, nullable=True, comment="持仓量变化") + oi_change_pct = Column(Numeric(8, 4), nullable=True, comment="持仓量变化%") + + # 振幅 + amplitude = Column(Numeric(8, 4), nullable=True, comment="振幅%") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# ============================================ +# 期货分钟线历史数据表 - 分表 +# ============================================ + +class FuturesKLine1MBase(Base): + """期货1分钟K线 - 基础表""" + __tablename__ = "futures_klines_1m_base" + __table_args__ = ( + Index("idx_futures_1m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine5MBase(Base): + """期货5分钟K线 - 基础表""" + __tablename__ = "futures_klines_5m_base" + __table_args__ = ( + Index("idx_futures_5m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine15MBase(Base): + """期货15分钟K线 - 基础表""" + __tablename__ = "futures_klines_15m_base" + __table_args__ = ( + Index("idx_futures_15m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine30MBase(Base): + """期货30分钟K线 - 基础表""" + __tablename__ = "futures_klines_30m_base" + __table_args__ = ( + Index("idx_futures_30m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine60MBase(Base): + """期货60分钟K线 - 基础表""" + __tablename__ = "futures_klines_60m_base" + __table_args__ = ( + Index("idx_futures_60m_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# ============================================ +# 期货周月线历史数据表 - 分表 +# ============================================ + +class FuturesKLine1WBase(Base): + """期货周线K线 - 基础表""" + __tablename__ = "futures_klines_1w_base" + __table_args__ = ( + Index("idx_futures_1w_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1MonBase(Base): + """期货月线K线 - 基础表""" + __tablename__ = "futures_klines_1month_base" + __table_args__ = ( + Index("idx_futures_1month_base_symbol_ts", "symbol_id", "ts"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# ============================================ +# 期货实时行情表 +# ============================================ + +class FuturesRealTimeQuote(Base): + """期货实时行情快照""" + __tablename__ = "futures_realtime_quotes" + + symbol_id = Column(String(20), primary_key=True, comment="合约代码") + update_time = Column(DateTime, nullable=False, comment="更新时间") + + # 价格数据 + last_price = Column(Numeric(18, 4), nullable=True, comment="最新价") + open = Column(Numeric(18, 4), nullable=True, comment="开盘价") + high = Column(Numeric(18, 4), nullable=True, comment="最高价") + low = Column(Numeric(18, 4), nullable=True, comment="最低价") + pre_close = Column(Numeric(18, 4), nullable=True, comment="昨收") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + + # 成交量额 + volume = Column(BigInteger, nullable=True, comment="成交量") + amount = Column(Numeric(20, 4), nullable=True, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + + # 买卖盘 + bid1 = Column(Numeric(18, 4), nullable=True, comment="买一价") + bid1_volume = Column(BigInteger, nullable=True, comment="买一量") + ask1 = Column(Numeric(18, 4), nullable=True, comment="卖一价") + ask1_volume = Column(BigInteger, nullable=True, comment="卖一量") + + # 涨跌停 + limit_up = Column(Numeric(18, 4), nullable=True, comment="涨停价") + limit_down = Column(Numeric(18, 4), nullable=True, comment="跌停价") + + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# ============================================ +# 向后兼容的模型别名 +# ============================================ + +# 股票表别名 +StockKLine1D = StockKLine1DBase + +# 期货表别名（兼容旧代码） +FuturesKLine1M = FuturesKLine1MBase +FuturesKLine5M = FuturesKLine5MBase +FuturesKLine1D = FuturesKLine1DBase +''' + +def main(): + models_path = Path(__file__).parent.parent / "app" / "repositories" / "models.py" + + with open(models_path, 'r', encoding='utf-8') as f: + original = f.read() + + # 移除旧的向后兼容别名（如果存在） + marker = '\n\n\n# ============================================\n# 向后兼容的模型别名\n# ============================================\n\n# 旧代码可能引用的模型，现在映射到新表\nStockKLine1D = StockKLine1DBase\n# 注意: 1M/5M/15M/30M/60M/1W/1Month 表命名未加Base后缀，已在原始定义中' + if marker in original: + original = original[:original.find(marker)] + + # 保存新内容 + with open(models_path, 'w', encoding='utf-8') as f: + f.write(original + futures_split_tables) + + print('Futures split tables appended successfully!') + +if __name__ == "__main__": + main() diff --git a/scripts/create_split_tables.py b/scripts/create_split_tables.py new file mode 100644 index 0000000..90526e6 --- /dev/null +++ b/scripts/create_split_tables.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +创建拆分后的新表结构 + +用法: + python scripts/create_split_tables.py + python scripts/create_split_tables.py --drop-existing +""" +import argparse +import sys +from pathlib import Path + +# 添加项目根目录到路径 +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from sqlalchemy import create_engine, text +from app.repositories.database import Base, engine +from app.core.config import get_config +from app.core.logger import info, error + + +def create_tables(drop_existing: bool = False): + """创建所有表""" + config = get_config() + + # 创建数据库连接 + db_url = f"mysql+pymysql://{config.database.user}:{config.database.password}@{config.database.host}:{config.database.port}/{config.database.database}" + engine_local = create_engine(db_url, echo=False) + + if drop_existing: + info("Dropping existing split tables...") + tables_to_drop = [ + # 股票分表 + "stock_klines_1d_base", "stock_klines_1d_quote", "stock_klines_1d_finance", + "stock_realtime_quotes", + # 期货分表 + "futures_klines_1d_base", "futures_klines_1d_quote", + "futures_klines_1m_base", "futures_klines_5m_base", + "futures_realtime_quotes", + ] + with engine_local.connect() as conn: + for table in tables_to_drop: + try: + conn.execute(text(f"DROP TABLE IF EXISTS {table}")) + info(f"Dropped table: {table}") + except Exception as e: + error(f"Failed to drop {table}: {e}") + conn.commit() + + info("Creating new split tables...") + + # 导入所有模型以确保它们被注册 + from app.repositories.models import ( + # 股票分表 + StockKLine1DBase, StockKLine1DQuote, StockKLine1DFinance, + StockKLine1M, StockKLine5M, StockKLine15M, StockKLine30M, StockKLine60M, + StockRealTimeQuote, + # 期货分表 + FuturesKLine1DBase, FuturesKLine1DQuote, + FuturesKLine1MBase, FuturesKLine5MBase, FuturesKLine15MBase, FuturesKLine30MBase, FuturesKLine60MBase, + FuturesRealTimeQuote, + ) + + # 创建表 + tables = [ + # 股票表 - 日线分表 + StockKLine1DBase.__table__, + StockKLine1DQuote.__table__, + StockKLine1DFinance.__table__, + # 股票表 - 分钟线 + StockKLine1M.__table__, + StockKLine5M.__table__, + StockKLine15M.__table__, + StockKLine30M.__table__, + StockKLine60M.__table__, + StockRealTimeQuote.__table__, + # 期货表 - 日线分表 + FuturesKLine1DBase.__table__, + FuturesKLine1DQuote.__table__, + # 期货表 - 分钟线 + FuturesKLine1MBase.__table__, + FuturesKLine5MBase.__table__, + FuturesKLine15MBase.__table__, + FuturesKLine30MBase.__table__, + FuturesKLine60MBase.__table__, + FuturesRealTimeQuote.__table__, + ] + + Base.metadata.create_all(bind=engine_local, tables=tables) + + info("Tables created successfully!") + + # 显示创建的表 + with engine_local.connect() as conn: + result = conn.execute(text(""" + SELECT TABLE_NAME, TABLE_COMMENT + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND ( + TABLE_NAME LIKE 'stock_klines_%' + OR TABLE_NAME LIKE 'futures_klines_%' + OR TABLE_NAME LIKE '%realtime_quotes' + ) + """)) + tables = result.fetchall() + + info("\nCreated split tables:") + stock_tables = [] + futures_tables = [] + for table_name, comment in tables: + if table_name.startswith('stock_'): + stock_tables.append(f" - {table_name}") + else: + futures_tables.append(f" - {table_name}") + + if stock_tables: + info("\n[Stock Tables]") + for t in stock_tables: + info(t) + + if futures_tables: + info("\n[Futures Tables]") + for t in futures_tables: + info(t) + + +def main(): + parser = argparse.ArgumentParser(description="Create split table structure") + parser.add_argument( + "--drop-existing", + action="store_true", + help="Drop existing tables before creating (WARNING: data will be lost!)" + ) + + args = parser.parse_args() + + if args.drop_existing: + confirm = input("WARNING: This will drop existing tables and ALL DATA will be lost!\nType 'yes' to continue: ") + if confirm != "yes": + print("Aborted.") + return + + create_tables(drop_existing=args.drop_existing) + + +if __name__ == "__main__": + main() diff --git a/scripts/fix_garbled_chinese.py b/scripts/fix_garbled_chinese.py new file mode 100644 index 0000000..4c04fba --- /dev/null +++ b/scripts/fix_garbled_chinese.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""修复 amazingdata_adapter.py 中的中文乱码""" + +# 读取文件 +with open('app/adapters/amazingdata_adapter.py', 'r', encoding='utf-8') as f: + content = f.read() + +# 常见乱码映射表 +garbled_map = { + # 常用词汇 + '淇℃伅': '信息', + '鏁版嵁': '数据', + '鑾峰彇': '获取', + '鏃ョ嚎': '日线', + '鍩虹': '基础', + '鏇村': '更多', + '鏃ュ巻': '日历', + '鏁版嵁搴�': '数据库', + '鏈嶅姟': '服务', + '绠＄悊': '管理', + '鐘熸搷': '后操', + '鍒濆鍖�': '初始化', + '璁块棶': '访问', + '鏃跺嚭閿�': '时出错', + '鍏抽棴': '关闭', + '鎵ц': '执行', + '鏌ヨ': '查询', + '娑ㄨ穼': '涨跌', + '鍋滀环': '停价', + '鏄惁': '是否', + '鐨�': '的', + '绛�': '等', + '鏉�': '来', + '浣�': '作', + '涔囩鐜�': '乖离率', + '鏃�': '时', + '璁＄畻': '计算', + '鎸囨爣': '指标', + '闇€瑕�': '需要', + '鍘嗗彶': '历史', + '鏃ユ湡': '日期', + '鑼冨洿': '范围', + '浠ヨ绠�': '以计算', + '鍧囩嚎': '均线', + '绛夛級': '等）', + '鍒ゆ柇': '判断', + '娑ㄥ仠': '涨停', + '浠疯穼鍋�': '跌停', + '鎵�': '所', + '灞�': '属', + '琛屼笟': '行业', + '鐘舵��': '状态', + '浠ｇ爜': '代码', + '鍒楄〃': '列表', + '鍚嶇О': '名称', + '浜ゆ槗': '交易', + '鏃�': '日', + '鏈�': '有', + '鍊�': '价', + '鍙�': '可', + '鑳�': '能', + '閿�': '错', + '璇�': '误', + '鎴�': '或', + '鍘�': '去', + '鎺�': '接', + '鍙�': '口', + '杩�': '返', + '鍥�': '回', + '鎹�': '据', + '闃�': '防', + '姝�': '止', + '閲嶅�?': '重复', + '鐧�': '登', + '褰�': '录', + '澶辫触': '失败', + '璇锋眰': '请求', + '鍙傛暟': '参数', + '鏃犳晥': '无效', + '鎵�': '锁', + '瀹�': '定', + '寮�': '开', + '閲�': '锁', + '閲婃斁': '释放', + '璧勬簮': '资源', + '鍔犺浇': '加载', + '閰嶇疆': '配置', + '鏂囦欢': '文件', + '涓嶅瓨鍦�': '不存在', + '浣跨敤': '使用', + '榛樿': '默认', + '瀹屾垚': '完成', + '鎴愬姛': '成功', + '濮�': '始', + '缁撴潫': '结束', + '澶勭悊': '处理', + '寮傚父': '异常', + '閿欒': '错误', + '鎻愮ず': '提示', + '纭': '确认', + '鍙栨秷': '取消', + '纭畾': '确定', + '鎴戠殑': '我的', + '璁剧疆': '设置', + '甯姪': '帮助', + '鍏充簬': '关于', + '閫�鍑�': '退出', + '鏂板缓': '新建', + '鎵撳紑': '打开', + '淇濆瓨': '保存', + '鍙︀瓨涓�': '另存为', + '缂栬緫': '编辑', + '鍓��': '剪切', + '澶嶅埗': '复制', + '绮樿创': '粘贴', + '鍏ㄩ��': '全选', + '鍒犻櫎': '删除', + '鎼滅储': '搜索', + '鏇挎崲': '替换', + '鎵惧埌': '找到', + '涓嬩竴涓�': '下一个', + '涓婁竴涓�': '上一个', + '鏇挎崲鍏ㄩ儴': '替换全部', + '鏌ョ湅': '查看', + '宸ュ叿鏍�': '工具栏', + '鐘舵�佹爮': '状态栏', + '绐楀彛': '窗口', + '鏂扮獥鍙�': '新窗口', + '鍨冨溇鏋�': '层叠', + '骞抽摵': '平铺', + '鎺掑垪鍥炬爣': '排列图标', + '鍏ㄩ儴閫夋嫨': '全部选择', + '鍏ㄩ儏': '全屏', + '鏈�澶у寲': '最大化', + '鏈�灏忓寲': '最小化', + '鎭㈠': '恢复', + '绉诲姩': '移动', + '澶у皬': '大小', + '鏈�灏�': '最小', + '鏈�澶�': '最大', + '鍓嶄竴涓�': '前一个', + '鍚庝竴涓�': '后一个', + '瑙ｆ瀽': '解析', + '浠ｇ爜': '代码', + '璧嬪€�': '赋值', + '璁块棶': '访问', + '缁熻': '统计', + '璇︽儏': '详情', + '鎻忚堪': '描述', + '澶囨敞': '备注', + '绫诲瀷': '类型', + '鏍煎紡': '格式', + '澶у皬': '大小', + '浣嶇疆': '位置', + '鏃堕暱': '时长', + '棰戦��': '频率', + '鍝嶅簲': '响应', + '璇锋眰': '请求', + '澶勭悊': '处理', + '缁撴灉': '结果', + '鐘舵��': '状态', + '娑堟伅': '消息', + '绾跨▼': '线程', + '杩涚▼': '进程', + '鏃ュ織': '日志', + '閰嶇疆': '配置', + '閫夐」': '选项', + '鍙傛暟': '参数', + '灞炴€�': '属性', + '鏂规硶': '方法', + '鍑芥暟': '函数', + '绫�': '类', + '妯″潡': '模块', + '鍖�': '包', + '瀛楃涓�': '字符串', + '鍒楄〃': '列表', + '瀛楀吀': '字典', + '鍏冪粍': '元组', + '闆嗗悎': '集合', + '瀵硅薄': '对象', + '瀹炰緥': '实例', + '缁ф壙': '继承', + '澶氭€�': '多态', + '灏佽': '封装', + '鎺ュ彛': '接口', + '鎶借薄': '抽象', + '绉佹湁': '私有', + '鍏紑': '公开', + '淇濇姢': '保护', + '闈欐€�': '静态', + '绫诲彉閲�': '类变量', + '瀹炰緥鍙橀噺': '实例变量', + '鏂规硶': '方法', + '鏋勯€犲嚱鏁�': '构造函数', + '鏋愭瀯鍑芥暟': '析构函数', + '瑁呴グ鍣�': '装饰器', + '鐗规€�': '特性', + '灞炴€�': '属性', + '鏂规硶': '方法', +} + +# 替换所有乱码 +for garbled, correct in garbled_map.items(): + content = content.replace(garbled, correct) + +# 保存 +with open('app/adapters/amazingdata_adapter.py', 'w', encoding='utf-8') as f: + f.write(content) + +print('Fixed all garbled Chinese characters!') diff --git a/scripts/fix_models.py b/scripts/fix_models.py new file mode 100644 index 0000000..c80365f --- /dev/null +++ b/scripts/fix_models.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""追加期货分表模型到 models.py""" + +append_content = ''' + + +# ============================================ +# 期货分表结构 (新增) +# ============================================ + +class FuturesKLine1DBase(Base): + """期货日线K线 - 基础表""" + __tablename__ = "futures_klines_1d_base" + __table_args__ = ( + Index("idx_futures_1d_base_symbol_ts", "symbol_id", "ts"), + Index("idx_futures_1d_base_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算价") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1DQuote(Base): + """期货日线K线 - 行情指标表""" + __tablename__ = "futures_klines_1d_quote" + __table_args__ = ( + Index("idx_futures_1d_quote_symbol_date", "symbol_id", "trade_date"), + ) + + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + trade_date = Column(String(10), nullable=False, index=True, comment="交易日 YYYY-MM-DD") + change_pct = Column(Numeric(8, 4), nullable=True, comment="涨跌幅%") + change_5d_pct = Column(Numeric(8, 4), nullable=True, comment="5日涨跌幅%") + change_10d_pct = Column(Numeric(8, 4), nullable=True, comment="10日涨跌幅%") + change_20d_pct = Column(Numeric(8, 4), nullable=True, comment="20日涨跌幅%") + ma5 = Column(Numeric(18, 4), nullable=True, comment="5日均线") + ma10 = Column(Numeric(18, 4), nullable=True, comment="10日均线") + ma20 = Column(Numeric(18, 4), nullable=True, comment="20日均线") + ma30 = Column(Numeric(18, 4), nullable=True, comment="30日均线") + ma60 = Column(Numeric(18, 4), nullable=True, comment="60日均线") + macd_dif = Column(Numeric(18, 6), nullable=True, comment="MACD DIF") + macd_dea = Column(Numeric(18, 6), nullable=True, comment="MACD DEA") + macd_bar = Column(Numeric(18, 6), nullable=True, comment="MACD BAR") + oi_change = Column(BigInteger, nullable=True, comment="持仓量变化") + oi_change_pct = Column(Numeric(8, 4), nullable=True, comment="持仓量变化%") + amplitude = Column(Numeric(8, 4), nullable=True, comment="振幅%") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine1MBase(Base): + """期货1分钟K线 - 基础表""" + __tablename__ = "futures_klines_1m_base" + __table_args__ = (Index("idx_futures_1m_base_symbol_ts", "symbol_id", "ts"),) + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesKLine5MBase(Base): + """期货5分钟K线 - 基础表""" + __tablename__ = "futures_klines_5m_base" + __table_args__ = (Index("idx_futures_5m_base_symbol_ts", "symbol_id", "ts"),) + id = Column(BigInteger, primary_key=True, autoincrement=True) + symbol_id = Column(String(20), nullable=False, index=True, comment="合约代码") + ts = Column(DateTime, nullable=False, comment="时间戳") + trade_date = Column(String(10), nullable=False, comment="交易日") + open = Column(Numeric(18, 4), nullable=False, comment="开盘价") + high = Column(Numeric(18, 4), nullable=False, comment="最高价") + low = Column(Numeric(18, 4), nullable=False, comment="最低价") + close = Column(Numeric(18, 4), nullable=False, comment="收盘价") + volume = Column(BigInteger, nullable=False, comment="成交量") + amount = Column(Numeric(20, 4), nullable=False, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + created_at = Column(DateTime, default=datetime.now, comment="创建时间") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +class FuturesRealTimeQuote(Base): + """期货实时行情快照""" + __tablename__ = "futures_realtime_quotes" + symbol_id = Column(String(20), primary_key=True, comment="合约代码") + update_time = Column(DateTime, nullable=False, comment="更新时间") + last_price = Column(Numeric(18, 4), nullable=True, comment="最新价") + open = Column(Numeric(18, 4), nullable=True, comment="开盘价") + high = Column(Numeric(18, 4), nullable=True, comment="最高价") + low = Column(Numeric(18, 4), nullable=True, comment="最低价") + pre_close = Column(Numeric(18, 4), nullable=True, comment="昨收") + pre_settlement = Column(Numeric(18, 4), nullable=True, comment="昨结算") + settlement = Column(Numeric(18, 4), nullable=True, comment="结算价") + volume = Column(BigInteger, nullable=True, comment="成交量") + amount = Column(Numeric(20, 4), nullable=True, comment="成交额") + open_interest = Column(BigInteger, nullable=True, comment="持仓量") + bid1 = Column(Numeric(18, 4), nullable=True, comment="买一价") + bid1_volume = Column(BigInteger, nullable=True, comment="买一量") + ask1 = Column(Numeric(18, 4), nullable=True, comment="卖一价") + ask1_volume = Column(BigInteger, nullable=True, comment="卖一量") + limit_up = Column(Numeric(18, 4), nullable=True, comment="涨停价") + limit_down = Column(Numeric(18, 4), nullable=True, comment="跌停价") + updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间") + + +# 向后兼容别名 +FuturesKLine1M = FuturesKLine1MBase +FuturesKLine5M = FuturesKLine5MBase +FuturesKLine1D = FuturesKLine1DBase +''' + +# 读取原始文件 +with open('app/repositories/models.py', 'r', encoding='utf-8') as f: + content = f.read() + +# 检查是否已存在 +if 'class FuturesKLine1DBase' not in content: + with open('app/repositories/models.py', 'w', encoding='utf-8') as f: + f.write(content + append_content) + print('Futures split tables appended successfully!') +else: + print('Futures split tables already exist.')