fix: 修复性能问题;优化查询缺失代码列表的逻辑

master
Lxy 1 month ago
parent 7307fe6182
commit dd9b57ad11

@ -417,6 +417,29 @@ async def get_missing_dates_for_code(
return ResponseModel(data=result) return ResponseModel(data=result)
@router.get("/missing-codes-by-date", response_model=ResponseModel)
async def get_missing_codes_by_date(
trade_date: str = Query(..., description="交易日 YYYYMMDD"),
security_type: str = Query("stock", description="证券类型: stock, future"),
period_type: str = Query("daily", description="周期类型: daily, min1, min5"),
contract_type: str = Query("all", description="合约类型: all, main"),
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""获取某个交易日缺失的代码列表"""
service = CacheService(db)
date_obj = parse_date(trade_date)
result = service.get_missing_codes_by_date(
date_obj,
security_type,
period_type,
contract_type
)
return ResponseModel(data=result)
@router.post("/fill-single-date", response_model=ResponseModel) @router.post("/fill-single-date", response_model=ResponseModel)
async def fill_single_date_data( async def fill_single_date_data(
request: dict, request: dict,
@ -501,3 +524,67 @@ async def fill_all_dates_for_code(
"missing_count": len(missing_dates), "missing_count": len(missing_dates),
"status": "processing" "status": "processing"
}) })
@router.post("/fill-date-batch", response_model=ResponseModel)
async def fill_missing_codes_for_date_batch(
request: dict,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""批量补齐某个交易日的所有缺失代码数据"""
trade_date = request.get("trade_date")
security_type = request.get("security_type", "stock")
period_type = request.get("period_type", "daily")
missing_codes = request.get("missing_codes", [])
if not trade_date:
return ResponseModel(code=400, message="缺少必要参数: trade_date")
if not missing_codes or len(missing_codes) == 0:
return ResponseModel(data={
"task_id": None,
"message": "没有缺失代码需要补齐",
"missing_count": 0
})
from app.utils.date_utils import parse_date
date_obj = parse_date(trade_date)
service = CacheService(db)
task = service._create_batch_fill_task(
trade_date=date_obj,
security_type=security_type,
period_type=period_type,
missing_codes=missing_codes
)
def run_batch_fill_task():
from app.db.session import SessionLocal
db_local = SessionLocal()
try:
service_local = CacheService(db_local)
service_local._execute_batch_fill_task(
task.id,
date_obj,
security_type,
period_type,
missing_codes
)
except Exception as e:
import logging
logging.getLogger(__name__).error(f"批量补齐交易日数据失败: {str(e)}")
finally:
db_local.close()
background_tasks.add_task(run_batch_fill_task)
return ResponseModel(data={
"task_id": task.id,
"task_name": task.task_name,
"trade_date": trade_date,
"missing_count": len(missing_codes),
"status": "processing"
})

@ -11,6 +11,7 @@ from sqlalchemy import and_, func
from app.models.cache import CacheTask, CacheTaskDetail from app.models.cache import CacheTask, CacheTaskDetail
from app.models.stock import StockKlineDaily from app.models.stock import StockKlineDaily
from app.models.future import FutureKlineDaily from app.models.future import FutureKlineDaily
from app.models.config import SDKConfig
from app.services.base_data_service import BaseDataService from app.services.base_data_service import BaseDataService
from app.services.stock_service import StockService from app.services.stock_service import StockService
from app.services.future_service import FutureService from app.services.future_service import FutureService
@ -174,6 +175,12 @@ class CacheService:
""" """
一键检测所有数据的缺失情况 一键检测所有数据的缺失情况
改进逻辑
1. 初步检测统计每个代码的数据条数找出疑似缺失的代码
2. 二次验证对疑似缺失的代码按交易日调用接口验证是否真的缺失
- 如果接口也没有数据说明该交易日股票未上市或停牌不算缺失
- 如果接口有数据但本地没有说明确实缺失
Args: Args:
security_type: 证券类型 (stock, future) security_type: 证券类型 (stock, future)
period_type: 周期类型 (daily, min1, etc.) period_type: 周期类型 (daily, min1, etc.)
@ -231,8 +238,9 @@ class CacheService:
market = "CFE" if security_type == "future" else "SH" market = "CFE" if security_type == "future" else "SH"
trading_days = self.base_service.get_trading_calendar(market, start_date, end_date) trading_days = self.base_service.get_trading_calendar(market, start_date, end_date)
expected_count = len(trading_days) expected_count = len(trading_days)
trading_days_set = set(trading_days)
missing_codes = [] preliminary_missing_codes = []
complete_codes = [] complete_codes = []
error_count = 0 error_count = 0
@ -241,11 +249,14 @@ class CacheService:
daily_stats[format_date(td)] = { daily_stats[format_date(td)] = {
"expected": len(code_list), "expected": len(code_list),
"actual": 0, "actual": 0,
"missing": 0 "missing": 0,
"verified_missing": 0
} }
push_progress(10, "running", message="查询数据库统计...") push_progress(10, "running", message="查询数据库统计...")
code_existing_dates = {}
if security_type == "stock" and period_type == "daily": if security_type == "stock" and period_type == "daily":
from sqlalchemy import func from sqlalchemy import func
@ -261,6 +272,21 @@ class CacheService:
code_counts = {r.code: r.count for r in code_count_query} code_counts = {r.code: r.count for r in code_count_query}
code_dates_query = self.db.query(
StockKlineDaily.code,
StockKlineDaily.trade_date
).filter(
and_(
StockKlineDaily.trade_date >= start_date,
StockKlineDaily.trade_date <= end_date
)
).all()
for r in code_dates_query:
if r.code not in code_existing_dates:
code_existing_dates[r.code] = set()
code_existing_dates[r.code].add(r.trade_date)
date_count_query = self.db.query( date_count_query = self.db.query(
func.date(StockKlineDaily.trade_date).label('trade_date'), func.date(StockKlineDaily.trade_date).label('trade_date'),
func.count(StockKlineDaily.id).label('count') func.count(StockKlineDaily.id).label('count')
@ -276,45 +302,38 @@ class CacheService:
if date_key in daily_stats: if date_key in daily_stats:
daily_stats[date_key]["actual"] = r.count daily_stats[date_key]["actual"] = r.count
push_progress(20, "running", message="分析数据完整性...") push_progress(20, "running", message="初步分析数据完整性...")
for i, code in enumerate(code_list): for i, code in enumerate(code_list):
actual_count = code_counts.get(code, 0) actual_count = code_counts.get(code, 0)
is_missing = actual_count < expected_count is_missing = actual_count < expected_count
if is_missing: if is_missing:
missing_codes.append({ existing_dates = code_existing_dates.get(code, set())
missing_dates = trading_days_set - existing_dates
preliminary_missing_codes.append({
"code": code, "code": code,
"actual_count": actual_count, "actual_count": actual_count,
"expected_count": expected_count, "expected_count": expected_count,
"missing_count": expected_count - actual_count, "missing_count": len(missing_dates),
"missing_ratio": (expected_count - actual_count) / expected_count if expected_count > 0 else 0 "missing_dates": sorted(list(missing_dates)),
"missing_ratio": len(missing_dates) / expected_count if expected_count > 0 else 0
}) })
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=True,
status="pending"
)
self.db.add(detail)
else: else:
complete_codes.append(code) complete_codes.append(code)
if (i + 1) % 500 == 0 or i == len(code_list) - 1: if (i + 1) % 500 == 0 or i == len(code_list) - 1:
task.success_count = len(missing_codes) + len(complete_codes) task.success_count = len(preliminary_missing_codes) + len(complete_codes)
task.error_count = error_count task.error_count = error_count
task.progress = min(100, int((i + 1) / len(code_list) * 100)) task.progress = min(50, int((i + 1) / len(code_list) * 30))
self.db.commit() self.db.commit()
push_progress( push_progress(
20 + int((i + 1) / len(code_list) * 70), 20 + int((i + 1) / len(code_list) * 30),
"running", "running",
processed=len(missing_codes) + len(complete_codes), processed=len(preliminary_missing_codes) + len(complete_codes),
missing=len(missing_codes), missing=len(preliminary_missing_codes),
complete=len(complete_codes) complete=len(complete_codes)
) )
@ -333,6 +352,21 @@ class CacheService:
code_counts = {r.code: r.count for r in code_count_query} code_counts = {r.code: r.count for r in code_count_query}
code_dates_query = self.db.query(
FutureKlineDaily.code,
FutureKlineDaily.trade_date
).filter(
and_(
FutureKlineDaily.trade_date >= start_date,
FutureKlineDaily.trade_date <= end_date
)
).all()
for r in code_dates_query:
if r.code not in code_existing_dates:
code_existing_dates[r.code] = set()
code_existing_dates[r.code].add(r.trade_date)
date_count_query = self.db.query( date_count_query = self.db.query(
func.date(FutureKlineDaily.trade_date).label('trade_date'), func.date(FutureKlineDaily.trade_date).label('trade_date'),
func.count(FutureKlineDaily.id).label('count') func.count(FutureKlineDaily.id).label('count')
@ -348,89 +382,160 @@ class CacheService:
if date_key in daily_stats: if date_key in daily_stats:
daily_stats[date_key]["actual"] = r.count daily_stats[date_key]["actual"] = r.count
push_progress(20, "running", message="分析数据完整性...") push_progress(20, "running", message="初步分析数据完整性...")
for i, code in enumerate(code_list): for i, code in enumerate(code_list):
actual_count = code_counts.get(code, 0) actual_count = code_counts.get(code, 0)
is_missing = actual_count < expected_count is_missing = actual_count < expected_count
if is_missing: if is_missing:
missing_codes.append({ existing_dates = code_existing_dates.get(code, set())
missing_dates = trading_days_set - existing_dates
preliminary_missing_codes.append({
"code": code, "code": code,
"actual_count": actual_count, "actual_count": actual_count,
"expected_count": expected_count, "expected_count": expected_count,
"missing_count": expected_count - actual_count, "missing_count": len(missing_dates),
"missing_ratio": (expected_count - actual_count) / expected_count if expected_count > 0 else 0 "missing_dates": sorted(list(missing_dates)),
"missing_ratio": len(missing_dates) / expected_count if expected_count > 0 else 0
}) })
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=True,
status="pending"
)
self.db.add(detail)
else: else:
complete_codes.append(code) complete_codes.append(code)
if (i + 1) % 500 == 0 or i == len(code_list) - 1: if (i + 1) % 500 == 0 or i == len(code_list) - 1:
task.success_count = len(missing_codes) + len(complete_codes) task.success_count = len(preliminary_missing_codes) + len(complete_codes)
task.error_count = error_count task.error_count = error_count
task.progress = min(100, int((i + 1) / len(code_list) * 100)) task.progress = min(50, int((i + 1) / len(code_list) * 30))
self.db.commit() self.db.commit()
push_progress( push_progress(
20 + int((i + 1) / len(code_list) * 70), 20 + int((i + 1) / len(code_list) * 30),
"running", "running",
processed=len(missing_codes) + len(complete_codes), processed=len(preliminary_missing_codes) + len(complete_codes),
missing=len(missing_codes), missing=len(preliminary_missing_codes),
complete=len(complete_codes) complete=len(complete_codes)
) )
else: else:
for i, code in enumerate(code_list): for i, code in enumerate(code_list):
actual_count = 0 preliminary_missing_codes.append({
is_missing = True
missing_codes.append({
"code": code, "code": code,
"actual_count": 0, "actual_count": 0,
"expected_count": expected_count, "expected_count": expected_count,
"missing_count": expected_count, "missing_count": expected_count,
"missing_dates": trading_days,
"missing_ratio": 1.0 "missing_ratio": 1.0
}) })
if (i + 1) % 500 == 0 or i == len(code_list) - 1:
task.success_count = len(preliminary_missing_codes)
task.error_count = error_count
task.progress = min(50, int((i + 1) / len(code_list) * 30))
self.db.commit()
push_progress(
20 + int((i + 1) / len(code_list) * 30),
"running",
processed=len(preliminary_missing_codes),
missing=len(preliminary_missing_codes)
)
logger.info(f"初步检测完成: 完整{len(complete_codes)}个, 疑似缺失{len(preliminary_missing_codes)}")
push_progress(50, "running", message="开始二次验证疑似缺失数据...")
verified_missing_codes = []
verified_complete_codes = []
adapter = sdk_manager.get_default_connection()
if not adapter:
logger.warning("SDK连接失败跳过二次验证使用初步检测结果")
verified_missing_codes = preliminary_missing_codes
else:
total_preliminary = len(preliminary_missing_codes)
for i, item in enumerate(preliminary_missing_codes):
code = item["code"]
missing_dates = item["missing_dates"]
truly_missing_dates = []
unavailable_dates = []
for trade_date in missing_dates:
try:
if security_type == "stock":
kline_data = adapter.get_stock_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
elif security_type == "future":
kline_data = adapter.get_future_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
else:
kline_data = []
if kline_data and len(kline_data) > 0:
truly_missing_dates.append(trade_date)
else:
unavailable_dates.append(trade_date)
except Exception as e:
logger.debug(f"验证{code} {format_date(trade_date)}时出错: {str(e)}")
unavailable_dates.append(trade_date)
truly_missing_count = len(truly_missing_dates)
if truly_missing_count > 0:
verified_missing_codes.append({
"code": code,
"actual_count": item["actual_count"],
"expected_count": item["expected_count"],
"missing_count": truly_missing_count,
"missing_dates": truly_missing_dates,
"unavailable_dates": unavailable_dates,
"missing_ratio": truly_missing_count / expected_count if expected_count > 0 else 0
})
detail = CacheTaskDetail( detail = CacheTaskDetail(
task_id=task.id, task_id=task.id,
code=code, code=code,
trade_date=start_date, trade_date=start_date,
expected_count=expected_count, expected_count=expected_count,
actual_count=0, actual_count=item["actual_count"],
is_missing=True, is_missing=True,
status="pending" status="pending"
) )
self.db.add(detail) self.db.add(detail)
else:
verified_complete_codes.append(code)
complete_codes.append(code)
if (i + 1) % 500 == 0 or i == len(code_list) - 1: if (i + 1) % 50 == 0 or i == total_preliminary - 1:
task.success_count = len(missing_codes) task.success_count = len(complete_codes)
task.error_count = error_count task.error_count = error_count
task.progress = min(100, int((i + 1) / len(code_list) * 100)) task.progress = min(100, 50 + int((i + 1) / total_preliminary * 50))
self.db.commit() self.db.commit()
push_progress( push_progress(
20 + int((i + 1) / len(code_list) * 70), 50 + int((i + 1) / total_preliminary * 50),
"running", "running",
processed=len(missing_codes), message=f"二次验证进度: {i + 1}/{total_preliminary}",
missing=len(missing_codes) processed=len(complete_codes),
missing=len(verified_missing_codes),
complete=len(complete_codes)
) )
for date_key in daily_stats: for date_key in daily_stats:
daily_stats[date_key]["missing"] = daily_stats[date_key]["expected"] - daily_stats[date_key]["actual"] daily_stats[date_key]["missing"] = daily_stats[date_key]["expected"] - daily_stats[date_key]["actual"]
missing_code_list = [m["code"] for m in missing_codes] verified_missing_code_list = [m["code"] for m in verified_missing_codes]
task.code_list = ",".join(missing_code_list[:500]) if missing_code_list else "" task.code_list = ",".join(verified_missing_code_list[:500]) if verified_missing_code_list else ""
task.status = "completed" task.status = "completed"
task.success_count = len(complete_codes) task.success_count = len(complete_codes)
@ -439,13 +544,25 @@ class CacheService:
self.db.commit() self.db.commit()
push_progress(100, "completed", push_progress(100, "completed",
message="检测完成", message="检测完成(含二次验证)",
complete_count=len(complete_codes), complete_count=len(complete_codes),
missing_count=len(missing_codes), missing_count=len(verified_missing_codes),
error_count=error_count error_count=error_count
) )
logger.info(f"检测完成: 完整{len(complete_codes)}个, 缺失{len(missing_codes)}个, 错误{error_count}") logger.info(f"检测完成(含二次验证): 完整{len(complete_codes)}个, 真实缺失{len(verified_missing_codes)}个, 错误{error_count}")
missing_codes_display = []
for m in verified_missing_codes[:100]:
missing_dates_str = [format_date(d) for d in m.get("missing_dates", [])]
missing_codes_display.append({
"code": m["code"],
"actual_count": m["actual_count"],
"expected_count": m["expected_count"],
"missing_count": m["missing_count"],
"missing_ratio": m["missing_ratio"],
"missing_dates_display": missing_dates_str[:10]
})
return { return {
"task_id": task.id, "task_id": task.id,
@ -455,7 +572,7 @@ class CacheService:
"progress": float(task.progress), "progress": float(task.progress),
"total_count": task.total_count, "total_count": task.total_count,
"complete_count": len(complete_codes), "complete_count": len(complete_codes),
"missing_count": len(missing_codes), "missing_count": len(verified_missing_codes),
"error_count": error_count, "error_count": error_count,
"expected_days": expected_count, "expected_days": expected_count,
"start_date": format_date(start_date), "start_date": format_date(start_date),
@ -463,8 +580,9 @@ class CacheService:
"security_type": security_type, "security_type": security_type,
"period_type": period_type, "period_type": period_type,
"daily_stats": daily_stats, "daily_stats": daily_stats,
"missing_codes": missing_codes[:100], "missing_codes": missing_codes_display,
"missing_code_list": missing_code_list "missing_code_list": verified_missing_code_list,
"verification_enabled": True
} }
except Exception as e: except Exception as e:
@ -1172,6 +1290,125 @@ class CacheService:
"missing_dates": missing_dates_list "missing_dates": missing_dates_list
} }
def get_missing_codes_by_date(
self,
trade_date: date,
security_type: str,
period_type: str,
contract_type: str = "all"
) -> Dict:
"""
获取某个交易日缺失的代码列表
Args:
trade_date: 交易日
security_type: 证券类型 (stock, future)
period_type: 周期类型 (daily, min1, etc.)
contract_type: 合约类型 (all, main)
Returns:
缺失代码列表详情
"""
code_list = self.get_all_codes(security_type, contract_type)
if not code_list:
raise ValueError(f"无法获取{security_type}代码列表")
logger.info(f"获取{trade_date}交易日缺失代码,共{len(code_list)}个代码")
existing_codes = set()
if security_type == "stock" and period_type == "daily":
records = self.db.query(StockKlineDaily.code).filter(
StockKlineDaily.trade_date == trade_date
).all()
existing_codes = set(r.code for r in records)
elif security_type == "future" and period_type == "daily":
records = self.db.query(FutureKlineDaily.code).filter(
FutureKlineDaily.trade_date == trade_date
).all()
existing_codes = set(r.code for r in records)
preliminary_missing_codes = [code for code in code_list if code not in existing_codes]
adapter = sdk_manager.get_default_connection()
truly_missing_codes = []
unavailable_codes = []
unverified_codes = []
sdk_connection_error = None
if adapter:
logger.info(f"SDK连接成功开始二次验证 {len(preliminary_missing_codes)} 个疑似缺失代码")
for code in preliminary_missing_codes:
try:
if security_type == "stock":
kline_data = adapter.get_stock_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
elif security_type == "future":
kline_data = adapter.get_future_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
else:
kline_data = []
if kline_data and len(kline_data) > 0:
truly_missing_codes.append(code)
logger.debug(f"验证结果: {code} {format_date(trade_date)} - 接口有数据,确认为缺失")
else:
unavailable_codes.append(code)
logger.debug(f"验证结果: {code} {format_date(trade_date)} - 接口无数据,可能未上市或停牌")
except Exception as e:
error_msg = str(e)
logger.warning(f"验证{code} {format_date(trade_date)}时出错: {error_msg}")
unavailable_codes.append(code)
else:
sdk_connection_error = "SDK连接失败无法进行二次验证"
from app.services.config_service import ConfigService
sdk_configs = self.db.query(SDKConfig).filter(SDKConfig.is_active == True).all()
if not sdk_configs:
sdk_connection_error = "SDK连接失败未配置任何SDK连接请前往配置管理添加SDK配置"
logger.error(sdk_connection_error)
else:
default_config = self.db.query(SDKConfig).filter(SDKConfig.is_default == True).first()
if not default_config:
sdk_connection_error = "SDK连接失败未设置默认SDK配置请前往配置管理设置默认配置"
logger.error(sdk_connection_error)
else:
sdk_connection_error = f"SDK连接失败默认配置 '{default_config.name}' 连接失败,请检查配置或测试连接"
logger.error(sdk_connection_error)
unverified_codes = preliminary_missing_codes
logger.warning(f"由于SDK连接失败{len(unverified_codes)} 个疑似缺失代码未能验证,暂不标记为真实缺失")
return {
"trade_date": format_date(trade_date),
"security_type": security_type,
"period_type": period_type,
"contract_type": contract_type,
"total_codes": len(code_list),
"existing_codes": len(existing_codes),
"missing_count": len(truly_missing_codes),
"unavailable_count": len(unavailable_codes),
"unverified_count": len(unverified_codes),
"missing_codes": truly_missing_codes[:100],
"missing_code_list": truly_missing_codes,
"unavailable_codes": unavailable_codes[:50],
"unverified_codes": unverified_codes[:50],
"verification_status": "verified" if adapter else "skipped",
"sdk_connection_error": sdk_connection_error
}
def fill_single_date_data( def fill_single_date_data(
self, self,
code: str, code: str,
@ -1252,3 +1489,104 @@ class CacheService:
error_count += 1 error_count += 1
logger.info(f"补齐完成: {code}, 成功{success_count}个, 失败{error_count}") logger.info(f"补齐完成: {code}, 成功{success_count}个, 失败{error_count}")
def _create_batch_fill_task(
self,
trade_date: date,
security_type: str,
period_type: str,
missing_codes: List[str]
) -> CacheTask:
"""创建批量补齐交易日数据任务记录"""
if not missing_codes:
raise ValueError("没有缺失代码需要补齐")
task = CacheTask(
task_name=f"批量补齐交易日数据 - {format_date(trade_date)} - {len(missing_codes)}个代码",
task_type="batch_fill_date",
security_type=security_type,
period_type=period_type,
start_date=trade_date,
end_date=trade_date,
code_list=",".join(missing_codes[:500]),
status="pending",
total_count=len(missing_codes),
started_at=datetime.utcnow()
)
self.db.add(task)
self.db.commit()
self.db.refresh(task)
return task
def _execute_batch_fill_task(
self,
task_id: int,
trade_date: date,
security_type: str,
period_type: str,
missing_codes: List[str]
):
"""执行批量补齐交易日数据任务"""
task = self.db.query(CacheTask).filter(CacheTask.id == task_id).first()
if not task:
return
task.status = "running"
self.db.commit()
adapter = sdk_manager.get_default_connection()
success_count = 0
error_count = 0
for i, code in enumerate(missing_codes):
try:
if adapter:
if security_type == "stock":
kline_data = adapter.get_stock_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
elif security_type == "future":
kline_data = adapter.get_future_kline(
code,
format_date(trade_date),
format_date(trade_date),
period_type
)
else:
kline_data = []
if kline_data and len(kline_data) > 0:
if security_type == "stock":
self.stock_service._save_kline_data(code, kline_data, period_type)
elif security_type == "future":
self.future_service._save_kline_data(code, kline_data, period_type)
success_count += 1
else:
error_count += 1
logger.debug(f"{code} {format_date(trade_date)} 无数据")
else:
error_count += 1
logger.warning(f"SDK连接失败无法补齐 {code}")
except Exception as e:
logger.error(f"补齐{code} {format_date(trade_date)}失败: {str(e)}")
error_count += 1
if (i + 1) % 20 == 0 or i == len(missing_codes) - 1:
task.success_count = success_count
task.error_count = error_count
task.progress = min(100, int((i + 1) / len(missing_codes) * 100))
self.db.commit()
task.status = "completed"
task.success_count = success_count
task.error_count = error_count
task.completed_at = datetime.utcnow()
self.db.commit()
logger.info(f"批量补齐完成: {format_date(trade_date)}, 成功{success_count}个, 失败{error_count}")

@ -107,3 +107,21 @@ export const fillAllDatesForCode = (data: {
}) => { }) => {
return cacheRequest.post('/cache/fill-code-all', data) return cacheRequest.post('/cache/fill-code-all', data)
} }
export const getMissingCodesByDate = (params: {
trade_date: string
security_type: string
period_type: string
contract_type?: string
}) => {
return request.get('/cache/missing-codes-by-date', { params })
}
export const fillMissingCodesForDateBatch = (data: {
trade_date: string
security_type: string
period_type: string
missing_codes: string[]
}) => {
return cacheRequest.post('/cache/fill-date-batch', data)
}

@ -175,9 +175,9 @@
<!-- 每日数据统计 --> <!-- 每日数据统计 -->
<el-card class="daily-card" v-if="detectResult && detectResult.daily_stats"> <el-card class="daily-card" v-if="detectResult && detectResult.daily_stats">
<template #header> <template #header>
<span>每日数据统计</span> <span>每日数据统计点击交易日查看缺失详情</span>
</template> </template>
<el-table :data="dailyStatsList" stripe height="300"> <el-table :data="dailyStatsList" stripe height="300" @row-click="handleDailyRowClick">
<el-table-column prop="date" label="日期" width="120" /> <el-table-column prop="date" label="日期" width="120" />
<el-table-column prop="expected" label="预期数量" width="100" /> <el-table-column prop="expected" label="预期数量" width="100" />
<el-table-column prop="actual" label="实际数量" width="100" /> <el-table-column prop="actual" label="实际数量" width="100" />
@ -196,9 +196,165 @@
/> />
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="操作" width="80">
<template #default="{ row }">
<el-button
type="primary"
size="small"
@click.stop="handleDailyRowClick(row)"
:disabled="row.missing === 0"
>
详情
</el-button>
</template>
</el-table-column>
</el-table> </el-table>
</el-card> </el-card>
<!-- 交易日缺失代码详情弹窗 -->
<el-dialog
v-model="dailyDetailDialogVisible"
:title="`交易日缺失详情 - ${selectedTradeDate?.date || ''}`"
width="900px"
destroy-on-close
>
<div v-if="loadingDailyDetail" style="text-align: center; padding: 20px;">
<el-icon class="is-loading" :size="40"><Loading /></el-icon>
<p>正在加载缺失代码数据...</p>
</div>
<div v-else-if="dailyMissingCodes">
<el-alert
v-if="dailyMissingCodes.sdk_connection_error"
type="error"
:title="dailyMissingCodes.sdk_connection_error"
show-icon
style="margin-bottom: 20px;"
>
<template #default>
<p>{{ dailyMissingCodes.sdk_connection_error }}</p>
<p style="margin-top: 10px;">
<el-button type="primary" size="small" @click="$router.push('/config')">
前往配置管理
</el-button>
</p>
</template>
</el-alert>
<el-alert
v-if="dailyMissingCodes.verification_status === 'skipped' && !dailyMissingCodes.sdk_connection_error"
type="warning"
title="二次验证已跳过"
show-icon
style="margin-bottom: 20px;"
>
<template #default>
<p>SDK连接失败无法进行二次验证以下数据为初步检测结果可能包含误判</p>
</template>
</el-alert>
<el-descriptions :column="6" border style="margin-bottom: 20px;">
<el-descriptions-item label="交易日">{{ dailyMissingCodes.trade_date }}</el-descriptions-item>
<el-descriptions-item label="总代码数">{{ dailyMissingCodes.total_codes }}</el-descriptions-item>
<el-descriptions-item label="已有数据">{{ dailyMissingCodes.existing_codes }}</el-descriptions-item>
<el-descriptions-item label="真实缺失">
<el-tag type="warning">{{ dailyMissingCodes.missing_count }}</el-tag>
</el-descriptions-item>
<el-descriptions-item label="不可用(未上市/停牌)">
<el-tag type="info">{{ dailyMissingCodes.unavailable_count }}</el-tag>
</el-descriptions-item>
<el-descriptions-item label="未验证">
<el-tag :type="dailyMissingCodes.unverified_count > 0 ? 'danger' : 'success'">
{{ dailyMissingCodes.unverified_count || 0 }}
</el-tag>
</el-descriptions-item>
</el-descriptions>
<div v-if="dailyMissingCodes.missing_count > 0">
<div style="margin-bottom: 15px; display: flex; justify-content: space-between; align-items: center;">
<span>缺失代码列表前100个</span>
<el-button type="success" @click="handleFillAllForDate" :loading="fillingAllForDate">
<el-icon><Download /></el-icon>
</el-button>
</div>
<el-alert
v-if="batchFillTask"
:title="`批量补齐进度: ${batchFillTask.progress}%`"
:type="batchFillTask.status === 'completed' ? 'success' : batchFillTask.status === 'failed' ? 'error' : 'info'"
show-icon
style="margin-bottom: 15px;"
>
<template #default>
<p>状态: {{ batchFillTask.status }}</p>
<p>成功: {{ batchFillTask.success_count }}, 失败: {{ batchFillTask.error_count }}</p>
</template>
</el-alert>
<el-table :data="dailyMissingCodes.missing_codes.map(c => ({ code: c }))" stripe height="400">
<el-table-column prop="code" label="代码" width="150" />
<el-table-column label="操作" width="100">
<template #default="{ row }">
<el-button
type="primary"
size="small"
@click="handleFillSingleCodeForDate(row)"
:loading="row.filling"
>
补齐
</el-button>
</template>
</el-table-column>
</el-table>
</div>
<div v-else-if="dailyMissingCodes.unverified_count > 0">
<el-alert type="warning" title="无确认缺失数据,但有未验证代码" show-icon />
</div>
<div v-else>
<el-alert type="success" title="该交易日数据完整" show-icon />
</div>
<div v-if="dailyMissingCodes.unverified_codes && dailyMissingCodes.unverified_codes.length > 0" style="margin-top: 20px;">
<el-divider content-position="left">未验证代码SDK连接失败无法确认是否缺失</el-divider>
<el-tag
v-for="code in dailyMissingCodes.unverified_codes"
:key="code"
type="danger"
size="small"
style="margin: 2px;"
>
{{ code }}
</el-tag>
<p style="color: #909399; margin-top: 10px;">
这些代码因SDK连接失败未能验证请先配置SDK连接后再重新检测
</p>
</div>
<div v-if="dailyMissingCodes.unavailable_codes && dailyMissingCodes.unavailable_codes.length > 0" style="margin-top: 20px;">
<el-divider content-position="left">不可用代码未上市或停牌无需补齐</el-divider>
<el-tag
v-for="code in dailyMissingCodes.unavailable_codes"
:key="code"
type="info"
size="small"
style="margin: 2px;"
>
{{ code }}
</el-tag>
</div>
</div>
<div v-else style="text-align: center; padding: 20px; color: #999;">
<p>暂无数据</p>
</div>
<template #footer>
<el-button @click="dailyDetailDialogVisible = false">关闭</el-button>
</template>
</el-dialog>
<!-- 缺失代码列表 --> <!-- 缺失代码列表 -->
<el-card class="missing-card" v-if="detectResult && detectResult.missing_codes && detectResult.missing_codes.length > 0"> <el-card class="missing-card" v-if="detectResult && detectResult.missing_codes && detectResult.missing_codes.length > 0">
<template #header> <template #header>
@ -326,7 +482,7 @@
import { ref, reactive, computed } from 'vue' import { ref, reactive, computed } from 'vue'
import { ElMessage } from 'element-plus' import { ElMessage } from 'element-plus'
import { Loading } from '@element-plus/icons-vue' import { Loading } from '@element-plus/icons-vue'
import { detectMissingData, batchCacheData, detectAllMissingData, cacheAllMissingData, getCacheTask, fillMissingData, getMissingDatesForCode, fillSingleDateData, fillAllDatesForCode } from '@/api/cache' import { detectMissingData, batchCacheData, detectAllMissingData, cacheAllMissingData, getCacheTask, fillMissingData, getMissingDatesForCode, fillSingleDateData, fillAllDatesForCode, getMissingCodesByDate, fillMissingCodesForDateBatch } from '@/api/cache'
const detecting = ref(false) const detecting = ref(false)
const caching = ref(false) const caching = ref(false)
@ -354,6 +510,12 @@ const missingDatesDetail = ref<any>(null)
const fillingSingleDate = ref(false) const fillingSingleDate = ref(false)
const fillingAllForCode = ref(false) const fillingAllForCode = ref(false)
const dailyDetailDialogVisible = ref(false)
const selectedTradeDate = ref<any>(null)
const loadingDailyDetail = ref(false)
const dailyMissingCodes = ref<any>(null)
const fillingAllForDate = ref(false)
let ws: WebSocket | null = null let ws: WebSocket | null = null
const hasMissing = computed(() => batchDetectResult.value.some(r => r.missingCount > 0)) const hasMissing = computed(() => batchDetectResult.value.some(r => r.missingCount > 0))
@ -782,6 +944,138 @@ const handleFillAllForCode = async () => {
fillingAllForCode.value = false fillingAllForCode.value = false
} }
} }
const handleDailyRowClick = async (row: any) => {
if (row.missing === 0) {
ElMessage.info('该交易日数据完整,无缺失')
return
}
selectedTradeDate.value = row
dailyDetailDialogVisible.value = true
loadingDailyDetail.value = true
dailyMissingCodes.value = null
try {
const res: any = await getMissingCodesByDate({
trade_date: row.date.replace(/-/g, ''),
security_type: form.securityType,
period_type: form.periodType,
contract_type: form.contractType
})
if (res.data) {
dailyMissingCodes.value = res.data
} else {
ElMessage.error(res.message || '获取缺失代码失败')
}
} catch (error: any) {
console.error(error)
const errorMsg = error.response?.data?.message || error.message || '网络请求失败'
ElMessage.error(`获取缺失代码失败: ${errorMsg}`)
} finally {
loadingDailyDetail.value = false
}
}
const handleFillSingleCodeForDate = async (row: any) => {
row.filling = true
try {
const res: any = await fillSingleDateData({
code: row.code,
trade_date: selectedTradeDate.value.date.replace(/-/g, ''),
security_type: form.securityType,
period_type: form.periodType
})
if (res.data) {
ElMessage.success(`${row.code} 补齐任务已启动`)
} else {
ElMessage.error(res.message || '补齐失败')
}
} catch (error: any) {
console.error(error)
const errorMsg = error.response?.data?.message || error.message || '网络请求失败'
ElMessage.error(`补齐失败: ${errorMsg}`)
} finally {
row.filling = false
}
}
const handleFillAllForDate = async () => {
if (!dailyMissingCodes.value || dailyMissingCodes.value.missing_count === 0) {
ElMessage.warning('没有缺失数据需要补齐')
return
}
fillingAllForDate.value = true
try {
const missingCodes = dailyMissingCodes.value.missing_code_list || dailyMissingCodes.value.missing_codes
const res: any = await fillMissingCodesForDateBatch({
trade_date: selectedTradeDate.value.date.replace(/-/g, ''),
security_type: form.securityType,
period_type: form.periodType,
missing_codes: missingCodes
})
if (res.data) {
if (res.data.message) {
ElMessage.info(res.data.message)
} else {
ElMessage.success(`批量补齐任务已启动,共 ${res.data.missing_count} 个代码任务ID: ${res.data.task_id}`)
if (res.data.task_id) {
pollBatchFillProgress(res.data.task_id)
}
}
} else {
ElMessage.error(res.message || '补齐任务启动失败')
}
} catch (error: any) {
console.error(error)
const errorMsg = error.response?.data?.message || error.message || '网络请求失败'
ElMessage.error(`补齐失败: ${errorMsg}`)
} finally {
fillingAllForDate.value = false
}
}
const batchFillTask = ref<any>(null)
const pollBatchFillProgress = async (taskId: number) => {
batchFillTask.value = {
task_id: taskId,
status: 'running',
progress: 0,
success_count: 0,
error_count: 0
}
const poll = async () => {
if (batchFillTask.value && (batchFillTask.value.status === 'running' || batchFillTask.value.status === 'pending')) {
try {
const res: any = await getCacheTask(taskId)
if (res.data && res.data.task) {
batchFillTask.value = res.data.task
}
if (batchFillTask.value.status === 'completed') {
ElMessage.success(`批量补齐完成:成功${batchFillTask.value.success_count}个,失败${batchFillTask.value.error_count}`)
} else if (batchFillTask.value.status === 'failed') {
ElMessage.error(`批量补齐失败`)
} else if (batchFillTask.value.status === 'running' || batchFillTask.value.status === 'pending') {
setTimeout(poll, 3000)
}
} catch (e) {
console.error(e)
}
}
}
setTimeout(poll, 2000)
}
</script> </script>
<style scoped> <style scoped>

Loading…
Cancel
Save