feat: 一键检测缓存完成,可用

master
Lxy 2 months ago
parent 8dc9f8eeac
commit 660f5ebc20

@ -9,16 +9,65 @@ from app.db.session import get_db
from app.schemas.base import ResponseModel, PaginatedResponse
from app.schemas.cache import (
DetectMissingRequest, DetectMissingResponse,
BatchCacheRequest, CacheTaskResponse, CacheStatusResponse
BatchCacheRequest, CacheTaskResponse, CacheStatusResponse,
AllDataRequest
)
from app.services.cache_service import CacheService
from app.core.security import get_current_user
from app.models.user import User
from app.utils.date_utils import parse_date
from app.utils.date_utils import parse_date, format_date
router = APIRouter()
@router.post("/detect-all-missing", response_model=ResponseModel)
async def detect_all_missing_data(
request: AllDataRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""一键检测所有数据的缺失情况"""
service = CacheService(db)
start = parse_date(request.start_date)
end = parse_date(request.end_date)
result = service.detect_all_missing_data(
request.security_type,
request.period_type,
start,
end
)
return ResponseModel(data=result)
@router.post("/cache-all-missing", response_model=ResponseModel)
async def cache_all_missing_data(
request: AllDataRequest,
db: Session = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""一键缓存所有缺失数据"""
service = CacheService(db)
start = parse_date(request.start_date)
end = parse_date(request.end_date)
task = service.cache_all_missing_data(
request.security_type,
request.period_type,
start,
end
)
return ResponseModel(data={
"task_id": task.id,
"task_name": task.task_name,
"status": task.status,
"total_count": task.total_count,
"progress": task.progress
})
@router.post("/detect-missing", response_model=ResponseModel)
async def detect_missing_data(
request: DetectMissingRequest,

@ -2,7 +2,7 @@
缓存任务模型
"""
from datetime import datetime, date
from sqlalchemy import Column, Integer, BigInteger, String, Numeric, Text, Date, DateTime, ForeignKey
from sqlalchemy import Column, Integer, BigInteger, String, Numeric, Text, Date, DateTime, ForeignKey, Boolean
from sqlalchemy.orm import relationship
from app.db.base import Base
@ -43,7 +43,7 @@ class CacheTaskDetail(Base):
trade_date = Column(Date, nullable=False)
expected_count = Column(Integer, default=0)
actual_count = Column(Integer, default=0)
is_missing = Column(Integer, default=0)
is_missing = Column(Boolean, default=False)
status = Column(String(20), default="pending") # pending, success, failed, skipped
error_message = Column(Text)
processed_at = Column(DateTime(timezone=True))

@ -106,3 +106,11 @@ class BatchCacheRequest(BaseModel):
start_date: str
end_date: str
code_list: List[str]
class AllDataRequest(BaseModel):
"""一键检测/缓存所有数据请求"""
security_type: str = Field(..., description="证券类型: stock, future")
period_type: str = Field(default="daily", description="周期类型: daily, min1, min5, etc.")
start_date: str = Field(..., description="开始日期 YYYYMMDD")
end_date: str = Field(..., description="结束日期 YYYYMMDD")

@ -13,6 +13,7 @@ from app.models.future import FutureKlineDaily
from app.services.base_data_service import BaseDataService
from app.services.stock_service import StockService
from app.services.future_service import FutureService
from app.services.sdk_manager import sdk_manager
from app.utils.date_utils import parse_date, format_date, get_market_from_code
from app.config import settings
@ -28,6 +29,365 @@ class CacheService:
self.stock_service = StockService(db)
self.future_service = FutureService(db)
def get_all_codes(self, security_type: str) -> List[str]:
"""
获取所有代码列表
Args:
security_type: 证券类型 (stock, future)
Returns:
代码列表
"""
adapter = sdk_manager.get_default_connection()
if not adapter:
raise RuntimeError("SDK连接失败")
if security_type == "stock":
return adapter.get_code_list("EXTRA_STOCK_A")
elif security_type == "future":
return adapter.get_code_list("EXTRA_FUTURE")
else:
return []
def detect_all_missing_data(
self,
security_type: str,
period_type: str,
start_date: date,
end_date: date
) -> Dict:
"""
一键检测所有数据的缺失情况
Args:
security_type: 证券类型 (stock, future)
period_type: 周期类型 (daily, min1, etc.)
start_date: 开始日期
end_date: 结束日期
Returns:
检测结果字典
"""
# 获取所有代码
code_list = self.get_all_codes(security_type)
if not code_list:
raise ValueError(f"无法获取{security_type}代码列表")
logger.info(f"获取到{len(code_list)}{security_type}代码")
# 创建检测任务
task = CacheTask(
task_name=f"一键检测所有数据 - {security_type} - {len(code_list)}个代码",
task_type="detect_all_missing",
security_type=security_type,
period_type=period_type,
start_date=start_date,
end_date=end_date,
code_list=",".join(code_list[:100]) + "...",
status="running",
total_count=len(code_list),
started_at=datetime.utcnow()
)
self.db.add(task)
self.db.commit()
self.db.refresh(task)
try:
# 获取交易日历
market = "CFE" if security_type == "future" else "SH"
trading_days = self.base_service.get_trading_calendar(market, start_date, end_date)
expected_count = len(trading_days)
missing_codes = []
complete_codes = []
error_count = 0
# 统计每个交易日的缺失情况
daily_stats = {}
for td in trading_days:
daily_stats[format_date(td)] = {
"expected": len(code_list),
"actual": 0,
"missing": 0
}
for i, code in enumerate(code_list):
try:
# 查询实际数据量
if security_type == "stock" and period_type == "daily":
records = self.db.query(StockKlineDaily).filter(
and_(
StockKlineDaily.code == code,
StockKlineDaily.trade_date >= start_date,
StockKlineDaily.trade_date <= end_date
)
).all()
actual_count = len(records)
# 更新每日统计
for r in records:
date_key = format_date(r.trade_date)
if date_key in daily_stats:
daily_stats[date_key]["actual"] += 1
elif security_type == "future" and period_type == "daily":
records = self.db.query(FutureKlineDaily).filter(
and_(
FutureKlineDaily.code == code,
FutureKlineDaily.trade_date >= start_date,
FutureKlineDaily.trade_date <= end_date
)
).all()
actual_count = len(records)
for r in records:
date_key = format_date(r.trade_date)
if date_key in daily_stats:
daily_stats[date_key]["actual"] += 1
else:
actual_count = 0
# 判断是否缺失
is_missing = actual_count < expected_count
if is_missing:
missing_codes.append({
"code": code,
"actual_count": actual_count,
"expected_count": expected_count,
"missing_count": expected_count - actual_count,
"missing_ratio": (expected_count - actual_count) / expected_count if expected_count > 0 else 0
})
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=True,
status="pending"
)
self.db.add(detail)
else:
complete_codes.append(code)
except Exception as e:
logger.error(f"检测{code}缺失数据失败: {str(e)}")
error_count += 1
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
status="failed",
error_message=str(e)
)
self.db.add(detail)
# 每100个代码更新一次进度
if (i + 1) % 100 == 0 or i == len(code_list) - 1:
task.success_count = len(missing_codes) + len(complete_codes)
task.error_count = error_count
task.progress = min(100, int((i + 1) / len(code_list) * 100))
self.db.commit()
# 计算每日缺失数
for date_key in daily_stats:
daily_stats[date_key]["missing"] = daily_stats[date_key]["expected"] - daily_stats[date_key]["actual"]
task.status = "completed"
task.success_count = len(complete_codes)
task.error_count = error_count
task.completed_at = datetime.utcnow()
self.db.commit()
logger.info(f"检测完成: 完整{len(complete_codes)}个, 缺失{len(missing_codes)}个, 错误{error_count}")
return {
"task_id": task.id,
"task_name": task.task_name,
"status": task.status,
"progress": float(task.progress),
"total_count": task.total_count,
"complete_count": len(complete_codes),
"missing_count": len(missing_codes),
"error_count": error_count,
"expected_days": expected_count,
"start_date": format_date(start_date),
"end_date": format_date(end_date),
"security_type": security_type,
"period_type": period_type,
"daily_stats": daily_stats,
"missing_codes": missing_codes[:100] # 只返回前100个缺失代码
}
except Exception as e:
task.status = "failed"
task.error_message = str(e)
task.completed_at = datetime.utcnow()
self.db.commit()
logger.error(f"一键检测缺失数据任务失败: {str(e)}")
return {
"task_id": task.id,
"task_name": task.task_name,
"status": task.status,
"error_message": str(e)
}
def cache_all_missing_data(
self,
security_type: str,
period_type: str,
start_date: date,
end_date: date
) -> CacheTask:
"""
一键缓存所有缺失数据
Args:
security_type: 证券类型
period_type: 周期类型
start_date: 开始日期
end_date: 结束日期
Returns:
缓存任务对象
"""
# 获取所有代码
code_list = self.get_all_codes(security_type)
if not code_list:
raise ValueError(f"无法获取{security_type}代码列表")
logger.info(f"获取到{len(code_list)}{security_type}代码,开始缓存")
# 创建缓存任务
task = CacheTask(
task_name=f"一键缓存所有数据 - {security_type} - {len(code_list)}个代码",
task_type="cache_all_data",
security_type=security_type,
period_type=period_type,
start_date=start_date,
end_date=end_date,
code_list=",".join(code_list[:100]) + "...",
status="running",
total_count=len(code_list),
started_at=datetime.utcnow()
)
self.db.add(task)
self.db.commit()
self.db.refresh(task)
try:
# 获取交易日历
market = "CFE" if security_type == "future" else "SH"
trading_days = self.base_service.get_trading_calendar(market, start_date, end_date)
expected_count = len(trading_days)
success_count = 0
skipped_count = 0
error_count = 0
for i, code in enumerate(code_list):
try:
# 先检查是否已有完整数据
if security_type == "stock" and period_type == "daily":
actual_count = self.db.query(StockKlineDaily).filter(
and_(
StockKlineDaily.code == code,
StockKlineDaily.trade_date >= start_date,
StockKlineDaily.trade_date <= end_date
)
).count()
elif security_type == "future" and period_type == "daily":
actual_count = self.db.query(FutureKlineDaily).filter(
and_(
FutureKlineDaily.code == code,
FutureKlineDaily.trade_date >= start_date,
FutureKlineDaily.trade_date <= end_date
)
).count()
else:
actual_count = 0
# 如果数据完整,跳过
if actual_count >= expected_count:
skipped_count += 1
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=False,
status="skipped"
)
self.db.add(detail)
continue
# 获取数据(会自动缓存)
if security_type == "stock":
self.stock_service.get_kline([code], start_date, end_date, period_type)
elif security_type == "future":
self.future_service.get_kline([code], start_date, end_date, period_type)
success_count += 1
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=True,
status="success",
processed_at=datetime.utcnow()
)
self.db.add(detail)
except Exception as e:
logger.error(f"缓存{code}数据失败: {str(e)}")
error_count += 1
detail = CacheTaskDetail(
task_id=task.id,
code=code,
trade_date=start_date,
status="failed",
error_message=str(e)
)
self.db.add(detail)
# 每50个代码更新一次进度
if (i + 1) % 50 == 0 or i == len(code_list) - 1:
task.success_count = success_count
task.error_count = error_count
task.progress = min(100, int((i + 1) / len(code_list) * 100))
self.db.commit()
logger.info(f"进度: {i + 1}/{len(code_list)}, 成功: {success_count}, 跳过: {skipped_count}, 错误: {error_count}")
task.status = "completed"
task.success_count = success_count
task.error_count = error_count
task.completed_at = datetime.utcnow()
self.db.commit()
logger.info(f"缓存完成: 成功{success_count}个, 跳过{skipped_count}个, 错误{error_count}")
except Exception as e:
task.status = "failed"
task.error_message = str(e)
task.completed_at = datetime.utcnow()
self.db.commit()
logger.error(f"一键缓存数据任务失败: {str(e)}")
return task
def detect_missing_data(
self,
security_type: str,
@ -111,7 +471,7 @@ class CacheService:
trade_date=start_date,
expected_count=expected_count,
actual_count=actual_count,
is_missing=1 if is_missing else 0,
is_missing=is_missing,
status="pending" if is_missing else "skipped"
)
self.db.add(detail)

@ -20,6 +20,24 @@ export const batchCacheData = (data: {
return request.post('/cache/batch-cache', data)
}
export const detectAllMissingData = (data: {
security_type: string
period_type: string
start_date: string
end_date: string
}) => {
return request.post('/cache/detect-all-missing', data)
}
export const cacheAllMissingData = (data: {
security_type: string
period_type: string
start_date: string
end_date: string
}) => {
return request.post('/cache/cache-all-missing', data)
}
export const getCacheTasks = (params?: { page?: number; page_size?: number }) => {
return request.get('/cache/tasks', { params })
}

@ -31,6 +31,19 @@
value-format="YYYYMMDD"
/>
</el-form-item>
<el-form-item>
<el-button type="warning" @click="handleDetectAll" :loading="detectingAll">
<el-icon><Search /></el-icon>
</el-button>
<el-button type="success" @click="handleCacheAll" :loading="cachingAll">
<el-icon><Download /></el-icon>
</el-button>
</el-form-item>
</el-form>
<el-divider content-position="left">批量检测指定代码</el-divider>
<el-form :model="form" label-width="100px">
<el-form-item label="代码列表">
<el-input
v-model="codeInput"
@ -50,11 +63,115 @@
</el-form>
</el-card>
<el-card class="result-card" v-if="detectResult.length > 0">
<!-- 检测结果汇总 -->
<el-card class="summary-card" v-if="detectResult">
<template #header>
<span>检测结果汇总</span>
</template>
<el-row :gutter="20">
<el-col :span="6">
<el-statistic title="检测总数" :value="detectResult.total_count || 0" />
</el-col>
<el-col :span="6">
<el-statistic title="数据完整" :value="detectResult.complete_count || 0" suffix="个">
<template #suffix>
<el-tag type="success" size="small">完整</el-tag>
</template>
</el-statistic>
</el-col>
<el-col :span="6">
<el-statistic title="数据缺失" :value="detectResult.missing_count || 0" suffix="个">
<template #suffix>
<el-tag type="warning" size="small">缺失</el-tag>
</template>
</el-statistic>
</el-col>
<el-col :span="6">
<el-statistic title="检测错误" :value="detectResult.error_count || 0" suffix="个">
<template #suffix>
<el-tag type="danger" size="small">错误</el-tag>
</template>
</el-statistic>
</el-col>
</el-row>
<el-row :gutter="20" style="margin-top: 20px;">
<el-col :span="6">
<el-statistic title="检测周期" :value="detectResult.expected_days || 0" suffix="天" />
</el-col>
<el-col :span="6">
<el-statistic title="开始日期" :value="detectResult.start_date || '-'" />
</el-col>
<el-col :span="6">
<el-statistic title="结束日期" :value="detectResult.end_date || '-'" />
</el-col>
<el-col :span="6">
<el-statistic title="检测状态">
<el-tag :type="getStatusType(detectResult.status)">
{{ detectResult.status || '-' }}
</el-tag>
</el-statistic>
</el-col>
</el-row>
</el-card>
<!-- 每日数据统计 -->
<el-card class="daily-card" v-if="detectResult && detectResult.daily_stats">
<template #header>
<span>每日数据统计</span>
</template>
<el-table :data="dailyStatsList" stripe height="300">
<el-table-column prop="date" label="日期" width="120" />
<el-table-column prop="expected" label="预期数量" width="100" />
<el-table-column prop="actual" label="实际数量" width="100" />
<el-table-column prop="missing" label="缺失数量" width="100">
<template #default="{ row }">
<el-tag :type="row.missing > 0 ? 'warning' : 'success'" size="small">
{{ row.missing }}
</el-tag>
</template>
</el-table-column>
<el-table-column label="完整率">
<template #default="{ row }">
<el-progress
:percentage="Math.round((row.actual / row.expected) * 100)"
:status="row.missing > row.expected * 0.5 ? 'exception' : row.missing > 0 ? 'warning' : 'success'"
/>
</template>
</el-table-column>
</el-table>
</el-card>
<!-- 缺失代码列表 -->
<el-card class="missing-card" v-if="detectResult && detectResult.missing_codes && detectResult.missing_codes.length > 0">
<template #header>
<span>缺失代码列表前100个</span>
</template>
<el-table :data="detectResult.missing_codes" stripe height="300">
<el-table-column prop="code" label="代码" width="120" />
<el-table-column prop="expected_count" label="预期天数" width="100" />
<el-table-column prop="actual_count" label="实际天数" width="100" />
<el-table-column prop="missing_count" label="缺失天数" width="100">
<template #default="{ row }">
<el-tag type="warning" size="small">{{ row.missing_count }}</el-tag>
</template>
</el-table-column>
<el-table-column label="缺失率">
<template #default="{ row }">
<el-progress
:percentage="Math.round(row.missing_ratio * 100)"
:status="row.missing_ratio > 0.5 ? 'exception' : 'warning'"
/>
</template>
</el-table-column>
</el-table>
</el-card>
<!-- 批量检测结果 -->
<el-card class="result-card" v-if="batchDetectResult.length > 0">
<template #header>
<span>检测结果</span>
<span>批量检测结果</span>
</template>
<el-table :data="detectResult" stripe>
<el-table :data="batchDetectResult" stripe>
<el-table-column prop="code" label="代码" width="120" />
<el-table-column prop="missingCount" label="缺失天数" width="100" />
<el-table-column label="缺失率">
@ -80,15 +197,27 @@
<script setup lang="ts">
import { ref, reactive, computed } from 'vue'
import { ElMessage } from 'element-plus'
import { detectMissingData, batchCacheData } from '@/api/cache'
import { detectMissingData, batchCacheData, detectAllMissingData, cacheAllMissingData } from '@/api/cache'
const detecting = ref(false)
const caching = ref(false)
const detectingAll = ref(false)
const cachingAll = ref(false)
const codeInput = ref('000001.SZ\n600000.SH')
const detectResult = ref<any[]>([])
const taskId = ref<number | null>(null)
const detectResult = ref<any>(null)
const batchDetectResult = ref<any[]>([])
const hasMissing = computed(() => detectResult.value.some(r => r.missingCount > 0))
const hasMissing = computed(() => batchDetectResult.value.some(r => r.missingCount > 0))
const dailyStatsList = computed(() => {
if (!detectResult.value || !detectResult.value.daily_stats) return []
return Object.entries(detectResult.value.daily_stats).map(([date, stats]) => ({
date,
expected: (stats as any).expected,
actual: (stats as any).actual,
missing: (stats as any).missing
}))
})
const form = reactive({
securityType: 'stock',
@ -111,6 +240,12 @@ function formatDate(date: Date) {
return date.toISOString().slice(0, 10).replace(/-/g, '')
}
function getStatusType(status: string) {
if (status === 'completed') return 'success'
if (status === 'running') return 'warning'
return 'danger'
}
const parseCodes = () => {
return codeInput.value
.split(/[\n,]/)
@ -118,6 +253,54 @@ const parseCodes = () => {
.filter(c => c.length > 0)
}
const handleDetectAll = async () => {
detectingAll.value = true
detectResult.value = null
try {
const res: any = await detectAllMissingData({
security_type: form.securityType,
period_type: form.periodType,
start_date: form.startDate,
end_date: form.endDate
})
if (res.data) {
detectResult.value = res.data
if (res.data.status === 'completed') {
ElMessage.success(`检测完成:完整${res.data.complete_count}个,缺失${res.data.missing_count}个,错误${res.data.error_count}`)
} else if (res.data.status === 'failed') {
ElMessage.error(`检测失败:${res.data.error_message}`)
}
}
} catch (error) {
console.error(error)
ElMessage.error('检测失败')
} finally {
detectingAll.value = false
}
}
const handleCacheAll = async () => {
cachingAll.value = true
try {
const res: any = await cacheAllMissingData({
security_type: form.securityType,
period_type: form.periodType,
start_date: form.startDate,
end_date: form.endDate
})
if (res.data) {
ElMessage.success(`缓存任务已启动,共${res.data.total_count}个代码`)
}
} catch (error) {
console.error(error)
ElMessage.error('缓存失败')
} finally {
cachingAll.value = false
}
}
const handleDetect = async () => {
const codes = parseCodes()
if (codes.length === 0) {
@ -136,8 +319,7 @@ const handleDetect = async () => {
})
if (res.data) {
taskId.value = res.data.task_id
detectResult.value = res.data.missing_codes.map((item: any) => ({
batchDetectResult.value = res.data.missing_codes.map((item: any) => ({
code: item.code,
missingCount: item.missing_dates.length,
missingRatio: item.missing_dates.length > 0
@ -175,7 +357,6 @@ const handleCache = async () => {
}
const showDetail = (row: any) => {
//
console.log(row.details)
}
</script>
@ -185,7 +366,19 @@ const showDetail = (row: any) => {
padding: 10px;
}
.summary-card {
margin-top: 20px;
}
.daily-card {
margin-top: 20px;
}
.missing-card {
margin-top: 20px;
}
.result-card {
margin-top: 20px;
}
</style>
</style>
Loading…
Cancel
Save