diff --git a/app/api/trade_review.py b/app/api/trade_review.py index fe9afd6..2c91d25 100644 --- a/app/api/trade_review.py +++ b/app/api/trade_review.py @@ -30,7 +30,7 @@ async def import_settlement( file: UploadFile = File(...), db: Session = Depends(get_analysis_db), ): - """上传并解析期货结算单文件，导入交易记录""" + """上传并解析期货结算单文件，导入交易记录（含逐条去重校验）""" if not file.filename.endswith(('.xls', '.xlsx')): raise HTTPException(status_code=400, detail="仅支持 .xls/.xlsx 格式的结算单文件") @@ -43,16 +43,129 @@ async def import_settlement( result = save_to_db(db, df_futures, df_options, file.filename) + total_skipped = result['futures_skipped'] + result['options_skipped'] + total_new = result['futures_count'] + result['options_count'] + total_parsed = result['futures_parsed'] + result['options_parsed'] + + if total_new == 0 and total_skipped > 0: + return { + "success": True, + "message": f"全部跳过：共解析 {total_parsed} 条记录，均已存在（去重规则：交易日+品种+时间+价格），未导入任何新数据", + "data": result, + } + + msg_parts = [f"导入成功：期货 {result['futures_count']} 条，期权 {result['options_count']} 条"] + if total_skipped > 0: + msg_parts.append(f"跳过重复 {total_skipped} 条") + msg_parts.append(f"交易日期: {result['trade_dates']}") + return { "success": True, - "message": f"导入成功：期货 {result['futures_count']} 条，期权 {result['options_count']} 条", + "message": "，".join(msg_parts), "data": result, } + except HTTPException: + raise except Exception as e: logger.exception("导入结算单失败") raise HTTPException(status_code=500, detail=f"导入失败: {str(e)}") +@router.post("/batch-import") +async def batch_import_settlement( + files: list[UploadFile] = File(...), + db: Session = Depends(get_analysis_db), +): + """批量上传并解析期货结算单文件，导入交易记录（含逐条去重校验）""" + if not files: + raise HTTPException(status_code=400, detail="请选择至少一个结算单文件") + + results = [] + total_futures = 0 + total_options = 0 + total_skipped_all = 0 + processed_count = 0 + failed_count = 0 + + for file in files: + if not file.filename.endswith(('.xls', '.xlsx')): + results.append({ + "filename": file.filename, + "success": False, + "message": "文件格式不支持，仅支持 .xls/.xlsx", + }) + failed_count += 1 + continue + + try: + content = await file.read() + df_futures, df_options = parse_settlement_file(content, file.filename) + + if df_futures.empty and df_options.empty: + results.append({ + "filename": file.filename, + "success": False, + "message": "文件中未找到有效的交易记录", + }) + failed_count += 1 + continue + + result = save_to_db(db, df_futures, df_options, file.filename) + + file_new = result['futures_count'] + result['options_count'] + file_skipped = result['futures_skipped'] + result['options_skipped'] + total_futures += result['futures_count'] + total_options += result['options_count'] + total_skipped_all += file_skipped + processed_count += 1 + + msg_parts = [] + if file_new > 0: + msg_parts.append(f"新导入 {file_new} 条（期货 {result['futures_count']}，期权 {result['options_count']}）") + if file_skipped > 0: + msg_parts.append(f"跳过重复 {file_skipped} 条") + if not msg_parts: + msg_parts.append("无有效记录") + + results.append({ + "filename": file.filename, + "success": True, + "message": "，".join(msg_parts), + "new_count": file_new, + "skipped_count": file_skipped, + "trade_dates": result['trade_dates'], + "batch_id": result['batch_id'], + }) + except Exception as e: + logger.exception(f"批量导入文件失败: {file.filename}") + results.append({ + "filename": file.filename, + "success": False, + "message": f"导入失败: {str(e)}", + }) + failed_count += 1 + + summary = ( + f"批量导入完成：共 {len(files)} 个文件，" + f"处理 {processed_count} 个，失败 {failed_count} 个，" + f"合计新导入期货 {total_futures} 条、期权 {total_options} 条，跳过重复 {total_skipped_all} 条" + ) + + return { + "success": True, + "message": summary, + "data": { + "total_files": len(files), + "processed_count": processed_count, + "failed_count": failed_count, + "total_futures": total_futures, + "total_options": total_options, + "total_skipped": total_skipped_all, + "details": results, + }, + } + + # ==================== 交易记录查询 ==================== @router.get("/records") @@ -312,6 +425,19 @@ def get_kline_with_trades( return {"success": False, "message": f"未找到 {kline_symbol} 的 {period} K线数据"} candles = json.loads(market_data.candles_json) + # 将字典格式K线转换为前端期望的数组格式 [date, open, close, low, high, volume] + if candles and isinstance(candles[0], dict): + candles = [ + [ + c.get("time", "")[:10], # 截取日期部分 YYYY-MM-DD + c.get("open", 0), + c.get("close", 0), + c.get("low", 0), + c.get("high", 0), + c.get("volume", 0), + ] + for c in candles + ] kline_symbol = market_data.symbol logger.info(f"[K线查询] 成功获取K线数据: symbol={kline_symbol}, candles_count={len(candles)}") finally: @@ -410,6 +536,13 @@ async def analyze_trade( ).first() if market_data and market_data.candles_json: candles = json.loads(market_data.candles_json) + # 将字典格式K线转换为数组格式 [date, open, close, low, high, volume] + if candles and isinstance(candles[0], dict): + candles = [ + [c.get("time", "")[:10], c.get("open", 0), c.get("close", 0), + c.get("low", 0), c.get("high", 0), c.get("volume", 0)] + for c in candles + ] kline_context[period] = candles[-50:] if len(candles) > 50 else candles finally: main_db.close() diff --git a/app/services/trade_parser.py b/app/services/trade_parser.py index 9b51d77..5bed555 100644 --- a/app/services/trade_parser.py +++ b/app/services/trade_parser.py @@ -5,6 +5,7 @@ import re import uuid import json +import logging from datetime import datetime from collections import defaultdict from pathlib import Path @@ -15,6 +16,8 @@ from sqlalchemy.orm import Session as DBSession from app.analysis_models import TradeRecord, TradeImportBatch +logger = logging.getLogger(__name__) + # 品种代码 -> 中文名 映射（反向映射：从合约代码前缀查中文名） _VARIETY_NAME_MAP = {} @@ -193,6 +196,35 @@ def _normalize_time(time_val) -> str: return s +def _build_dedup_key(trade_date: str, variety: str, trade_time: str, price: float) -> str: + """构建去重键：交易日+品种+交易时间+价格（价格保留2位小数避免浮点精度问题）""" + return f"{trade_date}|{variety}|{trade_time}|{round(price, 2)}" + + +def _load_existing_dedup_keys(db: DBSession, trade_dates: set) -> set: + """ + 根据涉及的交易日，批量加载已有记录的去重键集合 + """ + if not trade_dates: + logger.warning("[去重校验] trade_dates 为空，跳过查询") + return set() + logger.info(f"[去重校验] 查询已有记录，日期范围: {sorted(trade_dates)}") + existing = db.query( + TradeRecord.trade_date, + TradeRecord.variety, + TradeRecord.trade_time, + TradeRecord.price, + ).filter( + TradeRecord.trade_date.in_(trade_dates) + ).all() + logger.info(f"[去重校验] 从数据库加载到 {len(existing)} 条已有记录") + keys = {_build_dedup_key(r[0], r[1], r[2] or '', r[3] or 0.0) for r in existing} + if keys: + sample = list(keys)[:3] + logger.info(f"[去重校验] 去重键示例: {sample}") + return keys + + def save_to_db( db: DBSession, df_futures: pd.DataFrame, @@ -200,16 +232,46 @@ def save_to_db( filename: str, ) -> dict: """ - 将解析后的交易记录保存到数据库 - :return: {"batch_id": str, "futures_count": int, "options_count": int, "trade_dates": str} + 将解析后的交易记录保存到数据库（含逐条去重校验） + 去重规则：同一交易日 + 同一品种 + 同一交易时间 + 同一价格 => 视为重复，跳过 + :return: { + "batch_id": str, "futures_count": int, "options_count": int, "trade_dates": str, + "futures_skipped": int, "options_skipped": int, "total_parsed": int + } """ _load_variety_name_map() batch_id = str(uuid.uuid4()) all_dates = set() futures_count = 0 options_count = 0 + futures_skipped = 0 + options_skipped = 0 + futures_parsed = 0 + options_parsed = 0 + + # 第一遍：收集所有涉及的交易日，用于批量查询已有记录 + if not df_futures.empty: + for _, row in df_futures.iterrows(): + contract = str(row.get('合约', '')).strip() + if not contract or contract == '合计': + continue + trade_date = _normalize_date(row.get('实际成交日期', '')) + if trade_date: + all_dates.add(trade_date) + if not df_options.empty: + for _, row in df_options.iterrows(): + contract = str(row.get('品种合约', '')).strip() + if not contract or contract == '合计': + continue + trade_date = _normalize_date(row.get('成交日期', '')) + if trade_date: + all_dates.add(trade_date) + + # 批量加载已有记录的去重键 + existing_keys = _load_existing_dedup_keys(db, all_dates) records = [] + new_keys_in_batch = set() # 防止本次导入文件内部重复 # 处理期货记录 if not df_futures.empty: @@ -217,12 +279,24 @@ def save_to_db( contract = str(row.get('合约', '')).strip() if not contract or contract == '合计': continue + futures_parsed += 1 variety = extract_variety(contract) trade_date = _normalize_date(row.get('实际成交日期', '')) trade_time = _normalize_time(row.get('成交时间', '')) bs_flag = str(row.get('买/卖', '')).strip() oc_flag = str(row.get('开/平', '')).strip() + price = safe_float(row.get('成交价')) + + dedup_key = _build_dedup_key(trade_date, variety, trade_time, price) + if dedup_key in existing_keys or dedup_key in new_keys_in_batch: + futures_skipped += 1 + continue + # 记录前几条的去重键用于调试 + if futures_parsed <= 3: + logger.info(f"[去重校验] 期货记录 #{futures_parsed}: date={trade_date!r}, variety={variety!r}, time={trade_time!r}, price={price!r} => key={dedup_key!r}") + + new_keys_in_batch.add(dedup_key) rec = TradeRecord( trade_type='期货', symbol=contract, @@ -230,7 +304,7 @@ def save_to_db( symbol_name=_VARIETY_NAME_MAP.get(variety, ''), direction='买' if '买' in bs_flag else '卖', offset='开' if '开' in oc_flag else ('平' if '平' in oc_flag else ''), - price=safe_float(row.get('成交价')), + price=price, volume=safe_float(row.get('手数')), amount=safe_float(row.get('成交额')), close_pnl=safe_float(row.get('平仓盈亏')), @@ -242,8 +316,6 @@ def save_to_db( ) records.append(rec) futures_count += 1 - if trade_date: - all_dates.add(trade_date) # 处理期权记录 if not df_options.empty: @@ -251,11 +323,19 @@ def save_to_db( contract = str(row.get('品种合约', '')).strip() if not contract or contract == '合计': continue + options_parsed += 1 variety = extract_variety(contract) trade_date = _normalize_date(row.get('成交日期', '')) trade_time = _normalize_time(row.get('成交时间', '')) bs_flag = str(row.get('买/卖', '')).strip() + price = safe_float(row.get('权利金单价')) + + dedup_key = _build_dedup_key(trade_date, variety, trade_time, price) + if dedup_key in existing_keys or dedup_key in new_keys_in_batch: + options_skipped += 1 + continue + new_keys_in_batch.add(dedup_key) rec = TradeRecord( trade_type='期权', symbol=contract, @@ -263,7 +343,7 @@ def save_to_db( symbol_name=_VARIETY_NAME_MAP.get(variety, ''), direction='买' if '买' in bs_flag else '卖', offset='', - price=safe_float(row.get('权利金单价')), + price=price, volume=safe_float(row.get('成交量')), amount=safe_float(row.get('权利金')), close_pnl=0.0, @@ -275,8 +355,6 @@ def save_to_db( ) records.append(rec) options_count += 1 - if trade_date: - all_dates.add(trade_date) if records: db.add_all(records) @@ -301,11 +379,22 @@ def save_to_db( db.add(batch) db.commit() + logger.info( + f"[去重校验] 导入完成: 文件={filename}, " + f"期货 解析={futures_parsed} 新增={futures_count} 跳过={futures_skipped}, " + f"期权 解析={options_parsed} 新增={options_count} 跳过={options_skipped}, " + f"已有去重键数量={len(existing_keys)}" + ) + return { "batch_id": batch_id, "futures_count": futures_count, "options_count": options_count, "trade_dates": trade_dates_str, + "futures_skipped": futures_skipped, + "options_skipped": options_skipped, + "futures_parsed": futures_parsed, + "options_parsed": options_parsed, } diff --git a/app/static/futures_analysis.html b/app/static/futures_analysis.html index 0c03578..590a689 100644 --- a/app/static/futures_analysis.html +++ b/app/static/futures_analysis.html @@ -979,9 +979,13 @@
+ + ~ @@ -1128,6 +1132,20 @@
+ +
+
+
+ 批量导入结果 + +
+
+
+
+
+
+
+
diff --git a/app/static/futures_analysis.js b/app/static/futures_analysis.js index e16c92f..05df242 100644 --- a/app/static/futures_analysis.js +++ b/app/static/futures_analysis.js @@ -643,6 +643,12 @@ function initTradeReview() { }); document.getElementById('tr-file-input').addEventListener('change', trHandleFileImport); + // 批量导入按钮 + document.getElementById('tr-btn-batch-import').addEventListener('click', () => { + document.getElementById('tr-batch-file-input').click(); + }); + document.getElementById('tr-batch-file-input').addEventListener('change', trHandleBatchImport); + // 查询按钮 document.getElementById('tr-btn-query').addEventListener('click', () => { trCurrentPage = 1; @@ -821,7 +827,7 @@ async function trHandleFileImport(e) { trCurrentPage = 1; trLoadAllData(); } else { - alert(json.message || '导入失败'); + alert('导入失败: ' + (json.message || '未知错误')); } } catch (err) { alert('导入失败: ' + err.message); @@ -831,6 +837,79 @@ async function trHandleFileImport(e) { } } +async function trHandleBatchImport(e) { + const files = e.target.files; + if (!files || files.length === 0) return; + e.target.value = ''; + + const btn = document.getElementById('tr-btn-batch-import'); + btn.disabled = true; + btn.querySelector('span:last-child').textContent = '导入中...'; + + try { + const formData = new FormData(); + for (let i = 0; i < files.length; i++) { + formData.append('files', files[i]); + } + const res = await fetch(`${TR_API_BASE}/batch-import`, { method: 'POST', body: formData }); + const json = await res.json(); + + if (json.success) { + trShowBatchImportResult(json); + const data = json.data; + if (data.total_futures > 0 || data.total_options > 0) { + trCurrentPage = 1; + trLoadAllData(); + } + } else { + alert('批量导入失败: ' + (json.message || '未知错误')); + } + } catch (err) { + alert('批量导入失败: ' + err.message); + } finally { + btn.disabled = false; + btn.querySelector('span:last-child').textContent = '批量导入'; + } +} + +function trShowBatchImportResult(json) { + const modal = document.getElementById('tr-batch-import-modal'); + const summaryEl = document.getElementById('tr-batch-import-summary'); + const detailsEl = document.getElementById('tr-batch-import-details'); + const data = json.data; + + summaryEl.textContent = json.message; + + detailsEl.innerHTML = data.details.map(d => { + let statusIcon, statusColor; + if (!d.success) { + statusIcon = '❌'; + statusColor = 'var(--color-error, #ef4444)'; + } else if (d.new_count > 0 && d.skipped_count > 0) { + statusIcon = '⚡'; + statusColor = 'var(--color-warning, #f59e0b)'; + } else if (d.new_count > 0) { + statusIcon = '✅'; + statusColor = 'var(--color-success, #22c55e)'; + } else { + statusIcon = '⚠️'; + statusColor = 'var(--color-warning, #f59e0b)'; + } + return `
+
+ ${statusIcon} ${d.filename} +
+
${d.message}
+
`; + }).join(''); + + modal.classList.add('show'); +} + +function trCloseBatchImportModal() { + document.getElementById('tr-batch-import-modal').classList.remove('show'); +} + // ==================== 批次管理 ==================== async function trShowBatchModal() { diff --git a/data/futures_analysis.db b/data/futures_analysis.db index 5c5da99..9626960 100644 Binary files a/data/futures_analysis.db and b/data/futures_analysis.db differ