From 8089fff3343e3fe469906d56bdd19d9320d2c681 Mon Sep 17 00:00:00 2001 From: Lxy Date: Thu, 9 Jul 2026 01:03:04 +0800 Subject: [PATCH] =?UTF-8?q?tweak:=20=E6=89=B9=E9=87=8F=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E8=B7=A8=E6=96=87=E4=BB=B6=E5=8E=BB=E9=87=8D=EF=BC=8C=E5=8D=95?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=86=85=E5=85=81=E8=AE=B8=E5=A4=9A=E6=AC=A1?= =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/trade_review.py | 9 ++++++++- app/services/trade_parser.py | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/app/api/trade_review.py b/app/api/trade_review.py index 893e52b..236a937 100644 --- a/app/api/trade_review.py +++ b/app/api/trade_review.py @@ -101,6 +101,8 @@ async def batch_import_settlement( total_skipped_all = 0 processed_count = 0 failed_count = 0 + # 维护跨文件的去重键集合，用于文件间去重 + batch_existing_keys = set() for file in files: if not file.filename.endswith(('.xls', '.xlsx')): @@ -125,7 +127,12 @@ async def batch_import_settlement( failed_count += 1 continue - result = save_to_db(db, df_futures, df_options, file.filename) + # 传入已处理文件的去重键，实现文件间去重 + result = save_to_db(db, df_futures, df_options, file.filename, batch_existing_keys) + + # 将本次导入的新记录去重键加入集合，供后续文件去重使用 + if 'new_keys' in result: + batch_existing_keys.update(result['new_keys']) file_new = result['futures_count'] + result['options_count'] file_skipped = result['futures_skipped'] + result['options_skipped'] diff --git a/app/services/trade_parser.py b/app/services/trade_parser.py index d1bc05b..470f27c 100644 --- a/app/services/trade_parser.py +++ b/app/services/trade_parser.py @@ -270,13 +270,16 @@ def save_to_db( df_futures: pd.DataFrame, df_options: pd.DataFrame, filename: str, + batch_existing_keys: set = None, ) -> dict: """ 将解析后的交易记录保存到数据库（含逐条去重校验） 去重规则：同一交易日 + 同一品种 + 同一交易时间 + 同一价格 => 视为重复，跳过 + :param batch_existing_keys: 批量导入时，已处理文件的去重键集合（用于文件间去重） :return: { "batch_id": str, "futures_count": int, "options_count": int, "trade_dates": str, - "futures_skipped": int, "options_skipped": int, "total_parsed": int + "futures_skipped": int, "options_skipped": int, "total_parsed": int, + "new_keys": set # 本次导入新增的去重键（用于批量导入时传递给下一个文件） } """ _load_variety_name_map() @@ -309,8 +312,12 @@ def save_to_db( # 批量加载已有记录的去重键 existing_keys = _load_existing_dedup_keys(db, all_dates) + # 合并批量导入时已处理文件的去重键 + if batch_existing_keys: + existing_keys = existing_keys | batch_existing_keys records = [] + new_keys_in_batch = set() # 本次导入新增的去重键 skipped_details = [] # 记录被跳过的重复记录详情 # 处理期货记录 @@ -341,10 +348,11 @@ def save_to_db( }) continue - # 记录前几条的去重键用于调试 + # 记录前几笔期货去重键用于调试 if futures_parsed <= 3: logger.info(f"[去重校验] 期货记录 #{futures_parsed}: raw_date={raw_date!r}, trade_date={trade_date!r}, variety={variety!r}, time={trade_time!r}, price={price!r} => key={dedup_key!r}") + new_keys_in_batch.add(dedup_key) rec = TradeRecord( trade_type='期货', symbol=contract, @@ -394,6 +402,7 @@ def save_to_db( # 期权盈亏来自权利金（带正负号：卖为正，买为负） premium = safe_float(row.get('权利金')) + new_keys_in_batch.add(dedup_key) rec = TradeRecord( trade_type='期权', symbol=contract, @@ -454,6 +463,7 @@ def save_to_db( "futures_parsed": futures_parsed, "options_parsed": options_parsed, "skipped_details": skipped_details, # 被跳过的重复记录详情 + "new_keys": new_keys_in_batch, }