From e48671fb48f276d92cf38d55d6dee102aec0a5cd Mon Sep 17 00:00:00 2001 From: moon <2623532423@qq.com> Date: Wed, 24 Sep 2025 11:02:53 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86es=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E5=88=86=E6=95=B0=EF=BC=8C=E8=B7=9F=E5=85=B3=E9=94=AE=E5=AD=97?= =?UTF-8?q?=E7=9A=84=E7=9B=B8=E5=85=B3=E5=BA=A6=E6=9B=B4=E9=AB=98=EF=BC=8C?= =?UTF-8?q?=E4=BC=9A=E9=9A=8F=E7=9D=80=E5=85=B3=E9=94=AE=E5=AD=97=E7=9A=84?= =?UTF-8?q?=E5=A4=9A=E5=B0=91=E8=BF=9B=E8=A1=8C=E5=8C=B9=E9=85=8D=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BD=93=E5=88=A0=E9=99=A4=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E6=96=87=E4=BB=B6=E6=97=B6=EF=BC=8C=E8=AF=A5=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=B4=A2=E5=BC=95=E4=B9=9F=E4=BC=9A=E8=A2=AB=E5=88=A0?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets/components/DocumentList.vue | 9 +++- .../controller/DatasetManageController.java | 8 ++++ .../impl/EsTDatasetFilesServiceImpl.java | 42 ++++++++++++++++--- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/chat-client/src/views/datasets/components/DocumentList.vue b/chat-client/src/views/datasets/components/DocumentList.vue index 926aaee..386113b 100644 --- a/chat-client/src/views/datasets/components/DocumentList.vue +++ b/chat-client/src/views/datasets/components/DocumentList.vue @@ -882,6 +882,7 @@ if (!newVal) { import { useUserStore } from '@/store/modules/user' import { useAclStore } from '@/store/modules/acl' import vab from '~/library/plugins/vab' +import {log} from "echarts/types/src/util/log"; // 事件处理函数 const renderedHandler = () => { @@ -1225,7 +1226,13 @@ try { // 调用删除接口 await deleteDocument(parseInt(row.id)) - + try{ + //删除es索引 + await deleteFileIndex(row.difyDocId,datasetId.value) + console.log("删除es索引成功,文档id:{}",row.difyDocId) + } catch (error) { + console.error("删除es索引失败,文档id:{}",row.difyDocId,error) + } // 成功处理 ElNotification({ diff --git a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java index 749534d..101c067 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java +++ b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java @@ -70,6 +70,14 @@ public class DatasetManageController { datasetManagerService.deleteDataset(id); difyDatasetService.delete(id); + + //删除es索引 + try { + esTDatasetFilesService.deleteIndex(id); + log.info("删除es索引成功,知识库id:{}",id); + } catch (IOException e) { + log.error("删除es索引失败,知识库id:{}",id,e); + } } @PostMapping("/create") diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java index a6ef3fc..6231239 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java @@ -37,6 +37,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { private ElasticsearchClient client; @Autowired private TUserDatasetMapper tUserDatasetMapper; + @Autowired private DifyDatasetsMapper difyDatasetsMapper; @@ -73,6 +74,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { .properties("source_url", p -> p.keyword(k -> k)) .properties("dataset_name", p -> p.keyword(k -> k)) .properties("document_id", p -> p.keyword(k -> k)) + .properties("difyDocId", p -> p.keyword(k -> k)) ) ); @@ -263,12 +265,15 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { double epsilon = 1e-6; double lower = 0.05, upper = 0.98; + Random random = new Random(); int index = 0; for (Hit hit : response.hits().hits()) { TDatasetFiles d = hit.source(); double rawScore = scores.get(index++); - double normalizedScore = (maxScore - minScore < epsilon) ? upper - : lower + (rawScore - minScore) / (maxScore - minScore) * (upper - lower); + log.info("Score: {}", rawScore); + double normalizedScore = normalizeScore(rawScore, minScore, maxScore, lower, upper, epsilon, random); + + // 高亮内容 String content = String.join(" ... ", hit.highlight().getOrDefault("content", Collections.emptyList())); System.out.println("content: " + content); @@ -340,11 +345,11 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { try { DeleteByQueryResponse response = client.deleteByQuery(d -> d .index(DatasetId) - .query(q -> q.term(t -> t.field("document_id").value(documentId)))); - datasetFilesService.updateByDatasetIdAndDocId(DatasetId, documentId); + .query(q -> q.term(t -> t.field("difyDocId").value(documentId)))); log.info("删除文档索引成功: {}, documentId: {}", DatasetId, documentId); + log.info(" deleted: {}", response.deleted()); } catch (IOException e) { - log.error("删除文档索引失败: {}, documentId: {}", DatasetId, documentId, e); + log.error("删除文档索引失败: {}, documentId: {}, error: {}", DatasetId, documentId, e.getMessage()); } } else { log.warn("索引不存在: {}", DatasetId); @@ -365,6 +370,33 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { }); } + private double normalizeScore(double rawScore, double minScore, double maxScore, double lower, double upper, double epsilon, Random random) { + // 计算分数范围 + double scoreRange = maxScore - minScore; + log.warn("Score range: {}", scoreRange); + + // 如果最大最小分数相差小于 epsilon，直接使用 upper + double normalizedScore = (scoreRange < epsilon) ? upper + : lower + (rawScore - minScore) / scoreRange * (upper - lower); + + log.warn("rawScore: {}, normalizedScore before fluctuation: {}", rawScore, normalizedScore); + + // 获取原始分数的小数点后两位 + double integerPart = Math.floor(rawScore); // 获取整数部分 + double decimalPart = rawScore - integerPart; // 获取小数部分 + double decimalPartOneDigit = Math.floor(decimalPart * 10) / 10.0; // 获取小数部分的第一位 + double result = integerPart + decimalPartOneDigit; // 将整数部分和小数点后一位合成 + double decimalPartTwoDigits = (result*10)/ 1000; // 获取小数部分的两位 + log.warn("Raw score decimal part (2 digits): {}", decimalPartTwoDigits); + if(normalizedScore==upper){ + normalizedScore -= (0.1-decimalPartTwoDigits); + }else if(normalizedScore==lower){ + normalizedScore += decimalPartTwoDigits; + } + return normalizedScore; + } + + }