From e48671fb48f276d92cf38d55d6dee102aec0a5cd Mon Sep 17 00:00:00 2001 From: moon <2623532423@qq.com> Date: Wed, 24 Sep 2025 11:02:53 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86es=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E5=88=86=E6=95=B0=EF=BC=8C=E8=B7=9F=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E5=AD=97=E7=9A=84=E7=9B=B8=E5=85=B3=E5=BA=A6=E6=9B=B4=E9=AB=98?= =?UTF-8?q?=EF=BC=8C=E4=BC=9A=E9=9A=8F=E7=9D=80=E5=85=B3=E9=94=AE=E5=AD=97?= =?UTF-8?q?=E7=9A=84=E5=A4=9A=E5=B0=91=E8=BF=9B=E8=A1=8C=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E5=BD=93=E5=88=A0=E9=99=A4=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E6=96=87=E4=BB=B6=E6=97=B6=EF=BC=8C=E8=AF=A5?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=B4=A2=E5=BC=95=E4=B9=9F=E4=BC=9A=E8=A2=AB?= =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets/components/DocumentList.vue | 9 +++- .../controller/DatasetManageController.java | 8 ++++ .../impl/EsTDatasetFilesServiceImpl.java | 42 ++++++++++++++++--- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/chat-client/src/views/datasets/components/DocumentList.vue b/chat-client/src/views/datasets/components/DocumentList.vue index 926aaee..386113b 100644 --- a/chat-client/src/views/datasets/components/DocumentList.vue +++ b/chat-client/src/views/datasets/components/DocumentList.vue @@ -882,6 +882,7 @@ if (!newVal) { import { useUserStore } from '@/store/modules/user' import { useAclStore } from '@/store/modules/acl' import vab from '~/library/plugins/vab' +import {log} from "echarts/types/src/util/log"; // 事件处理函数 const renderedHandler = () => { @@ -1225,7 +1226,13 @@ try { // 调用删除接口 await deleteDocument(parseInt(row.id)) - + try{ + //删除es索引 + await deleteFileIndex(row.difyDocId,datasetId.value) + console.log("删除es索引成功,文档id:{}",row.difyDocId) + } catch (error) { + console.error("删除es索引失败,文档id:{}",row.difyDocId,error) + } // 成功处理 ElNotification({ diff --git a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java index 749534d..101c067 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java +++ b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java @@ -70,6 +70,14 @@ public class DatasetManageController { datasetManagerService.deleteDataset(id); difyDatasetService.delete(id); + + //删除es索引 + try { + esTDatasetFilesService.deleteIndex(id); + log.info("删除es索引成功,知识库id:{}",id); + } catch (IOException e) { + log.error("删除es索引失败,知识库id:{}",id,e); + } } @PostMapping("/create") diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java index a6ef3fc..6231239 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java @@ -37,6 +37,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { private ElasticsearchClient client; @Autowired private TUserDatasetMapper tUserDatasetMapper; + @Autowired private DifyDatasetsMapper difyDatasetsMapper; @@ -73,6 +74,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { .properties("source_url", p -> p.keyword(k -> k)) .properties("dataset_name", p -> p.keyword(k -> k)) .properties("document_id", p -> p.keyword(k -> k)) + .properties("difyDocId", p -> p.keyword(k -> k)) ) ); @@ -263,12 +265,15 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { double epsilon = 1e-6; double lower = 0.05, upper = 0.98; + Random random = new Random(); int index = 0; for (Hit hit : response.hits().hits()) { TDatasetFiles d = hit.source(); double rawScore = scores.get(index++); - double normalizedScore = (maxScore - minScore < epsilon) ? upper - : lower + (rawScore - minScore) / (maxScore - minScore) * (upper - lower); + log.info("Score: {}", rawScore); + double normalizedScore = normalizeScore(rawScore, minScore, maxScore, lower, upper, epsilon, random); + + // 高亮内容 String content = String.join(" ... ", hit.highlight().getOrDefault("content", Collections.emptyList())); System.out.println("content: " + content); @@ -340,11 +345,11 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { try { DeleteByQueryResponse response = client.deleteByQuery(d -> d .index(DatasetId) - .query(q -> q.term(t -> t.field("document_id").value(documentId)))); - datasetFilesService.updateByDatasetIdAndDocId(DatasetId, documentId); + .query(q -> q.term(t -> t.field("difyDocId").value(documentId)))); log.info("删除文档索引成功: {}, documentId: {}", DatasetId, documentId); + log.info(" deleted: {}", response.deleted()); } catch (IOException e) { - log.error("删除文档索引失败: {}, documentId: {}", DatasetId, documentId, e); + log.error("删除文档索引失败: {}, documentId: {}, error: {}", DatasetId, documentId, e.getMessage()); } } else { log.warn("索引不存在: {}", DatasetId); @@ -365,6 +370,33 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { }); } + private double normalizeScore(double rawScore, double minScore, double maxScore, double lower, double upper, double epsilon, Random random) { + // 计算分数范围 + double scoreRange = maxScore - minScore; + log.warn("Score range: {}", scoreRange); + + // 如果最大最小分数相差小于 epsilon，直接使用 upper + double normalizedScore = (scoreRange < epsilon) ? upper + : lower + (rawScore - minScore) / scoreRange * (upper - lower); + + log.warn("rawScore: {}, normalizedScore before fluctuation: {}", rawScore, normalizedScore); + + // 获取原始分数的小数点后两位 + double integerPart = Math.floor(rawScore); // 获取整数部分 + double decimalPart = rawScore - integerPart; // 获取小数部分 + double decimalPartOneDigit = Math.floor(decimalPart * 10) / 10.0; // 获取小数部分的第一位 + double result = integerPart + decimalPartOneDigit; // 将整数部分和小数点后一位合成 + double decimalPartTwoDigits = (result*10)/ 1000; // 获取小数部分的两位 + log.warn("Raw score decimal part (2 digits): {}", decimalPartTwoDigits); + if(normalizedScore==upper){ + normalizedScore -= (0.1-decimalPartTwoDigits); + }else if(normalizedScore==lower){ + normalizedScore += decimalPartTwoDigits; + } + return normalizedScore; + } + + } From 81ccde15f54ba5d26eed3e51d329b7471dc9b4e5 Mon Sep 17 00:00:00 2001 From: moon <2623532423@qq.com> Date: Wed, 24 Sep 2025 12:00:41 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=B4=A2=E5=BC=95=E5=92=8C=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=8E=A5=E5=8F=A3=EF=BC=8C=E5=85=A8=E5=9C=A8?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets/components/DocumentList.vue | 7 ------- .../controller/DatasetDocController.java | 19 +++++++++++++++++- .../controller/KnowledgeBaseController.java | 20 +++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/chat-client/src/views/datasets/components/DocumentList.vue b/chat-client/src/views/datasets/components/DocumentList.vue index 386113b..4148d60 100644 --- a/chat-client/src/views/datasets/components/DocumentList.vue +++ b/chat-client/src/views/datasets/components/DocumentList.vue @@ -1226,13 +1226,6 @@ try { // 调用删除接口 await deleteDocument(parseInt(row.id)) - try{ - //删除es索引 - await deleteFileIndex(row.difyDocId,datasetId.value) - console.log("删除es索引成功,文档id:{}",row.difyDocId) - } catch (error) { - console.error("删除es索引失败,文档id:{}",row.difyDocId,error) - } // 成功处理 ElNotification({ diff --git a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetDocController.java b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetDocController.java index 8d65427..8eaf076 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetDocController.java +++ b/chat-server/src/main/java/com/bjtds/brichat/controller/DatasetDocController.java @@ -9,6 +9,7 @@ import com.bjtds.brichat.entity.dto.PdfTaskDto; import com.bjtds.brichat.service.DatasetFilesService; import com.bjtds.brichat.service.DatasetsDocService; import com.bjtds.brichat.service.DeepAnalysisQueueService; +import com.bjtds.brichat.service.EsTDatasetFilesService; import com.bjtds.brichat.service.dify.DifyDatasetApiService; import com.bjtds.brichat.util.Constants; import com.bjtds.brichat.util.ResultUtils; @@ -27,6 +28,7 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; +import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -70,6 +72,9 @@ public class DatasetDocController { @Resource private DeepAnalysisQueueService deepAnalysisQueueService; + @Resource + private EsTDatasetFilesService esTDatasetFilesService; + @Value("${dify.url}") private String difyUrl; @@ -137,9 +142,14 @@ public class DatasetDocController { for (TDatasetFiles tdatasetFiles : allFilesByParentId) { if(tdatasetFiles.getDifyDocId() != null && !tdatasetFiles.getDifyDocId().isEmpty()){ difyDatasetService.deleteDocument(tdatasetFiles.getDifyDatasetId(),tdatasetFiles.getDifyDocId()); + //删除es索引 + try{ + esTDatasetFilesService.deleteDocIndex(tdatasetFiles.getDifyDatasetId(),tdatasetFiles.getDifyDocId()); + } catch (IOException e) { + log.error("删除es索引失败,文档id:{}",tdatasetFiles.getDifyDocId(),e); + } } datasetFilesService.physicalDeleteFile(tdatasetFiles.getId()); - } } @@ -147,7 +157,14 @@ public class DatasetDocController { datasetFilesService.deleteFile(fileId); } else{ if(fileInfo.getDifyDocId() != null && !fileInfo.getDifyDocId().isEmpty()){ + difyDatasetService.deleteDocument(fileInfo.getDifyDatasetId(),fileInfo.getDifyDocId()); + //删除es索引 + try{ + esTDatasetFilesService.deleteDocIndex(fileInfo.getDifyDatasetId(),fileInfo.getDifyDocId()); + } catch (IOException e) { + log.error("删除es索引失败,文档id:{}",fileInfo.getDifyDocId(),e); + } } datasetFilesService.physicalDeleteFile(fileId); } diff --git a/chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java b/chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java index 99d95b8..2ebe60e 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java +++ b/chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java @@ -141,16 +141,16 @@ public class KnowledgeBaseController { result.put("finished", finished); return ResultUtils.success(result); } - @ApiOperation("删除索引下的文件") - @DeleteMapping("/deleteFileIndex") - public ResultUtils deleteFileIndex(@RequestParam("documentId") String documentId,@RequestParam("datasetId")String datasetId) throws Exception { - try{ - esTDatasetFilesService.deleteDocIndex(datasetId,documentId); - return ResultUtils.success("索引删除成功"); - } catch (IOException e) { - return ResultUtils.error("索引删除失败: " + e.getMessage()); - } - } +// @ApiOperation("删除索引下的文件") +// @DeleteMapping("/deleteFileIndex") +// public ResultUtils deleteFileIndex(@RequestParam("documentId") String documentId,@RequestParam("datasetId")String datasetId) throws Exception { +// try{ +// esTDatasetFilesService.deleteDocIndex(datasetId,documentId); +// return ResultUtils.success("索引删除成功"); +// } catch (IOException e) { +// return ResultUtils.error("索引删除失败: " + e.getMessage()); +// } +// } @ApiOperation("删除所有索引") @DeleteMapping("/deleteAllIndex") public ResultUtils deleteAllIndex() throws Exception {