diff --git a/chat-client/src/main.ts b/chat-client/src/main.ts index a9109d2..4b2e42b 100644 --- a/chat-client/src/main.ts +++ b/chat-client/src/main.ts @@ -14,7 +14,6 @@ import { setupRouter } from '@/router' */ import { baseURL, pwa } from './config' import { isExternal } from '@/utils/validate' - const app = createApp(App) if (process.env.NODE_ENV === 'production' && !isExternal(baseURL)) { @@ -56,6 +55,8 @@ if (process.env.NODE_ENV === 'production' && !isExternal(baseURL)) { if (pwa) require('./registerServiceWorker') + + setupVab(app) setupI18n(app) setupStore(app) diff --git a/chat-client/src/views/chatweb/intelRetrieval/index.vue b/chat-client/src/views/chatweb/intelRetrieval/index.vue index abc6491..4355e79 100644 --- a/chat-client/src/views/chatweb/intelRetrieval/index.vue +++ b/chat-client/src/views/chatweb/intelRetrieval/index.vue @@ -69,12 +69,12 @@

- {{ result.segmentDto?.documentDto?.name || result.title || "未知文档" }} + {{ result.segmentDto?.documentDto?.name || result.retrievalDto?.name || "未知文档" }}

@@ -87,9 +87,9 @@
-

+

-
+
{{ expandedItems.includes(getGlobalIndex(index)) ? '收起' : '展开更多' }}
@@ -124,7 +124,7 @@ text class="action-btn" @click="handlePreview(result)" - v-if="getSourceUrl(result.segmentDto?.documentDto?.docMetadata || result.source_url)" + v-if="getSourceUrl(result.segmentDto?.documentDto?.docMetadata || result.retrievalDto?.sourceUrl)" > 预览 @@ -134,7 +134,7 @@ text class="action-btn" @click="handleDownload(result)" - v-if="getSourceUrl(result.segmentDto?.documentDto?.docMetadata || result.source_url)" + v-if="getSourceUrl(result.segmentDto?.documentDto?.docMetadata || result.retrievalDto?.sourceUrl)" > 下载 @@ -258,8 +258,7 @@
- - 绑定知识库 + retrievalResult = knowledgeBaseService.retrieval(knowledgeBaseDto); + List retrievalResult = knowledgeBaseService.retrieval(knowledgeBaseDto); return ResultUtils.success(retrievalResult); } - @ApiOperation("Es全文检索") - @GetMapping("/search") - public ResultUtils search(@RequestBody KnowledgeBaseDto knowledgeBaseDto) throws Exception { - try { - String keyword = knowledgeBaseDto.getQuery(); - List DatasetId = knowledgeBaseDto.getSelectedKnowledgeBaseIds(); - List> results = esKnowledgeService.search(keyword, DatasetId); - return ResultUtils.success(results); - } catch (IOException e) { - return ResultUtils.error("检索失败"); - } - } + @ApiOperation("返回所有索引信息") @GetMapping("/getAllIndexInfos") diff --git a/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java b/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java index a89771c..abab018 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java +++ b/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java @@ -6,12 +6,14 @@ import lombok.Data; @Data public class RecordDto { /**分段信息*/ - @JSONField(name = "segment") - private SegmentDto segmentDto; - @JSONField(name = "child_chunks") - private String childChunks; + @JSONField(name = "retrieval") + private RetrievalDto retrievalDto; + /**置信度* 例如0.99*/ private String score; - @JSONField(name = "tsne_position") - private String tsnePosition; + + + + @JSONField(name = "segment") + private SegmentDto segmentDto; } diff --git a/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RetrievalDto.java b/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RetrievalDto.java new file mode 100644 index 0000000..cfceaf6 --- /dev/null +++ b/chat-server/src/main/java/com/bjtds/brichat/entity/dto/RetrievalDto.java @@ -0,0 +1,18 @@ +package com.bjtds.brichat.entity.dto; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RetrievalDto { + private String id; + private String name; + private String content; + private String datasetId; + private String datasetName; + private String sourceUrl; +} diff --git a/chat-server/src/main/java/com/bjtds/brichat/entity/esmodel/KnowledgeDoc.java b/chat-server/src/main/java/com/bjtds/brichat/entity/esmodel/KnowledgeDoc.java index d726b10..e7ef3d9 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/entity/esmodel/KnowledgeDoc.java +++ b/chat-server/src/main/java/com/bjtds/brichat/entity/esmodel/KnowledgeDoc.java @@ -11,7 +11,6 @@ import java.util.List; @NoArgsConstructor @AllArgsConstructor public class KnowledgeDoc { - private String id; private String title; private String content; private String fileName; diff --git a/chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.java b/chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.java index 580d3ae..38423a3 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.java +++ b/chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.java @@ -25,4 +25,8 @@ public interface TUserDatasetMapper { void deleteDataset(String datasetId); + String getDatasetName(String datasetId); + + + } \ No newline at end of file diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/EsTDatasetFilesService.java b/chat-server/src/main/java/com/bjtds/brichat/service/EsTDatasetFilesService.java index c2ca6b6..002f88d 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/EsTDatasetFilesService.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/EsTDatasetFilesService.java @@ -1,6 +1,7 @@ package com.bjtds.brichat.service; import com.bjtds.brichat.entity.dataset.TDatasetFiles; +import com.bjtds.brichat.entity.dto.RecordDto; import com.bjtds.brichat.entity.esmodel.IndexInfo; import com.bjtds.common.utils.Pagination; @@ -9,13 +10,11 @@ import java.util.List; import java.util.Map; public interface EsTDatasetFilesService { - boolean checkIndexExists(String DatasetId) throws IOException; - // 清除索引 void deleteIndex(String DatasetId) throws IOException; void createIndex(String DatasetId) throws IOException; void addDoc(TDatasetFiles doc) throws IOException; - List> searchSingle(String keyword, String DatasetId) throws IOException; - List> search(String keyword, List datasetIds) throws IOException; + List searchSingle(String keyword, String DatasetId) throws IOException; + List search(String keyword, List datasetIds) throws IOException; Pagination getAllIndexInfos(Integer pageNo, Integer pageSize, String keyword) throws IOException; } diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/KnowledgeBaseService.java b/chat-server/src/main/java/com/bjtds/brichat/service/KnowledgeBaseService.java index ef791f5..ff57d44 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/KnowledgeBaseService.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/KnowledgeBaseService.java @@ -17,7 +17,7 @@ import java.util.UUID; */ public interface KnowledgeBaseService { - List retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception; + List retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception; List getWorkflowAndDatasetTableData() throws Exception; diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsKnowledgeServiceImpl.java b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsKnowledgeServiceImpl.java index a589f5e..e7f53c3 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsKnowledgeServiceImpl.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsKnowledgeServiceImpl.java @@ -85,7 +85,6 @@ public class EsKnowledgeServiceImpl implements EsKnowledgeService { futures.add(executor.submit(() -> { try { KnowledgeDoc subDoc = new KnowledgeDoc( - doc.getId() + "_part" + currentIndex, doc.getTitle(), chunk, doc.getFileName(), @@ -98,7 +97,6 @@ public class EsKnowledgeServiceImpl implements EsKnowledgeService { for (String datasetId : doc.getDataset_id()) { client.index(req -> req .index(datasetId) - .id(subDoc.getId()) .document(subDoc) ); } @@ -115,7 +113,6 @@ public class EsKnowledgeServiceImpl implements EsKnowledgeService { if (!splitted[0] && file.length() <= maxSize) { String content = EsFileParser.parseFile(file); KnowledgeDoc singleDoc = new KnowledgeDoc( - doc.getId(), doc.getTitle(), content, doc.getFileName(), @@ -129,7 +126,6 @@ public class EsKnowledgeServiceImpl implements EsKnowledgeService { for (String datasetId : doc.getDataset_id()) { client.index(req -> req .index(datasetId) - .id(singleDoc.getId()) .document(singleDoc) ); } diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java index 73e2efb..14eb3ea 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java @@ -6,10 +6,14 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import com.bjtds.brichat.entity.dataset.TDatasetFiles; +import com.bjtds.brichat.entity.dto.RecordDto; +import com.bjtds.brichat.entity.dto.RetrievalDto; import com.bjtds.brichat.entity.esmodel.IndexInfo; +import com.bjtds.brichat.mapper.opengauss.TUserDatasetMapper; import com.bjtds.brichat.service.EsTDatasetFilesService; import com.bjtds.brichat.util.EsFileSplitter; import com.bjtds.common.utils.Pagination; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import co.elastic.clients.elasticsearch.cat.indices.IndicesRecord; @@ -21,10 +25,13 @@ import java.util.stream.Collectors; @Service +@Slf4j public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { @Autowired private ElasticsearchClient client; + @Autowired + private TUserDatasetMapper tUserDatasetMapper; @@ -47,14 +54,11 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { client.indices().create(c -> c .index(DatasetId) .mappings(m -> m - .properties("title", p -> p.text(t -> t.analyzer("ik_max_word").searchAnalyzer("ik_max_word"))) + .properties("name", p -> p.text(t -> t.analyzer("ik_max_word").searchAnalyzer("ik_max_word"))) .properties("content", p -> p.text(t -> t.analyzer("ik_max_word").searchAnalyzer("ik_max_word"))) - .properties("fileName", p -> p.keyword(k -> k)) - .properties("filePath", p -> p.keyword(k -> k)) - .properties("fileType", p -> p.keyword(k -> k)) .properties("dataset_id", p -> p.keyword(k -> k)) .properties("source_url", p -> p.keyword(k -> k)) - .properties("dify_storage_path", p -> p.keyword(k -> k)) + .properties("dataset_name", p -> p.keyword(k -> k)) ) ); @@ -181,28 +185,26 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { // ); // return !response.hits().hits().isEmpty(); // } + @Override - public List> search(String keyword, List datasetIds) throws IOException { - // 线程池，最多 10 个线程并发 + public List search(String keyword, List datasetIds) throws IOException { ExecutorService executor = Executors.newFixedThreadPool(Math.min(datasetIds.size(), 4)); try { - // 多线程提交任务 - List>>> futures = datasetIds.stream() + List>> futures = datasetIds.stream() .map(datasetId -> CompletableFuture.supplyAsync(() -> { try { - return searchSingle(keyword, datasetId); // 单索引查询 + return searchSingle(keyword, datasetId); } catch (IOException e) { - e.printStackTrace(); - return Collections.>emptyList(); + log.error("索引查询失败: {}", datasetId, e); + return Collections.emptyList(); } }, executor)) .collect(Collectors.toList()); - // 等待所有查询完成，并合并结果 return futures.stream() - .map(CompletableFuture::join) // 阻塞等待 - .flatMap(List::stream) // 合并成一个 List + .map(CompletableFuture::join) + .flatMap(List::stream) .collect(Collectors.toList()); } finally { @@ -210,40 +212,32 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { } } - @Override - public boolean checkIndexExists(String DatasetId) throws IOException { - return client.indices().exists(e -> e.index(DatasetId)).value(); - } - - // ================= 搜索方法 ================= - @Override - public List> searchSingle(String keyword, String DatasetId) throws IOException { + public List searchSingle(String keyword, String DatasetId) throws IOException { boolean indexExists = client.indices().exists(e -> e.index(DatasetId)).value(); + if (!indexExists) { + log.warn("索引不存在: {}", DatasetId); + return Collections.emptyList(); + } SearchResponse response = client.search(s -> s .index(DatasetId) .query(q -> q.bool(b -> b - .should(s1 -> s1.match(m -> m.field("title").query(keyword))) // title 分词模糊搜索) - .should(s2 -> s2.match(m -> m.field("content").query(keyword))) // content 分词全文搜索 + .should(s1 -> s1.match(m -> m.field("title").query(keyword))) + .should(s2 -> s2.match(m -> m.field("content").query(keyword))) )) .size(500) .highlight(h -> h - .requireFieldMatch(false) // 不要求必须在同一个字段高亮 + .requireFieldMatch(false) .fields("content", f -> f) ), TDatasetFiles.class ); -// List> results = new ArrayList<>(); - Map> uniqueResults = new LinkedHashMap<>(); - List scores = new ArrayList<>(); - List> hits = response.hits().hits(); - - // 收集所有分片得分 - for (Hit hit : hits) { - scores.add(hit.score() != null ? hit.score() : 0.0); - } + Map uniqueResults = new LinkedHashMap<>(); + List scores = response.hits().hits().stream() + .map(hit -> hit.score() != null ? hit.score() : 0.0) + .collect(Collectors.toList()); double minScore = scores.stream().min(Double::compare).orElse(0.0); double maxScore = scores.stream().max(Double::compare).orElse(1.0); @@ -251,41 +245,36 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService { double lower = 0.05, upper = 0.98; int index = 0; - for (Hit hit : hits) { + for (Hit hit : response.hits().hits()) { TDatasetFiles d = hit.source(); double rawScore = scores.get(index++); double normalizedScore = (maxScore - minScore < epsilon) ? upper : lower + (rawScore - minScore) / (maxScore - minScore) * (upper - lower); - Map item = new HashMap<>(); - item.put("title", d.getName()); - item.put("filePath", d.getPath()); - item.put("size", d.getSize()); - item.put("fileType", d.getType()); - item.put("sourceUrl", d.getSourceUrl()); - item.put("isDeep", d.getIsDeep()); - item.put("difyDocId", d.getDifyDocId()); - item.put("indexingStatus", d.getIndexingStatus()); - item.put("difyDatasetId", d.getDifyDatasetId()); - item.put("difyStoragePath", d.getDifyStoragePath()); - item.put("score", normalizedScore); + String content = String.join(" ... ", hit.highlight().getOrDefault("content", Collections.emptyList())); + String datasetName = tUserDatasetMapper.getDatasetName(d.getDifyDatasetId()); + RetrievalDto retrievalDto = new RetrievalDto( + d.getId() != null ? d.getId().toString() : null, + d.getName(), + content, + d.getDifyDatasetId(), + datasetName, + d.getSourceUrl() + ); + RecordDto recordDto = new RecordDto(); + recordDto.setRetrievalDto(retrievalDto); + recordDto.setScore(String.format("%.4f", normalizedScore)); - if (hit.highlight() != null) { - String content = String.join(" ... ", hit.highlight().getOrDefault("content", Collections.emptyList())); - item.put("content", content); - } - - // 按 fileName 去重，只保留第一次出现的分片 -// results.add(item); - uniqueResults.putIfAbsent(d.getName(), item); + // 按 name 去重，只保留第一个 + uniqueResults.putIfAbsent(d.getName(), recordDto); } -// return results; return new ArrayList<>(uniqueResults.values()); } + @Override public Pagination getAllIndexInfos(Integer pageNo, Integer pageSize, String keyword) throws IOException { List indexInfos = new ArrayList<>(); diff --git a/chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java b/chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java index ebfa69e..64896a3 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java +++ b/chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java @@ -65,20 +65,26 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { private final ExecutorService executorService = Executors.newFixedThreadPool(10); @Override - public List retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception { + public List retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception { String datasetPath = difyUrl + Constants.DATABASE_API; List recordDtos = Lists.newArrayList(); List results = Lists.newArrayList(); - List datasetIds = difyDatasetsMapper.getDatasetIds(); + List datasetIds =Lists.newArrayList(); + if (knowledgeBaseDto.getSelectedKnowledgeBaseIds() != null && !knowledgeBaseDto.getSelectedKnowledgeBaseIds().isEmpty()) { + datasetIds.addAll(knowledgeBaseDto.getSelectedKnowledgeBaseIds()); + }else { + datasetIds.addAll(difyDatasetsMapper.getDatasetIds()); + } + log.info("selectedKnowledgeBaseIds:{}", datasetIds); log.info("datasetPath:{}", datasetPath); log.info("apiKey:{}", apiKey); log.info("开始并行查询 {} 个数据集", datasetIds.size()); if (knowledgeBaseDto.getSearchMethod().equals("keyword_search")) { - List> datasetFiles=esTDatasetFilesService.search(knowledgeBaseDto.getQuery(),datasetIds); - results.addAll(datasetFiles); - return results; + List datasetFiles=esTDatasetFilesService.search(knowledgeBaseDto.getQuery(),datasetIds); + recordDtos.addAll(datasetFiles); + return recordDtos; } // 使用 CompletableFuture 并行查询多个数据集 @@ -130,7 +136,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { } }); results.addAll(recordDtos); - return results; + return recordDtos; } @Override diff --git a/chat-server/src/main/java/com/bjtds/brichat/util/EsKnowledgeImporter.java b/chat-server/src/main/java/com/bjtds/brichat/util/EsKnowledgeImporter.java index 195e20a..f9af747 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/util/EsKnowledgeImporter.java +++ b/chat-server/src/main/java/com/bjtds/brichat/util/EsKnowledgeImporter.java @@ -217,7 +217,6 @@ public class EsKnowledgeImporter { String key = dataSourceInfo.getKey(); KnowledgeDoc doc = new KnowledgeDoc(); - doc.setId(UUID.randomUUID().toString()); doc.setTitle(document.getName()); doc.setContent(content); doc.setFileName(file.getName()); diff --git a/chat-server/src/main/java/com/bjtds/brichat/util/RetrievalUtil.java b/chat-server/src/main/java/com/bjtds/brichat/util/RetrievalUtil.java index d04b97e..bd97d84 100644 --- a/chat-server/src/main/java/com/bjtds/brichat/util/RetrievalUtil.java +++ b/chat-server/src/main/java/com/bjtds/brichat/util/RetrievalUtil.java @@ -4,6 +4,8 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.bjtds.brichat.entity.dto.KnowledgeBaseDto; import com.bjtds.brichat.entity.dto.RecordDto; +import com.bjtds.brichat.entity.dto.RetrievalDto; +import com.bjtds.brichat.mapper.opengauss.TUserDatasetMapper; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -26,9 +28,11 @@ public class RetrievalUtil { public void setHttpClient(CloseableHttpClient httpClient) { this.httpClient = httpClient; } + @Autowired + private TUserDatasetMapper tUserDatasetMapper; -public List getRetrieval(String datasetPath, String apiKey, String datasetId, KnowledgeBaseDto knowledgeBaseDto) throws Exception { + public List getRetrieval(String datasetPath, String apiKey, String datasetId, KnowledgeBaseDto knowledgeBaseDto) throws Exception { // 记录开始时间 long startTime = System.currentTimeMillis(); @@ -36,6 +40,11 @@ public List getRetrieval(String datasetPath, String apiKey, String da if (httpClient == null) { throw new IllegalStateException("HttpClient未初始化，请确保Spring容器已启动"); } + RetrievalDto retrievalDto = new RetrievalDto(); + retrievalDto.setDatasetId(datasetId); + + String datasetName = tUserDatasetMapper.getDatasetName(datasetId); + retrievalDto.setDatasetName(datasetName); String uri = datasetPath +"/"+ datasetId + "/retrieve"; log.info("uri:" + uri); @@ -90,7 +99,6 @@ public List getRetrieval(String datasetPath, String apiKey, String da // 正确地将 JSON 数组转换为 RecordDto 列表 List recordDtoList = JSON.parseArray(jsonResult.getJSONArray("records").toJSONString(), RecordDto.class); - // 检查解析后的数据 if (recordDtoList != null && !recordDtoList.isEmpty()) { RecordDto firstRecord = recordDtoList.get(0); @@ -100,6 +108,9 @@ public List getRetrieval(String datasetPath, String apiKey, String da } // log.info("第一条记录的完整内容: {}", firstRecord); } + recordDtoList.forEach(recordDto -> { + recordDto.setRetrievalDto(retrievalDto); + }); //关闭响应资源（不关闭HttpClient，由连接池管理） response.close(); diff --git a/chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.xml b/chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.xml index ceedf84..30972e3 100644 --- a/chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.xml +++ b/chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TUserDatasetMapper.xml @@ -55,4 +55,8 @@ delete from t_user_dataset where dataset_id = #{datasetId} + + \ No newline at end of file