WJB
/
ai-manus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'es检索' 

# Conflicts:
#	chat-client/src/views/datasets/components/DocumentList.vue
This commit is contained in:
wenjinbo 2025-09-24 10:05:14 +08:00
parent 1936081a3c fbad0df5de
commit bc730595c8
10 changed files with 68 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java
View File

@ -7,6 +7,7 @@ import com.bjtds.brichat.entity.dify.DatasetDto;
import com.bjtds.brichat.entity.dto.UserBindDatasetDto;
import com.bjtds.brichat.entity.dto.UserLinkDatasetDto;
import com.bjtds.brichat.service.DatasetManagerService;
import com.bjtds.brichat.service.EsTDatasetFilesService;
import com.bjtds.brichat.service.dify.DifyDatasetApiService;
import com.bjtds.brichat.util.ResultUtils;
import com.bjtds.common.utils.Pagination;
@ -18,6 +19,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
@Api(tags = "知识库管理")
@ -27,6 +29,9 @@ import javax.annotation.Resource;
@RequestMapping("/datasetManage")
public class DatasetManageController {
@Resource
private EsTDatasetFilesService esTDatasetFilesService;
@Resource
private DatasetManagerService datasetManagerService;
@ -70,6 +75,16 @@ public class DatasetManageController {
@PostMapping("/create")
public ResultUtils create(@RequestBody DatasetCreateRequest datasetCreateRequest) {
ResponseEntity<DatasetDto>res = difyDatasetApiService.createDataset(datasetCreateRequest.getName(), datasetCreateRequest.getDescription());
DatasetDto datasetDto = res.getBody();
String datasetId = datasetDto.getId();
//构建es索引
try {
esTDatasetFilesService.createIndex(datasetId);
log.info("创建es索引成功,知识库id:{}",datasetId);
} catch (IOException e) {
log.error("创建es索引失败,知识库id:{}",datasetId,e);
}
return ResultUtils.success(res.getBody());
}

2
chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java
View File

@ -99,7 +99,7 @@ public class KnowledgeBaseController {
// public ResultUtils createIndex(@RequestParam("documentId") String documentId) throws Exception {
//
// try{
// esTDatasetFilesImporter.importDocumentId(documentId);
// esTDatasetFilesImporter.importDocumentId(Integer.valueOf(documentId));
// return ResultUtils.success("索引创建成功");
// } catch (IOException e) {
// return ResultUtils.error("索引创建失败: " + e.getMessage());

4
chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java
View File

@ -1,9 +1,13 @@
package com.bjtds.brichat.entity.dto;
import com.alibaba.fastjson.annotation.JSONField;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecordDto {
/**分段信息*/
@JSONField(name = "retrieval")

7
chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.java
View File

@ -186,6 +186,13 @@ public interface TDatasetFilesMapper {
TDatasetFiles selectByDatasetIdAndDocId(@Param("difyDatasetId") String difyDatasetId, @Param("difyDocId") String difyDocId);
/**
* 根据文档ID查询文件
*
* @param difyDocId 文档ID
* @return 文件信息
*/
TDatasetFiles selectByDocId( String difyDocId);
}

4
chat-server/src/main/java/com/bjtds/brichat/service/DatasetFilesService.java
View File

@ -171,6 +171,10 @@ public interface DatasetFilesService {
*/
void updateByDatasetIdAndDocId(String difyDatasetId, String difyDocId);
/**
* 根据文档ID查询文件
*/
TDatasetFiles getFileByDocId(String difyDocId);

5
chat-server/src/main/java/com/bjtds/brichat/service/impl/DatasetFilesServiceImpl.java
View File

@ -455,5 +455,10 @@ public class DatasetFilesServiceImpl implements DatasetFilesService {
}
}
@Override
public TDatasetFiles getFileByDocId(String difyDocId) {
return datasetFilesMapper.selectByDocId(difyDocId);
}
}

1
chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java
View File

@ -59,6 +59,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
}
}
// 创建索引
@Override
public void createIndex(String DatasetId) throws IOException {
boolean exists = client.indices().exists(e -> e.index(DatasetId)).value();
if (!exists) {

16
chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.units.qual.C;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@ -67,8 +68,8 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
@Override
public List<RecordDto> retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception {
String datasetPath = difyUrl + Constants.DATABASE_API;
List<RecordDto> recordDtos = Lists.newArrayList();
List<Object> results = Lists.newArrayList();
List<RecordDto> recordDtos = new ArrayList<>();
List<RecordDto> results = Lists.newArrayList();
List<String> datasetIds =Lists.newArrayList();
if (knowledgeBaseDto.getSelectedKnowledgeBaseIds() != null && !knowledgeBaseDto.getSelectedKnowledgeBaseIds().isEmpty()) {
@ -84,6 +85,15 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
if (knowledgeBaseDto.getSearchMethod().equals("keyword_search")) {
List<RecordDto> datasetFiles=esTDatasetFilesService.search(knowledgeBaseDto.getQuery(),datasetIds);
recordDtos.addAll(datasetFiles);
recordDtos.sort((dto1, dto2) -> {
try {
double score1 = Double.parseDouble(dto1.getScore());
double score2 = Double.parseDouble(dto2.getScore());
return Double.compare(score2, score1);
} catch (NumberFormatException e) {
return 0;
}
});
return recordDtos;
}
@ -136,7 +146,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
}
});
results.addAll(recordDtos);
return recordDtos;
return results;
}
@Override

17
chat-server/src/main/java/com/bjtds/brichat/util/EsTDatasetFilesImporter.java
View File

@ -3,13 +3,12 @@ package com.bjtds.brichat.util;
import com.bjtds.brichat.entity.dataset.TDatasetFiles;
import com.bjtds.brichat.service.DatasetFilesService;
import com.bjtds.brichat.service.EsTDatasetFilesService;
import io.swagger.models.auth.In;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
@ -17,6 +16,7 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.util.List;
@Slf4j
@Service
public class EsTDatasetFilesImporter {
@ -25,7 +25,6 @@ public class EsTDatasetFilesImporter {
@Autowired
private DatasetFilesService datasetFilesService;
// private static final Logger log = LoggerFactory.getLogger(EsTDatasetFilesImporter.class);
@Autowired
private StringRedisTemplate redisTemplate;
@ -63,16 +62,16 @@ public class EsTDatasetFilesImporter {
if (document == null) continue;
String filePath = document.getDifyStoragePath();
if (filePath == null) {
log.info("documentId=" + document.getId() + " 不存在difyStoragePath，跳过");
log.warn("documentId=" + document.getId() + " 不存在difyStoragePath，跳过");
continue;
}
File file = new File(filePath);
if (!file.exists()) {
log.info(file.getAbsolutePath() + " 不存在，跳过");
log.warn(file.getAbsolutePath() + " 不存在，跳过");
continue;
}
if(Boolean.TRUE.equals(document.getIsEs())){
log.info("documentId=" + document.getId() + " 是ES索引文件，跳过");
log.warn("documentId=" + document.getId() + " 是ES索引文件，跳过");
continue;
}
@ -83,9 +82,9 @@ public class EsTDatasetFilesImporter {
redisTemplate.opsForValue().set("import:task:" + taskId + ":finished", String.valueOf(finished));
document.setIsEs(true);
datasetFilesService.updateFile(document);
log.info("documentId=" + document.getId() + " 索引构建成功");
log.debug("documentId=" + document.getId() + " 索引构建成功");
} catch (Exception e) {
log.info("documentId=" + document.getId() + " 索引构建失败: " + e.getMessage());
log.debug("documentId=" + document.getId() + " 索引构建失败: " + e.getMessage());
}
}
redisTemplate.opsForValue().set("import:task:" + taskId + ":status", "done");

9
chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.xml
View File

@ -281,4 +281,13 @@
AND dify_doc_id = #{difyDocId}
AND is_deleted = false
</select>
<!-- 根据文档ID查询文件 -->
<select id="selectByDocId" resultMap="TDatasetFilesResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dataset_files
WHERE dify_doc_id = #{difyDocId}
AND is_deleted = false
</select>
</mapper>