WJB
/
ai-manus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加构建了知识库，同时构建相对应的索引

This commit is contained in:
moon 2025-09-23 17:02:50 +08:00
parent d9ae2a07c1
commit fbad0df5de
11 changed files with 98 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
chat-client/src/views/datasets/components/DocumentList.vue
View File

@ -167,11 +167,11 @@
>
{{t('vabI18n.knowledge.document.buttons.download')}}
</el-button>
<el-button
type="warning"
:icon="Upload"
text
class="action-btn"
<el-button
type="warning"
:icon="Upload"
text
class="action-btn"
@click="handleFileReplaceFromRow(row)"
>
替换
@ -244,12 +244,12 @@
@success="handleUploadSuccess"
/>
</el-dialog>
<!-- 文件替换对话框 -->
<el-dialog
v-model="replaceDialogVisible"
title="替换文件"
width="900px"
<el-dialog
v-model="replaceDialogVisible"
title="替换文件"
width="900px"
:close-on-click-modal="false"
class="upload-dialog"
>
@ -257,9 +257,9 @@
<div class="target-file-info">
<h4>将要替换的文件：</h4>
<div class="file-info-card">
<img
:src="getFileTypeIcon(replaceTargetFile.fileType)"
alt="文件图标"
<img
:src="getFileTypeIcon(replaceTargetFile.fileType)"
alt="文件图标"
class="file-icon"
/>
<div class="file-details">
@ -272,7 +272,7 @@
</div>
</div>
</div>
<DocUpload
<DocUpload
:visible="replaceDialogVisible"
:dataset-id="datasetId"
:parent-id="currentParentId"
@ -507,7 +507,7 @@
import { useRoute, useRouter } from 'vue-router'
import { ref, reactive, computed, onMounted, onUnmounted } from 'vue'
import VueOfficePdf from '@vue-office/pdf'
import { getDatasetDocPage, uploadDocument, deleteDocument, downloadDocument, previewDocumentUrl, renameDocument, createFolder, CreateFolderReq} from '@/api/dataset'
import { getDatasetDocPage, uploadDocument, deleteDocument, downloadDocument, previewDocumentUrl, renameDocument, createFolder, CreateFolderReq,deleteFileIndex} from '@/api/dataset'
import {getSegmentList} from "@/api/Segment"
import DocUpload from './DocUpload.vue'
//引入VueOfficeDocx组件
@ -608,7 +608,7 @@ y: 0
// 面包屑导航相关
const currentParentId = ref<number | undefined>(undefined)
const breadcrumbPath = ref<Array<{id: number, name: string}>>([])
const isScroll = ref(true)
const isScroll = ref(false)
// 面包屑导航项接口
interface BreadcrumbItem {
@ -1005,7 +1005,7 @@ const handlePreview = async (row: FileItem) => {
// 显示抽屉
previewDrawerVisible.value = true
previewLoading.value = true
// 保存当前预览的文件信息
currentPreviewFile.value = row
previewFileType.value = row.fileType;
@ -1038,7 +1038,7 @@ const handlePreview = async (row: FileItem) => {
previewFileUrl.value = await fetchPreviewUrl(parseInt(row.id))
console.log("previewFileUrl (from API)", previewFileUrl.value)
}
// 针对 Excel 文件设置预览选项
if (['xls', 'xlsx', 'csv'].includes(row.fileType.toLowerCase())) {
// 根据文件类型设置 xls 选项
@ -1226,6 +1226,7 @@ try {
// 调用删除接口
await deleteDocument(parseInt(row.id))
// 成功处理
ElNotification({
title: t('vabI18n.knowledge.document.messages.deleteSuccessOk'),
@ -1274,10 +1275,10 @@ const handleSortChange = (sortInfo: any) => {
'createDate': 'created_at',
'charCount': 'size'
}
sortConfig.orderBy = fieldMapping[sortInfo.prop] || sortInfo.prop
sortConfig.orderDirection = sortInfo.order === 'ascending' ? 'ASC' : 'DESC'
// 重置到第一页并刷新数据
pagination.current = 1
fetchDocuments()
@ -1407,6 +1408,7 @@ const handleScroll = async (event: Event) => {
isScroll.value = false
return
}
isScroll.value = true
// 判断是否已经滚动到底部
if (container.scrollHeight - container.scrollTop - container.clientHeight < 10) {
// 如果没有在加载状态，则加载更多
@ -2599,7 +2601,7 @@ transform: scale(0.9) translateY(-5px);
.excel-preview-container {
height: 100%;
background: #fafbfc;
.vue-office-excel {
border-radius: 8px;
overflow: hidden;
@ -2614,7 +2616,7 @@ transform: scale(0.9) translateY(-5px);
background: #f8f9fa;
border-radius: 8px;
border-left: 4px solid #e6a23c;
.target-file-info {
h4 {
margin: 0 0 16px 0;
@ -2622,7 +2624,7 @@ transform: scale(0.9) translateY(-5px);
font-size: 16px;
font-weight: 600;
}
.file-info-card {
display: flex;
align-items: center;
@ -2631,30 +2633,30 @@ transform: scale(0.9) translateY(-5px);
background: white;
border-radius: 6px;
border: 1px solid #ebeef5;
.file-icon {
width: 32px;
height: 32px;
flex-shrink: 0;
object-fit: contain;
}
.file-details {
flex: 1;
.file-name {
font-size: 14px;
font-weight: 600;
color: #303133;
margin-bottom: 4px;
}
.file-meta {
display: flex;
gap: 16px;
font-size: 12px;
color: #909399;
span {
&:not(:last-child)::after {
content: '•';

15
chat-server/src/main/java/com/bjtds/brichat/controller/DatasetManageController.java
View File

@ -7,6 +7,7 @@ import com.bjtds.brichat.entity.dify.DatasetDto;
import com.bjtds.brichat.entity.dto.UserBindDatasetDto;
import com.bjtds.brichat.entity.dto.UserLinkDatasetDto;
import com.bjtds.brichat.service.DatasetManagerService;
import com.bjtds.brichat.service.EsTDatasetFilesService;
import com.bjtds.brichat.service.dify.DifyDatasetApiService;
import com.bjtds.brichat.util.ResultUtils;
import com.bjtds.common.utils.Pagination;
@ -18,6 +19,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
@Api(tags = "知识库管理")
@ -27,6 +29,9 @@ import javax.annotation.Resource;
@RequestMapping("/datasetManage")
public class DatasetManageController {
@Resource
private EsTDatasetFilesService esTDatasetFilesService;
@Resource
private DatasetManagerService datasetManagerService;
@ -70,6 +75,16 @@ public class DatasetManageController {
@PostMapping("/create")
public ResultUtils create(@RequestBody DatasetCreateRequest datasetCreateRequest) {
ResponseEntity<DatasetDto>res = difyDatasetApiService.createDataset(datasetCreateRequest.getName(), datasetCreateRequest.getDescription());
DatasetDto datasetDto = res.getBody();
String datasetId = datasetDto.getId();
//构建es索引
try {
esTDatasetFilesService.createIndex(datasetId);
log.info("创建es索引成功,知识库id:{}",datasetId);
} catch (IOException e) {
log.error("创建es索引失败,知识库id:{}",datasetId,e);
}
return ResultUtils.success(res.getBody());
}

2
chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java
View File

@ -99,7 +99,7 @@ public class KnowledgeBaseController {
// public ResultUtils createIndex(@RequestParam("documentId") String documentId) throws Exception {
//
// try{
// esTDatasetFilesImporter.importDocumentId(documentId);
// esTDatasetFilesImporter.importDocumentId(Integer.valueOf(documentId));
// return ResultUtils.success("索引创建成功");
// } catch (IOException e) {
// return ResultUtils.error("索引创建失败: " + e.getMessage());

4
chat-server/src/main/java/com/bjtds/brichat/entity/dto/RecordDto.java
View File

@ -1,9 +1,13 @@
package com.bjtds.brichat.entity.dto;
import com.alibaba.fastjson.annotation.JSONField;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecordDto {
/**分段信息*/
@JSONField(name = "retrieval")

7
chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.java
View File

@ -186,6 +186,13 @@ public interface TDatasetFilesMapper {
TDatasetFiles selectByDatasetIdAndDocId(@Param("difyDatasetId") String difyDatasetId, @Param("difyDocId") String difyDocId);
/**
* 根据文档ID查询文件
*
* @param difyDocId 文档ID
* @return 文件信息
*/
TDatasetFiles selectByDocId( String difyDocId);
}

4
chat-server/src/main/java/com/bjtds/brichat/service/DatasetFilesService.java
View File

@ -171,6 +171,10 @@ public interface DatasetFilesService {
*/
void updateByDatasetIdAndDocId(String difyDatasetId, String difyDocId);
/**
* 根据文档ID查询文件
*/
TDatasetFiles getFileByDocId(String difyDocId);

5
chat-server/src/main/java/com/bjtds/brichat/service/impl/DatasetFilesServiceImpl.java
View File

@ -455,5 +455,10 @@ public class DatasetFilesServiceImpl implements DatasetFilesService {
}
}
@Override
public TDatasetFiles getFileByDocId(String difyDocId) {
return datasetFilesMapper.selectByDocId(difyDocId);
}
}

1
chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java
View File

@ -59,6 +59,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
}
}
// 创建索引
@Override
public void createIndex(String DatasetId) throws IOException {
boolean exists = client.indices().exists(e -> e.index(DatasetId)).value();
if (!exists) {

18
chat-server/src/main/java/com/bjtds/brichat/service/impl/KnowledgeBaseServiceImpl.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.units.qual.C;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@ -67,8 +68,8 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
@Override
public List<RecordDto> retrieval(KnowledgeBaseDto knowledgeBaseDto) throws Exception {
String datasetPath = difyUrl + Constants.DATABASE_API;
List<RecordDto> recordDtos = Lists.newArrayList();
List<Object> results = Lists.newArrayList();
List<RecordDto> recordDtos = new ArrayList<>();
List<RecordDto> results = Lists.newArrayList();
List<String> datasetIds =Lists.newArrayList();
if (knowledgeBaseDto.getSelectedKnowledgeBaseIds() != null && !knowledgeBaseDto.getSelectedKnowledgeBaseIds().isEmpty()) {
@ -84,7 +85,16 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
if (knowledgeBaseDto.getSearchMethod().equals("keyword_search")) {
List<RecordDto> datasetFiles=esTDatasetFilesService.search(knowledgeBaseDto.getQuery(),datasetIds);
recordDtos.addAll(datasetFiles);
return recordDtos;
recordDtos.sort((dto1, dto2) -> {
try {
double score1 = Double.parseDouble(dto1.getScore());
double score2 = Double.parseDouble(dto2.getScore());
return Double.compare(score2, score1);
} catch (NumberFormatException e) {
return 0;
}
});
return recordDtos;
}
// 使用 CompletableFuture 并行查询多个数据集
@ -136,7 +146,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
}
});
results.addAll(recordDtos);
return recordDtos;
return results;
}
@Override

17
chat-server/src/main/java/com/bjtds/brichat/util/EsTDatasetFilesImporter.java
View File

@ -3,13 +3,12 @@ package com.bjtds.brichat.util;
import com.bjtds.brichat.entity.dataset.TDatasetFiles;
import com.bjtds.brichat.service.DatasetFilesService;
import com.bjtds.brichat.service.EsTDatasetFilesService;
import io.swagger.models.auth.In;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
@ -17,6 +16,7 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.util.List;
@Slf4j
@Service
public class EsTDatasetFilesImporter {
@ -25,7 +25,6 @@ public class EsTDatasetFilesImporter {
@Autowired
private DatasetFilesService datasetFilesService;
// private static final Logger log = LoggerFactory.getLogger(EsTDatasetFilesImporter.class);
@Autowired
private StringRedisTemplate redisTemplate;
@ -63,16 +62,16 @@ public class EsTDatasetFilesImporter {
if (document == null) continue;
String filePath = document.getDifyStoragePath();
if (filePath == null) {
log.info("documentId=" + document.getId() + " 不存在difyStoragePath，跳过");
log.warn("documentId=" + document.getId() + " 不存在difyStoragePath，跳过");
continue;
}
File file = new File(filePath);
if (!file.exists()) {
log.info(file.getAbsolutePath() + " 不存在，跳过");
log.warn(file.getAbsolutePath() + " 不存在，跳过");
continue;
}
if(Boolean.TRUE.equals(document.getIsEs())){
log.info("documentId=" + document.getId() + " 是ES索引文件，跳过");
log.warn("documentId=" + document.getId() + " 是ES索引文件，跳过");
continue;
}
@ -83,9 +82,9 @@ public class EsTDatasetFilesImporter {
redisTemplate.opsForValue().set("import:task:" + taskId + ":finished", String.valueOf(finished));
document.setIsEs(true);
datasetFilesService.updateFile(document);
log.info("documentId=" + document.getId() + " 索引构建成功");
log.debug("documentId=" + document.getId() + " 索引构建成功");
} catch (Exception e) {
log.info("documentId=" + document.getId() + " 索引构建失败: " + e.getMessage());
log.debug("documentId=" + document.getId() + " 索引构建失败: " + e.getMessage());
}
}
redisTemplate.opsForValue().set("import:task:" + taskId + ":status", "done");

9
chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.xml
View File

@ -281,4 +281,13 @@
AND dify_doc_id = #{difyDocId}
AND is_deleted = false
</select>
<!-- 根据文档ID查询文件 -->
<select id="selectByDocId" resultMap="TDatasetFilesResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dataset_files
WHERE dify_doc_id = #{difyDocId}
AND is_deleted = false
</select>
</mapper>