WJB
/
ai-manus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修改索引构建功能，修改些许bug

This commit is contained in:
moon 2025-09-22 10:10:41 +08:00
parent 73b332de51
commit 68555fdfb6
12 changed files with 218 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
chat-client/src/api/dataset.ts
View File

@ -254,3 +254,20 @@ export function createAllIndex() {
}
})
}
export const deleteFileIndex = (documentId: string,datasetId: string) => {
return request({
url: "/brichat-service/knowledge-base/deleteFileIndex",
method: "delete",
params: {
documentId,
datasetId,
}
})
}
export const deleteAllIndex = () => {
return request({
url: "/brichat-service/knowledge-base/deleteAllIndex",
method: "delete",
})
}

39
chat-client/src/views/permission/esindexManage/index.vue
View File

@ -6,6 +6,9 @@
<el-button type="primary" @click="handleCreateIndex">
<el-icon><Plus /></el-icon>构建索引
</el-button>
<el-button type="danger" @click="handleDeleteAllIndex">
<el-icon><Delete /></el-icon>删除所有索引
</el-button>
</div>
<div class="right">
<el-input
@ -38,7 +41,7 @@
>
<el-table-column type="selection" width="55" />
<el-table-column prop="indexName" label="索引名" width="200" />
<el-table-column prop="aliases" label="别名" :formatter="formatAliases" />
<el-table-column prop="databaseName" label="数据库名称" :formatter="formatAliases" />
<el-table-column prop="docsCount" label="文档数量" width="120" align="center" />
<el-table-column prop="storeSize" label="存储大小" width="120" align="center" />
<el-table-column prop="health" label="健康状态" width="120" align="center">
@ -87,7 +90,7 @@
<transition name="fade-scale">
<div v-if="dialogVisible" class="dialog-content">
<p><strong>索引名:</strong> {{ selectedIndex?.indexName || '-' }}</p>
<p><strong>别名:</strong> {{ selectedIndex?.aliases?.join(', ') || '-' }}</p>
<p><strong>别名:</strong> {{ selectedIndex?.databaseName || '-' }}</p>
<p><strong>文档数量:</strong> {{ selectedIndex?.docsCount || '-' }}</p>
<p><strong>存储大小:</strong> {{ selectedIndex?.storeSize || '-' }}</p>
<p><strong>健康状态:</strong>
@ -112,7 +115,7 @@
import { ref, reactive, onMounted } from "vue"
import { ElMessage, ElMessageBox } from "element-plus"
import { Plus, Delete } from "@element-plus/icons-vue"
import { getIndexInfo, createAllIndex ,deleteIndex,setIndextask,getIndexTask} from "@/api/dataset"
import { getIndexInfo, createAllIndex ,deleteIndex,setIndextask,getIndexTask,deleteAllIndex} from "@/api/dataset"
const isFullscreen = ref(false) //是否全屏
const tableLoading = ref(false)
@ -151,6 +154,7 @@ const loadIndexList = async () => {
try {
const res = await getIndexInfo(pagination.currentPage, pagination.pageSize, searchForm.indexName)
console.log(res.data)
indexList.value = res.data.content
pagination.total = res.data.total
@ -162,8 +166,14 @@ const loadIndexList = async () => {
}
}
const formatAliases = (row) =>
row.aliases && row.aliases.length ? row.aliases.join(", ") : "-"
const formatAliases = (row) => {
if (!row.databaseName) return "-";
if (Array.isArray(row.databaseName)) {
return row.databaseName.join(", ");
}
return row.databaseName; // 字符串直接返回
};
const getStatusType = (health) => {
if (health === "green") return "success"
@ -177,6 +187,19 @@ const handleSearch = () => {
loadIndexList()
}
const handleDeleteAllIndex = async () => {
tableLoading.value = true
try {
await deleteAllIndex()
ElMessage.success("索引删除成功")
loadIndexList()
} catch (error) {
ElMessage.error(error.message)
} finally {
tableLoading.value = false
}
}
const handleCurrentChange = (page) => {
pagination.currentPage = page
loadIndexList()
@ -216,10 +239,10 @@ const pollTaskStatus = async (taskId) => {
if (total > 0) progress.value = Math.floor((finished / total) * 100)
if (status === "done") {
ElMessage.success("索引构建完成")
ElMessage.success("索引构建完成"+`共构建${total}个文档，成功${finished}个`)
loadIndexList()
} else if (status === "failed") {
ElMessage.error("索引构建失败")
ElMessage.error("索引构建失败"+`共构建${total}个文档，成功${finished}个`)
} else {
// 继续轮询
timer = setTimeout(() => pollTaskStatus(taskId), 200)
@ -233,7 +256,7 @@ const handleCreateIndex = async () => {
try {
const res = await setIndextask()
const taskId = res.data
ElMessage.success(`任务已提交，taskId=${ taskId}`)
ElMessage.success(`开始构建索引`)
progress.value = 0
taskStatus.value = "processing"

30
chat-server/src/main/java/com/bjtds/brichat/controller/KnowledgeBaseController.java
View File

@ -40,16 +40,16 @@ public class KnowledgeBaseController {
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private EsKnowledgeServiceImpl esKnowledgeService;
// @Autowired
// private EsKnowledgeServiceImpl esKnowledgeService;
@Autowired
private EsTDatasetFilesService esTDatasetFilesService;
@Autowired
private EsKnowledgeImporter esKnowledgeImporter;
// @Autowired
// private EsKnowledgeImporter esKnowledgeImporter;
@Autowired
private EsTDatasetFilesImporter esTDatasetFilesImporter;
@ -99,7 +99,7 @@ public class KnowledgeBaseController {
public ResultUtils createIndex(@RequestParam("documentId") String documentId) throws Exception {
try{
esKnowledgeImporter.importDocumentId(documentId);
esTDatasetFilesImporter.importDocumentId(documentId);
return ResultUtils.success("索引创建成功");
} catch (IOException e) {
return ResultUtils.error("索引创建失败: " + e.getMessage());
@ -141,6 +141,26 @@ public class KnowledgeBaseController {
result.put("finished", finished);
return ResultUtils.success(result);
}
@ApiOperation("删除索引下的文件")
@DeleteMapping("/deleteFileIndex")
public ResultUtils deleteFileIndex(@RequestParam("documentId") String documentId,@RequestParam("datasetId")String datasetId) throws Exception {
try{
esTDatasetFilesService.deleteDocIndex(datasetId,documentId);
return ResultUtils.success("索引删除成功");
} catch (IOException e) {
return ResultUtils.error("索引删除失败: " + e.getMessage());
}
}
@ApiOperation("删除所有索引")
@DeleteMapping("/deleteAllIndex")
public ResultUtils deleteAllIndex() throws Exception {
try{
esTDatasetFilesService.deleteAllIndex();
return ResultUtils.success("索引删除成功");
} catch (IOException e) {
return ResultUtils.error("索引删除失败: " + e.getMessage());
}
}
@ApiOperation("返回关联表数据")

1
chat-server/src/main/java/com/bjtds/brichat/entity/dto/RetrievalDto.java
View File

@ -17,4 +17,5 @@ public class RetrievalDto {
private String datasetId;
private String datasetName;
private String sourceUrl;
private String documentId;
}

1
chat-server/src/main/java/com/bjtds/brichat/entity/esmodel/IndexInfo.java
View File

@ -10,6 +10,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class IndexInfo {
private String indexName;
private String databaseName;
private String health; // 健康状态
private String docsCount; // 文档数量
private String storeSize; //索引大小

5
chat-server/src/main/java/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.java
View File

@ -182,5 +182,10 @@ public interface TDatasetFilesMapper {
List<TDatasetFiles> selectAll();
List<TDatasetFiles> selectByDatasetId(@Param("difyDatasetId") String difyDatasetId);
TDatasetFiles selectByDatasetIdAndDocId(@Param("difyDatasetId") String difyDatasetId, @Param("difyDocId") String difyDocId);
}

15
chat-server/src/main/java/com/bjtds/brichat/service/DatasetFilesService.java
View File

@ -159,4 +159,19 @@ public interface DatasetFilesService {
* 获取所有文件
*/
List<TDatasetFiles> getAllFiles();
/**
* 根据数据集ID更新所有文件的索引
*/
void updateByDatasetId(String difyDatasetId);
/**
* 根据数据集ID和文档ID查询文件
*/
void updateByDatasetIdAndDocId(String difyDatasetId, String difyDocId);
}

2
chat-server/src/main/java/com/bjtds/brichat/service/EsTDatasetFilesService.java
View File

@ -17,4 +17,6 @@ public interface EsTDatasetFilesService {
List<RecordDto> searchSingle(String keyword, String DatasetId) throws IOException;
List<RecordDto> search(String keyword, List<String> datasetIds) throws IOException;
Pagination<IndexInfo> getAllIndexInfos(Integer pageNo, Integer pageSize, String keyword) throws IOException;
void deleteDocIndex(String DatasetId, String documentId) throws IOException;
void deleteAllIndex() throws IOException;
}

38
chat-server/src/main/java/com/bjtds/brichat/service/impl/DatasetFilesServiceImpl.java
View File

@ -4,7 +4,9 @@ import com.bjtds.brichat.entity.dataset.TDatasetFiles;
import com.bjtds.brichat.mapper.opengauss.TDatasetFilesMapper;
import com.bjtds.brichat.service.DatasetFilesService;
import com.bjtds.common.utils.Pagination;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
@ -17,6 +19,7 @@ import java.util.List;
*
* @author system
*/
@Slf4j
@Service
public class DatasetFilesServiceImpl implements DatasetFilesService {
@ -411,4 +414,39 @@ public class DatasetFilesServiceImpl implements DatasetFilesService {
public List<TDatasetFiles> getAllFiles() {
return datasetFilesMapper.selectAll();
}
/**
* 根据数据集ID查询所有文件
*/
@Override
@Async
public void updateByDatasetId(String difyDatasetId) {
List<TDatasetFiles> tDatasetFilesList = datasetFilesMapper.selectByDatasetId(difyDatasetId);
if (tDatasetFilesList != null) {
tDatasetFilesList.stream()
.peek(tDatasetFiles -> tDatasetFiles.setIsEs(false)) // 修改isEs
.forEach(tDatasetFiles -> {
datasetFilesMapper.updateById(tDatasetFiles); // 更新数据库
log.info("更新文件索引成功，文件ID：{}", tDatasetFiles.getId());
});
}
}
/**
* 根据数据集ID和文档ID查询文件
*/
@Override
public void updateByDatasetIdAndDocId(String difyDatasetId, String difyDocId) {
TDatasetFiles tDatasetFiles=datasetFilesMapper.selectByDatasetIdAndDocId(difyDatasetId, difyDocId);
if(tDatasetFiles!=null){
tDatasetFiles.setIsEs(false);
datasetFilesMapper.updateById(tDatasetFiles);
log.info("更新文件索引成功，文件ID：{}", tDatasetFiles.getId());
}else{
log.info("未找到文件，数据集ID：{}，文档ID：{}", difyDatasetId, difyDocId);
}
}
}

56
chat-server/src/main/java/com/bjtds/brichat/service/impl/EsTDatasetFilesServiceImpl.java
View File

@ -2,8 +2,11 @@ package com.bjtds.brichat.service.impl;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse;
import co.elastic.clients.elasticsearch.indices.GetIndexResponse;
import com.bjtds.brichat.entity.dataset.TDatasetFiles;
import co.elastic.clients.elasticsearch._types.mapping.DynamicMapping;
import com.bjtds.brichat.entity.dto.RecordDto;
@ -11,6 +14,7 @@ import com.bjtds.brichat.entity.dto.RetrievalDto;
import com.bjtds.brichat.entity.esmodel.IndexInfo;
import com.bjtds.brichat.mapper.opengauss.TUserDatasetMapper;
import com.bjtds.brichat.mapper.postgresql.DifyDatasetsMapper;
import com.bjtds.brichat.service.DatasetFilesService;
import com.bjtds.brichat.service.EsTDatasetFilesService;
import com.bjtds.brichat.util.EsFileSplitter;
import com.bjtds.common.utils.Pagination;
@ -36,6 +40,12 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
@Autowired
private DifyDatasetsMapper difyDatasetsMapper;
@Autowired
private DatasetFilesService datasetFilesService;
// 清除索引
@Override
@ -61,6 +71,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
.properties("dataset_id", p -> p.keyword(k -> k))
.properties("source_url", p -> p.keyword(k -> k))
.properties("dataset_name", p -> p.keyword(k -> k))
.properties("document_id", p -> p.keyword(k -> k))
)
);
@ -264,7 +275,8 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
content,
d.getDifyDatasetId(),
datasetName,
d.getSourceUrl()
d.getSourceUrl(),
d.getDifyDocId()
);
RecordDto recordDto = new RecordDto();
@ -283,8 +295,14 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
@Override
public Pagination<IndexInfo> getAllIndexInfos(Integer pageNo, Integer pageSize, String keyword) throws IOException {
List<IndexInfo> indexInfos = new ArrayList<>();
List<IndicesRecord> records= new ArrayList<>();
// 不使用 format()，直接调用 cat().indices()
List<IndicesRecord> records = client.cat().indices(r -> r).valueBody();
try {
records = client.cat().indices(r -> r).valueBody();
} catch (IOException e) {
log.error("获取索引列表失败", e);
return new Pagination<>();
}
if (keyword != null && !keyword.isEmpty()) {
records = records.stream()
.filter(r -> r.index() != null && r.index().contains(keyword))
@ -300,7 +318,8 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
String health = record.health();
String docsCount = record.docsCount() != null ? record.docsCount() : "0";
String storeSize = record.storeSize() != null ? record.storeSize() : "0b";
indexInfos.add(new IndexInfo(indexName, health, docsCount, storeSize));
String databaseName = difyDatasetsMapper.getDatasetNameById(indexName);
indexInfos.add(new IndexInfo(indexName, databaseName, health, docsCount, storeSize));
}
Pagination<IndexInfo> pagination = new Pagination<>();
pagination.setTotal(total);
@ -310,6 +329,37 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
return pagination;
}
@Override
public void deleteDocIndex(String DatasetId, String documentId) throws IOException {
boolean exists = client.indices().exists(e -> e.index(DatasetId)).value();
if (exists) {
try {
DeleteByQueryResponse response = client.deleteByQuery(d -> d
.index(DatasetId)
.query(q -> q.term(t -> t.field("document_id").value(documentId))));
datasetFilesService.updateByDatasetIdAndDocId(DatasetId, documentId);
log.info("删除文档索引成功: {}, documentId: {}", DatasetId, documentId);
} catch (IOException e) {
log.error("删除文档索引失败: {}, documentId: {}", DatasetId, documentId, e);
}
} else {
log.warn("索引不存在: {}", DatasetId);
}
}
@Override
public void deleteAllIndex() throws IOException {
GetIndexResponse allIndices = client.indices().get(g -> g.index("*"));
allIndices.result().keySet().forEach(indexName -> {
try {
DeleteIndexResponse response = client.indices().delete(d -> d.index(indexName));
datasetFilesService.updateByDatasetId(indexName);
log.info("删除索引：{} -> {}", indexName, response.acknowledged());
} catch (Exception e) {
log.error("删除索引失败：{} -> {}", indexName, e.getMessage());
}
});
}
}

12
chat-server/src/main/java/com/bjtds/brichat/util/EsTDatasetFilesImporter.java
View File

@ -28,13 +28,13 @@ public class EsTDatasetFilesImporter {
@Autowired
private StringRedisTemplate redisTemplate;
public void importDocumentId(Integer documentId) throws IOException {
public void importDocumentId(String documentId) throws IOException {
// 一次性获取完整的文档信息
TDatasetFiles datasetFiles = datasetFilesService.getFileById(documentId);
TDatasetFiles datasetFiles = datasetFilesService.getFileById(Integer.valueOf(documentId));
if (datasetFiles == null) {
throw new IllegalArgumentException("datasetFiles 不存在");
}
String filePath = datasetFiles.getDifyStoragePath();
String filePath = datasetFiles.getSourceUrl();
File file = new File(filePath);
if (!file.exists()) {
throw new IllegalArgumentException(filePath + " 不存在");
@ -60,9 +60,13 @@ public class EsTDatasetFilesImporter {
for (TDatasetFiles document : documents) {
if (document == null) continue;
String filePath = document.getDifyStoragePath();
if (filePath == null) {
log.debug("documentId=" + document.getId() + " 不存在difyStoragePath，跳过");
continue;
}
File file = new File(filePath);
if (!file.exists()) {
log.debug(filePath + " 不存在，跳过");
log.debug(file.getAbsolutePath() + " 不存在，跳过");
continue;
}
if(document.getIsEs()){

18
chat-server/src/main/resources/com/bjtds/brichat/mapper/opengauss/TDatasetFilesMapper.xml
View File

@ -263,4 +263,22 @@
WHERE type = 'file'
ORDER BY created_at DESC
</select>
<!-- 根据数据集ID查询所有文件 -->
<select id="selectByDatasetId" resultMap="TDatasetFilesResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dataset_files
WHERE dify_dataset_id = #{difyDatasetId}
AND is_deleted = false
ORDER BY created_at DESC
</select>
<!-- 根据数据集ID和文档ID查询文件 -->
<select id="selectByDatasetIdAndDocId" resultMap="TDatasetFilesResultMap">
SELECT <include refid="Base_Column_List"/>
FROM t_dataset_files
WHERE dify_dataset_id = #{difyDatasetId}
AND dify_doc_id = #{difyDocId}
AND is_deleted = false
</select>
</mapper>