feat: 文件解析在Dify解析成功后,在es构建索引。
This commit is contained in:
parent
aaf8a3b901
commit
801bbe4970
|
|
@ -119,10 +119,10 @@
|
||||||
<template v-if="row.type === 'file'">
|
<template v-if="row.type === 'file'">
|
||||||
<el-tag
|
<el-tag
|
||||||
:type="getStatusTagType(row.status)"
|
:type="getStatusTagType(row.status)"
|
||||||
:class="{ 'rotating-tag': ['preprocessing', 'indexing', 'waiting'].includes(row.status) }"
|
:class="{ 'rotating-tag': ['preprocessing', 'indexing', 'waiting', 'queued'].includes(row.status) }"
|
||||||
>
|
>
|
||||||
<el-icon
|
<el-icon
|
||||||
:class="{ 'is-loading': ['preprocessing', 'indexing', 'waiting'].includes(row.status) }"
|
:class="{ 'is-loading': ['preprocessing', 'indexing', 'waiting', 'queued'].includes(row.status) }"
|
||||||
>
|
>
|
||||||
<component :is="getStatusIcon(row.status)" />
|
<component :is="getStatusIcon(row.status)" />
|
||||||
</el-icon>
|
</el-icon>
|
||||||
|
|
@ -975,6 +975,7 @@ const statusMap: Record<string, string> = {
|
||||||
'preprocessing': '预处理',
|
'preprocessing': '预处理',
|
||||||
'indexing': '解析中',
|
'indexing': '解析中',
|
||||||
'waiting': '等待中',
|
'waiting': '等待中',
|
||||||
|
'queued': '排队中',
|
||||||
'failed': '解析失败',
|
'failed': '解析失败',
|
||||||
'completed': '解析成功'
|
'completed': '解析成功'
|
||||||
}
|
}
|
||||||
|
|
@ -987,6 +988,7 @@ const typeMap: Record<string, 'success' | 'primary' | 'warning' | 'info' | 'dang
|
||||||
'preprocessing': 'info',
|
'preprocessing': 'info',
|
||||||
'indexing': 'warning',
|
'indexing': 'warning',
|
||||||
'waiting': 'info',
|
'waiting': 'info',
|
||||||
|
'queued': 'primary',
|
||||||
'failed': 'danger',
|
'failed': 'danger',
|
||||||
'completed': 'success'
|
'completed': 'success'
|
||||||
}
|
}
|
||||||
|
|
@ -999,6 +1001,7 @@ const iconMap: Record<string, any> = {
|
||||||
'preprocessing': Loading,
|
'preprocessing': Loading,
|
||||||
'indexing': Loading,
|
'indexing': Loading,
|
||||||
'waiting': Loading,
|
'waiting': Loading,
|
||||||
|
'queued': Loading,
|
||||||
'failed': Close,
|
'failed': Close,
|
||||||
'completed': Check
|
'completed': Check
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -181,4 +181,6 @@ public interface TDatasetFilesMapper {
|
||||||
*/
|
*/
|
||||||
List<TDatasetFiles> selectAll();
|
List<TDatasetFiles> selectAll();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -232,7 +232,10 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
||||||
String batch= (String) res.getBody().get("batch");
|
String batch= (String) res.getBody().get("batch");
|
||||||
String documentId = document.get("id");
|
String documentId = document.get("id");
|
||||||
logger.info("获取文档ID完成: {}, 耗时: {} ms", documentId, System.currentTimeMillis() - stepStartTime);
|
logger.info("获取文档ID完成: {}, 耗时: {} ms", documentId, System.currentTimeMillis() - stepStartTime);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// 8. 记录文件信息到助手服务数据库
|
// 8. 记录文件信息到助手服务数据库
|
||||||
stepStartTime = System.currentTimeMillis();
|
stepStartTime = System.currentTimeMillis();
|
||||||
TDatasetFiles datasetFiles = new TDatasetFiles();
|
TDatasetFiles datasetFiles = new TDatasetFiles();
|
||||||
|
|
@ -262,7 +265,6 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
||||||
indexingTask.setFileName(file.getOriginalFilename());
|
indexingTask.setFileName(file.getOriginalFilename());
|
||||||
indexingTaskQueueService.addTaskToQueue(indexingTask);
|
indexingTaskQueueService.addTaskToQueue(indexingTask);
|
||||||
logger.info("文档任务已加入Redis队列,耗时: {} ms", System.currentTimeMillis() - stepStartTime);
|
logger.info("文档任务已加入Redis队列,耗时: {} ms", System.currentTimeMillis() - stepStartTime);
|
||||||
|
|
||||||
long totalTime = System.currentTimeMillis() - startTime;
|
long totalTime = System.currentTimeMillis() - startTime;
|
||||||
logger.info("文档上传处理全部完成,总耗时: {} ms,文件: {}", totalTime, originalFilename);
|
logger.info("文档上传处理全部完成,总耗时: {} ms,文件: {}", totalTime, originalFilename);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import com.bjtds.brichat.entity.dto.IndexingTaskDto;
|
||||||
import com.bjtds.brichat.enums.IndexingStatusEnum;
|
import com.bjtds.brichat.enums.IndexingStatusEnum;
|
||||||
import com.bjtds.brichat.mapper.postgresql.DifyDatasetsDocMapper;
|
import com.bjtds.brichat.mapper.postgresql.DifyDatasetsDocMapper;
|
||||||
import com.bjtds.brichat.service.DatasetFilesService;
|
import com.bjtds.brichat.service.DatasetFilesService;
|
||||||
|
import com.bjtds.brichat.service.EsKnowledgeService;
|
||||||
import com.bjtds.brichat.service.IndexingTaskQueueService;
|
import com.bjtds.brichat.service.IndexingTaskQueueService;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
@ -12,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -32,6 +34,9 @@ public class IndexingStatusTaskService {
|
||||||
|
|
||||||
@Resource
|
@Resource
|
||||||
private DifyDatasetsDocMapper difyDatasetsDocMapper;
|
private DifyDatasetsDocMapper difyDatasetsDocMapper;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private EsKnowledgeService esKnowledgeService;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 定时任务:每1秒检查一次文档索引状态
|
* 定时任务:每1秒检查一次文档索引状态
|
||||||
|
|
@ -66,7 +71,7 @@ public class IndexingStatusTaskService {
|
||||||
/**
|
/**
|
||||||
* 检查单个任务的状态
|
* 检查单个任务的状态
|
||||||
*/
|
*/
|
||||||
private void checkSingleTaskStatus(IndexingTaskDto task) {
|
private void checkSingleTaskStatus(IndexingTaskDto task) throws IOException {
|
||||||
try {
|
try {
|
||||||
logger.debug("检查任务状态: documentId={}, 当前状态={}",
|
logger.debug("检查任务状态: documentId={}, 当前状态={}",
|
||||||
task.getDocumentId(), task.getCurrentStatus());
|
task.getDocumentId(), task.getCurrentStatus());
|
||||||
|
|
@ -92,6 +97,19 @@ public class IndexingStatusTaskService {
|
||||||
if (statusEnum != null && statusEnum.isFinalStatus()) {
|
if (statusEnum != null && statusEnum.isFinalStatus()) {
|
||||||
indexingTaskQueueService.removeTaskFromQueue(task.getDocumentId());
|
indexingTaskQueueService.removeTaskFromQueue(task.getDocumentId());
|
||||||
logger.info("文档{}已达到最终状态{},从队列中移除", task.getDocumentId(), newStatus);
|
logger.info("文档{}已达到最终状态{},从队列中移除", task.getDocumentId(), newStatus);
|
||||||
|
|
||||||
|
|
||||||
|
//添加索引到ES
|
||||||
|
esKnowledgeService.createIndex(task.getDocumentId());
|
||||||
|
//修改文件状态
|
||||||
|
TDatasetFiles file = datasetFilesService.getFileById(task.getFileId());
|
||||||
|
if (file != null) {
|
||||||
|
file.setIsEs(true);
|
||||||
|
datasetFilesService.updateFile(file);
|
||||||
|
}
|
||||||
|
logger.info("文档{}解析完成,已添加索引到ES中", task.getDocumentId());
|
||||||
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// 更新任务信息
|
// 更新任务信息
|
||||||
task.setCurrentStatus(newStatus);
|
task.setCurrentStatus(newStatus);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue