fix:1文件名过长抛出，2限制上传doc，3.修改创建索引调用方法.4.延迟解析失效时间
This commit is contained in:
parent
73b332de51
commit
c9fb5d8021
|
|
@ -24,8 +24,7 @@
|
|||
:on-change="handleFileChange"
|
||||
:file-list="[]"
|
||||
:show-file-list="false"
|
||||
:limit="10"
|
||||
accept=".txt,.md,.markdown,.mdx,.pdf,.html,.htm,.xlsx,.xls,.doc,.docx,.csv,.vtt,.properties"
|
||||
accept=".txt,.md,.markdown,.mdx,.pdf,.html,.htm,.xlsx,.xls,.docx,.csv,.vtt,.properties"
|
||||
class="upload-area"
|
||||
:class="{ 'is-dragover': isDragOver }"
|
||||
@drop="handleDrop"
|
||||
|
|
@ -38,7 +37,7 @@
|
|||
</el-icon>
|
||||
<div class="upload-text">
|
||||
<h4>拖拽文件到此处 或 <span class="link-text">点击选择</span></h4>
|
||||
<p>支持 TXT、MD、PDF、Word、Excel、HTML、CSV 等格式</p>
|
||||
<p>支持 TXT、MD、PDF、DOCX、Excel、HTML、CSV 等格式（不支持DOC格式）</p>
|
||||
<p>单个文件不超过 100MB，最多可选择 10 个文件</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -330,7 +329,7 @@
|
|||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, computed } from 'vue'
|
||||
import { ref, reactive, computed, nextTick } from 'vue'
|
||||
import { ElMessage, ElNotification, ElLoading } from 'element-plus'
|
||||
import type { UploadFile as ElUploadFile } from 'element-plus'
|
||||
import {
|
||||
|
|
@ -373,6 +372,7 @@ const uploading = ref(false)
|
|||
const uploadFiles = ref<ElUploadFile[]>([])
|
||||
const isDragOver = ref(false)
|
||||
const uploadRef = ref()
|
||||
const isProcessingFileLimit = ref(false)
|
||||
|
||||
// 解析模式选项
|
||||
const analysisMode = [
|
||||
|
|
@ -452,14 +452,55 @@ const getStep3Icon = () => {
|
|||
|
||||
// 方法
|
||||
const handleFileChange = (file: ElUploadFile, fileList: ElUploadFile[]) => {
|
||||
// 防止无限循环
|
||||
if (isProcessingFileLimit.value) {
|
||||
return
|
||||
}
|
||||
|
||||
// 文件大小检查
|
||||
if (file.size && file.size > 100 * 1024 * 1024) {
|
||||
ElMessage.error('文件大小不能超过 100MB')
|
||||
return false
|
||||
}
|
||||
|
||||
// 文件数量检查，如果超过10个，只保留前10个
|
||||
if (fileList.length > 10) {
|
||||
isProcessingFileLimit.value = true
|
||||
|
||||
const truncatedList = fileList.slice(0, 10)
|
||||
const removedCount = fileList.length - 10
|
||||
|
||||
ElMessage.warning({
|
||||
message: `已选择 ${fileList.length} 个文件，超出限制。系统已自动保留前 10 个文件，移除了 ${removedCount} 个文件。`,
|
||||
duration: 4000,
|
||||
showClose: true
|
||||
})
|
||||
|
||||
// 更新我们的文件列表
|
||||
uploadFiles.value = truncatedList
|
||||
|
||||
// 清除并重新设置上传组件的文件列表
|
||||
nextTick(() => {
|
||||
if (uploadRef.value) {
|
||||
uploadRef.value.clearFiles()
|
||||
|
||||
// 重新添加前10个文件
|
||||
truncatedList.forEach(uploadFile => {
|
||||
if (uploadFile.raw) {
|
||||
uploadRef.value.handleStart(uploadFile.raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 重置标志
|
||||
setTimeout(() => {
|
||||
isProcessingFileLimit.value = false
|
||||
}, 100)
|
||||
})
|
||||
} else {
|
||||
// 更新文件列表
|
||||
uploadFiles.value = fileList
|
||||
}
|
||||
}
|
||||
|
||||
const handleDrop = () => {
|
||||
|
|
@ -579,7 +620,7 @@ const handleUpload = async () => {
|
|||
await deleteDocument(parseInt(props.replaceFileId))
|
||||
deleteLoading.close()
|
||||
|
||||
// 重新显示上传进度
|
||||
// 重新显示上传进度X
|
||||
const uploadLoading = ElLoading.service({
|
||||
lock: true,
|
||||
text: '正在上传新文件...',
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@
|
|||
:header-row-style="{ height: '50px' }"
|
||||
@selection-change="handleSelectionChange"
|
||||
@row-dblclick="handleRowDoubleClick"
|
||||
@sort-change="handleSortChange"
|
||||
>
|
||||
<el-table-column type="selection" width="55" />
|
||||
<el-table-column
|
||||
|
|
@ -97,7 +98,7 @@
|
|||
type="index"
|
||||
:index="indexMethod"
|
||||
/>
|
||||
<el-table-column prop="fileName" :label="t('vabI18n.knowledge.document.table.fileName')" min-width="250">
|
||||
<el-table-column prop="fileName" :label="t('vabI18n.knowledge.document.table.fileName')" min-width="250" sortable="custom" :sort-orders="['ascending', 'descending']">
|
||||
<template #default="{ row }">
|
||||
<div class="file-name-cell">
|
||||
<img
|
||||
|
|
@ -132,8 +133,8 @@
|
|||
<span v-else class="folder-status">--</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="createDate" :label="t('vabI18n.knowledge.document.table.createDate')" width="180" />
|
||||
<el-table-column prop="charCount" :label="t('vabI18n.knowledge.document.table.charCount')" width="120" align="right">
|
||||
<el-table-column prop="createDate" :label="t('vabI18n.knowledge.document.table.createDate')" width="180" sortable="custom" :sort-orders="['ascending', 'descending']" />
|
||||
<el-table-column prop="charCount" :label="t('vabI18n.knowledge.document.table.charCount')" width="120" align="right" sortable="custom" :sort-orders="['ascending', 'descending']">
|
||||
<template #default="{ row }">
|
||||
<span class="char-count" v-if="row.type === 'file'">{{ formatFileSize(row.charCount) }}</span>
|
||||
<span v-else class="folder-indicator">--</span>
|
||||
|
|
@ -243,7 +244,6 @@
|
|||
@success="handleUploadSuccess"
|
||||
/>
|
||||
</el-dialog>
|
||||
<<<<<<< HEAD
|
||||
|
||||
<!-- 文件替换对话框 -->
|
||||
<el-dialog
|
||||
|
|
@ -282,9 +282,6 @@
|
|||
/>
|
||||
</el-dialog>
|
||||
|
||||
=======
|
||||
|
||||
>>>>>>> 分段预览
|
||||
<!-- 预览抽屉 -->
|
||||
|
||||
<el-drawer
|
||||
|
|
@ -683,6 +680,12 @@ size: 20,
|
|||
total: 0
|
||||
})
|
||||
|
||||
// 排序数据
|
||||
const sortConfig = reactive({
|
||||
orderBy: 'name',
|
||||
orderDirection: 'ASC'
|
||||
})
|
||||
|
||||
// 文件列表数据
|
||||
const fileList = ref<FileItem[]>([])
|
||||
|
||||
|
|
@ -1039,8 +1042,8 @@ try {
|
|||
difyDatasetId: datasetId.value,
|
||||
parentId: currentParentId.value, // 使用当前文件夹ID
|
||||
fileName: searchKeyword.value || undefined,
|
||||
orderBy: 'name',
|
||||
orderDirection: 'ASC',
|
||||
orderBy: sortConfig.orderBy,
|
||||
orderDirection: sortConfig.orderDirection,
|
||||
pageNo: pagination.current,
|
||||
pageSize: pagination.size
|
||||
}
|
||||
|
|
@ -1194,6 +1197,25 @@ pagination.current = 1
|
|||
fetchDocuments()
|
||||
}
|
||||
|
||||
// 排序变化处理
|
||||
const handleSortChange = (sortInfo: any) => {
|
||||
if (sortInfo.prop && sortInfo.order) {
|
||||
// 映射前端字段到后端字段
|
||||
const fieldMapping: Record<string, string> = {
|
||||
'fileName': 'name',
|
||||
'createDate': 'created_at',
|
||||
'charCount': 'size'
|
||||
}
|
||||
|
||||
sortConfig.orderBy = fieldMapping[sortInfo.prop] || sortInfo.prop
|
||||
sortConfig.orderDirection = sortInfo.order === 'ascending' ? 'ASC' : 'DESC'
|
||||
|
||||
// 重置到第一页并刷新数据
|
||||
pagination.current = 1
|
||||
fetchDocuments()
|
||||
}
|
||||
}
|
||||
|
||||
// 处理文件夹打开
|
||||
const handleOpenFolder = (row: FileItem) => {
|
||||
// 设置当前文件夹ID并更新面包屑路径
|
||||
|
|
|
|||
|
|
@ -50,7 +50,11 @@ public class TDatasetFiles {
|
|||
* 文件大小(字节)，文件夹通常为0或子项总和
|
||||
*/
|
||||
private Long size;
|
||||
|
||||
/**
|
||||
* 字符数量
|
||||
*/
|
||||
// private String charCount;
|
||||
//
|
||||
/**
|
||||
* 所有者用户ID
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -247,7 +247,7 @@ public class DeepAnalysisQueueService {
|
|||
Map<Object, Object> taskInfo = redisTemplate.opsForHash().entries(taskInfoKey);
|
||||
Integer fileId = (Integer) taskInfo.get("fileId");
|
||||
if (fileId != null) {
|
||||
updateFileStatus(fileId, IndexingStatusEnum.ERROR.getCode());
|
||||
updateFileStatus(fileId, IndexingStatusEnum.FAILED.getCode());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
logger.error("更新文件状态失败: {}", ex.getMessage(), ex);
|
||||
|
|
|
|||
|
|
@ -162,13 +162,13 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
|||
System.currentTimeMillis() - stepStartTime, originalFilename, fileExtension);
|
||||
|
||||
// 2. 处理DOC文件转换
|
||||
if (documentConversionService.isDocFile(fileExtension)) {
|
||||
stepStartTime = System.currentTimeMillis();
|
||||
logger.info("检测到DOC文件，开始转换: {}", originalFilename);
|
||||
file = documentConversionService.convertDocToDocx(file, originalFilename);
|
||||
logger.info("DOC转换完成，耗时: {} ms，新文件名: {}",
|
||||
System.currentTimeMillis() - stepStartTime, file.getOriginalFilename());
|
||||
}
|
||||
// if (documentConversionService.isDocFile(fileExtension)) {
|
||||
// stepStartTime = System.currentTimeMillis();
|
||||
// logger.info("检测到DOC文件，开始转换: {}", originalFilename);
|
||||
// file = documentConversionService.convertDocToDocx(file, originalFilename);
|
||||
// logger.info("DOC转换完成，耗时: {} ms，新文件名: {}",
|
||||
// System.currentTimeMillis() - stepStartTime, file.getOriginalFilename());
|
||||
// }
|
||||
|
||||
|
||||
// 5. 如果是深度解析，提交到队列处理
|
||||
|
|
@ -228,6 +228,8 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
|||
// 7. 上传后处理
|
||||
stepStartTime = System.currentTimeMillis();
|
||||
Map<String, String> document = (Map<String, String>) res.getBody().get("document");
|
||||
|
||||
long wordCount = Long.parseLong(document.get("word_count"));
|
||||
//批次号
|
||||
String batch= (String) res.getBody().get("batch");
|
||||
String documentId = document.get("id");
|
||||
|
|
@ -249,6 +251,7 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
|||
String storagePath = difyUploadPath + "/" + getUUIDFilePath(documentId);
|
||||
datasetFiles.setDifyStoragePath(storagePath);
|
||||
datasetFiles.setSourceUrl(determineSourceUrl(sourceUrl, documentId));
|
||||
// datasetFiles.setCharCount(String.valueOf(wordCount));
|
||||
datasetFiles.setIsDeep(false);
|
||||
datasetFiles.setIndexingStatus(IndexingStatusEnum.INDEXING.getCode());
|
||||
TDatasetFiles savedFile = datasetFilesService.createFile(datasetFiles);
|
||||
|
|
@ -268,6 +271,8 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
|
|||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
logger.info("文档上传处理全部完成，总耗时: {} ms，文件: {}", totalTime, originalFilename);
|
||||
|
||||
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -132,15 +132,22 @@ public class DatasetFilesServiceImpl implements DatasetFilesService {
|
|||
if (!StringUtils.hasText(datasetFiles.getName())) {
|
||||
throw new IllegalArgumentException("文件/文件夹名称不能为空");
|
||||
}
|
||||
if (StringUtils.hasText(datasetFiles.getName()) && datasetFiles.getName().length() > 100) {
|
||||
throw new IllegalArgumentException("文件夹名称过长,需要小于100个字符");
|
||||
}
|
||||
if (!StringUtils.hasText(datasetFiles.getType())) {
|
||||
throw new IllegalArgumentException("类型不能为空");
|
||||
}
|
||||
if (!"file".equals(datasetFiles.getType()) && !"folder".equals(datasetFiles.getType())) {
|
||||
throw new IllegalArgumentException("类型必须是 file 或 folder");
|
||||
}
|
||||
|
||||
|
||||
// if (!StringUtils.hasText(datasetFiles.getDifyDatasetId())) {
|
||||
// throw new IllegalArgumentException("知识库ID不能为空");
|
||||
// }
|
||||
|
||||
|
||||
if (datasetFiles.getOwnerId() == null) {
|
||||
throw new IllegalArgumentException("所有者ID不能为空");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ public class EsTDatasetFilesServiceImpl implements EsTDatasetFilesService {
|
|||
doc.getDifyStoragePath(),
|
||||
doc.getSourceUrl(),
|
||||
doc.getIsDeep(),
|
||||
|
||||
true
|
||||
|
||||
);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import com.bjtds.brichat.service.DatasetFilesService;
|
|||
import com.bjtds.brichat.service.EsKnowledgeService;
|
||||
import com.bjtds.brichat.service.EsTDatasetFilesService;
|
||||
import com.bjtds.brichat.service.IndexingTaskQueueService;
|
||||
import com.bjtds.brichat.util.EsTDatasetFilesImporter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
|
|
@ -37,7 +38,7 @@ public class IndexingStatusTaskService {
|
|||
private DifyDatasetsDocMapper difyDatasetsDocMapper;
|
||||
|
||||
@Resource
|
||||
private EsTDatasetFilesService esTDatasetFilesService;
|
||||
private EsTDatasetFilesImporter esTDatasetFilesImporter;
|
||||
|
||||
/**
|
||||
* 定时任务：每1秒检查一次文档索引状态
|
||||
|
|
@ -101,7 +102,7 @@ public class IndexingStatusTaskService {
|
|||
|
||||
|
||||
//添加索引到ES
|
||||
esTDatasetFilesService.createIndex(task.getDocumentId());
|
||||
esTDatasetFilesImporter.importDocumentId(task.getFileId());
|
||||
//修改文件状态
|
||||
TDatasetFiles file = datasetFilesService.getFileById(task.getFileId());
|
||||
if (file != null) {
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ public class PdfConversionTaskService {
|
|||
private static final String DEEP_ANALYSIS_PROCESSING = "deep_analysis:processing";
|
||||
|
||||
// PDF任务超时时间（毫秒）
|
||||
private static final long PDF_TASK_TIMEOUT_MS = 10 * 60 * 1000; // 10分钟
|
||||
private static final long PDF_TASK_TIMEOUT_MS = 1 * 60 * 60 * 1000; // 1小时
|
||||
|
||||
/**
|
||||
* 定时任务：每3秒检查一次PDF转换任务状态
|
||||
|
|
@ -208,9 +208,10 @@ public class PdfConversionTaskService {
|
|||
|
||||
String folderUrl = statusResponse.getFolderUrl();
|
||||
|
||||
String fileName = taskInfo.getName().split("\\.")[0] ;
|
||||
String absolutePath = outputPath + fileName + "/" + fileName+".md";
|
||||
|
||||
String originalName = taskInfo.getName();
|
||||
int lastDotIndex = originalName.lastIndexOf('.');
|
||||
String fileName = (lastDotIndex == -1) ? originalName : originalName.substring(0, lastDotIndex);
|
||||
String absolutePath = outputPath + fileName + "/" + fileName + ".md";
|
||||
|
||||
|
||||
logger.info("=== PDF转换任务完成 ===");
|
||||
|
|
|
|||
Loading…
Reference in New Issue