WJB
/
ai-manus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: WJB:c44b4d27285f031bb5ce102dd0062a375d5130cb
Branches Tags
WJB:main
WJB:dev-wuhan
..
compare: WJB:a03ef97f78bb0cc01e2240686a8567e4b5187be1
Branches Tags
WJB:main
WJB:dev-wuhan

No commits in common. "c44b4d27285f031bb5ce102dd0062a375d5130cb" and "a03ef97f78bb0cc01e2240686a8567e4b5187be1" have entirely different histories.
c44b4d2728 ... a03ef97f78

7 changed files with 43 additions and 277 deletions
Show Stats Expand all files Collapse all files

13
chat-client/.env.development
View File

@ -2,17 +2,14 @@
# 此文件修改后需要重启项目 # 此文件修改后需要重启项目
NODE_ENV=development NODE_ENV=development
#VUE_APP_BASE_URL='/vab-mock-server' #VUE_APP_BASE_URL='/vab-mock-server'
# VUE_APP_BASE_URL='http://localhost:10001' VUE_APP_BASE_URL='http://localhost:10001'
# VUE_APP_API_BASE_URL='http://localhost:8080' VUE_APP_API_BASE_URL='http://localhost:8080'
#北京服务器配置 #北京服务器配置
# VUE_APP_BASE_URL='http://192.168.1.211:80/brichat' # VUE_APP_BASE_URL='http://192.168.1.211:80/brichat'
# VUE_APP_API_BASE_URL='http://192.168.1.211:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.1.211:80/brichat'
# #武汉公司服务器 #武汉公司服务器
VUE_APP_BASE_URL='http://192.168.8.253:80/brichat' # VUE_APP_BASE_URL='http://192.168.8.253:80/brichat'
VUE_APP_API_BASE_URL='http://192.168.8.253:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.8.253:80/brichat'
#总部服务器配置 #总部服务器配置
# VUE_APP_BASE_URL='http://192.168.0.33:80/brichat' # VUE_APP_BASE_URL='http://192.168.0.33:80/brichat'
# VUE_APP_API_BASE_URL='http://192.168.0.33:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.0.33:80/brichat'
VUE_APP_GITHUB_USER_NAME=test
VUE_APP_SECRET_KEY=preview

8
chat-client/.env.production
View File

@ -8,12 +8,8 @@ VUE_APP_API_BASE_URL='http://localhost:10001'
# VUE_APP_BASE_URL='http://192.168.1.211:80/brichat' # VUE_APP_BASE_URL='http://192.168.1.211:80/brichat'
# VUE_APP_API_BASE_URL='http://192.168.1.211:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.1.211:80/brichat'
#武汉公司服务器 #武汉公司服务器
VUE_APP_BASE_URL='http://192.168.8.253:80/brichat' # VUE_APP_BASE_URL='http://192.168.8.253:80/brichat'
VUE_APP_API_BASE_URL='http://192.168.8.253:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.8.253:80/brichat'
# 总部服务器 # 总部服务器
# VUE_APP_BASE_URL='http://192.168.0.33:80/brichat' # VUE_APP_BASE_URL='http://192.168.0.33:80/brichat'
# VUE_APP_API_BASE_URL='http://192.168.0.33:80/brichat' # VUE_APP_API_BASE_URL='http://192.168.0.33:80/brichat'
VUE_APP_GITHUB_USER_NAME=test
VUE_APP_SECRET_KEY=preview

2
chat-client/src/views/datasets/components/DocumentList.vue
View File

@ -620,7 +620,7 @@ const handleUpload = async () => {
formData.append('request', new Blob([JSON.stringify({ formData.append('request', new Blob([JSON.stringify({
datasetId: datasetId.value, datasetId: datasetId.value,
indexingTechnique: uploadForm.indexingTechnique, indexingTechnique: uploadForm.indexingTechnique,
processRule: processRule, processRule: processRule
})], { })], {
type: 'application/json' type: 'application/json'
})) }))

7
chat-server/src/main/java/com/bjtds/brichat/controller/DatasetDocController.java
View File

@ -10,8 +10,6 @@ import com.bjtds.common.utils.Pagination;
import io.github.guoshiqiufeng.dify.core.pojo.DifyPageResult; import io.github.guoshiqiufeng.dify.core.pojo.DifyPageResult;
import io.github.guoshiqiufeng.dify.dataset.DifyDataset; import io.github.guoshiqiufeng.dify.dataset.DifyDataset;
import io.github.guoshiqiufeng.dify.dataset.dto.request.DatasetPageDocumentRequest; import io.github.guoshiqiufeng.dify.dataset.dto.request.DatasetPageDocumentRequest;
import io.github.guoshiqiufeng.dify.dataset.dto.request.DocumentCreateByFileRequest;
import io.github.guoshiqiufeng.dify.dataset.dto.response.DocumentCreateResponse;
import io.github.guoshiqiufeng.dify.dataset.dto.response.DocumentInfo; import io.github.guoshiqiufeng.dify.dataset.dto.response.DocumentInfo;
import io.github.guoshiqiufeng.dify.dataset.dto.response.UploadFileInfoResponse; import io.github.guoshiqiufeng.dify.dataset.dto.response.UploadFileInfoResponse;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
@ -40,10 +38,10 @@ import org.springframework.http.ResponseEntity;
public class DatasetDocController { public class DatasetDocController {
//开源组件调用API //开源组件
@Resource @Resource
private DifyDataset difyDatasetService; private DifyDataset difyDatasetService;
//内部手写API调用 //手动API调用
@Resource @Resource
private DifyDatasetApiService difyDatasetApiService; private DifyDatasetApiService difyDatasetApiService;
@ -83,7 +81,6 @@ public class DatasetDocController {
} }
@DeleteMapping("/delete") @DeleteMapping("/delete")
public void delete(@RequestParam("datasetId") String datasetId, public void delete(@RequestParam("datasetId") String datasetId,
@RequestParam("documentId")String documentId @RequestParam("documentId")String documentId

77
chat-server/src/main/java/com/bjtds/brichat/entity/dataset/DocumentUploadReq.java
View File

@ -1,116 +1,43 @@
package com.bjtds.brichat.entity.dataset; package com.bjtds.brichat.entity.dataset;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.github.guoshiqiufeng.dify.dataset.dto.request.document.SubChunkSegmentation;
import io.github.guoshiqiufeng.dify.dataset.enums.document.ParentModeEnum;
import lombok.Data; import lombok.Data;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
@Data @Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public class DocumentUploadReq implements Serializable { public class DocumentUploadReq implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private String datasetId; private String datasetId;
@JsonProperty("original_document_id")
@JsonAlias({"originalDocumentId"})
private String originalDocumentId;
@JsonProperty("indexing_technique")
@JsonAlias({"indexingTechnique"})
private String indexingTechnique; private String indexingTechnique;
@JsonProperty("doc_form")
@JsonAlias({"docForm"})
private String docForm;
@JsonProperty("doc_language")
@JsonAlias({"docLanguage"})
private String docLanguage = "english";
@JsonProperty("process_rule")
@JsonAlias({"processRule"})
private ProcessRule processRule; private ProcessRule processRule;
@JsonProperty("retrieval_model")
private RetrievalModel retrievalModel;
@Data @Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class ProcessRule implements Serializable { public static class ProcessRule implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private String mode;
private Rules rules; private Rules rules;
private String mode = "custom";
} }
@Data @Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class Rules implements Serializable { public static class Rules implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
@JsonProperty("pre_processing_rules")
@JsonAlias({"preProcessingRules"})
private List<PreProcessingRule> preProcessingRules; private List<PreProcessingRule> preProcessingRules;
private Segmentation segmentation; private Segmentation segmentation;
@JsonAlias({"parentMode"})
@JsonProperty("parent_mode")
private ParentModeEnum parentMode;
@JsonAlias({"subChunkSegmentation"})
@JsonProperty("subchunk_segmentation")
private SubChunkSegmentation subChunkSegmentation;
} }
@Data @Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class PreProcessingRule implements Serializable { public static class PreProcessingRule implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private String id; private String id;
private Boolean enabled; private boolean enabled;
} }
@Data @Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class Segmentation implements Serializable { public static class Segmentation implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private String separator; private String separator;
@JsonProperty("max_tokens")
@JsonAlias({"maxTokens"})
private Integer maxTokens; private Integer maxTokens;
// @JsonProperty("chunk_overlap")
// @JsonAlias({"chunkOverlap"})
// private Integer chunkOverlap;
//
// @JsonProperty("chunk_size")
// @JsonAlias({"chunkSize"})
// private Integer chunkSize;
}
@Data
@JsonInclude(JsonInclude.Include.NON_NULL)
public static class VariableSeparation implements Serializable {
private static final long serialVersionUID = 1L;
private String separator;
@JsonProperty("max_tokens")
@JsonAlias({"maxTokens"})
private Integer maxTokens;
@JsonProperty("chunk_overlap")
@JsonAlias({"chunkOverlap"})
private Integer chunkOverlap;
} }
} }

118
chat-server/src/main/java/com/bjtds/brichat/entity/dataset/RetrievalModel.java
View File

@ -1,118 +0,0 @@
package com.bjtds.brichat.entity.dataset;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* 检索模型配置
*/
@ApiModel("检索模型配置")
@JsonInclude(JsonInclude.Include.NON_NULL)
public class RetrievalModel {
@ApiModelProperty("检索方法：hybrid_search(混合检索)、semantic_search(语义检索)、full_text_search(全文检索)")
@JsonProperty("search_method")
private String searchMethod;
@ApiModelProperty("是否开启 rerank")
@JsonProperty("reranking_enable")
private Boolean rerankingEnable;
@ApiModelProperty("Rerank 模型配置")
@JsonProperty("reranking_model")
private RerankingModel rerankingModel;
@ApiModelProperty("召回条数")
@JsonProperty("top_k")
private Integer topK;
@ApiModelProperty("是否开启召回分数限制")
@JsonProperty("score_threshold_enabled")
private Boolean scoreThresholdEnabled;
@ApiModelProperty("召回分数限制")
@JsonProperty("score_threshold")
private Float scoreThreshold;
/**
* Rerank 模型配置内部类
*/
@ApiModel("Rerank模型配置")
public static class RerankingModel {
@ApiModelProperty("Rerank 模型的提供商")
@JsonProperty("reranking_provider_name")
private String rerankingProviderName;
@ApiModelProperty("Rerank 模型的名称")
@JsonProperty("reranking_model_name")
private String rerankingModelName;
public String getRerankingProviderName() {
return rerankingProviderName;
}
public void setRerankingProviderName(String rerankingProviderName) {
this.rerankingProviderName = rerankingProviderName;
}
public String getRerankingModelName() {
return rerankingModelName;
}
public void setRerankingModelName(String rerankingModelName) {
this.rerankingModelName = rerankingModelName;
}
}
// Getter and Setter methods
public String getSearchMethod() {
return searchMethod;
}
public void setSearchMethod(String searchMethod) {
this.searchMethod = searchMethod;
}
public Boolean getRerankingEnable() {
return rerankingEnable;
}
public void setRerankingEnable(Boolean rerankingEnable) {
this.rerankingEnable = rerankingEnable;
}
public RerankingModel getRerankingModel() {
return rerankingModel;
}
public void setRerankingModel(RerankingModel rerankingModel) {
this.rerankingModel = rerankingModel;
}
public Integer getTopK() {
return topK;
}
public void setTopK(Integer topK) {
this.topK = topK;
}
public Boolean getScoreThresholdEnabled() {
return scoreThresholdEnabled;
}
public void setScoreThresholdEnabled(Boolean scoreThresholdEnabled) {
this.scoreThresholdEnabled = scoreThresholdEnabled;
}
public Float getScoreThreshold() {
return scoreThreshold;
}
public void setScoreThreshold(Float scoreThreshold) {
this.scoreThreshold = scoreThreshold;
}
}

61
chat-server/src/main/java/com/bjtds/brichat/service/dify/impl/DifyDatasetApiServiceImpl.java
View File

@ -1,7 +1,6 @@
package com.bjtds.brichat.service.dify.impl; package com.bjtds.brichat.service.dify.impl;
import com.bjtds.brichat.entity.dataset.DocumentUploadReq; import com.bjtds.brichat.entity.dataset.DocumentUploadReq;
import com.bjtds.brichat.entity.dataset.RetrievalModel;
import com.bjtds.brichat.entity.dify.DatasetDto; import com.bjtds.brichat.entity.dify.DatasetDto;
import com.bjtds.brichat.entity.dify.DifyDatasetResponse; import com.bjtds.brichat.entity.dify.DifyDatasetResponse;
import com.bjtds.brichat.service.dify.DifyDatasetApiService; import com.bjtds.brichat.service.dify.DifyDatasetApiService;
@ -100,53 +99,24 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
@Override @Override
public ResponseEntity<Map> createDocumentByFile(DocumentUploadReq request, MultipartFile file) throws JsonProcessingException { public ResponseEntity<Map> createDocumentByFile(DocumentUploadReq request, MultipartFile file) throws JsonProcessingException {
// 参数验证
if (request == null) {
throw new IllegalArgumentException("请求参数不能为空");
}
if (file == null || file.isEmpty()) {
throw new IllegalArgumentException("上传文件不能为空");
}
String url = difyUrl + Constants.DATABASE_API + "/{dataset_id}/document/create-by-file"; String url = difyUrl + Constants.DATABASE_API + "/{dataset_id}/document/create-by-file";
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA); headers.setContentType(MediaType.MULTIPART_FORM_DATA);
headers.set("Authorization", Constants.BEARER + apiKey); headers.set("Authorization", Constants.BEARER +apiKey);
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>(); MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
// 设置默认值（如果未提供） // 1. JSON部分显式设置Content-Type
if (request.getDocLanguage() == null || request.getDocLanguage().trim().isEmpty()) { if (request != null) {
request.setDocLanguage("english"); //TODO 目前设置默认值
body.add("data", "{\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}");
// ObjectMapper mapper = new ObjectMapper();
// String json = mapper.writeValueAsString(request);
// body.add("data", json);
} }
// 2. 文件部分添加异常处理
// 创建临时请求对象，不包含datasetId（datasetId用于URL路径参数） if (file != null && !file.isEmpty()) {
DocumentUploadReq dataRequest = new DocumentUploadReq();
dataRequest.setIndexingTechnique(request.getIndexingTechnique());
dataRequest.setProcessRule(request.getProcessRule());
//设置检索模式（默认混合检索）
RetrievalModel retrievalModel = new RetrievalModel();
retrievalModel.setSearchMethod("hybrid_search");
retrievalModel.setRerankingEnable(true);
RetrievalModel.RerankingModel rerankingModel = new RetrievalModel.RerankingModel();
rerankingModel.setRerankingModelName("bge-reanker-v2-m3");
rerankingModel.setRerankingProviderName("langgenius/huggingface_tei/huggingface_tei");
retrievalModel.setTopK(3);
retrievalModel.setRerankingModel(rerankingModel);
retrievalModel.setScoreThresholdEnabled(false);
retrievalModel.setScoreThreshold(0.5f);
dataRequest.setRetrievalModel(retrievalModel);
// 序列化请求数据为JSON
ObjectMapper mapper = new ObjectMapper();
String json = mapper.writeValueAsString(dataRequest);
body.add("data", json);
// 添加文件
try { try {
body.add("file", new ByteArrayResource(file.getBytes()) { body.add("file", new ByteArrayResource(file.getBytes()) {
@Override @Override
@ -155,17 +125,17 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
} }
}); });
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException("文件读取失败: " + e.getMessage(), e); throw new RuntimeException("文件读取失败", e);
}
} }
HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers); HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
// 设置路径变量 // 3. 使用明确的路径变量Map
Map<String, String> uriVariables = new HashMap<>(); Map<String, String> uriVariables = new HashMap<>();
uriVariables.put("dataset_id", request.getDatasetId()); uriVariables.put("dataset_id", request.getDatasetId());
// 发送请求 // 4. 发送请求
try {
return restTemplate.exchange( return restTemplate.exchange(
url, url,
HttpMethod.POST, HttpMethod.POST,
@ -173,9 +143,6 @@ public class DifyDatasetApiServiceImpl implements DifyDatasetApiService {
Map.class, Map.class,
uriVariables uriVariables
); );
} catch (Exception e) {
throw new RuntimeException("文档上传失败: " + e.getMessage(), e);
}
} }