WJB
/
ai-manus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 统一管理http资源

This commit is contained in:
wenjinbo 2025-09-28 17:30:43 +08:00
parent 0d80efd8e2
commit 996bb9481a
7 changed files with 509 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
chat-server/pom.xml
View File

@ -309,13 +309,26 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.5</version>
<version>4.4.15</version>
</dependency>
<!-- Feign HttpClient依赖 -->
<!-- <dependency>-->
<!-- <groupId>io.github.openfeign</groupId>-->
<!-- <artifactId>feign-httpclient</artifactId>-->
<!-- <version>11.8</version>-->
<!-- </dependency>-->
<!-- Spring Boot Actuator用于健康检查 -->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-actuator</artifactId>-->
<!-- </dependency>-->
</dependencies>
<dependencyManagement>

53
chat-server/src/main/java/com/bjtds/brichat/config/EsConfig.java
View File

@ -5,14 +5,19 @@ import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.elasticsearch.client.RestClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Configuration
public class EsConfig {
private static final Logger logger = LoggerFactory.getLogger(EsConfig.class);
@Value("${elasticsearch.host}")
private String host;
@ -22,14 +27,60 @@ public class EsConfig {
@Value("${elasticsearch.scheme}")
private String scheme;
// 复用HttpClientConfig中的连接池配置参数
@Value("${http.pool.max-total:200}")
private int maxTotal;
@Value("${http.pool.max-per-route:50}")
private int maxPerRoute;
@Value("${http.timeout.connect:10000}")
private int connectTimeout;
@Value("${http.timeout.socket:30000}")
private int socketTimeout;
@Value("${http.timeout.request:5000}")
private int connectionRequestTimeout;
@Bean
public ElasticsearchClient elasticsearchClient() {
// 为Elasticsearch配置独立的RestClient，但使用相同的连接池参数
RestClient restClient = RestClient.builder(
new HttpHost(host, port, scheme)
).build();
)
.setRequestConfigCallback(requestConfigBuilder -> {
return requestConfigBuilder
.setConnectTimeout(connectTimeout)
.setSocketTimeout(socketTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout);
})
.setHttpClientConfigCallback(httpClientBuilder -> {
return httpClientBuilder
.setMaxConnTotal(maxTotal)
.setMaxConnPerRoute(maxPerRoute);
})
.build();
ElasticsearchTransport transport = new RestClientTransport(
restClient, new JacksonJsonpMapper()
);
logger.info("Elasticsearch客户端已配置独立连接池 - 主机: {}:{}, 最大连接数: {}, 每路由最大连接数: {}",
host, port, maxTotal, maxPerRoute);
return new ElasticsearchClient(transport);
}
// @Bean
// public ElasticsearchClient elasticsearchClient() {
// RestClient restClient = RestClient.builder(
// new HttpHost(host, port, scheme)
// ).build();
// ElasticsearchTransport transport = new RestClientTransport(
// restClient, new JacksonJsonpMapper()
// );
// return new ElasticsearchClient(transport);
// }
}

96
chat-server/src/main/java/com/bjtds/brichat/config/HttpClientConfig.java
View File

@ -6,8 +6,13 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PreDestroy;
import java.util.concurrent.TimeUnit;
/**
* HTTP客户端连接池配置
@ -16,36 +21,112 @@ import javax.annotation.PreDestroy;
@Configuration
public class HttpClientConfig {
private static final Logger logger = LoggerFactory.getLogger(HttpClientConfig.class);
private PoolingHttpClientConnectionManager connectionManager;
private CloseableHttpClient httpClient;
// 连接池配置参数
@Value("${http.pool.max-total:200}")
private int maxTotal;
@Value("${http.pool.max-per-route:50}")
private int maxPerRoute;
@Value("${http.timeout.connect:10000}")
private int connectTimeout;
@Value("${http.timeout.socket:30000}")
private int socketTimeout;
@Value("${http.timeout.request:5000}")
private int connectionRequestTimeout;
@Value("${http.pool.validate-after-inactivity:30000}")
private int validateAfterInactivity;
@Bean
public CloseableHttpClient httpClient() {
// 创建连接池管理器
connectionManager = new PoolingHttpClientConnectionManager();
// 设置最大连接数
connectionManager.setMaxTotal(200);
connectionManager.setMaxTotal(maxTotal);
// 设置每个路由的最大连接数
connectionManager.setDefaultMaxPerRoute(50);
connectionManager.setDefaultMaxPerRoute(maxPerRoute);
// 设置连接在池中的最大空闲时间
connectionManager.setValidateAfterInactivity(validateAfterInactivity);
// 设置连接超时、读取超时等配置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(10000) // 连接超时10秒
.setSocketTimeout(30000) // 读取超时30秒
.setConnectionRequestTimeout(5000) // 从连接池获取连接超时5秒
.setConnectTimeout(connectTimeout) // 连接超时
.setSocketTimeout(socketTimeout) // 读取超时
.setConnectionRequestTimeout(connectionRequestTimeout) // 从连接池获取连接超时
.build();
// 创建HttpClient
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
.setConnectionManagerShared(false)
.evictExpiredConnections() // 自动清理过期连接
.evictIdleConnections(60, TimeUnit.SECONDS) // 清理60秒空闲连接
.build();
logger.info("HttpClient连接池配置完成 - 最大连接数: {}, 每路由最大连接数: {}, 连接超时: {}ms, 读取超时: {}ms",
maxTotal, maxPerRoute, connectTimeout, socketTimeout);
return httpClient;
}
/**
* 定期清理空闲和过期连接
*/
@Scheduled(fixedRate = 30000) // 每30秒执行一次
public void cleanupConnections() {
if (connectionManager != null) {
// 清理过期连接
connectionManager.closeExpiredConnections();
// 清理空闲超过60秒的连接
connectionManager.closeIdleConnections(60, TimeUnit.SECONDS);
// 记录连接池状态
logConnectionPoolStats();
}
}
/**
* 记录连接池状态
*/
private void logConnectionPoolStats() {
if (connectionManager != null) {
int totalStats = connectionManager.getTotalStats().getAvailable() +
connectionManager.getTotalStats().getLeased();
int available = connectionManager.getTotalStats().getAvailable();
int leased = connectionManager.getTotalStats().getLeased();
int pending = connectionManager.getTotalStats().getPending();
logger.debug("HTTP连接池状态 - 总连接数: {}, 可用: {}, 已使用: {}, 等待: {}",
totalStats, available, leased, pending);
}
}
/**
* 获取连接池状态信息
*/
public String getConnectionPoolStatus() {
if (connectionManager != null) {
return String.format("总连接数: %d, 可用: %d, 已使用: %d, 等待: %d",
connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased(),
connectionManager.getTotalStats().getAvailable(),
connectionManager.getTotalStats().getLeased(),
connectionManager.getTotalStats().getPending());
}
return "连接池未初始化";
}
/**
* 应用关闭时清理资源
*/
@ -53,13 +134,16 @@ public class HttpClientConfig {
public void destroy() {
try {
if (httpClient != null) {
logger.info("正在关闭HttpClient连接池...");
httpClient.close();
logger.info("HttpClient连接池已关闭");
}
if (connectionManager != null) {
connectionManager.close();
logger.info("HTTP连接管理器已关闭");
}
} catch (Exception e) {
// 忽略关闭异常
logger.error("关闭HTTP连接池时发生异常", e);
}
}
}

105
chat-server/src/main/java/com/bjtds/brichat/config/RestTemplateConfig.java
View File

@ -1,18 +1,119 @@
package com.bjtds.brichat.config;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PreDestroy;
import java.util.concurrent.TimeUnit;
@Configuration
public class RestTemplateConfig {
private static final Logger logger = LoggerFactory.getLogger(RestTemplateConfig.class);
private PoolingHttpClientConnectionManager connectionManager;
private CloseableHttpClient httpClient;
// RestTemplate连接池配置参数
@Value("${rest-template.pool.max-total:300}")
private int maxTotal;
@Value("${rest-template.pool.max-per-route:100}")
private int maxPerRoute;
@Value("${rest-template.timeout.connect:15000}")
private int connectTimeout;
@Value("${rest-template.timeout.socket:60000}")
private int socketTimeout;
@Value("${rest-template.timeout.request:10000}")
private int connectionRequestTimeout;
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
// 创建连接池管理器
connectionManager = new PoolingHttpClientConnectionManager();
// 设置最大连接数
connectionManager.setMaxTotal(maxTotal);
// 设置每个路由的最大连接数
connectionManager.setDefaultMaxPerRoute(maxPerRoute);
// 设置连接验证间隔
connectionManager.setValidateAfterInactivity(30000);
// 设置连接超时、读取超时等配置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout) // 连接超时
.setSocketTimeout(socketTimeout) // 读取超时
.setConnectionRequestTimeout(connectionRequestTimeout) // 从连接池获取连接超时
.build();
// 创建HttpClient
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
.setConnectionManagerShared(false)
.evictExpiredConnections() // 自动清理过期连接
.evictIdleConnections(60, TimeUnit.SECONDS) // 清理60秒空闲连接
.build();
// 创建HttpComponentsClientHttpRequestFactory
HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory(httpClient);
// 创建RestTemplate
RestTemplate restTemplate = new RestTemplate(factory);
logger.info("RestTemplate连接池配置完成 - 最大连接数: {}, 每路由最大连接数: {}, 连接超时: {}ms, 读取超时: {}ms",
maxTotal, maxPerRoute, connectTimeout, socketTimeout);
return restTemplate;
}
/**
* 获取RestTemplate连接池状态
*/
public String getRestTemplateConnectionPoolStatus() {
if (connectionManager != null) {
return String.format("RestTemplate连接池 - 总连接数: %d, 可用: %d, 已使用: %d, 等待: %d",
connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased(),
connectionManager.getTotalStats().getAvailable(),
connectionManager.getTotalStats().getLeased(),
connectionManager.getTotalStats().getPending());
}
return "RestTemplate连接池未初始化";
}
/**
* 应用关闭时清理资源
*/
@PreDestroy
public void destroy() {
try {
if (httpClient != null) {
logger.info("正在关闭RestTemplate连接池...");
httpClient.close();
logger.info("RestTemplate连接池已关闭");
}
if (connectionManager != null) {
connectionManager.close();
logger.info("RestTemplate连接管理器已关闭");
}
} catch (Exception e) {
logger.error("关闭RestTemplate连接池时发生异常", e);
}
}
}

212
chat-server/src/main/java/com/bjtds/brichat/util/HttpConnectionPoolTestUtil.java Normal file
View File

@ -0,0 +1,212 @@
package com.bjtds.brichat.util;
import org.apache.http.impl.client.CloseableHttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
/**
* HTTP连接池性能测试工具类
* 用于验证连接池优化效果
*/
@Component
public class HttpConnectionPoolTestUtil {
private static final Logger logger = LoggerFactory.getLogger(HttpConnectionPoolTestUtil.class);
@Autowired
private RestTemplate restTemplate;
@Autowired
private CloseableHttpClient httpClient;
/**
* 并发测试HTTP连接池性能
*
* @param targetUrl 测试目标URL
* @param concurrency 并发数
* @param totalRequests 总请求数
* @return 测试结果
*/
public TestResult performConcurrencyTest(String targetUrl, int concurrency, int totalRequests) {
logger.info("开始HTTP连接池并发测试 - URL: {}, 并发数: {}, 总请求数: {}",
targetUrl, concurrency, totalRequests);
ExecutorService executor = Executors.newFixedThreadPool(concurrency);
CountDownLatch latch = new CountDownLatch(totalRequests);
List<Future<RequestResult>> futures = new ArrayList<>();
long startTime = System.currentTimeMillis();
// 提交并发请求任务
for (int i = 0; i < totalRequests; i++) {
Future<RequestResult> future = executor.submit(() -> {
try {
long requestStart = System.currentTimeMillis();
// 使用RestTemplate发送请求（可以根据需要切换为HttpClient）
String response = restTemplate.getForObject(targetUrl, String.class);
long requestEnd = System.currentTimeMillis();
long responseTime = requestEnd - requestStart;
return new RequestResult(true, responseTime, null);
} catch (Exception e) {
logger.warn("请求失败: {}", e.getMessage());
return new RequestResult(false, 0, e.getMessage());
} finally {
latch.countDown();
}
});
futures.add(future);
}
try {
// 等待所有请求完成
latch.await(120, TimeUnit.SECONDS); // 最多等待2分钟
} catch (InterruptedException e) {
logger.error("等待测试完成时被中断", e);
}
long endTime = System.currentTimeMillis();
long totalTime = endTime - startTime;
// 收集结果
TestResult result = collectResults(futures, totalTime, concurrency, totalRequests);
executor.shutdown();
logger.info("HTTP连接池并发测试完成 - 总耗时: {}ms, 成功率: {}%, 平均响应时间: {}ms",
result.getTotalTime(), result.getSuccessRate(), result.getAverageResponseTime());
return result;
}
/**
* 收集测试结果
*/
private TestResult collectResults(List<Future<RequestResult>> futures, long totalTime,
int concurrency, int totalRequests) {
int successCount = 0;
int failureCount = 0;
long totalResponseTime = 0;
long maxResponseTime = 0;
long minResponseTime = Long.MAX_VALUE;
List<String> errorMessages = new ArrayList<>();
for (Future<RequestResult> future : futures) {
try {
RequestResult result = future.get(1, TimeUnit.SECONDS);
if (result.isSuccess()) {
successCount++;
totalResponseTime += result.getResponseTime();
maxResponseTime = Math.max(maxResponseTime, result.getResponseTime());
minResponseTime = Math.min(minResponseTime, result.getResponseTime());
} else {
failureCount++;
if (result.getErrorMessage() != null) {
errorMessages.add(result.getErrorMessage());
}
}
} catch (Exception e) {
failureCount++;
errorMessages.add("获取结果超时: " + e.getMessage());
}
}
double successRate = (double) successCount / totalRequests * 100;
long averageResponseTime = successCount > 0 ? totalResponseTime / successCount : 0;
double qps = (double) successCount / (totalTime / 1000.0);
return TestResult.builder()
.totalRequests(totalRequests)
.successCount(successCount)
.failureCount(failureCount)
.successRate(successRate)
.totalTime(totalTime)
.averageResponseTime(averageResponseTime)
.maxResponseTime(maxResponseTime)
.minResponseTime(minResponseTime == Long.MAX_VALUE ? 0 : minResponseTime)
.qps(qps)
.concurrency(concurrency)
.errorMessages(errorMessages)
.build();
}
/**
* 请求结果
*/
public static class RequestResult {
private final boolean success;
private final long responseTime;
private final String errorMessage;
public RequestResult(boolean success, long responseTime, String errorMessage) {
this.success = success;
this.responseTime = responseTime;
this.errorMessage = errorMessage;
}
public boolean isSuccess() { return success; }
public long getResponseTime() { return responseTime; }
public String getErrorMessage() { return errorMessage; }
}
/**
* 测试结果
*/
public static class TestResult {
private int totalRequests;
private int successCount;
private int failureCount;
private double successRate;
private long totalTime;
private long averageResponseTime;
private long maxResponseTime;
private long minResponseTime;
private double qps;
private int concurrency;
private List<String> errorMessages;
public static Builder builder() {
return new Builder();
}
// Getters
public int getTotalRequests() { return totalRequests; }
public int getSuccessCount() { return successCount; }
public int getFailureCount() { return failureCount; }
public double getSuccessRate() { return successRate; }
public long getTotalTime() { return totalTime; }
public long getAverageResponseTime() { return averageResponseTime; }
public long getMaxResponseTime() { return maxResponseTime; }
public long getMinResponseTime() { return minResponseTime; }
public double getQps() { return qps; }
public int getConcurrency() { return concurrency; }
public List<String> getErrorMessages() { return errorMessages; }
public static class Builder {
private TestResult result = new TestResult();
public Builder totalRequests(int totalRequests) { result.totalRequests = totalRequests; return this; }
public Builder successCount(int successCount) { result.successCount = successCount; return this; }
public Builder failureCount(int failureCount) { result.failureCount = failureCount; return this; }
public Builder successRate(double successRate) { result.successRate = successRate; return this; }
public Builder totalTime(long totalTime) { result.totalTime = totalTime; return this; }
public Builder averageResponseTime(long averageResponseTime) { result.averageResponseTime = averageResponseTime; return this; }
public Builder maxResponseTime(long maxResponseTime) { result.maxResponseTime = maxResponseTime; return this; }
public Builder minResponseTime(long minResponseTime) { result.minResponseTime = minResponseTime; return this; }
public Builder qps(double qps) { result.qps = qps; return this; }
public Builder concurrency(int concurrency) { result.concurrency = concurrency; return this; }
public Builder errorMessages(List<String> errorMessages) { result.errorMessages = errorMessages; return this; }
public TestResult build() { return result; }
}
}
}

2
chat-server/src/main/resources/application-wuhan.yml
View File

@ -52,7 +52,7 @@ dify:
email: bjtds@bjtds.com # 请替换为实际的 Dify 服务邮箱，若不需要调用 server相关接口可不填
password: 123456Aa # 请替换为实际的 Dify 服务密码，若不需要调用 server相关接口可不填
dataset:
api-key: ${dify-dataset-api-key:dataset-0Hij9IwoWYbJe1vvwVh8y7DS} # 请替换为实际的知识库api-key, 若不需要调用知识库可不填
api-key: ${dify-dataset-api-key:dataset-fSaqYAlxDdkyyFRWA9zg1Jaw} # 请替换为实际的知识库api-key, 若不需要调用知识库可不填
# PDF转换服务配置
ocr:

35
chat-server/src/main/resources/application.yml
View File

@ -42,6 +42,41 @@ spring:
jackson:
time-zone: GMT+8
# HTTP连接池配置
http:
pool:
max-total: 300 # 连接池最大连接数
max-per-route: 100 # 每个路由的最大连接数
validate-after-inactivity: 30000 # 连接验证间隔(毫秒)
timeout:
connect: 15000 # 连接超时(毫秒)
socket: 60000 # 读取超时(毫秒)
request: 10000 # 从连接池获取连接超时(毫秒)
# RestTemplate连接池配置
rest-template:
pool:
max-total: 300 # RestTemplate连接池最大连接数
max-per-route: 100 # 每个路由的最大连接数
timeout:
connect: 15000 # 连接超时(毫秒)
socket: 60000 # 读取超时(毫秒)
request: 10000 # 从连接池获取连接超时(毫秒)
# Feign客户端连接池配置
feign:
httpclient:
enabled: true
max-connections: 400 # Feign最大连接数
max-connections-per-route: 150 # 每个路由的最大连接数
connection-timeout: 20000 # 连接超时(毫秒)
connection-timer-repeat: 15000 # 连接请求超时(毫秒)
client:
config:
default:
connectTimeout: 20000
readTimeout: 90000
# Redis配置已移至对应的环境配置文件
redis:
database: 0