From f6d53c6c01ecfa4808369a9edfc84c8db55755ab Mon Sep 17 00:00:00 2001 From: GrapeS Date: Wed, 23 Sep 2026 18:51:34 +0800 Subject: [PATCH 1/3] fix(http): time out idle connections at the connection limit Connections that send nothing could hold every maxHttpConnectNumber slot until the connector's 30-second idle timeout and lock other clients out. When the limit is reached, the open connections now time out after 10 seconds of inactivity. Connections still being accepted at that moment keep the connector's default idle timeout. --- .../tron/common/application/HttpService.java | 8 +- .../common/jetty/ConnectionLimitTest.java | 148 ++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java diff --git a/framework/src/main/java/org/tron/common/application/HttpService.java b/framework/src/main/java/org/tron/common/application/HttpService.java index 1dea271ec69..6f369c1a0d3 100644 --- a/framework/src/main/java/org/tron/common/application/HttpService.java +++ b/framework/src/main/java/org/tron/common/application/HttpService.java @@ -43,6 +43,10 @@ public abstract class HttpService extends AbstractService { protected long maxRequestSize = 4 * 1024 * 1024; // 4MB + // Once maxHttpConnectNumber is reached, open connections time out after this much + // inactivity, so connections that send nothing cannot hold every slot. + private static final long CONNECTION_LIMIT_IDLE_TIMEOUT_MS = 10_000; + @VisibleForTesting public long getMaxRequestSize() { return this.maxRequestSize; @@ -80,7 +84,9 @@ protected void initServer() { this.apiServer = new Server(this.port); int maxHttpConnectNumber = Args.getInstance().getMaxHttpConnectNumber(); if (maxHttpConnectNumber > 0) { - this.apiServer.addBean(new ConnectionLimit(maxHttpConnectNumber, this.apiServer)); + ConnectionLimit connectionLimit = new ConnectionLimit(maxHttpConnectNumber, this.apiServer); + connectionLimit.setIdleTimeout(CONNECTION_LIMIT_IDLE_TIMEOUT_MS); + this.apiServer.addBean(connectionLimit); } this.apiServer.setErrorHandler(new OversizedRequestErrorHandler()); } diff --git a/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java new file mode 100644 index 00000000000..ef220749f44 --- /dev/null +++ b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java @@ -0,0 +1,148 @@ +package org.tron.common.jetty; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.AbstractConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.tron.common.TestConstants; +import org.tron.common.application.HttpService; +import org.tron.common.utils.PublicMethod; +import org.tron.core.config.args.Args; + +/** + * Tests the connection limit configured in {@link HttpService}: once connections that send + * nothing hold every slot, the server closes them after the limit's idle timeout and serves + * new clients, instead of waiting for the connector's 30-second idle timeout. + */ +public class ConnectionLimitTest { + + private static final int MAX_CONNECTIONS = 2; + + // Below the connector's default 30-second idle timeout, above the limit's 10-second one + // applied twice (Jetty half-closes an idle connection before closing it). + private static final int TIMEOUT_MS = 25_000; + + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static TestHttpService httpService; + private static int port; + + public static class OkServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setStatus(HttpServletResponse.SC_OK); + resp.getWriter().print("ok"); + } + } + + static class TestHttpService extends HttpService { + TestHttpService(int port) { + this.port = port; + this.contextPath = "/"; + } + + @Override + protected void addServlet(ServletContextHandler context) { + context.addServlet(new ServletHolder(new OkServlet()), "/*"); + } + + int connectedEndPoints() { + return ((AbstractConnector) apiServer.getConnectors()[0]).getConnectedEndPoints().size(); + } + } + + @BeforeClass + public static void setup() throws Exception { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, + TestConstants.TEST_CONF); + Args.getInstance().setMaxHttpConnectNumber(MAX_CONNECTIONS); + port = PublicMethod.chooseRandomPort(); + httpService = new TestHttpService(port); + httpService.start().get(10, TimeUnit.SECONDS); + } + + @AfterClass + public static void teardown() throws Exception { + try { + if (httpService != null) { + httpService.stop(); + } + } finally { + Args.clearParam(); + } + } + + @Test(timeout = 60_000) + public void testIdleConnectionsDoNotLockOutClients() throws Exception { + List idleSockets = new ArrayList<>(); + try { + for (int i = 0; i < MAX_CONNECTIONS; i++) { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("localhost", port), TIMEOUT_MS); + idleSockets.add(socket); + awaitConnectedEndPoints(i + 1); + } + + Assert.assertEquals("HTTP/1.1 200 OK", get()); + for (Socket socket : idleSockets) { + assertClosedByServer(socket); + } + } finally { + for (Socket socket : idleSockets) { + socket.close(); + } + } + } + + private static void awaitConnectedEndPoints(int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + TIMEOUT_MS; + while (httpService.connectedEndPoints() < expected) { + Assert.assertTrue("server did not accept connection " + expected, + System.currentTimeMillis() < deadline); + Thread.sleep(10); + } + } + + private static String get() throws IOException { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress("localhost", port), TIMEOUT_MS); + socket.setSoTimeout(TIMEOUT_MS); + OutputStream out = socket.getOutputStream(); + out.write("GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + BufferedReader in = new BufferedReader( + new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII)); + return in.readLine(); + } + } + + private static void assertClosedByServer(Socket socket) throws IOException { + socket.setSoTimeout(TIMEOUT_MS); + try { + Assert.assertEquals(-1, socket.getInputStream().read()); + } catch (SocketException e) { + // a reset also means the server dropped the connection + } + } +} From 8e6f136b31973286573fb628f03588521326bf1c Mon Sep 17 00:00:00 2001 From: GrapeS Date: Mon, 12 Oct 2026 16:32:51 +0800 Subject: [PATCH 2/3] test(http): shorten the idle timeout in ConnectionLimitTest --- .../org/tron/common/application/HttpService.java | 15 +++++++++++---- .../tron/common/jetty/ConnectionLimitTest.java | 15 ++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/framework/src/main/java/org/tron/common/application/HttpService.java b/framework/src/main/java/org/tron/common/application/HttpService.java index 6f369c1a0d3..a61cd4c6b46 100644 --- a/framework/src/main/java/org/tron/common/application/HttpService.java +++ b/framework/src/main/java/org/tron/common/application/HttpService.java @@ -43,9 +43,16 @@ public abstract class HttpService extends AbstractService { protected long maxRequestSize = 4 * 1024 * 1024; // 4MB - // Once maxHttpConnectNumber is reached, open connections time out after this much - // inactivity, so connections that send nothing cannot hold every slot. - private static final long CONNECTION_LIMIT_IDLE_TIMEOUT_MS = 10_000; + // Once maxHttpConnectNumber is reached, Jetty applies this idle timeout to every open + // connection, so connections that send nothing cannot hold every slot. While the limit is + // reached, a keep-alive connection that stays idle this long between two requests is closed + // as well; time spent processing a request does not count as idle. + private long connectionLimitIdleTimeoutMs = 10_000; + + @VisibleForTesting + protected void setConnectionLimitIdleTimeoutMs(long connectionLimitIdleTimeoutMs) { + this.connectionLimitIdleTimeoutMs = connectionLimitIdleTimeoutMs; + } @VisibleForTesting public long getMaxRequestSize() { @@ -85,7 +92,7 @@ protected void initServer() { int maxHttpConnectNumber = Args.getInstance().getMaxHttpConnectNumber(); if (maxHttpConnectNumber > 0) { ConnectionLimit connectionLimit = new ConnectionLimit(maxHttpConnectNumber, this.apiServer); - connectionLimit.setIdleTimeout(CONNECTION_LIMIT_IDLE_TIMEOUT_MS); + connectionLimit.setIdleTimeout(connectionLimitIdleTimeoutMs); this.apiServer.addBean(connectionLimit); } this.apiServer.setErrorHandler(new OversizedRequestErrorHandler()); diff --git a/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java index ef220749f44..2231bbc8f5e 100644 --- a/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java +++ b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java @@ -31,15 +31,19 @@ /** * Tests the connection limit configured in {@link HttpService}: once connections that send * nothing hold every slot, the server closes them after the limit's idle timeout and serves - * new clients, instead of waiting for the connector's 30-second idle timeout. + * new clients, instead of waiting for the connector's 30-second idle timeout. The test shortens + * the limit's idle timeout to one second (production uses ten) so it finishes in a few seconds. */ public class ConnectionLimitTest { private static final int MAX_CONNECTIONS = 2; - // Below the connector's default 30-second idle timeout, above the limit's 10-second one - // applied twice (Jetty half-closes an idle connection before closing it). - private static final int TIMEOUT_MS = 25_000; + private static final long LIMIT_IDLE_TIMEOUT_MS = 1_000; + + // Well above the limit's idle timeout applied twice (Jetty half-closes an idle connection + // before closing it), and below the connector's default 30-second idle timeout, so the test + // fails if only the connector's timeout frees the slots. + private static final int TIMEOUT_MS = 10_000; @ClassRule public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -59,6 +63,7 @@ static class TestHttpService extends HttpService { TestHttpService(int port) { this.port = port; this.contextPath = "/"; + setConnectionLimitIdleTimeoutMs(LIMIT_IDLE_TIMEOUT_MS); } @Override @@ -92,7 +97,7 @@ public static void teardown() throws Exception { } } - @Test(timeout = 60_000) + @Test(timeout = 30_000) public void testIdleConnectionsDoNotLockOutClients() throws Exception { List idleSockets = new ArrayList<>(); try { From 87d9bd44c4ccf761e25697f0c59e840fb87d427c Mon Sep 17 00:00:00 2001 From: GrapeS Date: Tue, 13 Oct 2026 09:59:45 +0800 Subject: [PATCH 3/3] test(http): assert a freed slot instead of every idle socket closing --- .../common/jetty/ConnectionLimitTest.java | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java index 2231bbc8f5e..c67c416f7d3 100644 --- a/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java +++ b/framework/src/test/java/org/tron/common/jetty/ConnectionLimitTest.java @@ -6,7 +6,7 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.Socket; -import java.net.SocketException; +import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -30,9 +30,11 @@ /** * Tests the connection limit configured in {@link HttpService}: once connections that send - * nothing hold every slot, the server closes them after the limit's idle timeout and serves - * new clients, instead of waiting for the connector's 30-second idle timeout. The test shortens - * the limit's idle timeout to one second (production uses ten) so it finishes in a few seconds. + * nothing hold every slot, the limit's idle timeout closes idle connections and frees a slot for + * a new client, instead of waiting for the connector's 30-second idle timeout. As soon as a slot + * is free, Jetty restores the connector's timeout on the remaining connections, so only one idle + * connection is guaranteed to be closed. The test shortens the limit's idle timeout to one + * second (production uses ten) so it finishes in a few seconds. */ public class ConnectionLimitTest { @@ -42,9 +44,12 @@ public class ConnectionLimitTest { // Well above the limit's idle timeout applied twice (Jetty half-closes an idle connection // before closing it), and below the connector's default 30-second idle timeout, so the test - // fails if only the connector's timeout frees the slots. + // fails if only the connector's timeout frees a slot. private static final int TIMEOUT_MS = 10_000; + // How long a connection that is still open is read before it counts as not closed. + private static final int CLOSE_CHECK_TIMEOUT_MS = 1_000; + @ClassRule public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -108,10 +113,10 @@ public void testIdleConnectionsDoNotLockOutClients() throws Exception { awaitConnectedEndPoints(i + 1); } + // The new request is only served once the limit's idle timeout frees a slot. Assert.assertEquals("HTTP/1.1 200 OK", get()); - for (Socket socket : idleSockets) { - assertClosedByServer(socket); - } + Assert.assertTrue("no idle connection was closed by the server", + idleSockets.stream().anyMatch(ConnectionLimitTest::isClosedByServer)); } finally { for (Socket socket : idleSockets) { socket.close(); @@ -142,12 +147,17 @@ private static String get() throws IOException { } } - private static void assertClosedByServer(Socket socket) throws IOException { - socket.setSoTimeout(TIMEOUT_MS); + // A connection the server closed reads end-of-stream, or fails with a reset, right away; one + // that is still open times out. + private static boolean isClosedByServer(Socket socket) { try { - Assert.assertEquals(-1, socket.getInputStream().read()); - } catch (SocketException e) { + socket.setSoTimeout(CLOSE_CHECK_TIMEOUT_MS); + return socket.getInputStream().read() == -1; + } catch (SocketTimeoutException e) { + return false; + } catch (IOException e) { // a reset also means the server dropped the connection + return true; } } }