diff --git a/docker-java-transport-httpclient5/src/main/java/com/github/dockerjava/httpclient5/ApacheDockerHttpClientImpl.java b/docker-java-transport-httpclient5/src/main/java/com/github/dockerjava/httpclient5/ApacheDockerHttpClientImpl.java index c97a2bc453..eb05ed399b 100644 --- a/docker-java-transport-httpclient5/src/main/java/com/github/dockerjava/httpclient5/ApacheDockerHttpClientImpl.java +++ b/docker-java-transport-httpclient5/src/main/java/com/github/dockerjava/httpclient5/ApacheDockerHttpClientImpl.java @@ -52,6 +52,12 @@ class ApacheDockerHttpClientImpl implements DockerHttpClient { + /** + * Idle lifetime of a pooled connection when the peer advertises no {@code Keep-Alive} timeout. + * Visible for the test that brackets it. + */ + static final TimeValue CONNECTION_KEEP_ALIVE = TimeValue.ofSeconds(2); + private final CloseableHttpClient httpClient; private final HttpHost host; private final String pathPrefix; @@ -132,6 +138,24 @@ protected ApacheDockerHttpClientImpl( .setConnectionManager(connectionManager) .setDefaultRequestConfig(RequestConfig.custom() .setResponseTimeout(responseTimeout != null ? Timeout.of(responseTimeout.toNanos(), TimeUnit.NANOSECONDS) : null) + // How long a pooled connection may sit idle when the peer does not say. This is a + // fallback, not a ceiling: DefaultConnectionKeepAliveStrategy returns a response's + // Keep-Alive timeout if there is one and only consults this otherwise -- which is + // what we want, since a peer advertising its own window is telling the truth about + // its own connection. + // + // Both daemons that matter advertise nothing. Docker then never closes the + // connection, so any value is safe. Podman closes it once idle for twice its + // service_timeout -- that setting defaults to 5s, so the window is 10s -- and + // without this the strategy falls back to HttpClient5's 3-minute default and hands + // out connections podman dropped long ago. That is + // testcontainers-java#7310 / #7593. + // + // Kept well under 10s because podman's window is configurable downwards. The cost + // is a reconnect per gap longer than this, which is cheap over a socket or pipe but + // is a full TLS handshake for tcp:// with DOCKER_TLS_VERIFY. IdleConnectionReuseTest + // pins its timings either side of this value, so changing it means changing them. + .setConnectionKeepAlive(CONNECTION_KEEP_ALIVE) .build()) .disableConnectionState() .build(); diff --git a/docker-java-transport-httpclient5/src/test/java/com/github/dockerjava/httpclient5/IdleConnectionReuseTest.java b/docker-java-transport-httpclient5/src/test/java/com/github/dockerjava/httpclient5/IdleConnectionReuseTest.java new file mode 100644 index 0000000000..c665ac39f0 --- /dev/null +++ b/docker-java-transport-httpclient5/src/test/java/com/github/dockerjava/httpclient5/IdleConnectionReuseTest.java @@ -0,0 +1,450 @@ +package com.github.dockerjava.httpclient5; + +import com.github.dockerjava.transport.DockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient.Request; +import com.github.dockerjava.transport.DockerHttpClient.Request.Method; +import com.github.dockerjava.transport.DockerHttpClient.Response; +import org.apache.hc.core5.http.NoHttpResponseException; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assume.assumeTrue; + +/** + * A pooled connection the daemon closed while it was idle gets handed back out, so the next request + * is written to a dead socket and no response ever comes back. + * + *

Reported as {@code NoHttpResponseException: localhost:2375 failed to respond} — the + * {@code localhost:2375} being the synthetic authority {@link ApacheDockerHttpClientImpl} gives + * every socket transport, not a TCP endpoint anyone connected to. These tests speak {@code tcp://} + * to a local server so they stay portable; the pool behaviour under test is transport-agnostic. + * + *

Two thresholds decide what happens, and every test here is placed relative to them: + *

+ * + *

Whether a request survives a dead connection also depends on its method. HttpClient5 retries + * idempotent ones, so a GET recovers silently while a POST — {@code /containers/create}, + * {@code /exec} — surfaces the failure. That asymmetry is why the bug reads as random flakiness, + * and why the reproductions people post are usually a create or an exec. + * + * @see testcontainers-java#7310 + * @see testcontainers-java#7593 + */ +public class IdleConnectionReuseTest { + + private static final long KEEP_ALIVE_MS = ApacheDockerHttpClientImpl.CONNECTION_KEEP_ALIVE.toMilliseconds(); + + /** + * Deliberately shorter than {@link ApacheDockerHttpClientImpl#CONNECTION_KEEP_ALIVE}, so that + * the window where the daemon has closed a connection the client still trusts is reachable. + * Podman's real window is 10s (twice its 5s {@code service_timeout}). + * + *

Floored well above zero on purpose: this reaches {@code Socket#setSoTimeout}, where zero + * means infinite, so a bare division would turn a shrunken keep-alive into a server + * that never drops a connection and tests that hang instead of failing. + */ + private static final long SERVER_IDLE_MS = Math.max(200, KEEP_ALIVE_MS / 4); + + /** Connection alive at both ends. Derived, so it cannot drift above the window it must sit in. */ + private static final long BEFORE_EITHER_GIVES_UP_MS = SERVER_IDLE_MS / 4; + + /** Daemon has closed it; the client does not know yet. */ + private static final long AFTER_SERVER_BEFORE_CLIENT_MS = (SERVER_IDLE_MS + KEEP_ALIVE_MS) / 2; + + /** Both have given up; the client must open a new one. */ + private static final long AFTER_CLIENT_GIVES_UP_MS = KEEP_ALIVE_MS + 1_000; + + /** + * Every test here depends on the daemon giving up before the client does. If + * {@link ApacheDockerHttpClientImpl#CONNECTION_KEEP_ALIVE} were ever lowered past + * {@link #SERVER_IDLE_MS}, the two windows would cross and the suite would report failures that + * say nothing about the code. Skip rather than mislead. + */ + @BeforeClass + public static void windowsMustNotCross() { + assumeTrue( + "server idle window " + SERVER_IDLE_MS + "ms must stay under the client keep-alive of " + + KEEP_ALIVE_MS + "ms", + SERVER_IDLE_MS < KEEP_ALIVE_MS + ); + } + + /** + * Skip, rather than fail, when the machine stalled long enough to invalidate the timing a test + * relies on. These tests are placed relative to a 2s production constant, so a GC pause or a + * loaded CI box between two requests can expire a pooled connection and turn an assertion into a + * report of a regression that did not happen. + * + *

Measured from the moment the first request returned, because that is when the connection + * was released and its expiry clock started + * ({@code PoolingHttpClientConnectionManager.release} -> {@code updateExpiry}). Starting any + * earlier would charge the first request's own cost — classloading, opening the connection — + * against the window, and skip runs that were never actually at risk. + * + * @param releasedNanos taken immediately after the first request of the pair returned + */ + private static void assumeNoStallPastTheKeepAliveWindow(long releasedNanos) { + long elapsedMs = (System.nanoTime() - releasedNanos) / 1_000_000L; + assumeTrue( + "stalled " + elapsedMs + "ms, past the " + KEEP_ALIVE_MS + "ms keep-alive window", + elapsedMs < KEEP_ALIVE_MS + ); + } + + /** + * The control: inside both windows the connection is reused, so a second connection appearing + * in the tests below means something expired rather than the server refusing to keep any. + */ + @Test + public void reusesPooledConnectionWhileBothEndsStillTrustIt() throws Exception { + try (IdleClosingServer server = new IdleClosingServer(); + DockerHttpClient client = clientFor(server)) { + + execute(client, Method.GET, "/_ping"); + long released = System.nanoTime(); + Thread.sleep(BEFORE_EITHER_GIVES_UP_MS); + assumeNoStallPastTheKeepAliveWindow(released); + execute(client, Method.POST, "/containers/create"); + + assertThat(server.connectionsAccepted()) + .as("connections accepted — the second request reused the first one") + .isEqualTo(1); + } + } + + /** + * The bug, and the test a fix has to turn green. Without a keep-alive fallback the client still + * trusts the connection three seconds after the daemon dropped it, writes the POST into it, and + * gets nothing back — {@code NoHttpResponseException}. Nothing about the request deserves to + * fail: the daemon is up and answering, as the other tests show against this same server. + * + *

The only test here that needs no stall guard: it requires the gap to be longer than + * the keep-alive window, and a stalled machine can only make it longer still. + */ + @Test + public void postSurvivesConnectionThatBothEndsHaveGivenUpOn() throws Exception { + try (IdleClosingServer server = new IdleClosingServer(); + DockerHttpClient client = clientFor(server)) { + + execute(client, Method.GET, "/_ping"); + Thread.sleep(AFTER_CLIENT_GIVES_UP_MS); + execute(client, Method.POST, "/containers/create"); + + assertThat(server.connectionsAccepted()) + .as("connections accepted — the expired one had to be replaced") + .isEqualTo(2); + assertThat(server.bytesWrittenIntoDeadConnections()) + .as("bytes written into the dead connection — the client discarded it up front") + .isZero(); + } + } + + /** + * Inside the keep-alive window the client still hands out the dead connection, and this is the + * one case where that is survivable: the GET is idempotent, so HttpClient5 retries it on a fresh + * connection and the caller sees nothing. Guards that recovery against being lost. + * + *

The sleep has to sit between the two thresholds for this to mean anything. Longer, and the + * client would discard the connection up front and the retry path would never be entered. + */ + @Test + public void getRetriesOntoAFreshConnectionWhenTheDaemonClosedTheIdleOne() throws Exception { + try (IdleClosingServer server = new IdleClosingServer(); + DockerHttpClient client = clientFor(server)) { + + execute(client, Method.GET, "/_ping"); + long released = System.nanoTime(); + Thread.sleep(AFTER_SERVER_BEFORE_CLIENT_MS); + assumeNoStallPastTheKeepAliveWindow(released); + execute(client, Method.GET, "/_ping"); + + assertThat(server.bytesWrittenIntoDeadConnections()) + .as("bytes written into the dead connection — proves it was leased out, " + + "so the retry is what recovered rather than a proactive replacement") + .isPositive(); + assertThat(server.connectionsAccepted()) + .as("connections accepted — the retry opened a second one") + .isEqualTo(2); + } + } + + /** + * The gap the keep-alive fallback does not close, pinned so nobody mistakes it for covered. A + * connection that dies inside the window — a daemon restart, a relay restart, a network + * blip, or a peer whose idle timeout is shorter than ours — is still handed out, and a POST on + * it still fails, because {@code setValidateAfterInactivity(NEG_ONE_SECOND)} disables + * revalidation on lease and HttpClient5 will not retry a non-idempotent method. + * + *

Closing it needs either a bounded revalidation (which needs the custom sockets to honour + * {@code setSoTimeout} first — that is why revalidation was disabled, see #1726) or a retry + * strategy for {@code NoHttpResponseException} on a reused connection. Delete this test when one + * of those lands. + */ + @Test + public void knownGap_postStillFailsWhenTheConnectionDiesInsideTheKeepAliveWindow() throws Exception { + try (IdleClosingServer server = new IdleClosingServer(); + DockerHttpClient client = clientFor(server)) { + + execute(client, Method.GET, "/_ping"); + long released = System.nanoTime(); + Thread.sleep(AFTER_SERVER_BEFORE_CLIENT_MS); + assumeNoStallPastTheKeepAliveWindow(released); + + assertThatThrownBy(() -> execute(client, Method.POST, "/containers/create")) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(NoHttpResponseException.class) + .hasMessageContaining("failed to respond"); + } + } + + private DockerHttpClient clientFor(IdleClosingServer server) { + String loopback = InetAddress.getLoopbackAddress().getHostAddress(); + return new ApacheDockerHttpClient.Builder() + .dockerHost(URI.create("tcp://" + loopback + ":" + server.port())) + .build(); + } + + /** + * The body has to be read to the end for the connection to reach the pool at all: + * {@code ApacheResponse.close()} calls {@code request.abort()}, which discards a connection + * still mid-response. Draining first is what {@code docker-java-core} does on every call, and it + * is the only reason a connection is ever pooled to be reused — or, here, reused after the + * daemon has closed it. + */ + private void execute(DockerHttpClient client, Method method, String path) { + Request request = Request.builder() + .method(method) + .path(path) + .bodyBytes(method == Method.POST ? "{}".getBytes(StandardCharsets.UTF_8) : null) + .build(); + + try (Response response = client.execute(request)) { + assertThat(response.getStatusCode()).as("status code of %s %s", method, path).isEqualTo(200); + drainFully(response.getBody()); + } + } + + private static void drainFully(InputStream body) { + try { + byte[] buffer = new byte[64]; + while (body.read(buffer) != -1) { + // to the end, so the connection is released to the pool + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * A minimal HTTP/1.1 server standing in for the Docker API socket: keep-alive responses that, + * like podman, advertise no {@code Keep-Alive} timeout, and a connection dropped once it has sat + * idle for {@link #SERVER_IDLE_MS}. + * + *

Dropping it means FIN and then absorbing whatever still arrives, rather than closing + * outright. A full close makes the peer's next write draw a reset, which is the "broken pipe" + * shape of this bug; FIN-and-absorb is what the relay in front of the daemon does on Windows and + * macOS, and it is what produces the reported empty response. + */ + private static final class IdleClosingServer implements Closeable { + + private static final String RESPONSE = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK"; + + private static final String CONTENT_LENGTH = "Content-Length:"; + + /** + * Applied once a request has started arriving, so a slow or descheduled client mid-message + * is never mistaken for an idle connection. Only the wait for a request's first byte is + * bounded by {@link #SERVER_IDLE_MS}. + */ + private static final int MID_REQUEST_TIMEOUT_MS = 30_000; + + private final ServerSocket serverSocket; + + private final ExecutorService acceptor = Executors.newSingleThreadExecutor(IdleClosingServer::daemon); + + private final ExecutorService connections = Executors.newCachedThreadPool(IdleClosingServer::daemon); + + private final Collection open = new ConcurrentLinkedQueue<>(); + + private final AtomicInteger accepted = new AtomicInteger(); + + private final AtomicInteger absorbedBytes = new AtomicInteger(); + + private volatile boolean closed; + + IdleClosingServer() throws IOException { + serverSocket = new ServerSocket(0, 0, InetAddress.getLoopbackAddress()); + acceptor.submit(this::acceptLoop); + } + + private static Thread daemon(Runnable runnable) { + Thread thread = new Thread(runnable, "idle-closing-server"); + thread.setDaemon(true); + return thread; + } + + int port() { + return serverSocket.getLocalPort(); + } + + int connectionsAccepted() { + return accepted.get(); + } + + /** + * Bytes the client sent on a connection this server had already given up on. Non-zero means + * the client leased a dead connection out of its pool; zero means it never did. + */ + int bytesWrittenIntoDeadConnections() { + return absorbedBytes.get(); + } + + private void acceptLoop() { + while (!serverSocket.isClosed()) { + try { + Socket socket = serverSocket.accept(); + accepted.incrementAndGet(); + open.add(socket); + connections.submit(() -> serve(socket)); + } catch (IOException closedNow) { + return; + } + } + } + + private void serve(Socket socket) { + try { + InputStream in = socket.getInputStream(); + OutputStream out = socket.getOutputStream(); + while (readRequest(socket, in)) { + // No Keep-Alive header, matching podman: the client is never told when this + // connection will go away. + out.write(RESPONSE.getBytes(StandardCharsets.US_ASCII)); + out.flush(); + } + socket.shutdownOutput(); + absorb(in); + } catch (IOException hungUp) { + // nothing useful to do + } finally { + // Drop it before closing: close() only needs the sockets still parked in absorb(), + // and keeping finished ones would grow `open` for the server's whole lifetime. + open.remove(socket); + closeQuietly(socket); + } + } + + /** + * @return false once the connection has sat idle for {@link #SERVER_IDLE_MS} without a new + * request starting, or the peer has gone — either way it should be dropped + */ + private boolean readRequest(Socket socket, InputStream in) throws IOException { + socket.setSoTimeout((int) SERVER_IDLE_MS); + String requestLine; + do { + try { + requestLine = readLine(in); + } catch (SocketTimeoutException idle) { + return false; + } + if (requestLine == null) { + return false; + } + } while (requestLine.isEmpty()); + + // A request has started; from here a stall is the client being slow, not idle. + socket.setSoTimeout(MID_REQUEST_TIMEOUT_MS); + int contentLength = 0; + for (String line = readLine(in); !"".equals(line); line = readLine(in)) { + if (line == null) { + return false; + } + if (line.regionMatches(true, 0, CONTENT_LENGTH, 0, CONTENT_LENGTH.length())) { + contentLength = Integer.parseInt(line.substring(CONTENT_LENGTH.length()).trim()); + } + } + return drain(in, contentLength); + } + + private void absorb(InputStream in) throws IOException { + while (!closed) { + try { + if (in.read() == -1) { + return; + } + // discard, but record: the client is writing into a connection that will never + // answer, which is only possible if its pool handed the dead one back out + absorbedBytes.incrementAndGet(); + } catch (SocketTimeoutException keepAbsorbing) { + // never reset the connection, however long the client takes + } + } + } + + private String readLine(InputStream in) throws IOException { + StringBuilder line = new StringBuilder(); + int c; + while ((c = in.read()) != -1) { + if (c == '\n') { + int end = line.length(); + if (end > 0 && line.charAt(end - 1) == '\r') { + line.setLength(end - 1); + } + return line.toString(); + } + line.append((char) c); + } + return null; + } + + private boolean drain(InputStream in, int bytes) throws IOException { + for (int i = 0; i < bytes; i++) { + if (in.read() == -1) { + return false; + } + } + return true; + } + + @Override + public void close() { + closed = true; + closeQuietly(serverSocket); + for (Socket socket : open) { + closeQuietly(socket); + } + acceptor.shutdownNow(); + connections.shutdownNow(); + } + + private static void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException ignored) { + // nothing useful to do + } + } + } +} diff --git a/docker-java/src/test/java/com/github/dockerjava/cmd/IdleConnectionReuseIT.java b/docker-java/src/test/java/com/github/dockerjava/cmd/IdleConnectionReuseIT.java new file mode 100644 index 0000000000..5122cc56ac --- /dev/null +++ b/docker-java/src/test/java/com/github/dockerjava/cmd/IdleConnectionReuseIT.java @@ -0,0 +1,237 @@ +package com.github.dockerjava.cmd; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.transport.NamedPipeSocket; +import com.github.dockerjava.transport.UnixSocket; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static com.github.dockerjava.core.DockerRule.DEFAULT_IMAGE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.Assume.assumeTrue; + +/** + * A POST issued after the daemon has closed an idle pooled connection must still succeed. + * + *

This is the real-daemon counterpart to {@code IdleConnectionReuseTest}, which covers the same + * behaviour against a fake server over {@code tcp://}. What that test cannot cover is the transports + * this actually gets reported on — {@code unix://} and {@code npipe://} — so this one runs whatever + * {@code DOCKER_HOST} points at. + * + *

It skips against Docker, and that is the point. The failure needs a daemon that + * hangs up on idle connections. Dockerd never does, so on Docker this test would pass whether or not + * the bug is present — a false green. Podman closes after twice its {@code service_timeout} (10s by + * default), so the bug is reachable there. Rather than guess from the daemon's identity — a podman + * with {@code service_timeout=0} keeps connections open too — {@link #hangsUpOnIdleConnections} + * measures the actual behaviour on a raw socket and the test skips when it does not reproduce. + * + *

Budget roughly {@code docker.java.test.probeMs} (12s) to observe the hang-up plus + * {@code docker.java.test.idleMs} (15s) to provoke it. Note that failsafe is configured with + * {@code rerunFailingTestsCount=5}, so a genuine failure is retried; the probe result is cached per + * JVM so those retries do not re-probe. + * + * @see testcontainers-java#7310 + */ +public class IdleConnectionReuseIT extends CmdIT { + + public static final Logger LOG = LoggerFactory.getLogger(IdleConnectionReuseIT.class); + + /** + * How long to provoke the daemon for. Must exceed its idle window; podman's default is 10s. + * Override with {@code -Ddocker.java.test.idleMs=...} for a daemon tuned differently. + */ + private static final int IDLE_MS = Integer.getInteger("docker.java.test.idleMs", 15_000); + + /** + * How long to wait for the hang-up before concluding there is no idle timeout. Kept separate + * from {@link #IDLE_MS}, and shorter: this is paid on every run, including the runs that go on to + * skip, so it should not carry the provocation sleep's safety margin. + */ + private static final int PROBE_TIMEOUT_MS = Integer.getInteger("docker.java.test.probeMs", 12_000); + + private static final String PING_REQUEST = + "GET /_ping HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Connection: keep-alive\r\n" + + "\r\n"; + + /** Cached so failsafe's reruns, and any further tests here, do not each pay the probe. */ + private static Boolean daemonHangsUp; + + @Test + public void postSucceedsAfterTheDaemonClosedTheIdleConnection() throws Exception { + URI dockerHost = dockerRule.getConfig().getDockerHost(); + + assumeTrue( + "the raw-socket probe speaks plaintext HTTP, and DOCKER_HOST is " + dockerHost + + " with an SSL config present. Skipping conservatively: a non-null SSLConfig is " + + "taken as TLS in use, which may be stricter than necessary", + !("tcp".equals(dockerHost.getScheme()) && dockerRule.getConfig().getSSLConfig() != null) + ); + + boolean canHangUp; + try { + canHangUp = hangsUpOnIdleConnections(dockerHost); + } catch (IOException unreachable) { + // Environmental: the daemon would not talk to us. Skip rather than fail — but note that + // a bug in the probe itself is a RuntimeException and is deliberately left to propagate, + // so this test cannot rot into a permanent silent skip. + LOG.warn("probe against {} could not complete; skipping", dockerHost, unreachable); + canHangUp = false; + } + assumeTrue( + "this daemon (" + dockerHost + ") kept an idle connection open for " + PROBE_TIMEOUT_MS + + "ms, so it cannot exhibit the bug — expected for Docker, and for podman with " + + "service_timeout=0", + canHangUp + ); + + DockerClient client = dockerRule.getClient(); + + // Opens a connection and returns it to the pool. + client.pingCmd().exec(); + + LOG.info("idling {}ms so the daemon closes the pooled connection", IDLE_MS); + Thread.sleep(IDLE_MS); + + // The non-idempotent request. HttpClient5 does not retry POST, so if the pool hands out the + // connection the daemon just closed, this is where it surfaces as + // "NoHttpResponseException: ... failed to respond". + CreateContainerResponse container = client.createContainerCmd(DEFAULT_IMAGE) + .withCmd("true") + .exec(); + + assertThat(container.getId(), not(is(emptyString()))); + } + + private static synchronized boolean hangsUpOnIdleConnections(URI dockerHost) throws IOException { + if (daemonHangsUp == null) { + daemonHangsUp = probeForHangUp(dockerHost); + } + return daemonHangsUp; + } + + /** + * Measures the daemon rather than trusting its name: opens a raw connection, completes one + * request on it, then waits to be hung up on. + * + *

All of it runs on a worker thread, request included. {@link NamedPipeSocket} and + * {@link UnixSocket} ignore {@code setSoTimeout} — the same defect that keeps stale-connection + * validation disabled, see #1726 — so no read here can be bounded in place, and a daemon that + * accepts a connection without answering would otherwise hang the build indefinitely. + * + * @return true if the daemon closed the idle connection within {@link #PROBE_TIMEOUT_MS} + * @throws IOException if the daemon could not be reached or would not answer + */ + private static boolean probeForHangUp(URI dockerHost) throws IOException { + ExecutorService worker = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "idle-connection-probe"); + thread.setDaemon(true); + return thread; + }); + try (Socket socket = openRawSocket(dockerHost)) { + Future hangUp = worker.submit(() -> { + OutputStream out = socket.getOutputStream(); + out.write(PING_REQUEST.getBytes(StandardCharsets.US_ASCII)); + out.flush(); + + InputStream in = socket.getInputStream(); + if (!readPastResponseHeaders(in)) { + throw new IOException("daemon closed the connection before answering /_ping"); + } + + long idleSince = System.nanoTime(); + byte[] discard = new byte[256]; + while (in.read(discard) != -1) { + // trailing body bytes; keep waiting for the close + } + return (System.nanoTime() - idleSince) / 1_000_000L; + }); + + try { + long afterMs = hangUp.get(PROBE_TIMEOUT_MS, TimeUnit.MILLISECONDS); + LOG.info("{} hung up on an idle connection after {}ms", dockerHost, afterMs); + return true; + } catch (TimeoutException stillOpen) { + LOG.info("{} kept an idle connection open for {}ms", dockerHost, PROBE_TIMEOUT_MS); + return false; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new IOException(cause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } finally { + // The socket is already closed by then, which is what unblocks a read still in flight. + worker.shutdownNow(); + } + } + + /** + * @return false if the connection ended before a complete set of response headers arrived + */ + private static boolean readPastResponseHeaders(InputStream in) throws IOException { + ByteArrayOutputStream seen = new ByteArrayOutputStream(); + int c; + while ((c = in.read()) != -1) { + seen.write(c); + if (seen.size() >= 4) { + byte[] bytes = seen.toByteArray(); + int end = bytes.length; + if (bytes[end - 4] == '\r' && bytes[end - 3] == '\n' + && bytes[end - 2] == '\r' && bytes[end - 1] == '\n') { + return true; + } + } + } + return false; + } + + private static Socket openRawSocket(URI dockerHost) throws IOException { + String scheme = dockerHost.getScheme(); + if ("unix".equals(scheme)) { + return UnixSocket.get(dockerHost.getPath()); + } + if ("npipe".equals(scheme)) { + NamedPipeSocket socket = new NamedPipeSocket(dockerHost.getPath()); + // The endpoint is ignored: NamedPipeSocket connects to the path it was constructed with. + // Passing an unresolved placeholder rather than null, so this does not depend on the + // override tolerating a null argument that java.net.Socket documents as an NPE. + socket.connect(InetSocketAddress.createUnresolved("localhost", 0), 10_000); + return socket; + } + if ("tcp".equals(scheme)) { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress(dockerHost.getHost(), dockerHost.getPort()), 10_000); + return socket; + } + throw new IOException("unsupported DOCKER_HOST scheme: " + scheme); + } +}