From a8358f91e14ab75a6bd673b685106ed4a6f59c98 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:54:55 +0100 Subject: [PATCH 01/57] feat(qwp): add checkError, hasRowInProgress and pendingRowCount probes to Sender Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 38 +++++++ .../qwp/client/QwpWebSocketSender.java | 15 +++ .../io/questdb/client/impl/PooledSender.java | 15 +++ .../qwp/client/SenderProbeMethodsTest.java | 103 ++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 7617aa643..67142c594 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -415,6 +415,21 @@ default Sender charColumn(CharSequence name, char value) { throw new LineSenderException("current protocol version does not support char"); } + /** + * Throws the error this sender has latched, if any: the terminal error of + * the WebSocket I/O loop (server rejection, exhausted reconnect budget, + * authentication failure) or a connection error. Publishes nothing and + * never blocks. Used by the connection pool to decide whether a returned + * sender may be handed to the next borrower. + *
+ * Only the WebSocket QWP transport latches errors; on other transports this + * is a no-op. + * + * @throws LineSenderException if a terminal or connection error is latched + */ + default void checkError() { + } + /** * Close this Sender. *
@@ -639,6 +654,18 @@ default long getAckedFsn() { return -1L; } + /** + * Whether a row has been started with {@link #table(CharSequence)} and not + * yet finished with {@link #at}, {@link #atNow()} or {@link #cancelRow()}. + * Only the WebSocket QWP transport tracks this; other transports return + * {@code false}. + * + * @return {@code true} while a row is in progress + */ + default boolean hasRowInProgress() { + return false; + } + /** * Add a column with a 32-bit signed integer value. * @@ -705,6 +732,17 @@ default Sender long256Column(CharSequence name, long l0, long l1, long l2, long Sender longColumn(CharSequence name, long value); + /** + * Number of rows appended with {@link #at}/{@link #atNow()} that have not + * been shipped to the server yet. Only the WebSocket QWP transport counts + * pending rows; other transports return {@code 0}. + * + * @return rows buffered and not yet flushed + */ + default int pendingRowCount() { + return 0; + } + /** * Advisory request to start a fresh symbol-dictionary epoch. The reset * runs at the next {@code table(...)} call that finds all published data diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f591b565f..609c45242 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1656,6 +1656,11 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { return this; } + @Override + public void checkError() { + checkNotClosed(); + } + /** * Closes the sender: flushes user-thread state into the engine, drains * acked data within {@code close_flush_timeout}, stops the I/O loop, @@ -2899,6 +2904,11 @@ public long getTotalServerErrors() { return counters.serverErrors.get(); } + @Override + public boolean hasRowInProgress() { + return currentTableBuffer != null && currentTableBuffer.hasInProgressRow(); + } + /** * Adds an INT column value to the current row. * @@ -3133,6 +3143,11 @@ public CursorWebSocketSendLoop.ReconnectFactory newBackgroundReconnectFactory( return new ReconnectSupplier(stopFlag, "drainer stop requested during connect"); } + @Override + public int pendingRowCount() { + return pendingRowCount; + } + /** * Test seam: installs the per-attempt WebSocket client factory override * consulted by {@code newWebSocketClient()} inside the connect walk. diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 095f64abd..8bb153c49 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -131,6 +131,11 @@ public Sender charColumn(CharSequence name, char value) { return this; } + @Override + public void checkError() { + slot.live(generation).checkError(); + } + /** * Flushes pending rows and returns this lease's slot to the pool. Does not * actually close the underlying {@link Sender}; that only happens when the @@ -276,6 +281,11 @@ public long getAckedFsn() { return slot.live(generation).getAckedFsn(); } + @Override + public boolean hasRowInProgress() { + return slot.live(generation).hasRowInProgress(); + } + @Override public Sender intColumn(CharSequence name, int value) { slot.live(generation).intColumn(name, value); @@ -330,6 +340,11 @@ public Sender longColumn(CharSequence name, long value) { return this; } + @Override + public int pendingRowCount() { + return slot.live(generation).pendingRowCount(); + } + @Override public void reset() { slot.live(generation).reset(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java new file mode 100644 index 000000000..f6354ae25 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -0,0 +1,103 @@ +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * The three probe methods the pool uses at release: {@code checkError()}, + * {@code hasRowInProgress()} and {@code pendingRowCount()}. Exercised on a + * real WebSocket sender against the in-repo test server, which acks every + * frame. + */ +public class SenderProbeMethodsTest { + + @Test + public void testHealthySenderReportsBufferState() throws Exception { + try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.checkError(); // healthy: no throw + Assert.assertFalse(sender.hasRowInProgress()); + Assert.assertEquals(0, sender.pendingRowCount()); + + sender.table("t").longColumn("v", 1L); // row started, not committed + Assert.assertTrue(sender.hasRowInProgress()); + Assert.assertEquals(0, sender.pendingRowCount()); + + sender.cancelRow(); + Assert.assertFalse(sender.hasRowInProgress()); + + sender.table("t").longColumn("v", 2L).atNow(); + Assert.assertFalse(sender.hasRowInProgress()); + Assert.assertEquals(1, sender.pendingRowCount()); + + sender.flush(); + Assert.assertEquals(0, sender.pendingRowCount()); + } + } + } + + @Test + public void testCheckErrorThrowsLatchedTerminal() throws Exception { + try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { + server.setRejectWithStatus(401, "Unauthorized"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // async connect: build() returns, the I/O thread hits the 401 wall and latches a terminal + try (Sender sender = Sender.fromConfig(config(server.getPort()) + + "initial_connect_retry=async;reconnect_max_duration_millis=300;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { + long deadline = System.currentTimeMillis() + 5_000; + LineSenderException seen = null; + while (System.currentTimeMillis() < deadline) { + try { + sender.checkError(); + } catch (LineSenderException e) { + seen = e; + break; + } + Thread.sleep(20); + } + Assert.assertNotNull("checkError() must throw the latched auth terminal", seen); + } + } + } + + @Test + public void testDefaultsOnNonWebSocketTransport() { + try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { + sender.checkError(); // default: no-op + Assert.assertFalse(sender.hasRowInProgress()); // default: false + Assert.assertEquals(0, sender.pendingRowCount()); // default: 0 + } + } + + private static String config(int port) { + return "ws::addr=localhost:" + port + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;"; + } + + private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { + private TestWebSocketServer.ClientHandler currentClient; + private long nextSeq; + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + nextSeq = 0; + } + try { + client.sendBinary(QwpWireTestUtils.buildAck(nextSeq++)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} From 0f9155efe00be580849d29efe9bdc0921448d852 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:00:55 +0100 Subject: [PATCH 02/57] feat(pool)!: pooled close() releases the slot without flushing; flush_on_close restores the old path Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/QuestDB.java | 11 +- .../io/questdb/client/impl/PooledSender.java | 99 ++++--- .../io/questdb/client/impl/SenderPool.java | 20 +- .../client/test/impl/RecordingSender.java | 247 ++++++++++++++++++ .../impl/SenderPoolCloseSemanticsTest.java | 228 ++++++++++++++++ .../client/test/impl/SenderPoolTest.java | 8 +- 6 files changed, 569 insertions(+), 44 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/RecordingSender.java create mode 100644 core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index cf669a166..9c0efada5 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -137,8 +137,15 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token /** * Borrows a {@link Sender} from the pool. The caller MUST call * {@link Sender#close()} on the returned instance to release it back to - * the pool. {@code close()} on a pooled Sender flushes pending rows - * before returning to the pool; a real disconnect only happens at + * the pool. Close releases, flush ships, drain waits: {@code close()} on a + * pooled Sender returns the slot and by default ships nothing, the rows go + * out when an auto-flush threshold trips on a later append or when the + * housekeeper settles the idle slot (at most one {@code auto_flush_interval} + * later). Call {@link Sender#flush()} to ship now and + * {@link Sender#drain(long)} to wait for the server. Set + * {@code flush_on_close=on} to have {@code close()} flush, as before 1.4. + * Note that in this client "flush" means published to the store-and-forward + * ring, not acknowledged by the server. A real disconnect only happens at * {@link #close()} on this {@code QuestDB} handle. *

* Allocation: zero at steady state -- the returned instance is a diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 8bb153c49..a6de0fba0 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -25,6 +25,7 @@ package io.questdb.client.impl; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.std.Decimal128; @@ -44,9 +45,11 @@ * {@link Sender} call to the reused {@link SenderSlot}'s delegate, validating * that generation first via {@link SenderSlot#live(long)}. *

- * Behaviour difference from a raw Sender: {@link #close()} flushes the buffer - * and returns the slot to the pool. The underlying Sender is only truly closed - * when {@link io.questdb.client.QuestDB#close()} shuts the pool down. + * Behaviour difference from a raw Sender: {@link #close()} returns the slot to + * the pool and, by default, ships nothing; rows ship by threshold or by the + * pool's housekeeper. With {@code flush_on_close=on} close flushes first. The + * underlying Sender is only truly closed when + * {@link io.questdb.client.QuestDB#close()} shuts the pool down. *

* Because the slot is reused across borrows, this wrapper -- not the slot -- * carries the lease identity. A stale handle (held after {@link #close()}, with @@ -137,56 +140,58 @@ public void checkError() { } /** - * Flushes pending rows and returns this lease's slot to the pool. Does not - * actually close the underlying {@link Sender}; that only happens when the - * owning {@code QuestDB} is closed. If the pool is already shutting down - * when this lease comes home, the underlying Sender is torn down here, on - * the calling thread (see {@link SenderPool#close()}). + * Returns this lease's slot to the pool. By default nothing is shipped: + * the rows stay in the slot's buffer and go out when an auto-flush + * threshold trips on a later append or when the pool's housekeeper settles + * the idle slot. With {@code flush_on_close=on} the rows are flushed first, + * as before 1.4. Does not close the underlying {@link Sender}; that only + * happens when the owning {@code QuestDB} is closed. If the pool is already + * shutting down when this lease comes home, the underlying Sender is torn + * down here, on the calling thread, which ships whatever it still buffers. + *

+ * A row left in progress is cancelled, the slot is returned, and then a + * {@link LineSenderException} is thrown so the mistake is visible. A + * delegate that has latched a terminal error is retired instead of being + * returned, and the error is rethrown here. *

* Idempotent for sequential reuse: a stale generation (the lease was - * already returned and the slot possibly re-borrowed) is a no-op, and the - * pool re-checks the generation under its lock, so a duplicate close can - * never re-enqueue a slot a different borrower now owns. Concurrent - * double-close of ONE lease from two threads is outside the Sender - * threading contract; the flush below revalidates the lease via - * {@link SenderSlot#live(long)} so a racing duplicate typically fails with - * {@code IllegalStateException} rather than flushing into a re-borrowed - * slot, but two threads racing inside a single lease are not fully - * protected. + * already returned and the slot possibly re-borrowed) is a no-op. */ @Override public void close() { if (generation != slot.generation()) { return; } - // Track normal completion rather than catching a specific throwable - // type. flush() can exit abnormally with an Error (AssertionError - // under -ea, OutOfMemoryError, ...) as well as a RuntimeException; - // keying the recycle decision off normal completion treats every - // abnormal exit as unrecyclable, which is the fail-safe default. - boolean flushed = false; + if (slot.pool().isFlushOnClose()) { + closeFlushing(); + return; + } + boolean cancelledRow = false; + boolean recycle = false; try { - // live(), not delegate(): re-validating the generation right - // before the flush narrows the duplicate-close race documented - // above. A stale duplicate throws here, and the finally routes it - // to discardBroken, which drops it on the same stale-generation - // check under the pool lock. - slot.live(generation).flush(); - flushed = true; + // live(), not delegate(): re-validate the generation right before + // touching the delegate, narrowing the duplicate-close race. + Sender delegate = slot.live(generation); + delegate.checkError(); + if (delegate.hasRowInProgress()) { + delegate.cancelRow(); + cancelledRow = true; + } + recycle = true; } finally { - if (flushed) { + if (recycle) { slot.pool().giveBack(this); } else { - // flush() did not complete normally. Sender does not clear its - // buffer on flush failure (see Sender Javadoc), and WebSocket - // transport latches the failure for good. Either way the slot - // is unsafe to recycle: the next borrower would inherit the - // failed rows or a dead connection. The original throwable - // propagates naturally once this finally returns -- no explicit - // rethrow needed. + // The probe or the cancel threw: the delegate has latched an + // error (or the generation went stale). Retire it; the + // original throwable propagates once this finally returns. slot.pool().discardBroken(this); } } + if (cancelledRow) { + throw new LineSenderException("row in progress at close(): the row was cancelled and the sender returned to the pool; " + + "finish rows with at(), atNow() or cancelRow() before close()"); + } } @Override @@ -419,4 +424,22 @@ public boolean hasSameSlotForTesting(PooledSender that) { SenderSlot slot() { return slot; } + + /** + * The {@code flush_on_close=on} path: today's close. Ships the buffer, then + * returns the slot; any abnormal exit of the flush retires the slot. + */ + private void closeFlushing() { + boolean flushed = false; + try { + slot.live(generation).flush(); + flushed = true; + } finally { + if (flushed) { + slot.pool().giveBack(this); + } else { + slot.pool().discardBroken(this); + } + } + } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index bc93309d3..33d5cdce9 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -251,6 +251,7 @@ public final class SenderPool implements AutoCloseable { // production; lifecycle tests use it to acknowledge distinct retries // without inspecting transient Condition queue membership. private volatile Runnable creationWaitRetryHook; + private volatile boolean flushOnClose; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -1238,7 +1239,10 @@ public PooledSender borrow() { throw new LineSenderException("QuestDB handle is closed"); } if (!available.isEmpty()) { - SenderSlot s = available.pollFirst(); + // LIFO: the most recently returned slot is the warmest and + // its buffer is the fullest; cold slots sink to the head + // where reapIdle() visits them first. + SenderSlot s = available.pollLast(); // Stamp a fresh lease id under the lock so the PooledSender // wrapper handed out can be told apart from any prior, // now-stale borrow of the same slot. @@ -1952,6 +1956,16 @@ public int totalSize() { } } + /** + * Whether a pooled {@code close()} ships the lease's rows before releasing + * the slot (the pre-1.4 behaviour) or only releases it. Off by default: + * rows then ship when an auto-flush threshold trips on a later append, or + * when the housekeeper settles the idle slot. + */ + public boolean isFlushOnClose() { + return flushOnClose; + } + /** * Snapshot of the number of SF slots currently retired because a * delegate {@code close()} returned with the slot flock still held after @@ -1972,6 +1986,10 @@ public int leakedSlotCount() { } } + public void setFlushOnClose(boolean flushOnClose) { + this.flushOnClose = flushOnClose; + } + private SenderSlot createSlot(IntFunction factory, int slotIndex) { Sender delegate = factory.apply(slotIndex); try { diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java new file mode 100644 index 000000000..714ffb522 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -0,0 +1,247 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.std.bytes.DirectByteSlice; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * In-memory {@link Sender} double for pool tests. Records calls, models the + * buffer state the pool asks about, and can be told to fail or to block. + * Never touches the network. Not final: one test subclasses it to make a + * single method throw. + */ +class RecordingSender implements Sender { + final AtomicInteger cancelCount = new AtomicInteger(); + final AtomicInteger closeCount = new AtomicInteger(); + final AtomicInteger deferredFlushCount = new AtomicInteger(); + final AtomicInteger flushCount = new AtomicInteger(); + volatile boolean deferredFramesOpen; + volatile RuntimeException flushError; // thrown by flush() when set + volatile CountDownLatch flushGate; // flush() blocks on it when set + volatile RuntimeException latchedError; // thrown by checkError() when set + volatile int pendingRows; + volatile boolean rowInProgress; + volatile long rowsAppended; + + @Override + public void at(long timestamp, ChronoUnit unit) { + rowInProgress = false; + pendingRows++; + rowsAppended++; + } + + @Override + public void at(Instant timestamp) { + at(0L, ChronoUnit.MICROS); + } + + @Override + public void atNow() { + at(0L, ChronoUnit.MICROS); + } + + @Override + public Sender boolColumn(CharSequence name, boolean value) { + return this; + } + + @Override + public DirectByteSlice bufferView() { + return null; + } + + @Override + public void cancelRow() { + cancelCount.incrementAndGet(); + rowInProgress = false; + } + + @Override + public void checkError() { + RuntimeException e = latchedError; + if (e != null) { + throw e; + } + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + @Override + public Sender doubleArray(CharSequence name, double[] values) { + return this; + } + + @Override + public Sender doubleArray(CharSequence name, double[][] values) { + return this; + } + + @Override + public Sender doubleArray(CharSequence name, double[][][] values) { + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray values) { + return this; + } + + @Override + public Sender doubleColumn(CharSequence name, double value) { + return this; + } + + @Override + public void flush() { + awaitGate(); + RuntimeException e = flushError; + if (e != null) { + throw e; + } + // Like the real sender: publishes nothing when there is nothing to + // ship and no open group to commit, so count only real work. + if (pendingRows > 0 || deferredFramesOpen) { + flushCount.incrementAndGet(); + } + pendingRows = 0; + deferredFramesOpen = false; + } + + /* Phase B (Task 9): re-enable these three overrides once Sender gains + flushDeferred(), hasDeferredFrames() and rowsAppended(). + + @Override + public long flushDeferred() { + awaitGate(); + boolean hadRows = pendingRows > 0; + if (!hadRows) { + return -1L; // nothing pending: the real sender publishes nothing + } + pendingRows = 0; + deferredFramesOpen = true; + return deferredFlushCount.incrementAndGet(); + } + + @Override + public boolean hasDeferredFrames() { + return deferredFramesOpen; + } + + @Override + public long rowsAppended() { + return rowsAppended; + } + */ + + @Override + public boolean hasRowInProgress() { + return rowInProgress; + } + + @Override + public Sender longArray(CharSequence name, long[] values) { + return this; + } + + @Override + public Sender longArray(CharSequence name, long[][] values) { + return this; + } + + @Override + public Sender longArray(CharSequence name, long[][][] values) { + return this; + } + + @Override + public Sender longArray(CharSequence name, LongArray values) { + return this; + } + + @Override + public Sender longColumn(CharSequence name, long value) { + return this; + } + + @Override + public int pendingRowCount() { + return pendingRows; + } + + @Override + public void reset() { + pendingRows = 0; + rowInProgress = false; + } + + @Override + public Sender stringColumn(CharSequence name, CharSequence value) { + return this; + } + + @Override + public Sender symbol(CharSequence name, CharSequence value) { + return this; + } + + @Override + public Sender table(CharSequence table) { + rowInProgress = true; + return this; + } + + @Override + public Sender timestampColumn(CharSequence name, long value, ChronoUnit unit) { + return this; + } + + @Override + public Sender timestampColumn(CharSequence name, Instant value) { + return this; + } + + private void awaitGate() { + CountDownLatch gate = flushGate; + if (gate != null) { + try { + gate.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java new file mode 100644 index 000000000..d0de18d55 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -0,0 +1,228 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.impl.PooledSender; +import io.questdb.client.impl.SenderPool; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +/** + * The pooled close() contract after the release-mode change: close releases, + * flush ships, drain waits. Drives {@link SenderPool} through its + * {@code senderFactory} seam with {@link RecordingSender} doubles, so nothing + * touches the network. The config string is parsed for store-and-forward + * settings only. + */ +public class SenderPoolCloseSemanticsTest { + private static final String CONFIG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + @Test + public void testCloseDoesNotFlushByDefault() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Assert.assertFalse(pool.isFlushOnClose()); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + Assert.assertEquals("close() must not ship rows", 0, delegate.flushCount.get()); + Assert.assertEquals("rows stay in the slot's buffer", 1, delegate.pendingRowCount()); + Assert.assertEquals("slot must be back on the free list", 1, pool.availableSize()); + } + } + + @Test + public void testFlushOnCloseRestoresShippingAtClose() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + pool.setFlushOnClose(true); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + Assert.assertEquals(1, delegate.flushCount.get()); + Assert.assertEquals(0, delegate.pendingRowCount()); + Assert.assertEquals(1, pool.availableSize()); + } + } + + @Test + public void testLatchedErrorAtCloseRetiresSlotAndThrows() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + RecordingSender delegate = built.get(0); + delegate.latchedError = new LineSenderException("latched terminal"); + try { + first.close(); + Assert.fail("close() must rethrow the latched terminal"); + } catch (LineSenderException e) { + Assert.assertEquals("latched terminal", e.getMessage()); + } + Assert.assertEquals("dead slot must be retired", 0, pool.totalSize()); + Assert.assertEquals("retired delegate must be closed", 1, delegate.closeCount.get()); + try (Sender second = pool.borrow()) { + Assert.assertEquals("next borrow must build a fresh delegate", 2, built.size()); + Assert.assertNotSame(delegate, ((PooledSender) second).getDelegateForTesting()); + } + } + } + + @Test + public void testPartialRowAtCloseIsCancelledAndSlotKept() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + first.table("t").longColumn("v", 1L); // no at(): row in progress + try { + first.close(); + Assert.fail("close() with a row in progress must throw after releasing"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("row in progress")); + } + RecordingSender delegate = built.get(0); + Assert.assertEquals(1, delegate.cancelCount.get()); + Assert.assertEquals("slot stays in service", 1, pool.availableSize()); + try (Sender second = pool.borrow()) { + Assert.assertSame("same slot is reused", delegate, ((PooledSender) second).getDelegateForTesting()); + } + } + } + + @Test + public void testCancelRowThrowingAtCloseRetiresSlot() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + RecordingSender delegate = built.get(0); + first.table("t"); + // A terminal latched between the probe and the cancel: model it as + // cancelRow() itself failing. The lease must still come home. + RecordingSender throwingOnCancel = new RecordingSender() { + @Override + public void cancelRow() { + throw new LineSenderException("latched during cancel"); + } + }; + // swap the delegate the slot holds, white-box + setDelegate(first, throwingOnCancel); + throwingOnCancel.rowInProgress = true; + try { + first.close(); + Assert.fail("close() must propagate the cancel failure"); + } catch (LineSenderException e) { + Assert.assertEquals("latched during cancel", e.getMessage()); + } + Assert.assertEquals("slot must be retired, not stranded", 0, pool.totalSize()); + Assert.assertEquals(1, throwingOnCancel.closeCount.get()); + Assert.assertEquals(0, delegate.closeCount.get()); + } + } + + @Test + public void testStaleCloseDoesNotTouchReborrowedSlot() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + first.close(); + Sender second = pool.borrow(); + second.table("t").longColumn("v", 1L); // second's row in progress + first.close(); // stale: must be a no-op + RecordingSender delegate = built.get(0); + Assert.assertEquals("stale close must not cancel the live borrower's row", 0, delegate.cancelCount.get()); + Assert.assertTrue(delegate.hasRowInProgress()); + Assert.assertEquals("stale close must not re-enqueue the slot", 0, pool.availableSize()); + second.atNow(); + second.close(); + Assert.assertEquals(1, pool.availableSize()); + } + } + + @Test + public void testLeaseReturnedAfterPoolCloseClosesDelegate() { + List built = new ArrayList<>(); + SenderPool pool = newPool(1, 1, built); + Sender s = pool.borrow(); + s.table("t").longColumn("v", 1L).atNow(); + pool.close(); // bounded wait, lease outstanding + Assert.assertEquals("borrowed slot is left for the returning lease", 1, pool.totalSize()); + s.close(); + RecordingSender delegate = built.get(0); + Assert.assertEquals("returning lease must close its delegate, which ships the rows", 1, delegate.closeCount.get()); + Assert.assertEquals(0, pool.totalSize()); + } + + @Test + public void testFreeListIsLifo() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(2, 2, built)) { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + Object slotB = slotOf(b); + a.close(); + b.close(); + try (Sender next = pool.borrow()) { + Assert.assertSame("most recently returned slot is borrowed first", slotB, slotOf(next)); + } + } + } + + static SenderPool newPool(int min, int max, List built) { + return new SenderPool(CONFIG, min, max, 150L, Long.MAX_VALUE, Long.MAX_VALUE, idx -> { + RecordingSender s = new RecordingSender(); + built.add(s); + return s; + }); + } + + static Object slotOf(Sender pooledWrapper) { + try { + Field f = PooledSender.class.getDeclaredField("slot"); + f.setAccessible(true); + return f.get(pooledWrapper); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + private static void setDelegate(Sender pooledWrapper, Sender delegate) { + try { + Object slot = slotOf(pooledWrapper); + Field f = slot.getClass().getDeclaredField("delegate"); + f.setAccessible(true); + f.set(slot, delegate); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java index 1aedb3ea1..792f2622d 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java @@ -44,9 +44,9 @@ *

* Tests never call methods that would attempt a network round-trip * (no {@code flush}, no row builders that auto-flush). Pooled - * {@link io.questdb.client.impl.PooledSender#close()} does call - * {@code delegate.flush()}, but on an empty buffer that path is a no-op for - * HTTP transport. + * {@link io.questdb.client.impl.PooledSender#close()} only flushes when + * {@code setFlushOnClose(true)} is set; the two tests that need the flushing + * path set it. */ public class SenderPoolTest { @@ -82,6 +82,7 @@ public void testBrokenSenderIsNotReturnedToPool() throws Exception { // is also unusable. Either way, the next borrower must get a fresh // slot, not the broken one. try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + pool.setFlushOnClose(true); // this test is about the flushing close path Sender first = pool.borrow(); Object firstSlot = slotOf(first); first.table("t").longColumn("v", 1).atNow(); @@ -138,6 +139,7 @@ public void testLeaseReturnedAfterCloseTearsDownItsOwnDelegate() { // acquireTimeout is kept short: it doubles as close()'s bounded // lease-wait budget, which this test intentionally exhausts. try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 150, Long.MAX_VALUE, Long.MAX_VALUE)) { + pool.setFlushOnClose(true); // this test is about the flushing close path Sender s = pool.borrow(); // A row in the buffer forces flush() to attempt a real HTTP // request on close(); against the unreachable DEAD_HTTP target From 030c53abcd851941e96975b14cf0ee6d45aec46f Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:02:35 +0100 Subject: [PATCH 03/57] test(pool): opt the flush-failure discard test into flush_on_close Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 38561c46a..fbb1f35ca 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -172,6 +172,7 @@ public void flushErrorDiscardsBrokenSenderInsteadOfRecycling() throws Exception try (SenderPool pool = newPool(CFG, 1, 1, 1_000, factory)) { Sender first = pool.borrow(); + pool.setFlushOnClose(true); // this test is about the flushing close path // Capture the underlying slot before close(): borrow() always hands // out a FRESH PooledSender wrapper, so assertNotSame(first, second) // on the wrappers is vacuously true and proves nothing -- it stays From 5c9f15ccd8b377b2fe0f794aec503e0d744c36c0 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:11:01 +0100 Subject: [PATCH 04/57] test(pool): alphabetical test order, no reflection in the cancel-failure test, stronger assertions Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 4 +- .../client/test/impl/RecordingSender.java | 8 +- .../impl/SenderPoolCloseSemanticsTest.java | 126 ++++++++---------- .../test/impl/SenderPoolErrorSafetyTest.java | 2 +- 4 files changed, 65 insertions(+), 75 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index a6de0fba0..c8f76f17b 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -155,7 +155,9 @@ public void checkError() { * returned, and the error is rethrown here. *

* Idempotent for sequential reuse: a stale generation (the lease was - * already returned and the slot possibly re-borrowed) is a no-op. + * already returned and the slot possibly re-borrowed) is a no-op. Concurrent + * double-close of ONE lease from two threads is outside the Sender + * threading contract. */ @Override public void close() { diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index 714ffb522..b2c856205 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -38,14 +38,14 @@ /** * In-memory {@link Sender} double for pool tests. Records calls, models the * buffer state the pool asks about, and can be told to fail or to block. - * Never touches the network. Not final: one test subclasses it to make a - * single method throw. + * Never touches the network. Not final, so a test can override a single method. */ class RecordingSender implements Sender { final AtomicInteger cancelCount = new AtomicInteger(); final AtomicInteger closeCount = new AtomicInteger(); final AtomicInteger deferredFlushCount = new AtomicInteger(); final AtomicInteger flushCount = new AtomicInteger(); + volatile RuntimeException cancelError; // thrown by cancelRow() when set volatile boolean deferredFramesOpen; volatile RuntimeException flushError; // thrown by flush() when set volatile CountDownLatch flushGate; // flush() blocks on it when set @@ -84,6 +84,10 @@ public DirectByteSlice bufferView() { @Override public void cancelRow() { cancelCount.incrementAndGet(); + RuntimeException e = cancelError; + if (e != null) { + throw e; + } rowInProgress = false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index d0de18d55..732e75571 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -46,6 +46,27 @@ public class SenderPoolCloseSemanticsTest { private static final String CONFIG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + @Test + public void testCancelRowThrowingAtCloseRetiresSlot() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + RecordingSender delegate = built.get(0); + // A terminal latched between the probe and the cancel: model it as + // cancelRow() itself failing. The lease must still come home. + delegate.rowInProgress = true; + delegate.cancelError = new LineSenderException("latched during cancel"); + try { + first.close(); + Assert.fail("close() must propagate the cancel failure"); + } catch (LineSenderException e) { + Assert.assertEquals("latched during cancel", e.getMessage()); + } + Assert.assertEquals("slot must be retired, not stranded", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + } + @Test public void testCloseDoesNotFlushByDefault() { List built = new ArrayList<>(); @@ -76,6 +97,21 @@ public void testFlushOnCloseRestoresShippingAtClose() { } } + @Test + public void testFreeListIsLifo() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(2, 2, built)) { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + Object slotB = slotOf(b); + a.close(); + b.close(); + try (Sender next = pool.borrow()) { + Assert.assertSame("most recently returned slot is borrowed first", slotB, slotOf(next)); + } + } + } + @Test public void testLatchedErrorAtCloseRetiresSlotAndThrows() { List built = new ArrayList<>(); @@ -98,6 +134,21 @@ public void testLatchedErrorAtCloseRetiresSlotAndThrows() { } } + @Test + public void testLeaseReturnedAfterPoolCloseClosesDelegate() { + List built = new ArrayList<>(); + SenderPool pool = newPool(1, 1, built); + Sender s = pool.borrow(); + s.table("t").longColumn("v", 1L).atNow(); + pool.close(); // bounded wait, lease outstanding + Assert.assertEquals("borrowed slot is left for the returning lease", 1, pool.totalSize()); + s.close(); + RecordingSender delegate = built.get(0); + Assert.assertEquals("returning lease must close its delegate", 1, delegate.closeCount.get()); + Assert.assertEquals("the pooled close itself must not flush", 0, delegate.flushCount.get()); + Assert.assertEquals(0, pool.totalSize()); + } + @Test public void testPartialRowAtCloseIsCancelledAndSlotKept() { List built = new ArrayList<>(); @@ -119,36 +170,6 @@ public void testPartialRowAtCloseIsCancelledAndSlotKept() { } } - @Test - public void testCancelRowThrowingAtCloseRetiresSlot() { - List built = new ArrayList<>(); - try (SenderPool pool = newPool(1, 1, built)) { - Sender first = pool.borrow(); - RecordingSender delegate = built.get(0); - first.table("t"); - // A terminal latched between the probe and the cancel: model it as - // cancelRow() itself failing. The lease must still come home. - RecordingSender throwingOnCancel = new RecordingSender() { - @Override - public void cancelRow() { - throw new LineSenderException("latched during cancel"); - } - }; - // swap the delegate the slot holds, white-box - setDelegate(first, throwingOnCancel); - throwingOnCancel.rowInProgress = true; - try { - first.close(); - Assert.fail("close() must propagate the cancel failure"); - } catch (LineSenderException e) { - Assert.assertEquals("latched during cancel", e.getMessage()); - } - Assert.assertEquals("slot must be retired, not stranded", 0, pool.totalSize()); - Assert.assertEquals(1, throwingOnCancel.closeCount.get()); - Assert.assertEquals(0, delegate.closeCount.get()); - } - } - @Test public void testStaleCloseDoesNotTouchReborrowedSlot() { List built = new ArrayList<>(); @@ -157,8 +178,11 @@ public void testStaleCloseDoesNotTouchReborrowedSlot() { first.close(); Sender second = pool.borrow(); second.table("t").longColumn("v", 1L); // second's row in progress - first.close(); // stale: must be a no-op RecordingSender delegate = built.get(0); + delegate.latchedError = new LineSenderException("must not be probed by a stale close"); + first.close(); // stale: must be a no-op, must not probe + Assert.assertEquals("stale close must not retire the live slot", 1, pool.totalSize()); + delegate.latchedError = null; Assert.assertEquals("stale close must not cancel the live borrower's row", 0, delegate.cancelCount.get()); Assert.assertTrue(delegate.hasRowInProgress()); Assert.assertEquals("stale close must not re-enqueue the slot", 0, pool.availableSize()); @@ -168,35 +192,6 @@ public void testStaleCloseDoesNotTouchReborrowedSlot() { } } - @Test - public void testLeaseReturnedAfterPoolCloseClosesDelegate() { - List built = new ArrayList<>(); - SenderPool pool = newPool(1, 1, built); - Sender s = pool.borrow(); - s.table("t").longColumn("v", 1L).atNow(); - pool.close(); // bounded wait, lease outstanding - Assert.assertEquals("borrowed slot is left for the returning lease", 1, pool.totalSize()); - s.close(); - RecordingSender delegate = built.get(0); - Assert.assertEquals("returning lease must close its delegate, which ships the rows", 1, delegate.closeCount.get()); - Assert.assertEquals(0, pool.totalSize()); - } - - @Test - public void testFreeListIsLifo() { - List built = new ArrayList<>(); - try (SenderPool pool = newPool(2, 2, built)) { - Sender a = pool.borrow(); - Sender b = pool.borrow(); - Object slotB = slotOf(b); - a.close(); - b.close(); - try (Sender next = pool.borrow()) { - Assert.assertSame("most recently returned slot is borrowed first", slotB, slotOf(next)); - } - } - } - static SenderPool newPool(int min, int max, List built) { return new SenderPool(CONFIG, min, max, 150L, Long.MAX_VALUE, Long.MAX_VALUE, idx -> { RecordingSender s = new RecordingSender(); @@ -214,15 +209,4 @@ static Object slotOf(Sender pooledWrapper) { throw new AssertionError(e); } } - - private static void setDelegate(Sender pooledWrapper, Sender delegate) { - try { - Object slot = slotOf(pooledWrapper); - Field f = slot.getClass().getDeclaredField("delegate"); - f.setAccessible(true); - f.set(slot, delegate); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index fbb1f35ca..941b5ba93 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -171,8 +171,8 @@ public void flushErrorDiscardsBrokenSenderInsteadOfRecycling() throws Exception IntFunction factory = slotIndex -> flushThrowingSender(); try (SenderPool pool = newPool(CFG, 1, 1, 1_000, factory)) { - Sender first = pool.borrow(); pool.setFlushOnClose(true); // this test is about the flushing close path + Sender first = pool.borrow(); // Capture the underlying slot before close(): borrow() always hands // out a FRESH PooledSender wrapper, so assertNotSame(first, second) // on the wrappers is vacuously true and proves nothing -- it stays From e15d20a1c1045a72ec08deed8ce3dba6e343f38b Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:14:20 +0100 Subject: [PATCH 05/57] feat(pool): settleIdle ships rows left in idle slots through an internal lease Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/SenderPool.java | 112 +++++++++++ .../test/impl/SenderPoolSettleTest.java | 175 ++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 33d5cdce9..8e34fb3c6 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1709,6 +1709,71 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna this.afterStartupRecoveryJoinHook = afterJoinHook; } + /** + * Ships rows left behind in idle slots. A pooled {@code close()} does not + * flush by default, and auto-flush thresholds are only evaluated on append, + * so a slot nobody borrows again would keep its last rows until the pool + * closes. The housekeeper calls this on its settle cadence. + *

+ * Each candidate slot is taken off the free list under the pool lock and + * stamped with a fresh generation, which makes it an internal lease: a + * borrower arriving meanwhile takes another slot or waits, and + * {@link #close()} waits for it like any outstanding lease. The flush runs + * outside the lock. A delegate whose flush failed is probed with + * {@link Sender#checkError()}: a latched error retires the slot, anything + * else keeps the rows for the next settle. The slot's idle clock is not + * touched, so reaping is unaffected. + * + * @return the number of slots whose flush completed + */ + public int settleIdle() { + if (closed) { + return 0; + } + ArrayList leases = null; + lock.lock(); + try { + if (closed) { + return 0; + } + Iterator it = available.iterator(); + while (it.hasNext()) { + SenderSlot s = it.next(); + if (hasUnshippedWork(s.delegate())) { + it.remove(); + s.bumpGeneration(); + if (leases == null) { + leases = new ArrayList<>(); + } + leases.add(new PooledSender(s, s.generation())); + } + } + } finally { + lock.unlock(); + } + if (leases == null) { + return 0; + } + int settled = 0; + for (int i = 0, n = leases.size(); i < n; i++) { + PooledSender lease = leases.get(i); + boolean flushed = false; + try { + lease.flush(); + flushed = true; + } catch (Throwable ignored) { + // Transient (ring full, recycle in flight) or terminal: the + // requeue below tells them apart with the probe. + } finally { + requeueSettled(lease); + } + if (flushed) { + settled++; + } + } + return settled; + } + private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { if (recoveryThread == null || recoveryThread == Thread.currentThread()) { return; @@ -1778,6 +1843,53 @@ private void waitForStartupRecoveryRetry() { } } + /** + * What {@link #settleIdle()} looks for in an idle slot. Phase A: buffered + * rows. Phase B adds deferred frames awaiting their commit. + */ + private static boolean hasUnshippedWork(Sender delegate) { + return delegate.pendingRowCount() > 0; + } + + /** + * Returns a slot held by {@link #settleIdle()} to the free list without + * resetting its idle clock, or retires it when the delegate has latched an + * error or the pool is shutting down. Mirrors {@link #giveBack} for the + * internal lease. + */ + private void requeueSettled(PooledSender lease) { + SenderSlot s = lease.slot(); + boolean healthy = false; + try { + s.delegate().checkError(); + healthy = true; + } catch (Throwable ignored) { + // latched: retire below + } + if (!healthy) { + retireLease(lease, " during settle"); + return; + } + lock.lock(); + try { + if (!closed) { + if (s.generation() != lease.generation()) { + return; + } + s.bumpGeneration(); + assert !available.contains(s) : "slot already present in available deque on requeueSettled"; + available.addLast(s); + slotReleased.signal(); + return; + } + } finally { + lock.unlock(); + } + // Pool is shutting down: tear the delegate down here, on the settle + // thread, exactly as giveBack does for a borrower's lease. + retireLease(lease, " during settle after pool close"); + } + /** * Retires one lease on the calling (borrower's) thread: validates the * lease generation under the lock, removes the slot from {@code all}, diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java new file mode 100644 index 000000000..088d79cb7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -0,0 +1,175 @@ +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.impl.SenderPool; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * {@link SenderPool#settleIdle()}: ships rows left in idle slots through an + * internal lease, retires dead delegates, and never shares a slot with a + * concurrent borrower. + */ +public class SenderPoolSettleTest { + + @Test + public void testSettleShipsRowsLeftInIdleSlot() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + long idleSince = idleSinceOf(pool); + Assert.assertEquals(0, delegate.flushCount.get()); + Thread.sleep(5); + Assert.assertEquals(1, pool.settleIdle()); + Assert.assertEquals(1, delegate.flushCount.get()); + Assert.assertEquals(0, delegate.pendingRowCount()); + Assert.assertEquals("slot must be back on the free list", 1, pool.availableSize()); + Assert.assertEquals("settling must not reset the idle clock", idleSince, idleSinceOf(pool)); + Assert.assertEquals("nothing left to settle", 0, pool.settleIdle()); + } + } + + @Test + public void testSettleSkipsSlotsWithoutRows() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(2, 2, built)) { + Assert.assertEquals(0, pool.settleIdle()); + Assert.assertEquals(0, built.get(0).flushCount.get()); + Assert.assertEquals(0, built.get(1).flushCount.get()); + } + } + + @Test + public void testSettleRetiresDeadDelegate() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + delegate.flushError = new LineSenderException("wire is dead"); + delegate.latchedError = new LineSenderException("latched terminal"); + Assert.assertEquals(0, pool.settleIdle()); + Assert.assertEquals("dead delegate must be retired", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + } + + @Test + public void testSettleKeepsSlotAfterTransientFlushFailure() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + delegate.flushError = new LineSenderException("ring full, try later"); + Assert.assertEquals(0, pool.settleIdle()); + Assert.assertEquals("healthy delegate stays in the pool", 1, pool.totalSize()); + Assert.assertEquals(1, pool.availableSize()); + Assert.assertEquals("rows stay for the next settle", 1, delegate.pendingRowCount()); + delegate.flushError = null; + Assert.assertEquals(1, pool.settleIdle()); + Assert.assertEquals(0, delegate.pendingRowCount()); + } + } + + @Test + public void testBorrowerNeverSharesSlotWithSettle() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + CountDownLatch gate = new CountDownLatch(1); + delegate.flushGate = gate; + Thread settler = new Thread(pool::settleIdle, "settler"); + settler.start(); + // wait until the settle holds the slot (free list empty) + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals("settle must hold the slot as an internal lease", 0, pool.availableSize()); + // a borrower arriving now must wait (max=1), not get the slot being flushed + try { + pool.borrow(); + Assert.fail("borrow must time out while the only slot is being settled"); + } catch (LineSenderException expected) { + // acquire timeout (150 ms in newPool) + } + gate.countDown(); + settler.join(2_000); + Assert.assertFalse(settler.isAlive()); + Assert.assertEquals(1, delegate.flushCount.get()); + try (Sender s = pool.borrow()) { + Assert.assertNotNull(s); + } + } + } + + @Test + public void testPoolCloseDuringSettleRetiresSlotOnSettleThread() throws Exception { + List built = new ArrayList<>(); + SenderPool pool = new SenderPool("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;", 1, 1, 2_000L, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> { + RecordingSender rs = new RecordingSender(); + built.add(rs); + return rs; + }); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + CountDownLatch gate = new CountDownLatch(1); + delegate.flushGate = gate; + AtomicReference settleFailure = new AtomicReference<>(); + Thread settler = new Thread(() -> { + try { + pool.settleIdle(); + } catch (Throwable t) { + settleFailure.set(t); + } + }, "settler"); + settler.start(); + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals(0, pool.availableSize()); + Thread closer = new Thread(pool::close, "closer"); + closer.start(); + Thread.sleep(50); // close() is now waiting for the internal lease + gate.countDown(); + settler.join(2_000); + closer.join(2_000); + Assert.assertFalse(settler.isAlive()); + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(settleFailure.get()); + Assert.assertEquals("the settle thread must retire the slot once the pool is closing", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + + @SuppressWarnings("unchecked") + private static long idleSinceOf(SenderPool pool) throws Exception { + Field availableF = SenderPool.class.getDeclaredField("available"); + availableF.setAccessible(true); + Object slot = ((java.util.ArrayDeque) availableF.get(pool)).peekLast(); + Method m = slot.getClass().getDeclaredMethod("idleSinceMillis"); + m.setAccessible(true); + return (long) m.invoke(slot); + } +} From 52cc2b1450a6c8d8eb66db7435f69062d7dcb1d9 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:23:03 +0100 Subject: [PATCH 06/57] fix(pool): deterministic settle tests, retire on Error, log retired settle slots Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/SenderPool.java | 41 ++++-- .../client/test/impl/RecordingSender.java | 5 +- .../test/impl/SenderPoolSettleTest.java | 129 ++++++++++++------ 3 files changed, 123 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 8e34fb3c6..f71ed2a97 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1590,7 +1590,8 @@ public void close() { inFlightCreations, creationWaitMillis); } // Bounded graceful wait for outstanding leases. A slot is borrowed - // iff it is in `all` but not in `available`; retireLease's + // iff it is in `all` but not in `available`, which includes the + // internal leases settleIdle() holds while it flushes; retireLease's // delegate-close section (running outside the lock on a returning // borrower's thread) is tracked by pendingLeaseTeardowns so this // method does not return while a teardown is still in flight. @@ -1719,9 +1720,11 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna * stamped with a fresh generation, which makes it an internal lease: a * borrower arriving meanwhile takes another slot or waits, and * {@link #close()} waits for it like any outstanding lease. The flush runs - * outside the lock. A delegate whose flush failed is probed with - * {@link Sender#checkError()}: a latched error retires the slot, anything - * else keeps the rows for the next settle. The slot's idle clock is not + * outside the lock. After the flush, whether it succeeded or not, the + * delegate is probed with {@link Sender#checkError()}: a latched error + * retires the slot, anything else returns it to the free list, and rows a + * failed flush left behind wait for the next settle. An {@link Error} + * thrown by the flush is not transient and retires the slot. The slot's idle clock is not * touched, so reaping is unaffected. * * @return the number of slots whose flush completed @@ -1758,14 +1761,27 @@ public int settleIdle() { for (int i = 0, n = leases.size(); i < n; i++) { PooledSender lease = leases.get(i); boolean flushed = false; + Throwable fatal = null; try { lease.flush(); flushed = true; - } catch (Throwable ignored) { + } catch (Exception ignored) { // Transient (ring full, recycle in flight) or terminal: the // requeue below tells them apart with the probe. - } finally { - requeueSettled(lease); + } catch (Throwable t) { + // An Error (AssertionError, OutOfMemoryError) is not transient. + fatal = t; + } + try { + if (fatal != null) { + LOG.warn("settleIdle: flush threw an Error, retiring slot {}", lease.slot().slotIndex(), fatal); + retireLease(lease, " during settle"); + } else { + requeueSettled(lease); + } + } catch (Throwable t) { + // Keep going: the remaining leases must not stay out of `available`. + LOG.warn("settleIdle: could not requeue or retire slot {}", lease.slot().slotIndex(), t); } if (flushed) { settled++; @@ -1867,6 +1883,14 @@ private void requeueSettled(PooledSender lease) { // latched: retire below } if (!healthy) { + long pendingRows = 0; + try { + pendingRows = s.delegate().pendingRowCount(); + } catch (Throwable ignored) { + // best effort: the count is only for the log line + } + LOG.warn("settleIdle: delegate of slot {} latched an error, retiring it with {} pending row(s)", + s.slotIndex(), pendingRows); retireLease(lease, " during settle"); return; } @@ -1894,7 +1918,8 @@ private void requeueSettled(PooledSender lease) { * Retires one lease on the calling (borrower's) thread: validates the * lease generation under the lock, removes the slot from {@code all}, * closes the delegate OUTSIDE the lock, and reclaims the SF slot index. - * Shared by {@link #discardBroken} (broken delegate) and by + * Shared by {@link #discardBroken} (broken delegate), by + * {@link #requeueSettled} (settle lease with a dead delegate) and by * {@link #giveBack} when the pool is shutting down (delegated teardown -- * see {@link #close()}). *

diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index b2c856205..c22e888f8 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -33,6 +33,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** @@ -242,7 +243,9 @@ private void awaitGate() { CountDownLatch gate = flushGate; if (gate != null) { try { - gate.await(); + if (!gate.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("flushGate was never released"); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index 088d79cb7..3992cb2aa 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -8,10 +8,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -31,7 +32,7 @@ public void testSettleShipsRowsLeftInIdleSlot() throws Exception { RecordingSender delegate = built.get(0); long idleSince = idleSinceOf(pool); Assert.assertEquals(0, delegate.flushCount.get()); - Thread.sleep(5); + Thread.sleep(20); Assert.assertEquals(1, pool.settleIdle()); Assert.assertEquals(1, delegate.flushCount.get()); Assert.assertEquals(0, delegate.pendingRowCount()); @@ -98,20 +99,23 @@ public void testBorrowerNeverSharesSlotWithSettle() throws Exception { delegate.flushGate = gate; Thread settler = new Thread(pool::settleIdle, "settler"); settler.start(); - // wait until the settle holds the slot (free list empty) - long deadline = System.currentTimeMillis() + 2_000; - while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { - Thread.sleep(5); - } - Assert.assertEquals("settle must hold the slot as an internal lease", 0, pool.availableSize()); - // a borrower arriving now must wait (max=1), not get the slot being flushed try { - pool.borrow(); - Assert.fail("borrow must time out while the only slot is being settled"); - } catch (LineSenderException expected) { - // acquire timeout (150 ms in newPool) + // wait until the settle holds the slot (free list empty) + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals("settle must hold the slot as an internal lease", 0, pool.availableSize()); + // a borrower arriving now must wait (max=1), not get the slot being flushed + try { + pool.borrow(); + Assert.fail("borrow must time out while the only slot is being settled"); + } catch (LineSenderException expected) { + // acquire timeout (150 ms in newPool) + } + } finally { + gate.countDown(); } - gate.countDown(); settler.join(2_000); Assert.assertFalse(settler.isAlive()); Assert.assertEquals(1, delegate.flushCount.get()); @@ -130,44 +134,83 @@ public void testPoolCloseDuringSettleRetiresSlotOnSettleThread() throws Exceptio built.add(rs); return rs; }); - try (Sender s = pool.borrow()) { - s.table("t").longColumn("v", 1L).atNow(); - } - RecordingSender delegate = built.get(0); - CountDownLatch gate = new CountDownLatch(1); - delegate.flushGate = gate; - AtomicReference settleFailure = new AtomicReference<>(); - Thread settler = new Thread(() -> { + try { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + CountDownLatch gate = new CountDownLatch(1); + delegate.flushGate = gate; + AtomicReference settleFailure = new AtomicReference<>(); + AtomicInteger settleResult = new AtomicInteger(-1); + Thread settler = new Thread(() -> { + try { + settleResult.set(pool.settleIdle()); + } catch (Throwable t) { + settleFailure.set(t); + } + }, "settler"); + Thread closer = new Thread(pool::close, "closer"); + settler.start(); try { - pool.settleIdle(); - } catch (Throwable t) { - settleFailure.set(t); + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals(0, pool.availableSize()); + closer.start(); + // close() parks in its timed wait for the internal lease + deadline = System.currentTimeMillis() + 2_000; + while (closer.getState() != Thread.State.TIMED_WAITING && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals("close() must be waiting for the internal lease", + Thread.State.TIMED_WAITING, closer.getState()); + Assert.assertTrue("close() must be waiting for the internal lease", closer.isAlive()); + } finally { + gate.countDown(); } - }, "settler"); - settler.start(); - long deadline = System.currentTimeMillis() + 2_000; - while (pool.availableSize() != 0 && System.currentTimeMillis() < deadline) { - Thread.sleep(5); + settler.join(2_000); + closer.join(2_000); + Assert.assertFalse(settler.isAlive()); + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(settleFailure.get()); + Assert.assertEquals(1, settleResult.get()); + Assert.assertEquals("the settle thread must retire the slot once the pool is closing", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } finally { + pool.close(); + } + } + + @Test + public void testSettleHandlesSeveralCandidatesIndependently() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(2, 2, built)) { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + a.table("t").longColumn("v", 1L).atNow(); + b.table("t").longColumn("v", 2L).atNow(); + a.close(); + b.close(); + RecordingSender dead = built.get(0); + RecordingSender healthy = built.get(1); + dead.flushError = new LineSenderException("wire is dead"); + dead.latchedError = new LineSenderException("latched terminal"); + Assert.assertEquals(1, pool.settleIdle()); + Assert.assertEquals(1, healthy.flushCount.get()); + Assert.assertEquals(1, pool.availableSize()); + Assert.assertEquals(1, pool.totalSize()); + Assert.assertEquals(1, dead.closeCount.get()); + Assert.assertEquals(0, healthy.closeCount.get()); } - Assert.assertEquals(0, pool.availableSize()); - Thread closer = new Thread(pool::close, "closer"); - closer.start(); - Thread.sleep(50); // close() is now waiting for the internal lease - gate.countDown(); - settler.join(2_000); - closer.join(2_000); - Assert.assertFalse(settler.isAlive()); - Assert.assertFalse(closer.isAlive()); - Assert.assertNull(settleFailure.get()); - Assert.assertEquals("the settle thread must retire the slot once the pool is closing", 0, pool.totalSize()); - Assert.assertEquals(1, delegate.closeCount.get()); } @SuppressWarnings("unchecked") private static long idleSinceOf(SenderPool pool) throws Exception { Field availableF = SenderPool.class.getDeclaredField("available"); availableF.setAccessible(true); - Object slot = ((java.util.ArrayDeque) availableF.get(pool)).peekLast(); + Object slot = ((ArrayDeque) availableF.get(pool)).peekLast(); Method m = slot.getClass().getDeclaredMethod("idleSinceMillis"); m.setAccessible(true); return (long) m.invoke(slot); From ddaaa465c13ee6bf9730ad2be2bf5b0fc020e420 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:25:18 +0100 Subject: [PATCH 07/57] feat(pool): housekeeper settles idle slots on its own cadence Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/QuestDBBuilder.java | 4 +- .../questdb/client/impl/PoolHousekeeper.java | 71 +++++++++++++------ .../io/questdb/client/impl/QuestDBImpl.java | 20 ++++-- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index 11364f9ba..b8868d3f0 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -238,7 +238,9 @@ public QuestDB build() { httpTokenProvider, errorHandler, connectionListener, - drainerListener + drainerListener, + false, + housekeeperIntervalMillis ); } diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 0778104b9..066fb5817 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -50,14 +50,23 @@ final class PoolHousekeeper { private final long intervalMillis; private final QueryClientPool queryPool; private final SenderPool senderPool; + private final long settleIntervalMillis; private final Object signalLock = new Object(); private final Thread thread; private volatile boolean stop; - PoolHousekeeper(SenderPool senderPool, QueryClientPool queryPool, long intervalMillis) { + /** + * @param intervalMillis reap cadence: idle and over-age slots + * @param settleIntervalMillis settle cadence: ship rows left in idle slots; + * the shorter of the configured auto-flush and + * commit intervals, or {@code intervalMillis} + * when both are off + */ + PoolHousekeeper(SenderPool senderPool, QueryClientPool queryPool, long intervalMillis, long settleIntervalMillis) { this.senderPool = senderPool; this.queryPool = queryPool; this.intervalMillis = intervalMillis; + this.settleIntervalMillis = Math.max(1L, settleIntervalMillis); this.thread = new Thread(this::runLoop, "questdb-pool-housekeeper"); this.thread.setDaemon(true); } @@ -152,6 +161,8 @@ static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { } private void runLoop() { + long nextSettleAt = System.currentTimeMillis() + settleIntervalMillis; + long nextReapAt = System.currentTimeMillis() + intervalMillis; while (!stop) { // Per-slot startup SF recovery, driven on THIS (the reap-loop) // thread, one stranded slot per iteration. Doing it here rather than @@ -182,34 +193,50 @@ private void runLoop() { return; } if (!recovering) { - try { - signalLock.wait(intervalMillis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; + long waitMillis = Math.min(nextSettleAt, nextReapAt) - System.currentTimeMillis(); + if (waitMillis > 0) { + try { + signalLock.wait(waitMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } } } } if (stop) { return; } - try { - senderPool.reapIdle(); - } catch (Throwable ignored) { - // Defensive, intentionally unreachable in normal operation: - // SenderPool.reapIdle() already swallows per-delegate close() - // failures internally. The outer catch is a belt-and-braces - // guard. Reaping must not propagate -- it's best-effort - // housekeeping. Catch Throwable (not just RuntimeException) so - // an Error from a delegate teardown can never kill this daemon - // thread and stop all future reaping for the life of the handle. + long now = System.currentTimeMillis(); + if (now >= nextSettleAt) { + try { + senderPool.settleIdle(); + } catch (Throwable ignored) { + // Best-effort: settleIdle() already guards per-slot + // failures; a Throwable here must never kill the daemon. + } + nextSettleAt = now + settleIntervalMillis; } - try { - queryPool.reapIdle(); - } catch (Throwable ignored) { - // Same rationale as the senderPool guard above: best-effort, - // must never propagate, and Throwable (not RuntimeException) so - // an Error from query-client teardown cannot kill the daemon. + if (now >= nextReapAt) { + try { + senderPool.reapIdle(); + } catch (Throwable ignored) { + // Defensive, intentionally unreachable in normal operation: + // SenderPool.reapIdle() already swallows per-delegate close() + // failures internally. The outer catch is a belt-and-braces + // guard. Reaping must not propagate -- it's best-effort + // housekeeping. Catch Throwable (not just RuntimeException) so + // an Error from a delegate teardown can never kill this daemon + // thread and stop all future reaping for the life of the handle. + } + try { + queryPool.reapIdle(); + } catch (Throwable ignored) { + // Same rationale as the senderPool guard above: best-effort, + // must never propagate, and Throwable (not RuntimeException) so + // an Error from query-client teardown cannot kill the daemon. + } + nextReapAt = now + intervalMillis; } } } diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index 574d6b59e..e9fe52669 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -72,7 +72,8 @@ public QuestDBImpl( this(ingestConfig, queryConfig, senderMin, senderMax, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, queryCloseTimeoutMillis, null, - errorHandler, connectionListener, drainerListener); + errorHandler, connectionListener, drainerListener, + false, housekeeperIntervalMillis); } public QuestDBImpl( @@ -90,12 +91,15 @@ public QuestDBImpl( HttpTokenProvider tokenProvider, SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, - BackgroundDrainerListener drainerListener + BackgroundDrainerListener drainerListener, + boolean flushOnClose, + long settleIntervalMillis ) { this(ingestConfig, queryConfig, senderMin, senderMax, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, queryCloseTimeoutMillis, null, null, - tokenProvider, errorHandler, connectionListener, drainerListener); + tokenProvider, errorHandler, connectionListener, drainerListener, + flushOnClose, settleIntervalMillis); } // Test-only constructor exposing the senderFactory and connectHook seams: @@ -122,7 +126,8 @@ public QuestDBImpl( this(ingestConfig, queryConfig, senderMin, senderMax, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, QueryClientPool.DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS, - senderFactory, connectHook, null, null, null, null); + senderFactory, connectHook, null, null, null, null, + false, housekeeperIntervalMillis); } // Full constructor adding the ingest-side errorHandler/connectionListener/ @@ -147,7 +152,9 @@ public QuestDBImpl( HttpTokenProvider tokenProvider, SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, - BackgroundDrainerListener drainerListener + BackgroundDrainerListener drainerListener, + boolean flushOnClose, + long settleIntervalMillis ) { SenderPool builtSenderPool = null; QueryClientPool builtQueryPool = null; @@ -161,11 +168,12 @@ public QuestDBImpl( // server; the housekeeper drives it via runStartupRecoveryStep(). true, errorHandler, connectionListener, drainerListener, tokenProvider); + builtSenderPool.setFlushOnClose(flushOnClose); builtQueryPool = new QueryClientPool( queryConfig, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, connectHook, null, tokenProvider); builtQueryPool.closeQueryTimeoutMillis(queryCloseTimeoutMillis); - builtHousekeeper = new PoolHousekeeper(builtSenderPool, builtQueryPool, housekeeperIntervalMillis); + builtHousekeeper = new PoolHousekeeper(builtSenderPool, builtQueryPool, housekeeperIntervalMillis, settleIntervalMillis); builtHousekeeper.start(); } catch (Throwable e) { // Catch Throwable, not just RuntimeException: this orchestrator is the From 81cba034ab2708bda9152c57946708833aad5118 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:28:39 +0100 Subject: [PATCH 08/57] feat(pool): flush_on_close key and builder option; settle cadence follows auto_flush_interval Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/QuestDBBuilder.java | 51 ++++++++++++++++++- .../io/questdb/client/impl/ConfigSchema.java | 1 + .../client/test/QuestDBBuilderTest.java | 46 +++++++++++++++++ .../test/impl/PoolConfigHonoredTest.java | 1 + 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index b8868d3f0..0f1bfcf60 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -60,6 +60,10 @@ public final class QuestDBBuilder { static final int DEFAULT_POOL_MAX = 4; static final int DEFAULT_POOL_MIN = 1; static final long DEFAULT_QUERY_CLOSE_TIMEOUT_MILLIS = 5_000; + // Settle cadence when auto_flush_interval is not set: the WebSocket + // sender's own default auto-flush interval, so rows left in an idle slot + // wait no longer than rows in a busy one. + static final long DEFAULT_SETTLE_INTERVAL_MILLIS = 100; // Every valid pool value is >= 0, so -1 unambiguously marks "not set // explicitly". The public pool setters are the only writers of these @@ -72,6 +76,8 @@ public final class QuestDBBuilder { private SenderConnectionListener connectionListener; private BackgroundDrainerListener drainerListener; private SenderErrorHandler errorHandler; + // null = not set explicitly; the string key or the default applies. + private Boolean flushOnClose; private long housekeeperIntervalMillis = UNSET; private HttpTokenProvider httpTokenProvider; private String config; @@ -166,6 +172,21 @@ public QuestDBBuilder errorHandler(SenderErrorHandler handler) { return this; } + /** + * Whether {@code close()} on a borrowed sender flushes its rows before + * returning the slot. Off by default: close releases, flush ships, drain + * waits, and rows left in a slot ship within one auto-flush interval. On + * restores the pre-1.4 behaviour, one frame and one WAL commit per close, + * which is the slow path for applications that borrow per message. In this + * client "flush" means published to the store-and-forward ring, not + * acknowledged by the server; use {@link Sender#drain(long)} for that. + * Overrides {@code flush_on_close} in the config string. + */ + public QuestDBBuilder flushOnClose(boolean flushOnClose) { + this.flushOnClose = flushOnClose; + return this; + } + /** * Builds the {@link QuestDB} handle. Validates both connect strings up * front -- so a malformed config fails here even when both pools have @@ -222,6 +243,10 @@ public QuestDB build() { resolvePoolLong(idleTimeoutMillis, "idle_timeout_ms", view, DEFAULT_IDLE_TIMEOUT_MILLIS, this::idleTimeoutMillis); resolvePoolLong(maxLifetimeMillis, "max_lifetime_ms", view, DEFAULT_MAX_LIFETIME_MILLIS, this::maxLifetimeMillis); resolvePoolLong(housekeeperIntervalMillis, "housekeeper_interval_ms", view, DEFAULT_HOUSEKEEPER_INTERVAL_MILLIS, this::housekeeperIntervalMillis); + if (flushOnClose == null) { + flushOnClose = view.getBool("flush_on_close", false); + } + long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis); return new QuestDBImpl( ingestConfig, @@ -239,8 +264,8 @@ public QuestDB build() { errorHandler, connectionListener, drainerListener, - false, - housekeeperIntervalMillis + flushOnClose, + settleIntervalMillis ); } @@ -459,6 +484,7 @@ public java.util.Map poolConfigSnapshotForTest() { m.put("idle_timeout_ms", idleTimeoutMillis); m.put("max_lifetime_ms", maxLifetimeMillis); m.put("housekeeper_interval_ms", housekeeperIntervalMillis); + m.put("flush_on_close", flushOnClose); return m; } @@ -473,6 +499,27 @@ private static String withDefaultAsyncConnect(String config) { return config.substring(0, sep + 2) + "initial_connect_retry=async;" + config.substring(sep + 2); } + /** + * The housekeeper's settle cadence: {@code auto_flush_interval} when set, + * {@link #DEFAULT_SETTLE_INTERVAL_MILLIS} when unset, and the reap cadence + * when the interval is off. (The WebSocket transport rejects + * {@code auto_flush=off}, so the interval key is the only switch.) + * Public only for the unit test, which lives in another package. + */ + @TestOnly + public static long resolveSettleIntervalMillis(ConfigView view, long housekeeperIntervalMillis) { + String interval = view.getStr("auto_flush_interval"); + if (interval == null) { + return DEFAULT_SETTLE_INTERVAL_MILLIS; + } + if ("off".equalsIgnoreCase(interval)) { + return housekeeperIntervalMillis; + } + // The Sender parse ran on the same string before this point and already + // rejected a non-positive or non-numeric value. + return Math.max(1L, Long.parseLong(interval.trim())); + } + private static void requireWebSocketSchema(CharSequence config, String role) { String schema = ConfigString.parse(config).schema(); if (!"ws".equals(schema) && !"wss".equals(schema)) { diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index b97256084..dc78f8c9d 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -124,6 +124,7 @@ public final class ConfigSchema { longRange("max_lifetime_ms", Side.POOL, OPEN, OPEN_MAX, false, false); longRange("housekeeper_interval_ms", Side.POOL, OPEN, OPEN_MAX, false, false); boolOnOff("lazy_connect", Side.POOL); // facade flag: tolerant non-blocking startup (async ingest + lazy reads) + boolOnOff("flush_on_close", Side.POOL); // pooled close() ships rows (on) or only releases the slot (off, default) // RESERVED -- accepted no-op (error-policy keys reserved by the spec). str("on_internal_error", Side.RESERVED); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index e338aa72c..a915a0393 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -29,6 +29,8 @@ import io.questdb.client.QuestDBBuilder; import io.questdb.client.Query; import io.questdb.client.Sender; +import io.questdb.client.impl.ConfigString; +import io.questdb.client.impl.ConfigView; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -241,6 +243,50 @@ public void testTlsRootsWithUnsafeOffRejectedAtBuildWithMinZero() { ); } + @Test + public void testFlushOnCloseBuilderCallOverridesString() { + QuestDBBuilder b = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=off;") + .flushOnClose(true); + try (QuestDB ignored = b.build()) { + Assert.assertEquals(Boolean.TRUE, b.poolConfigSnapshotForTest().get("flush_on_close")); + } + } + + @Test + public void testFlushOnCloseKeyParsedAndSnapshotted() { + QuestDBBuilder on = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;"); + try (QuestDB ignored = on.build()) { + Assert.assertEquals(Boolean.TRUE, on.poolConfigSnapshotForTest().get("flush_on_close")); + } + QuestDBBuilder dflt = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;"); + try (QuestDB ignored = dflt.build()) { + Assert.assertEquals("default is off", Boolean.FALSE, dflt.poolConfigSnapshotForTest().get("flush_on_close")); + } + } + + @Test + public void testFlushOnCloseRejectsInvalidValue() { + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=maybe;", + "flush_on_close"); + } + + @Test + public void testSettleIntervalFollowsAutoFlushInterval() { + // The WebSocket transport rejects auto_flush=off, so the interval key + // alone decides: unset -> the sender's 100 ms default, a number -> that + // number, "off" -> the housekeeper's reap cadence. + ConfigView dflt = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); + Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 5_000L)); + ConfigView explicit = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L)); + ConfigView intervalOff = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=off;")); + Assert.assertEquals(5_000L, QuestDBBuilder.resolveSettleIntervalMillis(intervalOff, 5_000L)); + } + @Test public void testMalformedPoolValueRejectedAtBuild() { // A non-numeric pool value is rejected at build()'s pool-key resolution, diff --git a/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java index a8c94e40d..70f93456e 100644 --- a/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java @@ -60,6 +60,7 @@ public void testEveryPoolKeyIsHonored() throws Exception { expected.put("idle_timeout_ms", 4321L); expected.put("max_lifetime_ms", 98765L); expected.put("housekeeper_interval_ms", 222L); + expected.put("flush_on_close", Boolean.TRUE); StringBuilder cfg = new StringBuilder("ws::addr=127.0.0.1:1;"); for (Map.Entry e : expected.entrySet()) { From f99cd3cf516aadaef4516b5d3c5ce665eea7a575 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:30:39 +0100 Subject: [PATCH 09/57] fix(pool): parse auto_flush_interval for the settle cadence with the connect-string number syntax Co-Authored-By: Claude Fable 5.1 --- .../java/io/questdb/client/QuestDBBuilder.java | 6 +++--- .../io/questdb/client/test/QuestDBBuilderTest.java | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index 0f1bfcf60..c7fce128e 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -515,9 +515,9 @@ public static long resolveSettleIntervalMillis(ConfigView view, long housekeeper if ("off".equalsIgnoreCase(interval)) { return housekeeperIntervalMillis; } - // The Sender parse ran on the same string before this point and already - // rejected a non-positive or non-numeric value. - return Math.max(1L, Long.parseLong(interval.trim())); + // getLong uses the connect-string number syntax (underscore separators + // included), the same one the Sender parse already accepted. + return Math.max(1L, view.getLong("auto_flush_interval", DEFAULT_SETTLE_INTERVAL_MILLIS)); } private static void requireWebSocketSchema(CharSequence config, String role) { diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index a915a0393..2d148e4f3 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -265,6 +265,16 @@ public void testFlushOnCloseKeyParsedAndSnapshotted() { try (QuestDB ignored = dflt.build()) { Assert.assertEquals("default is off", Boolean.FALSE, dflt.poolConfigSnapshotForTest().get("flush_on_close")); } + QuestDBBuilder t = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=true;"); + try (QuestDB ignored = t.build()) { + Assert.assertEquals(Boolean.TRUE, t.poolConfigSnapshotForTest().get("flush_on_close")); + } + QuestDBBuilder f = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=false;"); + try (QuestDB ignored = f.build()) { + Assert.assertEquals(Boolean.FALSE, f.poolConfigSnapshotForTest().get("flush_on_close")); + } } @Test @@ -285,6 +295,10 @@ public void testSettleIntervalFollowsAutoFlushInterval() { Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L)); ConfigView intervalOff = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=off;")); Assert.assertEquals(5_000L, QuestDBBuilder.resolveSettleIntervalMillis(intervalOff, 5_000L)); + ConfigView intervalOffUpper = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=OFF;")); + Assert.assertEquals(5_000L, QuestDBBuilder.resolveSettleIntervalMillis(intervalOffUpper, 5_000L)); + ConfigView underscored = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=1_000;")); + Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 5_000L)); } @Test From 8a22d6a394a3e22c34953900b09cca9c379961fc Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:33:52 +0100 Subject: [PATCH 10/57] test(pool): end-to-end close/flush/settle behaviour on the wire Co-Authored-By: Claude Fable 5.1 --- .../client/test/QuestDBFlushOnCloseTest.java | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java new file mode 100644 index 000000000..5adef191f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -0,0 +1,204 @@ +package io.questdb.client.test; + +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Close releases, flush ships, drain waits: observed on the wire through the + * in-repo test server. Each test counts the data frames (table count above + * zero) the server receives. + */ +public class QuestDBFlushOnCloseTest { + + @Test + public void testCloseShipsNothingUntilSettleOrFlush() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // settle cadence 60 s: nothing ships inside the test window + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + appendThreeRowsInThreeLeases(db); + Thread.sleep(300); + Assert.assertEquals("three closes must not ship", 0, log.dataFrames()); + try (Sender s = db.borrowSender()) { + s.flush(); + } + log.awaitDataFrames(1, 5_000); + Assert.assertEquals("explicit flush ships one frame holding all three rows", 1, log.dataFrames()); + } + } + } + + @Test + public void testFlushOnCloseShipsOneFramePerClose() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "flush_on_close=on;housekeeper_interval_ms=60000;"))) { + appendThreeRowsInThreeLeases(db); + log.awaitDataFrames(3, 5_000); + Assert.assertEquals(3, log.dataFrames()); + for (int i = 0; i < 3; i++) { + Assert.assertFalse("no commit window: frames are not flagged", log.deferFlag(i)); + } + } + } + } + + @Test + public void testSettleShipsOnDefaultCadenceWhenIntervalUnset() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // no auto_flush_interval: the settle cadence defaults to 100 ms, independent of the 60 s reap cadence + try (QuestDB db = QuestDB.connect(config(server.getPort(), "housekeeper_interval_ms=60000;"))) { + appendThreeRowsInThreeLeases(db); + log.awaitDataFrames(1, 2_000); + Thread.sleep(300); + Assert.assertEquals("the settle ships one frame and nothing else follows", 1, log.dataFrames()); + } + } + } + + @Test + public void testSettleShipsWithinAutoFlushInterval() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=100;housekeeper_interval_ms=60000;"))) { + appendThreeRowsInThreeLeases(db); + log.awaitDataFrames(1, 2_000); + Thread.sleep(300); + Assert.assertEquals("the settle ships one frame and nothing else follows", 1, log.dataFrames()); + } + } + } + + static void appendThreeRowsInThreeLeases(QuestDB db) { + for (int i = 0; i < 3; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", i).atNow(); + } + } + } + + static String config(int port, String extra) { + return "ws::addr=localhost:" + port + + ";sender_pool_min=1;sender_pool_max=1" + + ";query_pool_min=0;query_pool_max=1" + + ";initial_connect_retry=async" + + ";close_flush_timeout_millis=0;" + + extra; + } + + /** + * Parses every QWP frame in each WebSocket message (a read can carry more + * than one on replay), records its flags byte and table count, and acks + * cumulatively so the sender never backs up. Public: the sender-level + * probe test in a sub-package and the benchmark reuse it. + */ + public static final class FrameLog implements TestWebSocketServer.WebSocketServerHandler { + private static final int HEADER_SIZE = 12; + private static final int OFFSET_FLAGS = 5; + private static final int OFFSET_PAYLOAD_LENGTH = 8; + private static final int OFFSET_TABLE_COUNT = 6; + private final List flags = new ArrayList<>(); + private final List tableCounts = new ArrayList<>(); + private TestWebSocketServer.ClientHandler currentClient; + private long nextSeq; + + public void awaitCommitFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (commitFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("expected at least " + atLeast + " commit frame(s) within " + timeoutMillis + " ms, got " + commitFrames(), + commitFrames() >= atLeast); + } + + public void awaitDataFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (dataFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("expected at least " + atLeast + " data frame(s) within " + timeoutMillis + " ms, got " + dataFrames(), + dataFrames() >= atLeast); + } + + /** + * Count of rowless frames with the defer flag clear: the commit messages. + */ + public synchronized int commitFrames() { + int n = 0; + for (int i = 0, size = tableCounts.size(); i < size; i++) { + if (tableCounts.get(i) == 0 && (flags.get(i) & 0x01) == 0) { + n++; + } + } + return n; + } + + public synchronized int dataFrames() { + int n = 0; + for (int i = 0, size = tableCounts.size(); i < size; i++) { + if (tableCounts.get(i) > 0) { + n++; + } + } + return n; + } + + /** + * Defer flag of the i-th data frame (frames with a table count above zero). + */ + public synchronized boolean deferFlag(int dataFrameIndex) { + int seen = -1; + for (int i = 0, size = tableCounts.size(); i < size; i++) { + if (tableCounts.get(i) > 0 && ++seen == dataFrameIndex) { + return (flags.get(i) & 0x01) != 0; + } + } + throw new AssertionError("no data frame at index " + dataFrameIndex); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + nextSeq = 0; + } + int pos = 0; + int framesInMessage = 0; + while (pos + HEADER_SIZE <= data.length) { + int tableCount = (data[pos + OFFSET_TABLE_COUNT] & 0xFF) | ((data[pos + OFFSET_TABLE_COUNT + 1] & 0xFF) << 8); + long payloadLength = (data[pos + OFFSET_PAYLOAD_LENGTH] & 0xFFL) + | ((data[pos + OFFSET_PAYLOAD_LENGTH + 1] & 0xFFL) << 8) + | ((data[pos + OFFSET_PAYLOAD_LENGTH + 2] & 0xFFL) << 16) + | ((data[pos + OFFSET_PAYLOAD_LENGTH + 3] & 0xFFL) << 24); + flags.add(data[pos + OFFSET_FLAGS] & 0xFF); + tableCounts.add(tableCount); + framesInMessage++; + pos += HEADER_SIZE + (int) payloadLength; + } + nextSeq += Math.max(1, framesInMessage); + try { + client.sendBinary(QwpWireTestUtils.buildAck(nextSeq - 1)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} From 5f5f51a8daea3cf71e573a76ca0eb016a1dfedb2 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:35:45 +0100 Subject: [PATCH 11/57] fix(pool): drop the unreachable auto_flush_interval=off branch from the settle cadence Co-Authored-By: Claude Fable 5.1 --- .../java/io/questdb/client/QuestDBBuilder.java | 14 ++++---------- .../io/questdb/client/test/QuestDBBuilderTest.java | 10 +++------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index c7fce128e..d41a7e219 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -501,20 +501,14 @@ private static String withDefaultAsyncConnect(String config) { /** * The housekeeper's settle cadence: {@code auto_flush_interval} when set, - * {@link #DEFAULT_SETTLE_INTERVAL_MILLIS} when unset, and the reap cadence - * when the interval is off. (The WebSocket transport rejects - * {@code auto_flush=off}, so the interval key is the only switch.) + * {@link #DEFAULT_SETTLE_INTERVAL_MILLIS} when unset. The WebSocket + * transport rejects both {@code auto_flush=off} and + * {@code auto_flush_interval=off}, so the interval is a number or unset. + * {@code housekeeperIntervalMillis} is currently unused. * Public only for the unit test, which lives in another package. */ @TestOnly public static long resolveSettleIntervalMillis(ConfigView view, long housekeeperIntervalMillis) { - String interval = view.getStr("auto_flush_interval"); - if (interval == null) { - return DEFAULT_SETTLE_INTERVAL_MILLIS; - } - if ("off".equalsIgnoreCase(interval)) { - return housekeeperIntervalMillis; - } // getLong uses the connect-string number syntax (underscore separators // included), the same one the Sender parse already accepted. return Math.max(1L, view.getLong("auto_flush_interval", DEFAULT_SETTLE_INTERVAL_MILLIS)); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index 2d148e4f3..9d2a87923 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -286,17 +286,13 @@ public void testFlushOnCloseRejectsInvalidValue() { @Test public void testSettleIntervalFollowsAutoFlushInterval() { - // The WebSocket transport rejects auto_flush=off, so the interval key - // alone decides: unset -> the sender's 100 ms default, a number -> that - // number, "off" -> the housekeeper's reap cadence. + // The WebSocket transport rejects both auto_flush=off and + // auto_flush_interval=off, so the interval is unset (the sender's + // 100 ms default) or a number (underscore separators allowed). ConfigView dflt = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 5_000L)); ConfigView explicit = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L)); - ConfigView intervalOff = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=off;")); - Assert.assertEquals(5_000L, QuestDBBuilder.resolveSettleIntervalMillis(intervalOff, 5_000L)); - ConfigView intervalOffUpper = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=OFF;")); - Assert.assertEquals(5_000L, QuestDBBuilder.resolveSettleIntervalMillis(intervalOffUpper, 5_000L)); ConfigView underscored = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=1_000;")); Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 5_000L)); } From 0b1024062cf834e50dff24cdb79d2b233556d748 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:46:43 +0100 Subject: [PATCH 12/57] docs(pool): close releases, flush ships, drain waits; adjust tests that relied on close flushing Co-Authored-By: Claude Fable 5.1 --- README.md | 17 +++++++++++------ .../client/test/impl/SenderPoolSfTest.java | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a013a15a8..e09fc79ec 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ try (QuestDB db = QuestDB.connect("ws::addr=localhost:9000;")) { .doubleColumn("price", 2615.54) .doubleColumn("amount", 0.00044) .atNow(); - // close() flushes pending rows and returns the Sender to the pool. + // close() returns the Sender to the pool; rows ship by threshold or within auto_flush_interval. } } ``` @@ -98,8 +98,12 @@ snippets below borrow from its pools. ### Ingest Data -Borrow a `Sender`, write rows, and close it. `close()` flushes pending rows and returns the sender to the pool — the -underlying connection is only torn down when the `QuestDB` handle itself is closed. +Borrow a `Sender`, write rows, and close it. Close releases, flush ships, drain waits: `close()` returns the sender to +the pool and ships nothing by itself; rows go out when an auto-flush threshold trips on a later append or when the +pool's housekeeper settles the idle slot, at most one `auto_flush_interval` later (100 ms by default). Borrowing per +message is the intended pattern and batches like a sender held for the whole run. Call `flush()` to ship now and +`drain()` to wait for the server. The underlying connection is only torn down when the `QuestDB` handle itself is +closed. ```java import io.questdb.client.Sender; @@ -110,15 +114,15 @@ try (Sender sender = db.borrowSender()) { .doubleColumn("price", 42_500.50) .longColumn("size", 100) .atNow(); - // No explicit flush() needed: close() flushes for you. + // close() returns the sender; the pool ships the row by threshold or on its next settle. } ``` ### Sending Batches of Rows The sender buffers rows and sends them in batches. For high throughput, write many rows on one borrowed sender and -flush per batch rather than per row: each flush has a fixed cost, so batching raises throughput. `close()` flushes the -final partial batch. +flush per batch rather than per row: each flush has a fixed cost, so batching raises throughput. The final partial batch ships on the +pool's next settle; call `flush()` before `close()` if you need it on the wire now. ```java try (Sender sender = db.borrowSender()) { @@ -542,6 +546,7 @@ schema::key1=value1;key2=value2; | `acquire_timeout_ms` | `5000` | How long `borrowSender()`/`borrowQuery()` waits for a free slot | | `idle_timeout_ms` | `60000` | How long a pooled connection may stay idle before it is reaped | | `max_lifetime_ms` | `1800000` | Maximum lifetime of a pooled connection before it is recycled | +| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged | ### Query routing keys diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index 7744cd1a1..ae07944ae 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1410,6 +1410,7 @@ public void testBorrowerWokenWhenRecycleRetainedEngineReleasesSlot() throws Exce String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; Thread borrower = null; try (SenderPool pool = new SenderPool(config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + pool.setFlushOnClose(true); PooledSender lease = pool.borrow(); QwpWebSocketSender delegate = (QwpWebSocketSender) getDelegate(lease); lease.table("t").symbol("s", "a").longColumn("v", 1L).atNow(); From 74343ba4c298ebd8c92178889fb630893a66bcb4 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:50:01 +0100 Subject: [PATCH 13/57] feat(qwp): rowsAppended, flushDeferred and hasDeferredFrames on Sender Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 38 ++++++++++++++++++ .../qwp/client/QwpWebSocketSender.java | 39 +++++++++++++++++++ .../io/questdb/client/impl/PooledSender.java | 15 +++++++ .../qwp/client/SenderProbeMethodsTest.java | 30 ++++++++++++++ .../client/test/impl/RecordingSender.java | 14 +++---- 5 files changed, 127 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 67142c594..59e5febad 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -585,6 +585,21 @@ default long flushAndGetSequence() { return -1L; } + /** + * Ships the pending rows as a frame carrying the defer-commit flag: the + * server appends them to its WAL writers and withholds both the commit and + * the acknowledgement until a later {@link #flush()} on this sender, which + * closes the group. Used by the connection pool's commit window. Only the + * WebSocket QWP transport supports deferred commits. + * + * @return the published frame sequence number, or {@code -1} if nothing was pending + * @throws UnsupportedOperationException on transports without deferred commits + * @throws LineSenderException if the transport has latched a terminal error + */ + default long flushDeferred() { + throw new UnsupportedOperationException("deferred flush requires the WebSocket transport"); + } + /** * Add a GEOHASH column value from pre-packed bits and an explicit bit precision. *

@@ -654,6 +669,17 @@ default long getAckedFsn() { return -1L; } + /** + * Whether frames shipped with {@link #flushDeferred()}, or by auto-flush in + * {@code transaction=on} mode, are still waiting for the commit that a + * {@link #flush()} sends. Only the WebSocket QWP transport tracks this. + * + * @return {@code true} while a deferred group is open + */ + default boolean hasDeferredFrames() { + return false; + } + /** * Whether a row has been started with {@link #table(CharSequence)} and not * yet finished with {@link #at}, {@link #atNow()} or {@link #cancelRow()}. @@ -798,6 +824,18 @@ default void resetSymbolDictionary() { */ void reset(); + /** + * Monotonic count of rows finished with {@link #at}/{@link #atNow()} over + * this sender's life, whether or not they have shipped. The connection + * pool measures its commit window on deltas of this value. Only the + * WebSocket QWP transport counts; other transports return {@code 0}. + * + * @return rows appended so far + */ + default long rowsAppended() { + return 0L; + } + /** * Add a column with a 16-bit signed integer value. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 609c45242..bca0f7315 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -461,6 +461,7 @@ public class QwpWebSocketSender implements Sender { // while no engine is retained. private volatile CursorSendEngine retainedEngine; private int pendingRowCount; + private long rowsAppended; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a // setProgressHandler call before connect() swaps in a real handler. @@ -2243,6 +2244,33 @@ public long flushAndGetSequence() { return afterFsn > beforeFsn ? fsnEpochBase + afterFsn : -1L; } + @Override + public long flushDeferred() { + checkNotClosed(); + checkRecycleFailure(); + if (cursorEngine != null) { + cursorEngine.checkDurability(); + } + ensureNoInProgressRow(); + ensureConnected(); + + long beforeFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L; + + // Same publish path as flushAndGetSequence() with the flag forced on: + // the frame is appended server-side, not committed, and its ack is + // withheld until a later unflagged frame or commit message. The commit + // boundary is deliberately NOT advanced here. + flushPendingRows(true); + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + cursorSendLoop.checkError(); + checkConnectionError(); + + long afterFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L; + return afterFsn > beforeFsn ? fsnEpochBase + afterFsn : -1L; + } + /** * Flushes pending rows and blocks until the server has acknowledged * every frame published so far (the current published-FSN watermark), @@ -2904,6 +2932,11 @@ public long getTotalServerErrors() { return counters.serverErrors.get(); } + @Override + public boolean hasDeferredFrames() { + return hasDeferredMessages; + } + @Override public boolean hasRowInProgress() { return currentTableBuffer != null && currentTableBuffer.hasInProgressRow(); @@ -3220,6 +3253,11 @@ public void reset() { cachedTimestampNanosColumn = null; } + @Override + public long rowsAppended() { + return rowsAppended; + } + /** * Advisory request to start a fresh symbol-dictionary epoch. Sets * {@link #manualResetRequested}; if no flush is in flight @@ -6998,6 +7036,7 @@ private void sendRow() { firstPendingRowTimeNanos = System.nanoTime(); } pendingRowCount++; + rowsAppended++; // Advance pendingBytes by the bytes the just-committed row added to // the current table. nextRow() accumulated the total during its diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index c8f76f17b..c861a1ef8 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -271,6 +271,11 @@ public long flushAndGetSequence() { return slot.live(generation).flushAndGetSequence(); } + @Override + public long flushDeferred() { + return slot.live(generation).flushDeferred(); + } + @Override public Sender geoHashColumn(CharSequence name, long bits, int precisionBits) { slot.live(generation).geoHashColumn(name, bits, precisionBits); @@ -288,6 +293,11 @@ public long getAckedFsn() { return slot.live(generation).getAckedFsn(); } + @Override + public boolean hasDeferredFrames() { + return slot.live(generation).hasDeferredFrames(); + } + @Override public boolean hasRowInProgress() { return slot.live(generation).hasRowInProgress(); @@ -357,6 +367,11 @@ public void reset() { slot.live(generation).reset(); } + @Override + public long rowsAppended() { + return slot.live(generation).rowsAppended(); + } + @Override public void resetSymbolDictionary() { slot.live(generation).resetSymbolDictionary(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index f6354ae25..ddba8a3eb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -2,6 +2,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.QuestDBFlushOnCloseTest; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -17,6 +18,35 @@ */ public class SenderProbeMethodsTest { + @Test + public void testDeferredFlushShipsFlaggedFrameAndFlushCommits() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + Assert.assertEquals(0L, sender.rowsAppended()); + Assert.assertFalse(sender.hasDeferredFrames()); + Assert.assertEquals(-1L, sender.flushDeferred()); // nothing pending + + sender.table("t").longColumn("v", 1L).atNow(); + Assert.assertEquals(1L, sender.rowsAppended()); + Assert.assertTrue(sender.flushDeferred() >= 0L); + Assert.assertTrue("a deferred frame is open", sender.hasDeferredFrames()); + Assert.assertEquals(0, sender.pendingRowCount()); + + sender.table("t").longColumn("v", 2L).atNow(); + sender.flush(); // closes the group + Assert.assertFalse(sender.hasDeferredFrames()); + Assert.assertEquals(2L, sender.rowsAppended()); + + log.awaitDataFrames(2, 5_000); + Assert.assertTrue("first frame carries FLAG_DEFER_COMMIT", log.deferFlag(0)); + Assert.assertFalse("committing frame is unflagged", log.deferFlag(1)); + } + } + } + @Test public void testHealthySenderReportsBufferState() throws Exception { try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index c22e888f8..589c34862 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -146,9 +146,6 @@ public void flush() { deferredFramesOpen = false; } - /* Phase B (Task 9): re-enable these three overrides once Sender gains - flushDeferred(), hasDeferredFrames() and rowsAppended(). - @Override public long flushDeferred() { awaitGate(); @@ -166,12 +163,6 @@ public boolean hasDeferredFrames() { return deferredFramesOpen; } - @Override - public long rowsAppended() { - return rowsAppended; - } - */ - @Override public boolean hasRowInProgress() { return rowInProgress; @@ -213,6 +204,11 @@ public void reset() { rowInProgress = false; } + @Override + public long rowsAppended() { + return rowsAppended; + } + @Override public Sender stringColumn(CharSequence name, CharSequence value) { return this; From 9884f805f0a41a2d098a668e50e35226eaeb80a5 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:54:15 +0100 Subject: [PATCH 14/57] fix(qwp): sort rowsAppended, pin no-commit-message on the wire, document the defaults Co-Authored-By: Claude Fable 5.1 --- core/src/main/java/io/questdb/client/Sender.java | 11 ++++++----- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 11 ++++++----- .../java/io/questdb/client/impl/PooledSender.java | 8 ++++---- .../cutlass/qwp/client/SenderProbeMethodsTest.java | 8 ++++++++ 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 59e5febad..208841012 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -586,13 +586,14 @@ default long flushAndGetSequence() { } /** - * Ships the pending rows as a frame carrying the defer-commit flag: the + * Ships the pending rows as frames carrying the defer-commit flag: the * server appends them to its WAL writers and withholds both the commit and - * the acknowledgement until a later {@link #flush()} on this sender, which - * closes the group. Used by the connection pool's commit window. Only the - * WebSocket QWP transport supports deferred commits. + * the acknowledgement until the group is closed. A later {@link #flush()}, + * {@link #close()}, or (outside transactional mode) an auto-flush that + * ships an unflagged frame closes the group. Used by the connection pool's + * commit window. Only the WebSocket QWP transport supports deferred commits. * - * @return the published frame sequence number, or {@code -1} if nothing was pending + * @return the highest FSN published by this call, or {@code -1} if nothing was pending * @throws UnsupportedOperationException on transports without deferred commits * @throws LineSenderException if the transport has latched a terminal error */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bca0f7315..9b30be539 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -461,6 +461,7 @@ public class QwpWebSocketSender implements Sender { // while no engine is retained. private volatile CursorSendEngine retainedEngine; private int pendingRowCount; + // Monotonic for the sender's life; reset() does not clear it. Counts rows whether or not they shipped. private long rowsAppended; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a @@ -3253,11 +3254,6 @@ public void reset() { cachedTimestampNanosColumn = null; } - @Override - public long rowsAppended() { - return rowsAppended; - } - /** * Advisory request to start a fresh symbol-dictionary epoch. Sets * {@link #manualResetRequested}; if no flush is in flight @@ -3289,6 +3285,11 @@ public void resetSymbolDictionary() { } } + @Override + public long rowsAppended() { + return rowsAppended; + } + /** * Register an async listener for connection-state transitions: initial * connect, primary failover, endpoint attempt failures, the full address diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index c861a1ef8..db09bd891 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -368,13 +368,13 @@ public void reset() { } @Override - public long rowsAppended() { - return slot.live(generation).rowsAppended(); + public void resetSymbolDictionary() { + slot.live(generation).resetSymbolDictionary(); } @Override - public void resetSymbolDictionary() { - slot.live(generation).resetSymbolDictionary(); + public long rowsAppended() { + return slot.live(generation).rowsAppended(); } @Override diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index ddba8a3eb..8815c720a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -41,6 +41,7 @@ public void testDeferredFlushShipsFlaggedFrameAndFlushCommits() throws Exception Assert.assertEquals(2L, sender.rowsAppended()); log.awaitDataFrames(2, 5_000); + Assert.assertEquals("no separate commit message: the committing frame carries the rows", 0, log.commitFrames()); Assert.assertTrue("first frame carries FLAG_DEFER_COMMIT", log.deferFlag(0)); Assert.assertFalse("committing frame is unflagged", log.deferFlag(1)); } @@ -106,6 +107,13 @@ public void testDefaultsOnNonWebSocketTransport() { sender.checkError(); // default: no-op Assert.assertFalse(sender.hasRowInProgress()); // default: false Assert.assertEquals(0, sender.pendingRowCount()); // default: 0 + Assert.assertEquals(0L, sender.rowsAppended()); + Assert.assertFalse(sender.hasDeferredFrames()); + try { + sender.flushDeferred(); + Assert.fail("expected UnsupportedOperationException"); + } catch (UnsupportedOperationException expected) { + } } } From d2524157eaad219c45c19008a375a3d72a275542 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:57:20 +0100 Subject: [PATCH 15/57] feat(pool): commit window: deferred frames per close, one commit per window, settle commits idle groups Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 60 ++++--- .../io/questdb/client/impl/SenderPool.java | 37 ++++- .../io/questdb/client/impl/SenderSlot.java | 31 ++++ .../test/impl/SenderPoolCommitWindowTest.java | 148 ++++++++++++++++++ 4 files changed, 254 insertions(+), 22 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index db09bd891..1f2485bae 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -158,16 +158,19 @@ public void checkError() { * already returned and the slot possibly re-borrowed) is a no-op. Concurrent * double-close of ONE lease from two threads is outside the Sender * threading contract. + *

+ * With a commit window ({@link SenderPool#setCommitWindow}) and + * {@code flush_on_close=on}, each close ships a deferred frame; the close + * that fills the window (rows appended or age of the open group) flushes + * instead, which commits the group. Without {@code flush_on_close} the + * window still closes on that release, but nothing is shipped otherwise. */ @Override public void close() { if (generation != slot.generation()) { return; } - if (slot.pool().isFlushOnClose()) { - closeFlushing(); - return; - } + SenderPool pool = slot.pool(); boolean cancelledRow = false; boolean recycle = false; try { @@ -179,15 +182,16 @@ public void close() { delegate.cancelRow(); cancelledRow = true; } + ship(pool, delegate); recycle = true; } finally { if (recycle) { - slot.pool().giveBack(this); + pool.giveBack(this); } else { // The probe or the cancel threw: the delegate has latched an // error (or the generation went stale). Retire it; the // original throwable propagates once this finally returns. - slot.pool().discardBroken(this); + pool.discardBroken(this); } } if (cancelledRow) { @@ -263,12 +267,17 @@ public Sender floatColumn(CharSequence name, float value) { @Override public void flush() { - slot.live(generation).flush(); + Sender delegate = slot.live(generation); + delegate.flush(); + slot.markCommitted(delegate.rowsAppended()); } @Override public long flushAndGetSequence() { - return slot.live(generation).flushAndGetSequence(); + Sender delegate = slot.live(generation); + long sequence = delegate.flushAndGetSequence(); + slot.markCommitted(delegate.rowsAppended()); + return sequence; } @Override @@ -443,20 +452,31 @@ SenderSlot slot() { } /** - * The {@code flush_on_close=on} path: today's close. Ships the buffer, then - * returns the slot; any abnormal exit of the flush retires the slot. + * What a release ships, per the pool's settings: + *

    + *
  • commit window set and closing now: {@code flush()}, which ships + * pending rows unflagged and commits the group;
    • + *
  • otherwise {@code flush_on_close=on}: {@code flushDeferred()} with a + * window, {@code flush()} without one;
    • + *
  • otherwise nothing: rows wait for a threshold or the settle.
    • + *
*/ - private void closeFlushing() { - boolean flushed = false; - try { - slot.live(generation).flush(); - flushed = true; - } finally { - if (flushed) { - slot.pool().giveBack(this); - } else { - slot.pool().discardBroken(this); + private void ship(SenderPool pool, Sender delegate) { + if (pool.hasCommitWindow()) { + boolean closes = slot.windowCloses(delegate.rowsAppended(), delegate.hasDeferredFrames(), + System.nanoTime(), pool.commitRows(), pool.commitIntervalNanos()); + if (closes) { + delegate.flush(); + slot.markCommitted(delegate.rowsAppended()); + return; + } + if (pool.isFlushOnClose()) { + delegate.flushDeferred(); } + return; + } + if (pool.isFlushOnClose()) { + delegate.flush(); } } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index f71ed2a97..cadfeb4a1 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -251,6 +251,8 @@ public final class SenderPool implements AutoCloseable { // production; lifecycle tests use it to acknowledge distinct retries // without inspecting transient Condition queue membership. private volatile Runnable creationWaitRetryHook; + private volatile long commitIntervalNanos; + private volatile int commitRows; private volatile boolean flushOnClose; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the @@ -1764,6 +1766,7 @@ public int settleIdle() { Throwable fatal = null; try { lease.flush(); + lease.slot().markCommitted(lease.slot().delegate().rowsAppended()); flushed = true; } catch (Exception ignored) { // Transient (ring full, recycle in flight) or terminal: the @@ -1861,10 +1864,10 @@ private void waitForStartupRecoveryRetry() { /** * What {@link #settleIdle()} looks for in an idle slot. Phase A: buffered - * rows. Phase B adds deferred frames awaiting their commit. + * rows and deferred frames awaiting their commit. */ private static boolean hasUnshippedWork(Sender delegate) { - return delegate.pendingRowCount() > 0; + return delegate.pendingRowCount() > 0 || delegate.hasDeferredFrames(); } /** @@ -2093,6 +2096,21 @@ public int totalSize() { } } + /** Age per commit window in nanoseconds, or 0 when no window is configured. */ + public long commitIntervalNanos() { + return commitIntervalNanos; + } + + /** Rows per commit window, or 0 when no window is configured. */ + public int commitRows() { + return commitRows; + } + + /** Whether a commit window is configured: pooled closes ship deferred frames and commit per window. */ + public boolean hasCommitWindow() { + return commitRows > 0; + } + /** * Whether a pooled {@code close()} ships the lease's rows before releasing * the slot (the pre-1.4 behaviour) or only releases it. Off by default: @@ -2123,6 +2141,21 @@ public int leakedSlotCount() { } } + /** + * Configures the commit window. Both values positive enable it; {@code 0, 0} + * disables it. Must be set before the first borrow; the pool's delegates + * must have been built with {@code transaction=on} so that auto-flush + * inside a lease ships deferred frames too (QuestDBBuilder injects it). + */ + public void setCommitWindow(int commitRows, long commitIntervalMillis) { + if ((commitRows > 0) != (commitIntervalMillis > 0)) { + throw new IllegalArgumentException("commit window needs both rows and interval, or neither: rows=" + + commitRows + ", intervalMillis=" + commitIntervalMillis); + } + this.commitRows = commitRows; + this.commitIntervalNanos = TimeUnit.MILLISECONDS.toNanos(commitIntervalMillis); + } + public void setFlushOnClose(boolean flushOnClose) { this.flushOnClose = flushOnClose; } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 5d9ed6ff7..c7ea8f45f 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -48,6 +48,11 @@ final class SenderSlot { private final Sender delegate; private final SenderPool pool; private final int slotIndex; + // Commit-window state, guarded by the lease: only the thread holding the + // slot's current lease (a borrower or the settle step) reads or writes it. + // 0 = no group open since the last commit. + private long groupOpenedAtNanos; + private long rowsAppendedAtLastCommit; // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. // Guarded by the pool lock. The pool maintains this index across swap // removals so the delegate's release callback can recover this exact slot @@ -109,6 +114,15 @@ Sender live(long gen) { return delegate; } + /** + * Records that everything appended so far is committed. Called after a + * committing {@code flush()} on the delegate, by a release or by the settle. + */ + void markCommitted(long rowsAppended) { + rowsAppendedAtLastCommit = rowsAppended; + groupOpenedAtNanos = 0L; + } + void markIdleAt(long nowMillis) { idleSinceMillis = nowMillis; } @@ -128,4 +142,21 @@ void retiredIndex(int retiredIndex) { int slotIndex() { return slotIndex; } + + /** + * Whether the commit window should close at this release. Rows in the + * window are counted from appends, so the answer does not depend on + * whether rows shipped at close or by auto-flush. The first release that + * finds work after a commit opens the group's clock. + */ + boolean windowCloses(long rowsAppended, boolean deferredOpen, long nowNanos, int commitRows, long commitIntervalNanos) { + long inWindow = rowsAppended - rowsAppendedAtLastCommit; + if (inWindow <= 0 && !deferredOpen) { + return false; + } + if (groupOpenedAtNanos == 0L) { + groupOpenedAtNanos = nowNanos; + } + return inWindow >= commitRows || nowNanos - groupOpenedAtNanos >= commitIntervalNanos; + } } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java new file mode 100644 index 000000000..8bf1ed18a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java @@ -0,0 +1,148 @@ +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.impl.SenderPool; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * The pool's commit window with {@code flush_on_close=on}: each close ships a + * deferred frame, the close that fills the window ships a committing frame + * instead, and the settle step commits an open group left idle. + */ +public class SenderPoolCommitWindowTest { + + @Test + public void testAgeWindowClosesAtNextRelease() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(1_000, 30L); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender d = built.get(0); + Assert.assertEquals(1, d.deferredFlushCount.get()); + Thread.sleep(60); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 2L).atNow(); + } + Assert.assertEquals("group older than the interval: this close commits", 1, d.flushCount.get()); + Assert.assertEquals(1, d.deferredFlushCount.get()); + } + } + + @Test + public void testLeaseFlushResetsTheWindow() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(2, 60_000L); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 2L).atNow(); + s.flush(); // app commits explicitly: 2 rows, window would close here + } + RecordingSender d = built.get(0); + Assert.assertEquals(1, d.flushCount.get()); + // the close after the explicit flush found nothing to ship and the + // window restarted at 2 rows, so this third close is deferred + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 3L).atNow(); + } + Assert.assertEquals(2, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + } + } + + @Test + public void testReleaseWithoutRowsDoesNotOpenAWindow() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(1_000, 30L); + try (Sender s = pool.borrow()) { + Assert.assertNotNull(s); // no rows + } + Thread.sleep(60); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender d = built.get(0); + Assert.assertEquals("the empty release must not have started the clock", 1, d.deferredFlushCount.get()); + Assert.assertEquals(0, d.flushCount.get()); + } + } + + @Test + public void testRowsWindowClosesWithCommittingFlush() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(3, 60_000L); + RecordingSender d = null; + for (int i = 0; i < 3; i++) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", i).atNow(); + } + d = built.get(0); + } + Assert.assertEquals("first two closes ship deferred", 2, d.deferredFlushCount.get()); + Assert.assertEquals("third close fills the window and commits", 1, d.flushCount.get()); + Assert.assertFalse(d.hasDeferredFrames()); + // the window restarts: a fourth close is deferred again + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 4L).atNow(); + } + Assert.assertEquals(3, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + } + } + + @Test + public void testSettleCommitsIdleOpenGroup() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(1_000, 60_000L); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender d = built.get(0); + Assert.assertTrue(d.hasDeferredFrames()); + Assert.assertEquals(0, d.pendingRowCount()); + Assert.assertEquals("an open group is unshipped work for the settle", 1, pool.settleIdle()); + Assert.assertEquals(1, d.flushCount.get()); + Assert.assertFalse(d.hasDeferredFrames()); + Assert.assertEquals("window restarted by the settle", 0, pool.settleIdle()); + // the next close after a settle commit is deferred again, not committing + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 2L).atNow(); + } + Assert.assertEquals(2, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + } + } + + @Test + public void testWindowOffMeansEveryCloseCommits() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + Assert.assertFalse(pool.hasCommitWindow()); + for (int i = 0; i < 2; i++) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", i).atNow(); + } + } + RecordingSender d = built.get(0); + Assert.assertEquals(2, d.flushCount.get()); + Assert.assertEquals(0, d.deferredFlushCount.get()); + } + } +} From a3a333a14422c48748c6ddb52a8499cac210cefe Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:03:45 +0100 Subject: [PATCH 16/57] fix(pool): reject negative commit windows, pin the settle reset, tighten close() docs Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 14 ++++-- .../io/questdb/client/impl/SenderPool.java | 8 ++-- .../io/questdb/client/impl/SenderSlot.java | 3 +- .../test/impl/SenderPoolCommitWindowTest.java | 48 ++++++++++++++++++- .../test/impl/SenderPoolSettleTest.java | 24 ++++++++++ 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 1f2485bae..346b59ad3 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -140,7 +140,8 @@ public void checkError() { } /** - * Returns this lease's slot to the pool. By default nothing is shipped: + * Returns this lease's slot to the pool. By default nothing is shipped + * (see the commit-window note below): * the rows stay in the slot's buffer and go out when an auto-flush * threshold trips on a later append or when the pool's housekeeper settles * the idle slot. With {@code flush_on_close=on} the rows are flushed first, @@ -162,8 +163,10 @@ public void checkError() { * With a commit window ({@link SenderPool#setCommitWindow}) and * {@code flush_on_close=on}, each close ships a deferred frame; the close * that fills the window (rows appended or age of the open group) flushes - * instead, which commits the group. Without {@code flush_on_close} the - * window still closes on that release, but nothing is shipped otherwise. + * instead, which commits the group. Without {@code flush_on_close}, only + * the release that closes the window ships: it flushes, committing the + * group. The window's age is measured from the first release that found + * work. A failed ship retires the slot and rethrows. */ @Override public void close() { @@ -188,8 +191,9 @@ public void close() { if (recycle) { pool.giveBack(this); } else { - // The probe or the cancel threw: the delegate has latched an - // error (or the generation went stale). Retire it; the + // The probe, the cancel or the ship threw: the delegate has + // latched an error, the flush failed, or the generation went + // stale. Retire it; the // original throwable propagates once this finally returns. pool.discardBroken(this); } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index cadfeb4a1..1fb92e9d7 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -246,13 +246,13 @@ public final class SenderPool implements AutoCloseable { // production; regression tests release a retired slot here to prove that // the terminal pass re-probes returned capacity before throwing. private volatile Runnable borrowWaitExpiredHook; + private volatile long commitIntervalNanos; + private volatile int commitRows; // Test seam invoked after close() handles an interrupt and confirms the // original creation-wait deadline still permits another wait. Null in // production; lifecycle tests use it to acknowledge distinct retries // without inspecting transient Condition queue membership. private volatile Runnable creationWaitRetryHook; - private volatile long commitIntervalNanos; - private volatile int commitRows; private volatile boolean flushOnClose; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the @@ -2143,12 +2143,12 @@ public int leakedSlotCount() { /** * Configures the commit window. Both values positive enable it; {@code 0, 0} - * disables it. Must be set before the first borrow; the pool's delegates + * disables it; negative values are rejected. Must be set before the first borrow; the pool's delegates * must have been built with {@code transaction=on} so that auto-flush * inside a lease ships deferred frames too (QuestDBBuilder injects it). */ public void setCommitWindow(int commitRows, long commitIntervalMillis) { - if ((commitRows > 0) != (commitIntervalMillis > 0)) { + if (commitRows < 0 || commitIntervalMillis < 0 || (commitRows > 0) != (commitIntervalMillis > 0)) { throw new IllegalArgumentException("commit window needs both rows and interval, or neither: rows=" + commitRows + ", intervalMillis=" + commitIntervalMillis); } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index c7ea8f45f..2c742edfd 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -52,12 +52,13 @@ final class SenderSlot { // slot's current lease (a borrower or the settle step) reads or writes it. // 0 = no group open since the last commit. private long groupOpenedAtNanos; - private long rowsAppendedAtLastCommit; // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. // Guarded by the pool lock. The pool maintains this index across swap // removals so the delegate's release callback can recover this exact slot // without scanning every other retired slot. private int retiredIndex = -1; + // Commit-window state, guarded by the lease (see groupOpenedAtNanos). + private long rowsAppendedAtLastCommit; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java index 8bf1ed18a..3c88a994a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java @@ -1,3 +1,27 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + package io.questdb.client.test.impl; import io.questdb.client.Sender; @@ -104,12 +128,30 @@ public void testRowsWindowClosesWithCommittingFlush() { } } + @Test + public void testSetCommitWindowRejectsNegativeOrMixedValues() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + int[][] bad = {{-1, -1}, {0, -5}, {5, 0}, {0, 5}}; + for (int[] b : bad) { + try { + pool.setCommitWindow(b[0], b[1]); + Assert.fail("expected IllegalArgumentException for " + b[0] + ", " + b[1]); + } catch (IllegalArgumentException expected) { + Assert.assertFalse(pool.hasCommitWindow()); + } + } + pool.setCommitWindow(0, 0); + Assert.assertFalse(pool.hasCommitWindow()); + } + } + @Test public void testSettleCommitsIdleOpenGroup() { List built = new ArrayList<>(); try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { pool.setFlushOnClose(true); - pool.setCommitWindow(1_000, 60_000L); + pool.setCommitWindow(2, 60_000L); try (Sender s = pool.borrow()) { s.table("t").longColumn("v", 1L).atNow(); } @@ -120,7 +162,9 @@ public void testSettleCommitsIdleOpenGroup() { Assert.assertEquals(1, d.flushCount.get()); Assert.assertFalse(d.hasDeferredFrames()); Assert.assertEquals("window restarted by the settle", 0, pool.settleIdle()); - // the next close after a settle commit is deferred again, not committing + // the settle marked the commit at 1 row, so the next close (2 rows + // appended in total, 1 in the window) defers; had the mark been + // missed, the window would count 2 rows and this close would commit try (Sender s = pool.borrow()) { s.table("t").longColumn("v", 2L).atNow(); } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index 3992cb2aa..d44dda42d 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -1,3 +1,27 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + package io.questdb.client.test.impl; import io.questdb.client.Sender; From 4a15032cd2d5dc718c13dec52f8110b3f06fe124 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:06:54 +0100 Subject: [PATCH 17/57] feat(pool): commit_rows and commit_interval_ms keys; window requires flush_on_close=on Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/QuestDBBuilder.java | 100 +++++++++++++++++- .../io/questdb/client/impl/ConfigSchema.java | 2 + .../io/questdb/client/impl/QuestDBImpl.java | 17 ++- .../client/test/QuestDBBuilderTest.java | 82 +++++++++++++- .../test/impl/PoolConfigHonoredTest.java | 2 + 5 files changed, 190 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index d41a7e219..60d0e9f90 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -54,6 +54,8 @@ public final class QuestDBBuilder { static final long DEFAULT_ACQUIRE_TIMEOUT_MILLIS = 5_000; + static final long DEFAULT_COMMIT_INTERVAL_MILLIS = 100; + static final int DEFAULT_COMMIT_ROWS = 1000; static final long DEFAULT_HOUSEKEEPER_INTERVAL_MILLIS = 5_000; static final long DEFAULT_IDLE_TIMEOUT_MILLIS = 60_000; static final long DEFAULT_MAX_LIFETIME_MILLIS = 30 * 60_000L; @@ -73,6 +75,8 @@ public final class QuestDBBuilder { private long acquireTimeoutMillis = UNSET; // Optional ingest-side async callbacks. Null -> each pooled Sender uses its // loud-not-silent default. Applied to every Sender the pool builds. + private long commitIntervalMillis = UNSET; + private int commitRows = UNSET; private SenderConnectionListener connectionListener; private BackgroundDrainerListener drainerListener; private SenderErrorHandler errorHandler; @@ -86,6 +90,7 @@ public final class QuestDBBuilder { private long queryCloseTimeoutMillis = UNSET; private int queryPoolMax = UNSET; private int queryPoolMin = UNSET; + private String resolvedIngestConfig; private int senderPoolMax = UNSET; private int senderPoolMin = UNSET; @@ -120,6 +125,35 @@ public QuestDBBuilder queryCloseTimeoutMillis(long millis) { return this; } + /** + * Age of a commit window in milliseconds. Setting a commit window makes + * pooled closes ship frames flagged defer-commit: the server appends them + * and commits once per window, when either {@link #commitRows(int)} rows + * or this many milliseconds have accumulated since the last commit. Acks + * then arrive at the commit, so a {@link Sender#drain(long)} should follow + * a {@link Sender#flush()}. Requires {@code flush_on_close=on}. When only + * one of the two window settings is given, the other takes the auto-flush + * default (1000 rows, 100 ms). + */ + public QuestDBBuilder commitIntervalMillis(long millis) { + if (millis < 1) { + throw new IllegalArgumentException("commitIntervalMillis must be >= 1"); + } + this.commitIntervalMillis = millis; + return this; + } + + /** + * Rows per commit window. See {@link #commitIntervalMillis(long)}. + */ + public QuestDBBuilder commitRows(int rows) { + if (rows < 1) { + throw new IllegalArgumentException("commitRows must be >= 1"); + } + this.commitRows = rows; + return this; + } + /** * Sets the async connection-event listener applied to every pooled ingest * {@link Sender}. The listener observes connect / disconnect / failover @@ -246,7 +280,37 @@ public QuestDB build() { if (flushOnClose == null) { flushOnClose = view.getBool("flush_on_close", false); } - long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis); + resolvePoolInt(commitRows, "commit_rows", view, UNSET, this::setCommitRowsResolved); + resolvePoolLong(commitIntervalMillis, "commit_interval_ms", view, UNSET, this::setCommitIntervalResolved); + boolean windowSet = commitRows != UNSET || commitIntervalMillis != UNSET; + if (windowSet) { + if (commitRows == UNSET) { + commitRows = DEFAULT_COMMIT_ROWS; + } + if (commitIntervalMillis == UNSET) { + commitIntervalMillis = DEFAULT_COMMIT_INTERVAL_MILLIS; + } + if (!flushOnClose) { + throw new IllegalArgumentException( + "conflicting configuration: commit_rows/commit_interval_ms need flush_on_close=on in this version. " + + "The buffered variant (flush_on_close=off with a commit window) is not available yet; " + + "set flush_on_close=on or remove the commit window keys."); + } + String transaction = view.getStr("transaction"); + if (transaction != null && !"on".equalsIgnoreCase(transaction)) { + throw new IllegalArgumentException( + "conflicting configuration: a commit window needs transaction=on on the pooled senders, but " + + "transaction=" + transaction + " was set. Remove transaction= (the pool sets it) or set it to on."); + } + if (transaction == null) { + ingestConfig = injectAfterSchema(ingestConfig, "transaction=on;"); + } + } else { + commitRows = 0; + commitIntervalMillis = 0; + } + resolvedIngestConfig = ingestConfig; + long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis, commitIntervalMillis); return new QuestDBImpl( ingestConfig, @@ -265,7 +329,9 @@ public QuestDB build() { connectionListener, drainerListener, flushOnClose, - settleIntervalMillis + settleIntervalMillis, + commitRows, + commitIntervalMillis ); } @@ -485,14 +551,28 @@ public java.util.Map poolConfigSnapshotForTest() { m.put("max_lifetime_ms", maxLifetimeMillis); m.put("housekeeper_interval_ms", housekeeperIntervalMillis); m.put("flush_on_close", flushOnClose); + m.put("commit_rows", commitRows); + m.put("commit_interval_ms", commitIntervalMillis); return m; } + @TestOnly + public String ingestConfigForTest() { + return resolvedIngestConfig; + } + // Inject a non-blocking async initial connect right after the schema // separator so lazy_connect's build never blocks or fail-fast on a down // server. Only used when the user set no initial_connect_retry of their own // (resolveLazyConnect rejects an explicit blocking mode rather than silently // overriding it), so placement is immaterial -- there is no competing value. + // Inserts a key right after the "ws::" / "wss::" schema separator so the + // sender parse sees it as part of the ingest config. + private static String injectAfterSchema(String config, String keyValue) { + int sep = config.indexOf("::") + 2; + return config.substring(0, sep) + keyValue + config.substring(sep); + } + private static String withDefaultAsyncConnect(String config) { int sep = config.indexOf("::"); // sep >= 0: fromConfig() validated a ws/wss schema, so "::" is present. @@ -504,14 +584,16 @@ private static String withDefaultAsyncConnect(String config) { * {@link #DEFAULT_SETTLE_INTERVAL_MILLIS} when unset. The WebSocket * transport rejects both {@code auto_flush=off} and * {@code auto_flush_interval=off}, so the interval is a number or unset. - * {@code housekeeperIntervalMillis} is currently unused. + * With a commit window, the shorter of that and {@code commitIntervalMillis} + * (0 = no window). {@code housekeeperIntervalMillis} is currently unused. * Public only for the unit test, which lives in another package. */ @TestOnly - public static long resolveSettleIntervalMillis(ConfigView view, long housekeeperIntervalMillis) { + public static long resolveSettleIntervalMillis(ConfigView view, long housekeeperIntervalMillis, long commitIntervalMillis) { // getLong uses the connect-string number syntax (underscore separators // included), the same one the Sender parse already accepted. - return Math.max(1L, view.getLong("auto_flush_interval", DEFAULT_SETTLE_INTERVAL_MILLIS)); + long fromAutoFlush = Math.max(1L, view.getLong("auto_flush_interval", DEFAULT_SETTLE_INTERVAL_MILLIS)); + return commitIntervalMillis > 0 ? Math.min(fromAutoFlush, commitIntervalMillis) : fromAutoFlush; } private static void requireWebSocketSchema(CharSequence config, String role) { @@ -535,4 +617,12 @@ private void resolvePoolLong(long current, String key, ConfigView view, long dfl } setter.accept(view.has(key) ? view.getLong(key, UNSET) : dflt); } + + private void setCommitIntervalResolved(long millis) { + this.commitIntervalMillis = millis; + } + + private void setCommitRowsResolved(int rows) { + this.commitRows = rows; + } } diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index dc78f8c9d..f1c404c83 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -125,6 +125,8 @@ public final class ConfigSchema { longRange("housekeeper_interval_ms", Side.POOL, OPEN, OPEN_MAX, false, false); boolOnOff("lazy_connect", Side.POOL); // facade flag: tolerant non-blocking startup (async ingest + lazy reads) boolOnOff("flush_on_close", Side.POOL); // pooled close() ships rows (on) or only releases the slot (off, default) + intRange("commit_rows", Side.POOL, OPEN, OPEN_MAX, false, false); // commit window: rows per server commit + longRange("commit_interval_ms", Side.POOL, OPEN, OPEN_MAX, false, false); // commit window: age per server commit // RESERVED -- accepted no-op (error-policy keys reserved by the spec). str("on_internal_error", Side.RESERVED); diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index e9fe52669..f20e25522 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -73,7 +73,7 @@ public QuestDBImpl( acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, queryCloseTimeoutMillis, null, errorHandler, connectionListener, drainerListener, - false, housekeeperIntervalMillis); + false, housekeeperIntervalMillis, 0, 0L); } public QuestDBImpl( @@ -93,13 +93,15 @@ public QuestDBImpl( SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener, boolean flushOnClose, - long settleIntervalMillis + long settleIntervalMillis, + int commitRows, + long commitIntervalMillis ) { this(ingestConfig, queryConfig, senderMin, senderMax, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, queryCloseTimeoutMillis, null, null, tokenProvider, errorHandler, connectionListener, drainerListener, - flushOnClose, settleIntervalMillis); + flushOnClose, settleIntervalMillis, commitRows, commitIntervalMillis); } // Test-only constructor exposing the senderFactory and connectHook seams: @@ -127,7 +129,7 @@ public QuestDBImpl( acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, housekeeperIntervalMillis, QueryClientPool.DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS, senderFactory, connectHook, null, null, null, null, - false, housekeeperIntervalMillis); + false, housekeeperIntervalMillis, 0, 0L); } // Full constructor adding the ingest-side errorHandler/connectionListener/ @@ -154,7 +156,9 @@ public QuestDBImpl( SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener, boolean flushOnClose, - long settleIntervalMillis + long settleIntervalMillis, + int commitRows, + long commitIntervalMillis ) { SenderPool builtSenderPool = null; QueryClientPool builtQueryPool = null; @@ -169,6 +173,9 @@ public QuestDBImpl( true, errorHandler, connectionListener, drainerListener, tokenProvider); builtSenderPool.setFlushOnClose(flushOnClose); + if (commitRows > 0) { + builtSenderPool.setCommitWindow(commitRows, commitIntervalMillis); + } builtQueryPool = new QueryClientPool( queryConfig, queryMin, queryMax, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, connectHook, null, tokenProvider); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index 9d2a87923..2c7437de5 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -284,17 +284,93 @@ public void testFlushOnCloseRejectsInvalidValue() { "flush_on_close"); } + @Test + public void testCommitWindowKeysParsedWithDefaults() { + QuestDBBuilder rowsOnly = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=500;"); + try (QuestDB ignored = rowsOnly.build()) { + Assert.assertEquals(500, rowsOnly.poolConfigSnapshotForTest().get("commit_rows")); + Assert.assertEquals("interval defaults to the auto-flush default", 100L, rowsOnly.poolConfigSnapshotForTest().get("commit_interval_ms")); + } + QuestDBBuilder intervalOnly = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_interval_ms=250;"); + try (QuestDB ignored = intervalOnly.build()) { + Assert.assertEquals("rows default to the auto-flush default", 1000, intervalOnly.poolConfigSnapshotForTest().get("commit_rows")); + Assert.assertEquals(250L, intervalOnly.poolConfigSnapshotForTest().get("commit_interval_ms")); + } + QuestDBBuilder none = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;"); + try (QuestDB ignored = none.build()) { + Assert.assertEquals("no window", 0, none.poolConfigSnapshotForTest().get("commit_rows")); + Assert.assertEquals(0L, none.poolConfigSnapshotForTest().get("commit_interval_ms")); + } + } + + @Test + public void testCommitWindowBuilderCallsOverrideString() { + QuestDBBuilder b = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=500;") + .commitRows(7) + .commitIntervalMillis(9L); + try (QuestDB ignored = b.build()) { + Assert.assertEquals(7, b.poolConfigSnapshotForTest().get("commit_rows")); + Assert.assertEquals(9L, b.poolConfigSnapshotForTest().get("commit_interval_ms")); + } + } + + @Test + public void testSettleIntervalTakesShorterOfAutoFlushAndCommitIntervals() { + ConfigView v = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); + Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 40L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 900L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 0L)); + ConfigView unset = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); + Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 5_000L, 40L)); + Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 5_000L, 900L)); + } + + @Test + public void testWindowInjectsTransactionOnIntoIngestConfig() { + QuestDBBuilder b = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=10;"); + try (QuestDB ignored = b.build()) { + Assert.assertTrue(b.ingestConfigForTest().contains("transaction=on;")); + } + QuestDBBuilder none = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;"); + try (QuestDB ignored = none.build()) { + Assert.assertFalse(none.ingestConfigForTest().contains("transaction=")); + } + } + + @Test + public void testWindowWithTransactionOffIsRejected() { + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=10;transaction=off;", + "transaction"); + } + + @Test + public void testWindowWithoutFlushOnCloseIsRejected() { + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;commit_rows=10;", + "flush_on_close"); + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=off;commit_interval_ms=10;", + "flush_on_close"); + } + @Test public void testSettleIntervalFollowsAutoFlushInterval() { // The WebSocket transport rejects both auto_flush=off and // auto_flush_interval=off, so the interval is unset (the sender's // 100 ms default) or a number (underscore separators allowed). ConfigView dflt = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); - Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 5_000L)); + Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 5_000L, 0L)); ConfigView explicit = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); - Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L, 0L)); ConfigView underscored = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=1_000;")); - Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 5_000L)); + Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 5_000L, 0L)); } @Test diff --git a/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java index 70f93456e..1cb6fb2ed 100644 --- a/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/PoolConfigHonoredTest.java @@ -61,6 +61,8 @@ public void testEveryPoolKeyIsHonored() throws Exception { expected.put("max_lifetime_ms", 98765L); expected.put("housekeeper_interval_ms", 222L); expected.put("flush_on_close", Boolean.TRUE); + expected.put("commit_rows", 500); + expected.put("commit_interval_ms", 250L); StringBuilder cfg = new StringBuilder("ws::addr=127.0.0.1:1;"); for (Map.Entry e : expected.entrySet()) { From 27b7082f3611d05bb2d55ff6856eef32e3cd0dda Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:09:43 +0100 Subject: [PATCH 18/57] fix(pool): keep build() idempotent for the commit window; range-check string window keys Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/QuestDBBuilder.java | 82 ++++++++++--------- .../client/test/QuestDBBuilderTest.java | 32 +++++++- 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index 60d0e9f90..c35850b18 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -73,10 +73,10 @@ public final class QuestDBBuilder { private static final int UNSET = -1; private long acquireTimeoutMillis = UNSET; - // Optional ingest-side async callbacks. Null -> each pooled Sender uses its - // loud-not-silent default. Applied to every Sender the pool builds. private long commitIntervalMillis = UNSET; private int commitRows = UNSET; + // Optional ingest-side async callbacks. Null -> each pooled Sender uses its + // loud-not-silent default. Applied to every Sender the pool builds. private SenderConnectionListener connectionListener; private BackgroundDrainerListener drainerListener; private SenderErrorHandler errorHandler; @@ -90,6 +90,8 @@ public final class QuestDBBuilder { private long queryCloseTimeoutMillis = UNSET; private int queryPoolMax = UNSET; private int queryPoolMin = UNSET; + private int resolvedCommitRows; + private long resolvedCommitIntervalMillis; private String resolvedIngestConfig; private int senderPoolMax = UNSET; private int senderPoolMin = UNSET; @@ -280,15 +282,26 @@ public QuestDB build() { if (flushOnClose == null) { flushOnClose = view.getBool("flush_on_close", false); } - resolvePoolInt(commitRows, "commit_rows", view, UNSET, this::setCommitRowsResolved); - resolvePoolLong(commitIntervalMillis, "commit_interval_ms", view, UNSET, this::setCommitIntervalResolved); - boolean windowSet = commitRows != UNSET || commitIntervalMillis != UNSET; - if (windowSet) { - if (commitRows == UNSET) { - commitRows = DEFAULT_COMMIT_ROWS; + int windowRows = commitRows; + if (windowRows == UNSET && view.has("commit_rows")) { + windowRows = view.getInt("commit_rows", UNSET); + if (windowRows < 1) { + throw new IllegalArgumentException("commit_rows must be >= 1, got " + windowRows); } - if (commitIntervalMillis == UNSET) { - commitIntervalMillis = DEFAULT_COMMIT_INTERVAL_MILLIS; + } + long windowIntervalMillis = commitIntervalMillis; + if (windowIntervalMillis == UNSET && view.has("commit_interval_ms")) { + windowIntervalMillis = view.getLong("commit_interval_ms", UNSET); + if (windowIntervalMillis < 1) { + throw new IllegalArgumentException("commit_interval_ms must be >= 1, got " + windowIntervalMillis); + } + } + if (windowRows != UNSET || windowIntervalMillis != UNSET) { + if (windowRows == UNSET) { + windowRows = DEFAULT_COMMIT_ROWS; + } + if (windowIntervalMillis == UNSET) { + windowIntervalMillis = DEFAULT_COMMIT_INTERVAL_MILLIS; } if (!flushOnClose) { throw new IllegalArgumentException( @@ -306,11 +319,13 @@ public QuestDB build() { ingestConfig = injectAfterSchema(ingestConfig, "transaction=on;"); } } else { - commitRows = 0; - commitIntervalMillis = 0; + windowRows = 0; + windowIntervalMillis = 0; } + resolvedCommitRows = windowRows; + resolvedCommitIntervalMillis = windowIntervalMillis; resolvedIngestConfig = ingestConfig; - long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis, commitIntervalMillis); + long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis, windowIntervalMillis); return new QuestDBImpl( ingestConfig, @@ -330,8 +345,8 @@ public QuestDB build() { drainerListener, flushOnClose, settleIntervalMillis, - commitRows, - commitIntervalMillis + windowRows, + windowIntervalMillis ); } @@ -533,6 +548,11 @@ public QuestDBBuilder senderPoolSize(int size) { return this; } + @TestOnly + public String ingestConfigForTest() { + return resolvedIngestConfig; + } + /** * Snapshot of the resolved pool config, keyed by connect-string key name. * Valid after {@link #build()} has run pool-key resolution. Drives the @@ -551,21 +571,11 @@ public java.util.Map poolConfigSnapshotForTest() { m.put("max_lifetime_ms", maxLifetimeMillis); m.put("housekeeper_interval_ms", housekeeperIntervalMillis); m.put("flush_on_close", flushOnClose); - m.put("commit_rows", commitRows); - m.put("commit_interval_ms", commitIntervalMillis); + m.put("commit_rows", resolvedCommitRows); + m.put("commit_interval_ms", resolvedCommitIntervalMillis); return m; } - @TestOnly - public String ingestConfigForTest() { - return resolvedIngestConfig; - } - - // Inject a non-blocking async initial connect right after the schema - // separator so lazy_connect's build never blocks or fail-fast on a down - // server. Only used when the user set no initial_connect_retry of their own - // (resolveLazyConnect rejects an explicit blocking mode rather than silently - // overriding it), so placement is immaterial -- there is no competing value. // Inserts a key right after the "ws::" / "wss::" schema separator so the // sender parse sees it as part of the ingest config. private static String injectAfterSchema(String config, String keyValue) { @@ -573,10 +583,14 @@ private static String injectAfterSchema(String config, String keyValue) { return config.substring(0, sep) + keyValue + config.substring(sep); } + // Inject a non-blocking async initial connect right after the schema + // separator so lazy_connect's build never blocks or fail-fast on a down + // server. Only used when the user set no initial_connect_retry of their own + // (resolveLazyConnect rejects an explicit blocking mode rather than silently + // overriding it), so placement is immaterial -- there is no competing value. private static String withDefaultAsyncConnect(String config) { - int sep = config.indexOf("::"); - // sep >= 0: fromConfig() validated a ws/wss schema, so "::" is present. - return config.substring(0, sep + 2) + "initial_connect_retry=async;" + config.substring(sep + 2); + // fromConfig() validated a ws/wss schema, so "::" is present. + return injectAfterSchema(config, "initial_connect_retry=async;"); } /** @@ -617,12 +631,4 @@ private void resolvePoolLong(long current, String key, ConfigView view, long dfl } setter.accept(view.has(key) ? view.getLong(key, UNSET) : dflt); } - - private void setCommitIntervalResolved(long millis) { - this.commitIntervalMillis = millis; - } - - private void setCommitRowsResolved(int rows) { - this.commitRows = rows; - } } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index 2c7437de5..21d346a50 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -334,7 +334,14 @@ public void testWindowInjectsTransactionOnIntoIngestConfig() { QuestDBBuilder b = QuestDB.builder() .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=10;"); try (QuestDB ignored = b.build()) { - Assert.assertTrue(b.ingestConfigForTest().contains("transaction=on;")); + Assert.assertTrue(b.ingestConfigForTest().startsWith("ws::transaction=on;")); + } + QuestDBBuilder explicitOn = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=10;transaction=on;"); + try (QuestDB ignored = explicitOn.build()) { + String resolved = explicitOn.ingestConfigForTest(); + Assert.assertEquals(resolved, resolved.indexOf("transaction=on;"), resolved.lastIndexOf("transaction=on;")); + Assert.assertTrue(resolved.contains("transaction=on;")); } QuestDBBuilder none = QuestDB.builder() .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;"); @@ -360,6 +367,29 @@ public void testWindowWithoutFlushOnCloseIsRejected() { "flush_on_close"); } + @Test + public void testBuildTwiceWithoutWindowStaysWindowless() { + QuestDBBuilder b = QuestDB.builder() + .fromConfig("ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;"); + for (int i = 0; i < 2; i++) { + try (QuestDB ignored = b.build()) { + Assert.assertEquals(0, b.poolConfigSnapshotForTest().get("commit_rows")); + Assert.assertEquals(0L, b.poolConfigSnapshotForTest().get("commit_interval_ms")); + Assert.assertFalse(b.ingestConfigForTest().contains("transaction=")); + } + } + } + + @Test + public void testCommitWindowStringKeysBelowOneAreRejected() { + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_rows=0;", + "commit_rows"); + assertBuildRejected( + "ws::addr=127.0.0.1:1;sender_pool_min=0;query_pool_min=0;flush_on_close=on;commit_interval_ms=-5;", + "commit_interval_ms"); + } + @Test public void testSettleIntervalFollowsAutoFlushInterval() { // The WebSocket transport rejects both auto_flush=off and From 68036756e6a65b845a6b1fd3511cb8c5be99f129 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:13:16 +0100 Subject: [PATCH 19/57] test(pool): commit window on the wire: flags, age, settle commit, terminal and retriable rejections Co-Authored-By: Claude Fable 5.1 --- .../client/test/QuestDBCommitWindowTest.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java diff --git a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java new file mode 100644 index 000000000..e11b5b2ff --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java @@ -0,0 +1,223 @@ +package io.questdb.client.test; + +import io.questdb.client.LineSenderServerException; +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.test.QuestDBFlushOnCloseTest.FrameLog; +import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * The commit window as the server sees it: flagged frames per close, one + * unflagged frame per window, a rowless commit message for an idle tail, and + * the client's reaction to a rejected frame inside a group. + */ +public class QuestDBCommitWindowTest { + + @Test + public void testAgeWindowCommitsAtNextRelease() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=1000;commit_interval_ms=100;housekeeper_interval_ms=60000;auto_flush_interval=60000;"))) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + // Hold the only slot across the window: the settle skips a leased slot (it + // would otherwise commit the idle group on its 100 ms cadence), so the + // release below is the first thing to find the group older than the window. + try (Sender s = db.borrowSender()) { + Thread.sleep(150); + s.table("t").longColumn("v", 2L).atNow(); + } + log.awaitDataFrames(2, 5_000); + Assert.assertTrue(log.deferFlag(0)); + Assert.assertFalse(log.deferFlag(1)); + } + } + } + + @Test + public void testLongLeaseAutoFlushShipsFlaggedAndFlushCommits() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_rows=2;commit_rows=1000;commit_interval_ms=60000;housekeeper_interval_ms=60000;"))) { + try (Sender s = db.borrowSender()) { + for (int i = 0; i < 3; i++) { + s.table("t").longColumn("v", i).atNow(); // auto-flush at 2 rows: flagged + } + s.flush(); // row 3 rides in the committing frame + } + log.awaitDataFrames(2, 5_000); + Assert.assertTrue(log.deferFlag(0)); + Assert.assertFalse(log.deferFlag(1)); + } + } + } + + @Test + public void testRetriableCommitFailureReplaysGroupWithoutApplicationError() throws Exception { + RejectSecondFrameHandler handler = new RejectSecondFrameHandler(WebSocketResponse.STATUS_INTERNAL_ERROR); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=2;commit_interval_ms=60000;housekeeper_interval_ms=60000;") + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 2L).atNow(); // closes the window: the committing frame is rejected + } + handler.awaitSecondConnectionFrames(2, 5_000); + Assert.assertEquals("both frames replay on the new connection", 2, handler.secondConnectionDataFrames()); + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 3L).atNow(); // no exception reaches the application + } + } + } + } + + @Test + public void testRowsWindowShipsFlaggedFramesThenOneCommittingFrame() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=3;commit_interval_ms=60000;housekeeper_interval_ms=60000;"))) { + QuestDBFlushOnCloseTest.appendThreeRowsInThreeLeases(db); + log.awaitDataFrames(3, 5_000); + Assert.assertTrue(log.deferFlag(0)); + Assert.assertTrue(log.deferFlag(1)); + Assert.assertFalse("third close fills the window: unflagged, commits the group", log.deferFlag(2)); + Assert.assertEquals("no separate commit message when the closing frame carries rows", 0, log.commitFrames()); + } + } + } + + @Test + public void testSettleCommitsIdleGroupWithCommitMessage() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // settle cadence = min(auto_flush_interval, commit_interval_ms) = 100 ms + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=1000;commit_interval_ms=100;housekeeper_interval_ms=60000;"))) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + log.awaitDataFrames(1, 5_000); + Assert.assertTrue(log.deferFlag(0)); + log.awaitCommitFrames(1, 3_000); + } + } + } + + @Test + public void testTerminalOnSecondFrameSurfacesOnNextCallWithItsSequence() throws Exception { + RejectSecondFrameHandler handler = new RejectSecondFrameHandler(WebSocketResponse.STATUS_SCHEMA_MISMATCH); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=1000;commit_interval_ms=60000;housekeeper_interval_ms=60000;") + + "reconnect_max_duration_millis=500;reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { + // No try-with-resources: close() rethrows the instance the body threw, and + // the compiler-generated addSuppressed(self) would mask it with an + // IllegalArgumentException. + LineSenderServerException seen = null; + long deadline = System.currentTimeMillis() + 5_000; + for (long v = 1; seen == null && (v <= 2 || System.currentTimeMillis() < deadline); v++) { + Sender s = db.borrowSender(); + try { + s.table("t").longColumn("v", v).atNow(); + } catch (LineSenderServerException e) { + seen = e; + } + try { + s.close(); + } catch (LineSenderServerException e) { + seen = e; + } + Thread.sleep(20); + } + Assert.assertNotNull("the latched terminal must surface on a later call on that slot", seen); + Assert.assertEquals("it names the rejected frame", 1L, seen.getServerError().getMessageSequence()); + try (Sender s = db.borrowSender()) { + Assert.assertNotNull("the pool replaces the retired slot", s); + } + } + } + } + + private static String config(int port, String extra) { + return QuestDBFlushOnCloseTest.config(port, "flush_on_close=on;" + extra); + } + + /** + * First connection: withholds the ack of frame 0 (as the real server does + * for a flagged frame inside an open group) and NACKs frame 1 with the + * given status. Later connections: acks everything. Counts the frames the + * second connection receives, so a replay from the acked watermark, which + * is nothing, shows both frames again. + */ + private static final class RejectSecondFrameHandler implements TestWebSocketServer.WebSocketServerHandler { + private final byte rejectStatus; + private int connections; + private TestWebSocketServer.ClientHandler currentClient; + private TestWebSocketServer.ClientHandler firstClient; + private long nextSeq; + private int secondConnectionDataFrames; + + RejectSecondFrameHandler(byte rejectStatus) { + this.rejectStatus = rejectStatus; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + connections++; + nextSeq = 0; + if (firstClient == null) { + firstClient = client; + } + } + long seq = nextSeq++; + if (connections >= 2 && data.length >= 8 && ((data[6] & 0xFF) | ((data[7] & 0xFF) << 8)) > 0) { + secondConnectionDataFrames++; + } + try { + if (client == firstClient) { + if (seq == 1) { + client.sendBinary(QwpWireTestUtils.buildNack(seq, rejectStatus)); + } + // seq 0: ack withheld, the frame stays replayable client-side + } else { + client.sendBinary(QwpWireTestUtils.buildAck(seq)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + synchronized int secondConnectionDataFrames() { + return secondConnectionDataFrames; + } + + void awaitSecondConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (secondConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + } + } +} \ No newline at end of file From 2ce970617095909733fcf8a0dc5e48dc29c33b28 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:17:35 +0100 Subject: [PATCH 20/57] fix(pool): pooled close() wraps the delegate's error so try-with-resources keeps the primary exception close() threw the delegate's latched instance, which a lease body may already have thrown; try-with-resources then failed with a self-suppression IllegalArgumentException. close() now throws a new LineSenderException with the original as cause. Tests updated, terminal test restored to try-with-resources and hardened. Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 35 ++++++++++++----- .../client/test/QuestDBCommitWindowTest.java | 39 +++++++++---------- .../impl/SenderPoolCloseSemanticsTest.java | 12 ++++-- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 346b59ad3..a978050a7 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -153,7 +153,10 @@ public void checkError() { * A row left in progress is cancelled, the slot is returned, and then a * {@link LineSenderException} is thrown so the mistake is visible. A * delegate that has latched a terminal error is retired instead of being - * returned, and the error is rethrown here. + * returned, and a new {@link LineSenderException} is thrown here whose + * cause is the delegate's error. A lease body that already threw the typed + * error keeps it as the primary exception, with close's wrapper + * suppressed. *

* Idempotent for sequential reuse: a stale generation (the lease was * already returned and the slot possibly re-borrowed) is a no-op. Concurrent @@ -166,7 +169,7 @@ public void checkError() { * instead, which commits the group. Without {@code flush_on_close}, only * the release that closes the window ships: it flushes, committing the * group. The window's age is measured from the first release that found - * work. A failed ship retires the slot and rethrows. + * work. A failed ship retires the slot and throws the wrapper described above. */ @Override public void close() { @@ -176,28 +179,40 @@ public void close() { SenderPool pool = slot.pool(); boolean cancelledRow = false; boolean recycle = false; + RuntimeException failure = null; try { // live(), not delegate(): re-validate the generation right before // touching the delegate, narrowing the duplicate-close race. Sender delegate = slot.live(generation); - delegate.checkError(); - if (delegate.hasRowInProgress()) { - delegate.cancelRow(); - cancelledRow = true; + try { + delegate.checkError(); + if (delegate.hasRowInProgress()) { + delegate.cancelRow(); + cancelledRow = true; + } + ship(pool, delegate); + recycle = true; + } catch (RuntimeException e) { + // The delegate rethrows its latched instance, which a lease + // body may already have thrown: wrap it below, after the + // slot is retired. + failure = e; } - ship(pool, delegate); - recycle = true; } finally { if (recycle) { pool.giveBack(this); } else { // The probe, the cancel or the ship threw: the delegate has // latched an error, the flush failed, or the generation went - // stale. Retire it; the - // original throwable propagates once this finally returns. + // stale. Retire it; a wrapper around the original propagates + // once this finally returns. pool.discardBroken(this); } } + if (failure != null) { + String msg = failure.getMessage(); + throw new LineSenderException("pooled sender close failed: " + msg, failure); + } if (cancelledRow) { throw new LineSenderException("row in progress at close(): the row was cancelled and the sender returned to the pool; " + "finish rows with at(), atNow() or cancelRow() before close()"); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java index e11b5b2ff..6f9ca2c12 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java @@ -4,6 +4,7 @@ import io.questdb.client.QuestDB; import io.questdb.client.Sender; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.impl.PooledSender; import io.questdb.client.test.QuestDBFlushOnCloseTest.FrameLog; import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; @@ -60,6 +61,7 @@ public void testLongLeaseAutoFlushShipsFlaggedAndFlushCommits() throws Exception log.awaitDataFrames(2, 5_000); Assert.assertTrue(log.deferFlag(0)); Assert.assertFalse(log.deferFlag(1)); + Assert.assertEquals(0, log.commitFrames()); } } } @@ -70,7 +72,7 @@ public void testRetriableCommitFailureReplaysGroupWithoutApplicationError() thro try (TestWebSocketServer server = new TestWebSocketServer(handler)) { server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=2;commit_interval_ms=60000;housekeeper_interval_ms=60000;") + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=2;commit_interval_ms=60000;housekeeper_interval_ms=60000;auto_flush_interval=60000;") + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { try (Sender s = db.borrowSender()) { s.table("t").longColumn("v", 1L).atNow(); @@ -128,31 +130,26 @@ public void testTerminalOnSecondFrameSurfacesOnNextCallWithItsSequence() throws try (TestWebSocketServer server = new TestWebSocketServer(handler)) { server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=1000;commit_interval_ms=60000;housekeeper_interval_ms=60000;") + try (QuestDB db = QuestDB.connect(config(server.getPort(), "commit_rows=1000;commit_interval_ms=60000;housekeeper_interval_ms=60000;auto_flush_interval=60000;") + "reconnect_max_duration_millis=500;reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { - // No try-with-resources: close() rethrows the instance the body threw, and - // the compiler-generated addSuppressed(self) would mask it with an - // IllegalArgumentException. + Sender retiredDelegate = null; LineSenderServerException seen = null; long deadline = System.currentTimeMillis() + 5_000; for (long v = 1; seen == null && (v <= 2 || System.currentTimeMillis() < deadline); v++) { - Sender s = db.borrowSender(); - try { + // try-with-resources: the typed error stays primary, close()'s wrapper is suppressed + try (Sender s = db.borrowSender()) { + retiredDelegate = ((PooledSender) s).getDelegateForTesting(); s.table("t").longColumn("v", v).atNow(); } catch (LineSenderServerException e) { seen = e; } - try { - s.close(); - } catch (LineSenderServerException e) { - seen = e; - } Thread.sleep(20); } Assert.assertNotNull("the latched terminal must surface on a later call on that slot", seen); Assert.assertEquals("it names the rejected frame", 1L, seen.getServerError().getMessageSequence()); try (Sender s = db.borrowSender()) { - Assert.assertNotNull("the pool replaces the retired slot", s); + Assert.assertNotSame("the pool replaces the retired slot", retiredDelegate, ((PooledSender) s).getDelegateForTesting()); + s.table("t").longColumn("v", 4L).atNow(); } } } @@ -170,17 +167,24 @@ private static String config(int port, String extra) { * is nothing, shows both frames again. */ private static final class RejectSecondFrameHandler implements TestWebSocketServer.WebSocketServerHandler { - private final byte rejectStatus; private int connections; private TestWebSocketServer.ClientHandler currentClient; private TestWebSocketServer.ClientHandler firstClient; private long nextSeq; + private final byte rejectStatus; private int secondConnectionDataFrames; RejectSecondFrameHandler(byte rejectStatus) { this.rejectStatus = rejectStatus; } + void awaitSecondConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (secondConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + } + @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { if (currentClient != client) { @@ -212,12 +216,5 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien synchronized int secondConnectionDataFrames() { return secondConnectionDataFrames; } - - void awaitSecondConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { - long deadline = System.currentTimeMillis() + timeoutMillis; - while (secondConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { - Thread.sleep(10); - } - } } } \ No newline at end of file diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index 732e75571..6df9c9a77 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -55,12 +55,14 @@ public void testCancelRowThrowingAtCloseRetiresSlot() { // A terminal latched between the probe and the cancel: model it as // cancelRow() itself failing. The lease must still come home. delegate.rowInProgress = true; - delegate.cancelError = new LineSenderException("latched during cancel"); + LineSenderException original = new LineSenderException("latched during cancel"); + delegate.cancelError = original; try { first.close(); Assert.fail("close() must propagate the cancel failure"); } catch (LineSenderException e) { - Assert.assertEquals("latched during cancel", e.getMessage()); + Assert.assertSame(original, e.getCause()); + Assert.assertTrue(e.getMessage().contains("latched during cancel")); } Assert.assertEquals("slot must be retired, not stranded", 0, pool.totalSize()); Assert.assertEquals(1, delegate.closeCount.get()); @@ -118,12 +120,14 @@ public void testLatchedErrorAtCloseRetiresSlotAndThrows() { try (SenderPool pool = newPool(1, 1, built)) { Sender first = pool.borrow(); RecordingSender delegate = built.get(0); - delegate.latchedError = new LineSenderException("latched terminal"); + LineSenderException original = new LineSenderException("latched terminal"); + delegate.latchedError = original; try { first.close(); Assert.fail("close() must rethrow the latched terminal"); } catch (LineSenderException e) { - Assert.assertEquals("latched terminal", e.getMessage()); + Assert.assertSame(original, e.getCause()); + Assert.assertTrue(e.getMessage().contains("latched terminal")); } Assert.assertEquals("dead slot must be retired", 0, pool.totalSize()); Assert.assertEquals("retired delegate must be closed", 1, delegate.closeCount.get()); From e2cae4951672fc7eda3c1486af09ebd89f00968b Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:23:33 +0100 Subject: [PATCH 21/57] fix(pool): keep retryable on the close() wrapper; order handler members; tolerate a late-latched terminal in the test Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 7 ++++-- .../client/test/QuestDBCommitWindowTest.java | 23 ++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index a978050a7..80f8b7467 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -156,7 +156,7 @@ public void checkError() { * returned, and a new {@link LineSenderException} is thrown here whose * cause is the delegate's error. A lease body that already threw the typed * error keeps it as the primary exception, with close's wrapper - * suppressed. + * suppressed. The typed error and its classification live on the cause. *

* Idempotent for sequential reuse: a stale generation (the lease was * already returned and the slot possibly re-borrowed) is a no-op. Concurrent @@ -211,7 +211,10 @@ public void close() { } if (failure != null) { String msg = failure.getMessage(); - throw new LineSenderException("pooled sender close failed: " + msg, failure); + boolean retryable = failure instanceof LineSenderException && ((LineSenderException) failure).isRetryable(); + LineSenderException wrapper = new LineSenderException("pooled sender close failed: " + msg, retryable); + wrapper.initCause(failure); + throw wrapper; } if (cancelledRow) { throw new LineSenderException("row in progress at close(): the row was cancelled and the sender returned to the pool; " diff --git a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java index 6f9ca2c12..c81366f42 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java @@ -3,6 +3,7 @@ import io.questdb.client.LineSenderServerException; import io.questdb.client.QuestDB; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.impl.PooledSender; import io.questdb.client.test.QuestDBFlushOnCloseTest.FrameLog; @@ -142,6 +143,12 @@ public void testTerminalOnSecondFrameSurfacesOnNextCallWithItsSequence() throws s.table("t").longColumn("v", v).atNow(); } catch (LineSenderServerException e) { seen = e; + } catch (LineSenderException e) { + // the terminal latched after the last row call: close() wraps it + if (!(e.getCause() instanceof LineSenderServerException)) { + throw e; + } + seen = (LineSenderServerException) e.getCause(); } Thread.sleep(20); } @@ -167,24 +174,17 @@ private static String config(int port, String extra) { * is nothing, shows both frames again. */ private static final class RejectSecondFrameHandler implements TestWebSocketServer.WebSocketServerHandler { + private final byte rejectStatus; private int connections; private TestWebSocketServer.ClientHandler currentClient; private TestWebSocketServer.ClientHandler firstClient; private long nextSeq; - private final byte rejectStatus; private int secondConnectionDataFrames; RejectSecondFrameHandler(byte rejectStatus) { this.rejectStatus = rejectStatus; } - void awaitSecondConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { - long deadline = System.currentTimeMillis() + timeoutMillis; - while (secondConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { - Thread.sleep(10); - } - } - @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { if (currentClient != client) { @@ -213,6 +213,13 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } + void awaitSecondConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (secondConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + } + synchronized int secondConnectionDataFrames() { return secondConnectionDataFrames; } From 9ffb0410526107572fe75f4522a9f871812fe4d0 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:31:10 +0100 Subject: [PATCH 22/57] docs(pool): commit window keys; add the borrow-per-row benchmark Co-Authored-By: Claude Fable 5.1 --- README.md | 7 ++ .../test/PoolBorrowPerRowBenchmark.java | 79 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java diff --git a/README.md b/README.md index e09fc79ec..166222222 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,11 @@ The sender buffers rows and sends them in batches. For high throughput, write ma flush per batch rather than per row: each flush has a fixed cost, so batching raises throughput. The final partial batch ships on the pool's next settle; call `flush()` before `close()` if you need it on the wire now. +If you borrow per message and want each message in the store-and-forward ring the moment `close()` returns, set +`flush_on_close=on` together with a commit window (`commit_rows`, `commit_interval_ms`): every close ships one frame +flagged defer-commit, the server appends it, and the pool commits once per window instead of once per close. A +rejected frame rolls the open window back on the server and the client replays the healthy frames from the ring. + ```java try (Sender sender = db.borrowSender()) { for (int i = 0; i < trades.size(); i++) { @@ -547,6 +552,8 @@ schema::key1=value1;key2=value2; | `idle_timeout_ms` | `60000` | How long a pooled connection may stay idle before it is reaped | | `max_lifetime_ms` | `1800000` | Maximum lifetime of a pooled connection before it is recycled | | `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged | +| `commit_rows` | _(unset)_ | Commit window: with `flush_on_close=on`, closes ship frames flagged defer-commit and the server commits once per window of this many rows. Acks arrive at the commit; `drain()` after `flush()` | +| `commit_interval_ms` | _(unset)_ | Commit window age in milliseconds; setting either window key enables the window, the other defaults to 1000 rows / 100 ms | ### Query routing keys diff --git a/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java b/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java new file mode 100644 index 000000000..0ccb419a9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java @@ -0,0 +1,79 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +/** + * The customer's shape: one million rows, one borrow per row, against the + * ack-all test server. Run by hand; prints wall time per configuration. + * Target: the default is within noise of a single held lease; the window + * numbers answer whether frames or commits dominate. + */ +//@Ignore("manual benchmark") +public class PoolBorrowPerRowBenchmark { + private static final int ROWS = 1_000_000; + + @Test + public void run() throws Exception { + time("held lease, default", "", false); + time("borrow per row, default (buffer on close)", "", true); + time("borrow per row, flush_on_close=on (today's shape)", "flush_on_close=on;", true); + time("borrow per row, flush_on_close=on + window 1000/100", "flush_on_close=on;commit_rows=1000;commit_interval_ms=100;", true); + } + + private static void time(String label, String extra, boolean borrowPerRow) throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + server.awaitStart(5, TimeUnit.SECONDS); + String cfg = "ws::addr=localhost:" + server.getPort() + + ";sender_pool_min=1;sender_pool_max=1;query_pool_min=0;close_flush_timeout_millis=60000;" + extra; + long start = System.nanoTime(); + try (QuestDB db = QuestDB.connect(cfg)) { + if (borrowPerRow) { + for (int i = 0; i < ROWS; i++) { + try (Sender s = db.borrowSender()) { + s.table("bench").symbol("sym", "ABC").doubleColumn("px", i).longColumn("qty", i).atNow(); + } + } + } else { + try (Sender s = db.borrowSender()) { + for (int i = 0; i < ROWS; i++) { + s.table("bench").symbol("sym", "ABC").doubleColumn("px", i).longColumn("qty", i).atNow(); + } + } + } + } + long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + System.out.printf("%-60s %6d ms, %d data frames%n", label, millis, log.dataFrames()); + } + } +} From 58e7102424e25c0c227020fa1d07f4279df46b76 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:31:29 +0100 Subject: [PATCH 23/57] test(pool): restore @Ignore on the benchmark Co-Authored-By: Claude Fable 5.1 --- .../java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java b/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java index 0ccb419a9..0d75d4964 100644 --- a/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java +++ b/core/src/test/java/io/questdb/client/test/PoolBorrowPerRowBenchmark.java @@ -27,6 +27,7 @@ import io.questdb.client.QuestDB; import io.questdb.client.Sender; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Ignore; import org.junit.Test; import java.util.concurrent.TimeUnit; @@ -37,7 +38,7 @@ * Target: the default is within noise of a single held lease; the window * numbers answer whether frames or commits dominate. */ -//@Ignore("manual benchmark") +@Ignore("manual benchmark") public class PoolBorrowPerRowBenchmark { private static final int ROWS = 1_000_000; From 21bfb593122cf40332dcc0ddd8afdd3e10e0801e Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:50:30 +0100 Subject: [PATCH 24/57] fix(pool): surface shutdown and settle ship failures, keep the window mark on drain, typed close errors Co-Authored-By: Claude Fable 5.1 --- README.md | 6 +- .../main/java/io/questdb/client/Sender.java | 5 +- .../questdb/client/impl/PoolHousekeeper.java | 2 +- .../io/questdb/client/impl/PooledSender.java | 23 ++++++- .../io/questdb/client/impl/SenderPool.java | 41 ++++++++++++- .../io/questdb/client/impl/SenderSlot.java | 17 ++++-- .../client/test/QuestDBFlushOnCloseTest.java | 61 ++++++++++++++++++- .../client/test/impl/RecordingSender.java | 11 ++++ .../impl/SenderPoolCloseSemanticsTest.java | 47 +++++++++++++- .../test/impl/SenderPoolCommitWindowTest.java | 23 +++++++ 10 files changed, 218 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 166222222..d5a5e71d5 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ snippets below borrow from its pools. Borrow a `Sender`, write rows, and close it. Close releases, flush ships, drain waits: `close()` returns the sender to the pool and ships nothing by itself; rows go out when an auto-flush threshold trips on a later append or when the -pool's housekeeper settles the idle slot, at most one `auto_flush_interval` later (100 ms by default). Borrowing per +pool's housekeeper settles the idle slot, typically within one `auto_flush_interval` (100 ms by default). Borrowing per message is the intended pattern and batches like a sender held for the whole run. Call `flush()` to ship now and `drain()` to wait for the server. The underlying connection is only torn down when the `QuestDB` handle itself is closed. @@ -551,9 +551,9 @@ schema::key1=value1;key2=value2; | `acquire_timeout_ms` | `5000` | How long `borrowSender()`/`borrowQuery()` waits for a free slot | | `idle_timeout_ms` | `60000` | How long a pooled connection may stay idle before it is reaped | | `max_lifetime_ms` | `1800000` | Maximum lifetime of a pooled connection before it is recycled | -| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged | +| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged. Close-time errors are wrapped; the delegate's error is the cause | | `commit_rows` | _(unset)_ | Commit window: with `flush_on_close=on`, closes ship frames flagged defer-commit and the server commits once per window of this many rows. Acks arrive at the commit; `drain()` after `flush()` | -| `commit_interval_ms` | _(unset)_ | Commit window age in milliseconds; setting either window key enables the window, the other defaults to 1000 rows / 100 ms | +| `commit_interval_ms` | _(unset)_ | Commit window age in milliseconds; setting either window key enables the window, the other defaults to 1000 rows / 100 ms. An idle slot's open group is committed at the settle cadence, so the interval only bounds groups on busy slots | ### Query routing keys diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 208841012..2f6881da1 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -682,8 +682,9 @@ default boolean hasDeferredFrames() { } /** - * Whether a row has been started with {@link #table(CharSequence)} and not - * yet finished with {@link #at}, {@link #atNow()} or {@link #cancelRow()}. + * Whether a row is in progress: at least one column value has been set for + * the current row (just {@link #table(CharSequence)} does not count) and the + * row is not yet finished with {@link #at}, {@link #atNow()} or {@link #cancelRow()}. * Only the WebSocket QWP transport tracks this; other transports return * {@code false}. * diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 066fb5817..956af353a 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -161,7 +161,7 @@ static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { } private void runLoop() { - long nextSettleAt = System.currentTimeMillis() + settleIntervalMillis; + long nextSettleAt = System.currentTimeMillis() + this.settleIntervalMillis; long nextReapAt = System.currentTimeMillis() + intervalMillis; while (!stop) { // Per-slot startup SF recovery, driven on THIS (the reap-loop) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 80f8b7467..69252039e 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -24,6 +24,7 @@ package io.questdb.client.impl; +import io.questdb.client.LineSenderServerException; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; @@ -170,6 +171,11 @@ public void checkError() { * the release that closes the window ships: it flushes, committing the * group. The window's age is measured from the first release that found * work. A failed ship retires the slot and throws the wrapper described above. + *

+ * Close-time errors are wrapped: the delegate's error is the cause. A latched + * {@link LineSenderServerException} is rethrown as a new one carrying the same + * {@code SenderError}. While the pool is shutting down, the rows are flushed here + * so a ship failure propagates to the caller. */ @Override public void close() { @@ -190,7 +196,12 @@ public void close() { delegate.cancelRow(); cancelledRow = true; } - ship(pool, delegate); + if (pool.isClosed()) { + // Pool shutting down: ship here so a failure reaches the caller. + delegate.flush(); + } else { + ship(pool, delegate); + } recycle = true; } catch (RuntimeException e) { // The delegate rethrows its latched instance, which a lease @@ -210,6 +221,11 @@ public void close() { } } if (failure != null) { + if (failure instanceof LineSenderServerException) { + LineSenderServerException typed = new LineSenderServerException(((LineSenderServerException) failure).getServerError()); + typed.initCause(failure); + throw typed; + } String msg = failure.getMessage(); boolean retryable = failure instanceof LineSenderException && ((LineSenderException) failure).isRetryable(); LineSenderException wrapper = new LineSenderException("pooled sender close failed: " + msg, retryable); @@ -278,7 +294,10 @@ public Sender doubleColumn(CharSequence name, double value) { @Override public boolean drain(long timeoutMillis) { - return slot.live(generation).drain(timeoutMillis); + Sender delegate = slot.live(generation); + boolean drained = delegate.drain(timeoutMillis); + slot.markCommitted(delegate.rowsAppended()); + return drained; } @Override diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1fb92e9d7..41466cc0d 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1765,12 +1765,14 @@ public int settleIdle() { boolean flushed = false; Throwable fatal = null; try { + // PooledSender.flush() already marks the commit on the slot. lease.flush(); - lease.slot().markCommitted(lease.slot().delegate().rowsAppended()); + lease.slot().settleFailureLogged(false); flushed = true; - } catch (Exception ignored) { + } catch (Exception e) { // Transient (ring full, recycle in flight) or terminal: the // requeue below tells them apart with the probe. + logSettleFailure(lease.slot(), e); } catch (Throwable t) { // An Error (AssertionError, OutOfMemoryError) is not transient. fatal = t; @@ -1793,6 +1795,23 @@ public int settleIdle() { return settled; } + private void logSettleFailure(SenderSlot slot, Exception e) { + int idx = slot.slotIndex(); + int pending = 0; + try { + pending = slot.delegate().pendingRowCount(); + } catch (Throwable ignored) { + // best effort: the count is only for the log line + } + String error = e.toString(); + if (!slot.settleFailureLogged()) { + slot.settleFailureLogged(true); + LOG.warn("SenderPool: settle flush failed for slot {} with {} pending row(s), will retry: {}", idx, pending, error); + } else { + LOG.debug("SenderPool: settle flush failed for slot {} with {} pending row(s), will retry: {}", idx, pending, error); + } + } + private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { if (recoveryThread == null || recoveryThread == Thread.currentThread()) { return; @@ -1970,9 +1989,20 @@ private void retireLease(PooledSender ps, String context) { // Close the delegate outside the lock (releases the SF flock). Always // attempt it so native resources are freed even on the defensive path // where the wrapper had already left `all`. + int pendingAtRetire = 0; try { - s.delegate().close(); + pendingAtRetire = s.delegate().pendingRowCount(); } catch (Throwable ignored) { + // best effort: the count is only for the log line + } + try { + s.delegate().close(); + } catch (Throwable t) { + if (pendingAtRetire > 0) { + String error = t.toString(); + LOG.warn("SenderPool: slot {} retired with {} unshipped row(s); delegate close failed: {}", + s.slotIndex(), pendingAtRetire, error); + } // Best-effort teardown: a delegate close() can throw an Error // (e.g. an -ea AssertionError) as well as a RuntimeException. // Either way the accounting in the finally below MUST run, @@ -2111,6 +2141,11 @@ public boolean hasCommitWindow() { return commitRows > 0; } + /** Whether {@code close()} has started; a lease returning now is torn down instead of recycled. */ + boolean isClosed() { + return closed; + } + /** * Whether a pooled {@code close()} ships the lease's rows before releasing * the slot (the pre-1.4 behaviour) or only releases it. Off by default: diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 2c742edfd..0271a1830 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -50,8 +50,8 @@ final class SenderSlot { private final int slotIndex; // Commit-window state, guarded by the lease: only the thread holding the // slot's current lease (a borrower or the settle step) reads or writes it. - // 0 = no group open since the last commit. - private long groupOpenedAtNanos; + // Long.MIN_VALUE = no group open since the last commit. + private long groupOpenedAtNanos = Long.MIN_VALUE; // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. // Guarded by the pool lock. The pool maintains this index across swap // removals so the delegate's release callback can recover this exact slot @@ -59,6 +59,7 @@ final class SenderSlot { private int retiredIndex = -1; // Commit-window state, guarded by the lease (see groupOpenedAtNanos). private long rowsAppendedAtLastCommit; + private boolean settleFailureLogged; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its @@ -121,7 +122,7 @@ Sender live(long gen) { */ void markCommitted(long rowsAppended) { rowsAppendedAtLastCommit = rowsAppended; - groupOpenedAtNanos = 0L; + groupOpenedAtNanos = Long.MIN_VALUE; } void markIdleAt(long nowMillis) { @@ -140,6 +141,14 @@ void retiredIndex(int retiredIndex) { this.retiredIndex = retiredIndex; } + void settleFailureLogged(boolean logged) { + this.settleFailureLogged = logged; + } + + boolean settleFailureLogged() { + return settleFailureLogged; + } + int slotIndex() { return slotIndex; } @@ -155,7 +164,7 @@ boolean windowCloses(long rowsAppended, boolean deferredOpen, long nowNanos, int if (inWindow <= 0 && !deferredOpen) { return false; } - if (groupOpenedAtNanos == 0L) { + if (groupOpenedAtNanos == Long.MIN_VALUE) { groupOpenedAtNanos = nowNanos; } return inWindow >= commitRows || nowNanos - groupOpenedAtNanos >= commitIntervalNanos; diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 5adef191f..9b2c2f41e 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -1,3 +1,27 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + package io.questdb.client.test; import io.questdb.client.QuestDB; @@ -56,6 +80,20 @@ public void testFlushOnCloseShipsOneFramePerClose() throws Exception { } } + @Test + public void testHandleCloseShipsRowsLeftInSlot() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + QuestDB db = QuestDB.connect(config(server.getPort(), + "housekeeper_interval_ms=60000;auto_flush_interval=60000;close_flush_timeout_millis=5000;")); + appendThreeRowsInThreeLeases(db); + db.close(); + log.awaitDataFrames(1, 5_000); + } + } + @Test public void testSettleShipsOnDefaultCadenceWhenIntervalUnset() throws Exception { FrameLog log = new FrameLog(); @@ -67,7 +105,8 @@ public void testSettleShipsOnDefaultCadenceWhenIntervalUnset() throws Exception appendThreeRowsInThreeLeases(db); log.awaitDataFrames(1, 2_000); Thread.sleep(300); - Assert.assertEquals("the settle ships one frame and nothing else follows", 1, log.dataFrames()); + Assert.assertTrue("a settle tick landing mid-loop may split the rows, got " + log.dataFrames(), + log.dataFrames() >= 1 && log.dataFrames() <= 3); } } } @@ -82,7 +121,22 @@ public void testSettleShipsWithinAutoFlushInterval() throws Exception { appendThreeRowsInThreeLeases(db); log.awaitDataFrames(1, 2_000); Thread.sleep(300); - Assert.assertEquals("the settle ships one frame and nothing else follows", 1, log.dataFrames()); + Assert.assertTrue("a settle tick landing mid-loop may split the rows, got " + log.dataFrames(), + log.dataFrames() >= 1 && log.dataFrames() <= 3); + } + } + } + + @Test + public void testThresholdOnLaterLeaseShipsEarlierRows() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), + "auto_flush_rows=3;auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + appendThreeRowsInThreeLeases(db); + log.awaitDataFrames(1, 5_000); } } } @@ -188,6 +242,9 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien | ((data[pos + OFFSET_PAYLOAD_LENGTH + 1] & 0xFFL) << 8) | ((data[pos + OFFSET_PAYLOAD_LENGTH + 2] & 0xFFL) << 16) | ((data[pos + OFFSET_PAYLOAD_LENGTH + 3] & 0xFFL) << 24); + if (payloadLength < 0 || pos + HEADER_SIZE + payloadLength > data.length) { + break; + } flags.add(data[pos + OFFSET_FLAGS] & 0xFF); tableCounts.add(tableCount); framesInMessage++; diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index 589c34862..968a607ef 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -47,6 +47,7 @@ class RecordingSender implements Sender { final AtomicInteger deferredFlushCount = new AtomicInteger(); final AtomicInteger flushCount = new AtomicInteger(); volatile RuntimeException cancelError; // thrown by cancelRow() when set + volatile RuntimeException deferredFlushError; // thrown by flushDeferred() when set volatile boolean deferredFramesOpen; volatile RuntimeException flushError; // thrown by flush() when set volatile CountDownLatch flushGate; // flush() blocks on it when set @@ -130,6 +131,12 @@ public Sender doubleColumn(CharSequence name, double value) { return this; } + @Override + public boolean drain(long timeoutMillis) { + flush(); + return true; + } + @Override public void flush() { awaitGate(); @@ -149,6 +156,10 @@ public void flush() { @Override public long flushDeferred() { awaitGate(); + RuntimeException deferredError = deferredFlushError; + if (deferredError != null) { + throw deferredError; + } boolean hadRows = pendingRows > 0; if (!hadRows) { return -1L; // nothing pending: the real sender publishes nothing diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index 6df9c9a77..e3d009bfe 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -25,7 +25,9 @@ package io.questdb.client.test.impl; +import io.questdb.client.LineSenderServerException; import io.questdb.client.Sender; +import io.questdb.client.SenderError; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; @@ -149,7 +151,7 @@ public void testLeaseReturnedAfterPoolCloseClosesDelegate() { s.close(); RecordingSender delegate = built.get(0); Assert.assertEquals("returning lease must close its delegate", 1, delegate.closeCount.get()); - Assert.assertEquals("the pooled close itself must not flush", 0, delegate.flushCount.get()); + Assert.assertEquals("a lease closing into a shutting-down pool flushes so ship failures propagate", 1, delegate.flushCount.get()); Assert.assertEquals(0, pool.totalSize()); } @@ -174,6 +176,49 @@ public void testPartialRowAtCloseIsCancelledAndSlotKept() { } } + @Test + public void testFlushDeferredThrowingAtCloseRetiresSlot() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(1000, 60_000L); + Sender first = pool.borrow(); + RecordingSender delegate = built.get(0); + LineSenderException original = new LineSenderException("deferred ship failed"); + delegate.deferredFlushError = original; + first.table("t").longColumn("v", 1L).atNow(); + try { + first.close(); + Assert.fail("close() must propagate the deferred ship failure"); + } catch (LineSenderException e) { + Assert.assertSame(original, e.getCause()); + } + Assert.assertEquals("slot must be retired, not stranded", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + } + + @Test + public void testLatchedServerErrorAtCloseKeepsItsType() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + RecordingSender delegate = built.get(0); + SenderError error = SenderError.dataLoss("rejected", "/tmp/q"); + LineSenderServerException original = new LineSenderServerException(error); + delegate.latchedError = original; + try { + first.close(); + Assert.fail("close() must propagate the latched error"); + } catch (LineSenderServerException e) { + Assert.assertNotSame(original, e); + Assert.assertSame(error, e.getServerError()); + Assert.assertSame(original, e.getCause()); + } + Assert.assertEquals(0, pool.totalSize()); + } + } + @Test public void testStaleCloseDoesNotTouchReborrowedSlot() { List built = new ArrayList<>(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java index 3c88a994a..1e0fa6f7d 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java @@ -59,6 +59,29 @@ public void testAgeWindowClosesAtNextRelease() throws Exception { } } + @Test + public void testDrainResetsTheWindow() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(2, 60_000L); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 2L).atNow(); + Assert.assertTrue(s.drain(0)); // app commits explicitly: 2 rows, window would close here + } + RecordingSender d = built.get(0); + Assert.assertEquals(1, d.flushCount.get()); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 3L).atNow(); + } + Assert.assertEquals(2, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + } + } + @Test public void testLeaseFlushResetsTheWindow() { List built = new ArrayList<>(); From 0b2d5baa697ff463e3dc579c8a1a72e76ec38a73 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:26:44 +0100 Subject: [PATCH 25/57] test(pool): make the window-reset tests fail when drain() or flush() skips the mark With a window of 2 the close right after the explicit drain or flush reached the threshold on its own, performed a no-op committing flush and marked the slot, so the tests passed with the mark removed from the pooled drain() and flush(). Use a window of 3, so the stale counter would commit on the third row, and assert where the window closes after the reset. Co-Authored-By: Claude Fable 5.1 --- .../test/impl/SenderPoolCommitWindowTest.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java index 1e0fa6f7d..428a2f89e 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java @@ -64,21 +64,24 @@ public void testDrainResetsTheWindow() { List built = new ArrayList<>(); try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { pool.setFlushOnClose(true); - pool.setCommitWindow(2, 60_000L); - try (Sender s = pool.borrow()) { - s.table("t").longColumn("v", 1L).atNow(); - } + pool.setCommitWindow(3, 60_000L); + borrowAppendClose(pool, 1L); // deferred: the window is open with 1 row try (Sender s = pool.borrow()) { s.table("t").longColumn("v", 2L).atNow(); - Assert.assertTrue(s.drain(0)); // app commits explicitly: 2 rows, window would close here + Assert.assertTrue(s.drain(0)); // app commits at 2 rows: the window restarts here } RecordingSender d = built.get(0); + Assert.assertEquals(1, d.deferredFlushCount.get()); Assert.assertEquals(1, d.flushCount.get()); - try (Sender s = pool.borrow()) { - s.table("t").longColumn("v", 3L).atNow(); - } - Assert.assertEquals(2, d.deferredFlushCount.get()); + // 3 rows in total: a counter that ignored the drain would commit here + borrowAppendClose(pool, 3L); + borrowAppendClose(pool, 4L); + Assert.assertEquals(3, d.deferredFlushCount.get()); Assert.assertEquals(1, d.flushCount.get()); + // 3 rows since the drain: the window closes with a committing frame + borrowAppendClose(pool, 5L); + Assert.assertEquals(3, d.deferredFlushCount.get()); + Assert.assertEquals(2, d.flushCount.get()); } } @@ -87,23 +90,24 @@ public void testLeaseFlushResetsTheWindow() { List built = new ArrayList<>(); try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { pool.setFlushOnClose(true); - pool.setCommitWindow(2, 60_000L); - try (Sender s = pool.borrow()) { - s.table("t").longColumn("v", 1L).atNow(); - } + pool.setCommitWindow(3, 60_000L); + borrowAppendClose(pool, 1L); // deferred: the window is open with 1 row try (Sender s = pool.borrow()) { s.table("t").longColumn("v", 2L).atNow(); - s.flush(); // app commits explicitly: 2 rows, window would close here + s.flush(); // app commits at 2 rows: the window restarts here } RecordingSender d = built.get(0); + Assert.assertEquals(1, d.deferredFlushCount.get()); Assert.assertEquals(1, d.flushCount.get()); - // the close after the explicit flush found nothing to ship and the - // window restarted at 2 rows, so this third close is deferred - try (Sender s = pool.borrow()) { - s.table("t").longColumn("v", 3L).atNow(); - } - Assert.assertEquals(2, d.deferredFlushCount.get()); + // 3 rows in total: a counter that ignored the flush would commit here + borrowAppendClose(pool, 3L); + borrowAppendClose(pool, 4L); + Assert.assertEquals(3, d.deferredFlushCount.get()); Assert.assertEquals(1, d.flushCount.get()); + // 3 rows since the flush: the window closes with a committing frame + borrowAppendClose(pool, 5L); + Assert.assertEquals(3, d.deferredFlushCount.get()); + Assert.assertEquals(2, d.flushCount.get()); } } @@ -212,4 +216,10 @@ public void testWindowOffMeansEveryCloseCommits() { Assert.assertEquals(0, d.deferredFlushCount.get()); } } + + private static void borrowAppendClose(SenderPool pool, long v) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", v).atNow(); + } + } } From b97e99c407deedce5eec194191fd8fef1a40efe4 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:07:55 +0100 Subject: [PATCH 26/57] feat(qwp): bound the byte trigger by the server's batch cap when auto_flush_bytes is unset The WebSocket config-string default for auto_flush_bytes was 0, which the clamp read as an explicit opt-out, so nothing capped a batch's bytes. With cross-lease accumulation in the pool, 1000 wide rows exceed the server's cap and the batch cannot ship. Unset now means AUTO_FLUSH_BYTES_SERVER_CAP: no byte trigger before the handshake, 90 percent of the advertised cap after it, none on a node that advertises no cap. An explicit off stays off. Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 2 +- .../qwp/client/QwpWebSocketSender.java | 42 ++++++++++++------- .../client/test/QuestDBFlushOnCloseTest.java | 30 +++++++++++++ .../qwp/client/QwpWebSocketSenderTest.java | 18 ++++++++ .../qwp/client/SenderProbeMethodsTest.java | 23 ++++++++++ 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 2f6881da1..039d28cdd 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1131,7 +1131,7 @@ final class LineSenderBuilder { private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_UDP_PORT = 9007; private static final int DEFAULT_WEBSOCKET_PORT = 9000; - private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 0; + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = QwpWebSocketSender.AUTO_FLUSH_BYTES_SERVER_CAP; private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 1_000; // Cadence at which the SF cursor I/O loop sends a keepalive PING diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 9b30be539..b653d3569 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -98,7 +98,7 @@ * Configuration options: *

    *
  • {@code autoFlushRows} - Maximum rows per batch (default: 1000)
    • - *
  • {@code autoFlushBytes} - Maximum bytes per batch (default: disabled)
    • + *
  • {@code autoFlushBytes} - Maximum bytes per batch (default: bounded by the server's advertised cap, see {@link #AUTO_FLUSH_BYTES_SERVER_CAP})
    • *
  • {@code autoFlushIntervalNanos} - Maximum age before auto-flush (default: 100ms)
    • *
*

@@ -130,6 +130,14 @@ */ public class QwpWebSocketSender implements Sender { + /** + * {@code autoFlushBytes} value meaning "no byte budget of its own": flush at + * 90 percent of whatever batch cap the server advertises, and at no byte + * threshold while it advertises none. The config-string default for the + * WebSocket transport. An explicit {@code auto_flush_bytes=off} is {@code 0} + * and keeps byte flushing off whatever the server advertises. + */ + public static final int AUTO_FLUSH_BYTES_SERVER_CAP = -1; public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; // Soft per-batch byte budget. Trips a flush when raw column-buffer bytes // cross the threshold, well before the server's 16 MB wire cap. Wide-row @@ -681,12 +689,13 @@ private QwpWebSocketSender( this.autoFlushBytes = autoFlushBytes; // Until the handshake completes, honor the configured budget verbatim. // applyServerBatchSizeLimit() clamps this on connect once the server's - // X-QWP-Max-Batch-Size is known. - this.effectiveAutoFlushBytes = autoFlushBytes; + // X-QWP-Max-Batch-Size is known. A cap-bound sender (-1) has no budget of + // its own before that. + this.effectiveAutoFlushBytes = Math.max(0, autoFlushBytes); this.autoFlushIntervalNanos = autoFlushIntervalNanos; this.globalSymbolDictionary = new GlobalSymbolDictionary(); - int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); + int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, Math.max(0, autoFlushBytes) * 2); try { this.buffer0 = new MicrobatchBuffer(microbatchBufferSize); this.buffer1 = new MicrobatchBuffer(microbatchBufferSize); @@ -1368,7 +1377,7 @@ public static QwpWebSocketSender createForTesting(String host, int port, String * @param host server host (not connected) * @param port server port (not connected) * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit, -1 = 90 percent of the server's advertised cap) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return unconnected sender */ @@ -2423,7 +2432,9 @@ public int getActiveBackgroundDrainers() { } /** - * Returns the auto-flush byte threshold. + * Returns the configured auto-flush byte threshold: a positive budget, {@code 0} for off, + * or {@link #AUTO_FLUSH_BYTES_SERVER_CAP}. See {@link #getEffectiveAutoFlushBytes()} for + * the value in force. */ public int getAutoFlushBytes() { return autoFlushBytes; @@ -3870,21 +3881,24 @@ private static List singleEndpoint(String host, int port) { * in {@link #sendRow} fires against the freshly advertised value. The * byte trigger, however, is only adjusted when the user left it enabled: * an explicit {@code auto_flush_bytes=off} (autoFlushBytes == 0) is - * preserved even when the server advertises a cap, so applications that - * opted out keep the contract they asked for. + * preserved even when the server advertises a cap, and a sender built with + * {@link #AUTO_FLUSH_BYTES_SERVER_CAP} takes the 90 percent budget as its only byte trigger, + * so applications that opted out keep the contract they asked for. */ @TestOnly public void applyServerBatchSizeLimit(int advertisedMaxBatchSize) { serverMaxBatchSize = advertisedMaxBatchSize; - if (autoFlushBytes <= 0) { - // User opted out of byte-based auto-flush; respect that even when - // the server advertises a cap. The single-row guard still protects - // against oversize individual rows via serverMaxBatchSize. + if (autoFlushBytes == 0) { + // Explicit auto_flush_bytes=off: respect it even when the server + // advertises a cap. The single-row guard still protects against + // oversize individual rows via serverMaxBatchSize. effectiveAutoFlushBytes = 0; return; } if (advertisedMaxBatchSize <= 0) { - effectiveAutoFlushBytes = autoFlushBytes; + // Older build, no header: the configured budget verbatim, and no + // byte trigger at all for a sender bound to a cap it cannot see. + effectiveAutoFlushBytes = Math.max(0, autoFlushBytes); return; } // Cap at 90% of the server's hard limit. Raw column-buffer bytes are @@ -3892,7 +3906,7 @@ public void applyServerBatchSizeLimit(int advertisedMaxBatchSize) { // payload), but schema and dict-delta overhead can push the wire size // above the raw total in pathological cases. long safeBudget = (long) advertisedMaxBatchSize * 9 / 10; - if (autoFlushBytes < safeBudget) { + if (autoFlushBytes > 0 && autoFlushBytes < safeBudget) { effectiveAutoFlushBytes = autoFlushBytes; } else { effectiveAutoFlushBytes = (int) safeBudget; diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 9b2c2f41e..1299ef6a1 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -141,6 +141,30 @@ public void testThresholdOnLaterLeaseShipsEarlierRows() throws Exception { } } + @Test + public void testWideRowsAcrossLeasesShipAtTheServerCapNotAtTheRowThreshold() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(16 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // settle 60 s, row threshold 1000: only the cap-bound byte trigger can ship inside the test + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + String wide = wideValue(1000); + // initial_connect_retry=async: the cap is learned on the I/O thread's + // handshake, and rows appended before it carry no byte trigger. Keep + // appending (far below the 1000-row threshold) until a frame ships. + for (int i = 0; i < 200 && log.dataFrames() < 1; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").stringColumn("s", wide).atNow(); + } + Thread.sleep(10); + } + log.awaitDataFrames(1, 5_000); + } + } + } + static void appendThreeRowsInThreeLeases(QuestDB db) { for (int i = 0; i < 3; i++) { try (Sender s = db.borrowSender()) { @@ -158,6 +182,12 @@ static String config(int port, String extra) { + extra; } + public static String wideValue(int length) { + char[] chars = new char[length]; + java.util.Arrays.fill(chars, 'x'); + return new String(chars); + } + /** * Parses every QWP frame in each WebSocket message (a read can carry more * than one on replay), records its flags byte and table count, and acks diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 3911f5c05..2252cd95f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -122,6 +122,24 @@ public void testApplyServerBatchSizeLimit_optOutPreservedDespiteAdvertisement() }); } + @Test + public void testApplyServerBatchSizeLimit_serverCapBoundFollowsAdvertisedCap() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 9000, + /*autoFlushRows*/ 1000, + /*autoFlushBytes*/ QwpWebSocketSender.AUTO_FLUSH_BYTES_SERVER_CAP, + /*autoFlushIntervalNanos*/ 0L)) { + Assert.assertEquals("no byte trigger before the handshake", 0, sender.getEffectiveAutoFlushBytes()); + sender.applyServerBatchSizeLimit(16 * 1024 * 1024); + Assert.assertEquals("90% of the advertised cap", (long) (16 * 1024 * 1024) * 9 / 10, sender.getEffectiveAutoFlushBytes()); + Assert.assertEquals(16 * 1024 * 1024, sender.getServerMaxBatchSize()); + sender.applyServerBatchSizeLimit(0); + Assert.assertEquals("a node that advertises no cap: no byte trigger", 0, sender.getEffectiveAutoFlushBytes()); + } + }); + } + @Test public void testAtAfterCloseThrows() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 8815c720a..45a436f14 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -117,6 +117,29 @@ public void testDefaultsOnNonWebSocketTransport() { } } + @Test + public void testUnsetAutoFlushBytesShipsAtTheAdvertisedCap() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(16 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // no auto_flush_bytes key: the sender binds its byte trigger to 90% of the 16 KiB cap + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;")) { + char[] wide = new char[1000]; + java.util.Arrays.fill(wide, 'x'); + String value = new String(wide); + for (int i = 0; i < 20; i++) { + sender.table("t").stringColumn("s", value).atNow(); + } + // 20 KB of rows against a 14.7 KB budget: the byte trigger shipped at least once + log.awaitDataFrames(1, 5_000); + Assert.assertTrue("rows left after the byte flush", sender.pendingRowCount() > 0 && sender.pendingRowCount() < 20); + } + } + } + private static String config(int port) { return "ws::addr=localhost:" + port + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;"; } From 3cc15075cd7153b4e63708e3802d9977eeb51ebf Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:12:02 +0100 Subject: [PATCH 27/57] feat(qwp): table buffer savepoint: mark a row boundary and roll back to it rollbackToSavepoint truncates every column to the row count recorded by markSavepoint and removes the columns added since, reusing the per-column truncateTo that cancelCurrentRow already relies on. reset moves the mark to the empty buffer, clear drops it. Groundwork for a lease-scoped reset on the pooled sender. Co-Authored-By: Claude Fable 5.1 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 57 ++++++++++++ .../qwp/protocol/QwpTableBufferTest.java | 86 +++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 9f01f0ab8..931c16927 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -72,6 +72,11 @@ public class QwpTableBuffer implements QuietCloseable { private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access private int inProgressColumnCount; private int rowCount; + // Savepoint for rollbackToSavepoint(): the row count and column count at the + // last markSavepoint(). reset() moves it to "no rows, all current columns", + // since a flushed buffer's columns are known to the server; clear() zeroes it. + private int savepointColumnCount; + private int savepointRowCount; public QwpTableBuffer(String tableName) { this(tableName, null); @@ -130,6 +135,8 @@ public void clear() { committedColumnCount = 0; inProgressColumnCount = 0; rowCount = 0; + savepointRowCount = 0; + savepointColumnCount = 0; columnDefsCacheValid = false; cachedColumnDefs = null; } @@ -256,6 +263,14 @@ public String getTableName() { return tableName; } + /** + * Row count recorded by the last {@link #markSavepoint()}; {@code 0} when + * none was taken since the buffer was last reset or cleared. + */ + public int getSavepointRowCount() { + return savepointRowCount; + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -264,6 +279,16 @@ public boolean hasInProgressRow() { return inProgressColumnCount > 0; } + /** + * Remembers the current row count and column count. A later + * {@link #rollbackToSavepoint()} truncates the buffer back to this point. + * Take it between rows, not while a row is in progress. + */ + public void markSavepoint() { + savepointRowCount = rowCount; + savepointColumnCount = columns.size(); + } + /** * Advances to the next row. *

@@ -344,6 +369,8 @@ public void reset() { columnAccessCursor = 0; committedColumnCount = columns.size(); inProgressColumnCount = 0; + savepointRowCount = 0; + savepointColumnCount = columns.size(); rowCount = 0; } @@ -371,6 +398,36 @@ public void retainInProgressRow( committedColumnCount = columns.size(); } + /** + * Discards the rows and the columns added since {@link #markSavepoint()}, + * including a row in progress, and keeps everything before the mark. + * Columns the mark did not see are closed and removed: the server has not + * seen them either, so they must not pad the retained rows. Without a mark + * since the last {@link #reset()} this drops every row and keeps the column + * definitions, which is what {@code reset()} does. + */ + public void rollbackToSavepoint() { + columnAccessCursor = 0; + inProgressColumnCount = 0; + boolean columnsRemoved = false; + for (int i = columns.size() - 1; i >= savepointColumnCount; i--) { + ColumnBuffer col = columns.getQuick(i); + if (col != null) { + col.close(); + } + columns.remove(i); + columnsRemoved = true; + } + if (columnsRemoved) { + rebuildColumnAccessStructures(); + } + for (int i = 0, n = columns.size(); i < n; i++) { + fastColumns[i].truncateTo(savepointRowCount); + } + rowCount = savepointRowCount; + committedColumnCount = columns.size(); + } + public void rollbackUncommittedColumns() { if (columns.size() <= committedColumnCount) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index ab14c18d8..2695c1fe9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1388,6 +1388,23 @@ public void testNonAsciiColumnNameCaseInsensitive() throws Exception { }); } + @Test + public void testResetMovesTheSavepointToTheEmptyBuffer() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.markSavepoint(); + table.reset(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals("reset() emptied the savepoint", 0, table.getRowCount()); + assertEquals("columns the server already knows stay", 1, table.getColumnCount()); + } + }); + } + @Test public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { assertMemoryLeak(() -> { @@ -1546,4 +1563,73 @@ private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) } return result; } + + @Test + public void testRollbackToSavepointDiscardsAnInProgressRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + table.nextRow(); + table.markSavepoint(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); // row 2 started, not committed + assertTrue(table.hasInProgressRow()); + table.rollbackToSavepoint(); + assertEquals(1, table.getRowCount()); + assertFalse(table.hasInProgressRow()); + assertEquals(1, v.getSize()); + } + }); + } + + @Test + public void testRollbackToSavepointKeepsEarlierRowsAndDropsLaterColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + long bytesAtMark = table.nextRow(); + table.markSavepoint(); + assertEquals(1, table.getSavepointRowCount()); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("later"); + table.nextRow(); + assertEquals(2, table.getRowCount()); + assertEquals(2, table.getColumnCount()); + + table.rollbackToSavepoint(); + + assertEquals(1, table.getRowCount()); + assertEquals("the column added after the mark is gone", 1, table.getColumnCount()); + assertEquals(1, v.getSize()); + assertEquals(bytesAtMark, table.getBufferedBytes()); + assertFalse(table.hasInProgressRow()); + // still usable: the dropped column can come back + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(3L); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("again"); + table.nextRow(); + assertEquals(2, table.getRowCount()); + assertEquals(2, table.getColumnCount()); + assertEquals(2, v.getSize()); + } + }); + } + + @Test + public void testRollbackToSavepointWithoutMarkDropsEveryRowButKeepsColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + table.nextRow(); + table.reset(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals(0, v.getSize()); + assertEquals("no mark: behaves like reset(), definitions stay", 1, table.getColumnCount()); + } + }); + } } From 90d26546699e0d71b3d39b290b99d43a9b977405 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:13:36 +0100 Subject: [PATCH 28/57] fix(qwp): retainInProgressRow moves the table buffer savepoint like reset does A flush that keeps a row in progress ships the committed rows and sets rowCount to 0; leaving the savepoint where it was would make a later rollback truncate to rows that were already shipped. Also states the never-marked case in the rollback Javadoc. Co-Authored-By: Claude Fable 5.1 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 11 ++++++--- .../qwp/protocol/QwpTableBufferTest.java | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 931c16927..a37ff6a40 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -73,8 +73,9 @@ public class QwpTableBuffer implements QuietCloseable { private int inProgressColumnCount; private int rowCount; // Savepoint for rollbackToSavepoint(): the row count and column count at the - // last markSavepoint(). reset() moves it to "no rows, all current columns", - // since a flushed buffer's columns are known to the server; clear() zeroes it. + // last markSavepoint(). reset() and retainInProgressRow() move it to "no rows, + // all current columns", since a flushed buffer's columns are known to the + // server; clear() zeroes it. private int savepointColumnCount; private int savepointRowCount; @@ -395,6 +396,8 @@ public void retainInProgressRow( } } rowCount = 0; + savepointRowCount = 0; + savepointColumnCount = columns.size(); committedColumnCount = columns.size(); } @@ -404,7 +407,9 @@ public void retainInProgressRow( * Columns the mark did not see are closed and removed: the server has not * seen them either, so they must not pad the retained rows. Without a mark * since the last {@link #reset()} this drops every row and keeps the column - * definitions, which is what {@code reset()} does. + * definitions, which is what {@code reset()} does; on a buffer that was never + * marked or reset it also removes every column, since the server has seen + * none of them. */ public void rollbackToSavepoint() { columnAccessCursor = 0; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 2695c1fe9..e96278cb7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1468,6 +1468,29 @@ public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exc }); } + @Test + public void testRetainInProgressRowMovesTheSavepointToTheRetainedRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + table.nextRow(); + table.markSavepoint(); + assertEquals(1, table.getSavepointRowCount()); + v.addLong(2L); // row 2 in progress, not committed + table.retainInProgressRow(new int[]{1}, new int[]{1}, new int[]{0}, new int[]{0}); + assertEquals(0, table.getRowCount()); + assertEquals("the shipped rows are no longer a rollback target", 0, table.getSavepointRowCount()); + + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals(1, table.getColumnCount()); + } + }); + } + private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) { byte[] bytes = value.getBytes(StandardCharsets.UTF_8); long ptr = copyToNative(bytes); From 4f8fa65a9dd28a1457094b9bf1f156758ff2a9cb Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:20:11 +0100 Subject: [PATCH 29/57] fix(qwp): savepoint rollback keeps a retained column's geohash precision; tests for the retain and never-marked cases truncateTo clears geohashPrecision when it leaves a column empty, which is right for a cancelled first row but wrong for a server-known column a lease rolls back: the server has locked that precision. The rollback now restores it for every column below the mark. Adds the column half of the retainInProgressRow test, the never-marked rollback test that a table created after the mark relies on, and moves the new members to their alphabetical positions. Co-Authored-By: Claude Fable 5.1 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 25 ++- .../qwp/protocol/QwpTableBufferTest.java | 193 +++++++++++------- 2 files changed, 136 insertions(+), 82 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index a37ff6a40..9622e4379 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -258,18 +258,19 @@ public int getRowCount() { } /** - * Returns the table name. + * Row count recorded by the last {@link #markSavepoint()}; {@code 0} when + * none was taken since the buffer was last reset, cleared or + * retained a row in progress. */ - public String getTableName() { - return tableName; + public int getSavepointRowCount() { + return savepointRowCount; } /** - * Row count recorded by the last {@link #markSavepoint()}; {@code 0} when - * none was taken since the buffer was last reset or cleared. + * Returns the table name. */ - public int getSavepointRowCount() { - return savepointRowCount; + public String getTableName() { + return tableName; } /** @@ -407,7 +408,8 @@ public void retainInProgressRow( * Columns the mark did not see are closed and removed: the server has not * seen them either, so they must not pad the retained rows. Without a mark * since the last {@link #reset()} this drops every row and keeps the column - * definitions, which is what {@code reset()} does; on a buffer that was never + * definitions, which is what {@code reset()} does, including keeping a retained + * column's geohash precision; on a buffer that was never * marked or reset it also removes every column, since the server has seen * none of them. */ @@ -427,7 +429,12 @@ public void rollbackToSavepoint() { rebuildColumnAccessStructures(); } for (int i = 0, n = columns.size(); i < n; i++) { - fastColumns[i].truncateTo(savepointRowCount); + ColumnBuffer col = fastColumns[i]; + // truncateTo clears the geohash precision of a column it empties; a + // retained column is known to the server, which has locked it. + int geohashPrecisionBefore = col.geohashPrecision; + col.truncateTo(savepointRowCount); + col.geohashPrecision = geohashPrecisionBefore; } rowCount = savepointRowCount; committedColumnCount = columns.size(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index e96278cb7..18769d250 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1472,22 +1472,138 @@ public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exc public void testRetainInProgressRowMovesTheSavepointToTheRetainedRow() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer table = new QwpTableBuffer("t")) { - QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); - v.addLong(1L); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); table.nextRow(); table.markSavepoint(); assertEquals(1, table.getSavepointRowCount()); - v.addLong(2L); // row 2 in progress, not committed + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); // row 2 in progress + assertTrue(table.hasInProgressRow()); table.retainInProgressRow(new int[]{1}, new int[]{1}, new int[]{0}, new int[]{0}); assertEquals(0, table.getRowCount()); assertEquals("the shipped rows are no longer a rollback target", 0, table.getSavepointRowCount()); - table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); table.nextRow(); table.rollbackToSavepoint(); assertEquals(0, table.getRowCount()); assertEquals(1, table.getColumnCount()); } + + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.markSavepoint(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("x"); + // "s" was created by the in-progress row: sizeBefore -1 clears it to empty + table.retainInProgressRow(new int[]{1, -1}, new int[]{1, -1}, new int[]{0, 0}, new int[]{0, 0}); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals("the column the server now knows is not dropped", 2, table.getColumnCount()); + } + }); + } + + @Test + public void testRollbackToSavepointDiscardsAnInProgressRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + table.nextRow(); + table.markSavepoint(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); // row 2 started, not committed + assertTrue(table.hasInProgressRow()); + table.rollbackToSavepoint(); + assertEquals(1, table.getRowCount()); + assertFalse(table.hasInProgressRow()); + assertEquals(1, v.getSize()); + } + }); + } + + @Test + public void testRollbackToSavepointKeepsEarlierRowsAndDropsLaterColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + long bytesAtMark = table.nextRow(); + table.markSavepoint(); + assertEquals(1, table.getSavepointRowCount()); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("later"); + table.nextRow(); + assertEquals(2, table.getRowCount()); + assertEquals(2, table.getColumnCount()); + + table.rollbackToSavepoint(); + + assertEquals(1, table.getRowCount()); + assertEquals("the column added after the mark is gone", 1, table.getColumnCount()); + assertEquals(1, v.getSize()); + assertEquals(bytesAtMark, table.getBufferedBytes()); + assertFalse(table.hasInProgressRow()); + // still usable: the dropped column can come back + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(3L); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("again"); + table.nextRow(); + assertEquals(2, table.getRowCount()); + assertEquals(2, table.getColumnCount()); + assertEquals(2, v.getSize()); + } + }); + } + + @Test + public void testRollbackToSavepointKeepsGeoHashPrecisionOfARetainedColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer g = table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true); + g.addGeoHash(123L, 20); + table.nextRow(); + table.reset(); + table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true).addGeoHash(456L, 20); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals(20, g.getGeoHashPrecision()); + } + }); + } + + @Test + public void testRollbackToSavepointOnANeverMarkedBufferRemovesEveryColumn() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals(0, table.getColumnCount()); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.nextRow(); + assertEquals(1, table.getRowCount()); + assertEquals(1, table.getColumnCount()); + } + }); + } + + @Test + public void testRollbackToSavepointWithoutMarkDropsEveryRowButKeepsColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); + v.addLong(1L); + table.nextRow(); + table.reset(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.nextRow(); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals(0, v.getSize()); + assertEquals("no mark: behaves like reset(), definitions stay", 1, table.getColumnCount()); + } }); } @@ -1586,73 +1702,4 @@ private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) } return result; } - - @Test - public void testRollbackToSavepointDiscardsAnInProgressRow() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("t")) { - QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); - v.addLong(1L); - table.nextRow(); - table.markSavepoint(); - table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); // row 2 started, not committed - assertTrue(table.hasInProgressRow()); - table.rollbackToSavepoint(); - assertEquals(1, table.getRowCount()); - assertFalse(table.hasInProgressRow()); - assertEquals(1, v.getSize()); - } - }); - } - - @Test - public void testRollbackToSavepointKeepsEarlierRowsAndDropsLaterColumns() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("t")) { - QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); - v.addLong(1L); - long bytesAtMark = table.nextRow(); - table.markSavepoint(); - assertEquals(1, table.getSavepointRowCount()); - table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); - table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("later"); - table.nextRow(); - assertEquals(2, table.getRowCount()); - assertEquals(2, table.getColumnCount()); - - table.rollbackToSavepoint(); - - assertEquals(1, table.getRowCount()); - assertEquals("the column added after the mark is gone", 1, table.getColumnCount()); - assertEquals(1, v.getSize()); - assertEquals(bytesAtMark, table.getBufferedBytes()); - assertFalse(table.hasInProgressRow()); - // still usable: the dropped column can come back - table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(3L); - table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, false).addString("again"); - table.nextRow(); - assertEquals(2, table.getRowCount()); - assertEquals(2, table.getColumnCount()); - assertEquals(2, v.getSize()); - } - }); - } - - @Test - public void testRollbackToSavepointWithoutMarkDropsEveryRowButKeepsColumns() throws Exception { - assertMemoryLeak(() -> { - try (QwpTableBuffer table = new QwpTableBuffer("t")) { - QwpTableBuffer.ColumnBuffer v = table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false); - v.addLong(1L); - table.nextRow(); - table.reset(); - table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); - table.nextRow(); - table.rollbackToSavepoint(); - assertEquals(0, table.getRowCount()); - assertEquals(0, v.getSize()); - assertEquals("no mark: behaves like reset(), definitions stay", 1, table.getColumnCount()); - } - }); - } } From 557152d2cbd95d34aa2b3370870387362e73f0ef Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:24:15 +0100 Subject: [PATCH 30/57] feat(qwp): markSavepoint and resetToSavepoint on Sender; checkError reports a retained over-cap batch resetToSavepoint rolls every table buffer back to the row and column counts recorded by markSavepoint and restores the pending counters; with an empty savepoint it is reset(). A flush that ships the pending rows empties the savepoint. flushPendingRows remembers a BatchTooLargeForCapException until reset, rollback or a successful flush clears it, and checkError throws it, so a pool can tell a slot whose batch will never ship from a healthy one. SenderProbeMethodsTest now has its test methods in alphabetical order and uses QuestDBFlushOnCloseTest.wideValue(1000) instead of a hand-rolled char array. Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 28 ++- .../qwp/client/QwpWebSocketSender.java | 82 ++++++++- .../client/test/QuestDBFlushOnCloseTest.java | 13 ++ .../qwp/client/SenderProbeMethodsTest.java | 160 ++++++++++++++---- 4 files changed, 247 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 039d28cdd..5accdfdcd 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -759,6 +759,17 @@ default Sender long256Column(CharSequence name, long l0, long l1, long l2, long */ Sender longColumn(CharSequence name, long value); + /** + * Marks the current buffer state as the point {@link #resetToSavepoint()} + * rolls back to: rows appended after this call are discarded by that + * method, rows appended before it are kept. Any flush that ships the + * pending rows empties the savepoint, so a later {@code resetToSavepoint()} + * behaves like {@link #reset()}. The pool marks a savepoint when it hands a + * slot to a borrower. Pure bookkeeping: it runs no error check. Default + * no-op. + */ + default void markSavepoint() { + } /** * Number of rows appended with {@link #at}/{@link #atNow()} that have not @@ -819,13 +830,26 @@ default void resetSymbolDictionary() { * data format) and want to continue sending new data without retrying the problematic data. * After calling this method, you can start building a new row by calling {@link #table(CharSequence)}. *
- * Note: This method is only available for HTTP transport. TCP transport doesn't support - * this operation. + * Supported by the HTTP and WebSocket transports; TCP does not support it. On a pooled + * sender this discards only the rows appended through the current lease, see + * {@link #resetToSavepoint()}. * * @see #flush() */ void reset(); + /** + * Discards the rows appended since the last {@link #markSavepoint()} and + * keeps the rows buffered before it. Without a savepoint, or after a flush + * emptied it, this is {@link #reset()}. A pooled sender's {@code reset()} + * calls this, so a borrower can drop its own bad batch without discarding + * rows earlier borrowers left in the shared slot buffer. Default calls + * {@code reset()}. + */ + default void resetToSavepoint() { + reset(); + } + /** * Monotonic count of rows finished with {@link #at}/{@link #atNow()} over * this sender's life, whether or not they have shipped. The connection diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b653d3569..37a5ef1db 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -462,6 +462,11 @@ public class QwpWebSocketSender implements Sender { // Optional owning-pool notification, relayed after the engine confirms the // flock release and slotLockReleased is visible to pool re-probes. private volatile Runnable slotLockReleaseListener; + // Pre-flight batch-cap rejection of the last flush attempt, kept until reset(), + // resetToSavepoint() or a successful flush clears it. Not a terminal: the + // standalone contract lets the caller reset() and carry on. checkError() + // reports it so a pool can retire a slot whose batch no flush will ship. + private BatchTooLargeForCapException retainedBatchRejection; // Engine whose close() could not complete during sender close() — its // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() // re-probes it so a late flock release becomes visible to the owning pool. @@ -471,6 +476,11 @@ public class QwpWebSocketSender implements Sender { private int pendingRowCount; // Monotonic for the sender's life; reset() does not clear it. Counts rows whether or not they shipped. private long rowsAppended; + // Pending counters at the last markSavepoint(); zero once a flush ships the + // pending rows, at which point resetToSavepoint() is reset(). + private long savepointFirstPendingRowTimeNanos; + private long savepointPendingBytes; + private int savepointPendingRowCount; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a // setProgressHandler call before connect() swaps in a real handler. @@ -1670,6 +1680,10 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { @Override public void checkError() { checkNotClosed(); + BatchTooLargeForCapException rejected = retainedBatchRejection; + if (rejected != null) { + throw rejected; + } } /** @@ -3163,6 +3177,20 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { return this; } + @Override + public void markSavepoint() { + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.markSavepoint(); + } + } + savepointPendingRowCount = pendingRowCount; + savepointPendingBytes = pendingBytes; + savepointFirstPendingRowTimeNanos = firstPendingRowTimeNanos; + } + /** * Returns a {@link CursorWebSocketSendLoop.ReconnectFactory} that, on each * call, performs the multi-endpoint walk and returns a freshly connected @@ -3258,6 +3286,10 @@ public void reset() { pendingBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; + savepointPendingRowCount = 0; + savepointPendingBytes = 0; + savepointFirstPendingRowTimeNanos = 0; + retainedBatchRejection = null; currentTableBuffer = null; currentTableBufferSnapshotBytes = 0; currentTableName = null; @@ -3265,6 +3297,37 @@ public void reset() { cachedTimestampNanosColumn = null; } + @Override + public void resetToSavepoint() { + checkNotClosed(); + if (savepointPendingRowCount == 0) { + // No rows before the mark, or a flush since emptied it: every pending + // row is the caller's own and the full reset applies, watermark included. + reset(); + return; + } + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.rollbackToSavepoint(); + } + } + // The batch symbol watermark stays: ids the dropped rows registered ride + // along unused in the next delta section, which the server tolerates, + // whereas lowering it below an id a retained row references would ship + // rows pointing at symbols the server never received. + pendingRowCount = savepointPendingRowCount; + pendingBytes = savepointPendingBytes; + firstPendingRowTimeNanos = savepointFirstPendingRowTimeNanos; + currentTableBuffer = null; + currentTableBufferSnapshotBytes = 0; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + retainedBatchRejection = null; + } + /** * Advisory request to start a fresh symbol-dictionary epoch. Sets * {@link #manualResetRequested}; if no flush is in flight @@ -5100,6 +5163,10 @@ private void flushPendingRows(boolean deferCommit) { ? 0 : currentTableBuffer.getBufferedBytes(); pendingRowCount = 0; firstPendingRowTimeNanos = 0; + savepointPendingRowCount = 0; + savepointPendingBytes = 0; + savepointFirstPendingRowTimeNanos = 0; + retainedBatchRejection = null; return; } @@ -5163,7 +5230,13 @@ private void flushPendingRows(boolean deferCommit) { && !deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline && !splitFramesFit(cap, deltaBaseline) && splitFramesFit(cap, currentBatchMaxSymbolId)) { - publishDictionaryChunks(cap, deltaBaseline + 1, currentBatchMaxSymbolId); + try { + publishDictionaryChunks(cap, deltaBaseline + 1, currentBatchMaxSymbolId); + } catch (BatchTooLargeForCapException e) { + // Thrown before the try block below, which would latch it. + retainedBatchRejection = e; + throw e; + } dictionaryChunksAwaitCommit = true; deltaBaseline = currentBatchMaxSymbolId; combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); @@ -5199,6 +5272,9 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { resetTableBuffersAfterFlush(); } catch (Throwable t) { + if (t instanceof BatchTooLargeForCapException) { + retainedBatchRejection = (BatchTooLargeForCapException) t; + } if (dictionaryChunksAwaitCommit) { commitOrphanedDictionaryChunks(t); } @@ -5448,6 +5524,10 @@ private void resetTableBuffersAfterFlush() { currentTableBufferSnapshotBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; + savepointPendingRowCount = 0; + savepointPendingBytes = 0; + savepointFirstPendingRowTimeNanos = 0; + retainedBatchRejection = null; armIfEligible(); } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 1299ef6a1..ff296d1d4 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -287,5 +287,18 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien throw new RuntimeException(e); } } + + /** + * Table count of the i-th data frame (frames with a table count above zero). + */ + public synchronized int tableCount(int dataFrameIndex) { + int seen = -1; + for (int i = 0, size = tableCounts.size(); i < size; i++) { + if (tableCounts.get(i) > 0 && ++seen == dataFrameIndex) { + return tableCounts.get(i); + } + } + throw new AssertionError("no data frame at index " + dataFrameIndex); + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 45a436f14..38c4866b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -2,6 +2,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.test.QuestDBFlushOnCloseTest; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; @@ -18,6 +20,82 @@ */ public class SenderProbeMethodsTest { + @Test + public void testCheckErrorReportsARetainedOverCapBatchUntilReset() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + Assert.assertEquals("the cap is known once fromConfig returns", 4 * 1024, ((QwpWebSocketSender) sender).getServerMaxBatchSize()); + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 6; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + try { + sender.flush(); + Assert.fail("six 1 KB rows do not fit a 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // retained for retry, as the message says + } + try { + sender.checkError(); + Assert.fail("the probe must report the retained batch"); + } catch (BatchTooLargeForCapException expected) { + // the pool retires a slot on this + } + sender.reset(); + sender.checkError(); // cleared with the batch + sender.table("t").longColumn("v", 1L).atNow(); + sender.flush(); + log.awaitDataFrames(1, 5_000); + } + } + } + + @Test + public void testCheckErrorThrowsLatchedTerminal() throws Exception { + try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { + server.setRejectWithStatus(401, "Unauthorized"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // async connect: build() returns, the I/O thread hits the 401 wall and latches a terminal + try (Sender sender = Sender.fromConfig(config(server.getPort()) + + "initial_connect_retry=async;reconnect_max_duration_millis=300;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { + long deadline = System.currentTimeMillis() + 5_000; + LineSenderException seen = null; + while (System.currentTimeMillis() < deadline) { + try { + sender.checkError(); + } catch (LineSenderException e) { + seen = e; + break; + } + Thread.sleep(20); + } + Assert.assertNotNull("checkError() must throw the latched auth terminal", seen); + } + } + } + + @Test + public void testDefaultsOnNonWebSocketTransport() { + try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { + sender.checkError(); // default: no-op + Assert.assertFalse(sender.hasRowInProgress()); // default: false + Assert.assertEquals(0, sender.pendingRowCount()); // default: 0 + Assert.assertEquals(0L, sender.rowsAppended()); + Assert.assertFalse(sender.hasDeferredFrames()); + try { + sender.flushDeferred(); + Assert.fail("expected UnsupportedOperationException"); + } catch (UnsupportedOperationException expected) { + } + } + } + @Test public void testDeferredFlushShipsFlaggedFrameAndFlushCommits() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); @@ -48,6 +126,24 @@ public void testDeferredFlushShipsFlaggedFrameAndFlushCommits() throws Exception } } + @Test + public void testFlushEmptiesTheSavepoint() throws Exception { + try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.table("t").longColumn("v", 1L).atNow(); + sender.markSavepoint(); + sender.table("t").longColumn("v", 2L).atNow(); + sender.flush(); // ships both: the savepoint is now empty + sender.table("t").longColumn("v", 3L).atNow(); + sender.resetToSavepoint(); + Assert.assertEquals("a flush inside the lease makes every pending row the lease's own", 0, sender.pendingRowCount()); + sender.checkError(); + } + } + } + @Test public void testHealthySenderReportsBufferState() throws Exception { try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { @@ -76,43 +172,43 @@ public void testHealthySenderReportsBufferState() throws Exception { } @Test - public void testCheckErrorThrowsLatchedTerminal() throws Exception { - try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { - server.setRejectWithStatus(401, "Unauthorized"); + public void testResetToSavepointKeepsRowsBufferedBeforeTheMark() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - // async connect: build() returns, the I/O thread hits the 401 wall and latches a terminal - try (Sender sender = Sender.fromConfig(config(server.getPort()) - + "initial_connect_retry=async;reconnect_max_duration_millis=300;" - + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { - long deadline = System.currentTimeMillis() + 5_000; - LineSenderException seen = null; - while (System.currentTimeMillis() < deadline) { - try { - sender.checkError(); - } catch (LineSenderException e) { - seen = e; - break; - } - Thread.sleep(20); - } - Assert.assertNotNull("checkError() must throw the latched auth terminal", seen); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.table("t1").longColumn("v", 1L).atNow(); + sender.table("t1").longColumn("v", 2L).atNow(); + sender.markSavepoint(); + sender.table("t1").longColumn("v", 3L).stringColumn("extra", "x").atNow(); + sender.table("t2").longColumn("v", 1L).atNow(); + Assert.assertEquals(4, sender.pendingRowCount()); + + sender.resetToSavepoint(); + + Assert.assertEquals("the two rows before the mark stay", 2, sender.pendingRowCount()); + sender.flush(); + log.awaitDataFrames(1, 5_000); + Assert.assertEquals("t2 was created after the mark and is gone", 1, log.tableCount(0)); + // the buffer is still usable, including the column the rollback removed + sender.table("t1").longColumn("v", 4L).stringColumn("extra", "y").atNow(); + sender.flush(); + log.awaitDataFrames(2, 5_000); } } } @Test - public void testDefaultsOnNonWebSocketTransport() { - try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { - sender.checkError(); // default: no-op - Assert.assertFalse(sender.hasRowInProgress()); // default: false - Assert.assertEquals(0, sender.pendingRowCount()); // default: 0 - Assert.assertEquals(0L, sender.rowsAppended()); - Assert.assertFalse(sender.hasDeferredFrames()); - try { - sender.flushDeferred(); - Assert.fail("expected UnsupportedOperationException"); - } catch (UnsupportedOperationException expected) { + public void testResetToSavepointWithoutMarkIsReset() throws Exception { + try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.table("t").longColumn("v", 1L).atNow(); + sender.table("t").longColumn("v", 2L).atNow(); + sender.resetToSavepoint(); + Assert.assertEquals(0, sender.pendingRowCount()); } } } @@ -127,9 +223,7 @@ public void testUnsetAutoFlushBytesShipsAtTheAdvertisedCap() throws Exception { // no auto_flush_bytes key: the sender binds its byte trigger to 90% of the 16 KiB cap try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + ";auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;")) { - char[] wide = new char[1000]; - java.util.Arrays.fill(wide, 'x'); - String value = new String(wide); + String value = QuestDBFlushOnCloseTest.wideValue(1000); for (int i = 0; i < 20; i++) { sender.table("t").stringColumn("s", value).atNow(); } From 142dc250d6671f9d2d63df908f57d9355aa90690 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:37:36 +0100 Subject: [PATCH 31/57] fix(qwp): resetToSavepoint restores the symbol watermark and drops unsent column definitions Rows a lease rolls back may have raised currentBatchMaxSymbolId; keeping that value made the retained rows re-encode the dropped rows' dictionary entries, so a dictionary section the cap rejected was rejected again on the next borrower after the probe had been cleared. The mark now records the watermark and the rollback restores it: rows before the mark reference only ids at or below it. The empty-savepoint branch used reset(), which keeps column definitions the lease created while the rollback branch removes them; resetToSavepoint now rolls back every buffer in both branches, so a column the server never saw does not ship null-padded to it. Tests cover the single-symbol latch and the pool's mark-flush-rollback-ship case. Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 3 +- .../qwp/client/QwpWebSocketSender.java | 133 ++++++++++-------- .../qwp/client/SelfSufficientFramesTest.java | 10 +- .../qwp/client/SenderProbeMethodsTest.java | 35 +++++ 4 files changed, 117 insertions(+), 64 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 5accdfdcd..d5e0a28c8 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -841,7 +841,8 @@ default void resetSymbolDictionary() { /** * Discards the rows appended since the last {@link #markSavepoint()} and * keeps the rows buffered before it. Without a savepoint, or after a flush - * emptied it, this is {@link #reset()}. A pooled sender's {@code reset()} + * emptied it, this discards every pending row like {@link #reset()}, and + * also removes the column definitions the server has not seen yet. A pooled sender's {@code reset()} * calls this, so a borrower can drop its own bad batch without discarding * rows earlier borrowers left in the shared slot buffer. Default calls * {@code reset()}. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 37a5ef1db..ccd23b398 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -478,6 +478,7 @@ public class QwpWebSocketSender implements Sender { private long rowsAppended; // Pending counters at the last markSavepoint(); zero once a flush ships the // pending rows, at which point resetToSavepoint() is reset(). + private int savepointBatchMaxSymbolId; private long savepointFirstPendingRowTimeNanos; private long savepointPendingBytes; private int savepointPendingRowCount; @@ -3186,6 +3187,7 @@ public void markSavepoint() { buf.markSavepoint(); } } + savepointBatchMaxSymbolId = currentBatchMaxSymbolId; savepointPendingRowCount = pendingRowCount; savepointPendingBytes = pendingBytes; savepointFirstPendingRowTimeNanos = firstPendingRowTimeNanos; @@ -3272,60 +3274,7 @@ public void reset() { buf.reset(); } } - // Drop the batch's symbol watermark along with the rows that raised it. A - // later flush encodes the delta section as - // [sentMaxSymbolId+1 .. currentBatchMaxSymbolId], so a watermark left behind - // by the discarded batch keeps re-encoding that batch's symbols: a single-row - // batch after reset() would still carry the whole abandoned range and hit the - // very cap rejection reset() is documented to clear, leaving the sender unable - // to flush anything at all. -1 is the same value resetTableBuffersAfterFlush - // leaves behind after a successful flush, and sendCommitMessage already reads - // it as an empty delta. - currentBatchMaxSymbolId = -1; - reclaimUnsentSymbolIds(); - pendingBytes = 0; - pendingRowCount = 0; - firstPendingRowTimeNanos = 0; - savepointPendingRowCount = 0; - savepointPendingBytes = 0; - savepointFirstPendingRowTimeNanos = 0; - retainedBatchRejection = null; - currentTableBuffer = null; - currentTableBufferSnapshotBytes = 0; - currentTableName = null; - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - } - - @Override - public void resetToSavepoint() { - checkNotClosed(); - if (savepointPendingRowCount == 0) { - // No rows before the mark, or a flush since emptied it: every pending - // row is the caller's own and the full reset applies, watermark included. - reset(); - return; - } - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); - if (buf != null) { - buf.rollbackToSavepoint(); - } - } - // The batch symbol watermark stays: ids the dropped rows registered ride - // along unused in the next delta section, which the server tolerates, - // whereas lowering it below an id a retained row references would ship - // rows pointing at symbols the server never received. - pendingRowCount = savepointPendingRowCount; - pendingBytes = savepointPendingBytes; - firstPendingRowTimeNanos = savepointFirstPendingRowTimeNanos; - currentTableBuffer = null; - currentTableBufferSnapshotBytes = 0; - currentTableName = null; - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - retainedBatchRejection = null; + discardPendingBatchState(); } /** @@ -3359,6 +3308,41 @@ public void resetSymbolDictionary() { } } + @Override + public void resetToSavepoint() { + checkNotClosed(); + // Always roll back, also with an empty savepoint: after a flush each + // buffer's mark sits at "no rows, server-known columns", so the rollback + // drops the column definitions the lease created that the server has not + // seen, which reset() would keep and ship null-padded. + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.rollbackToSavepoint(); + } + } + if (savepointPendingRowCount == 0) { + // No rows before the mark, or a flush since emptied it: every pending + // row is the caller's own and the sender-level clears of reset() apply. + discardPendingBatchState(); + return; + } + // Rows before the mark reference only symbol ids at or below the + // watermark recorded at the mark, and the watermark only grows within a + // batch, so restoring it drops the ids the discarded rows registered. + currentBatchMaxSymbolId = savepointBatchMaxSymbolId; + pendingRowCount = savepointPendingRowCount; + pendingBytes = savepointPendingBytes; + firstPendingRowTimeNanos = savepointFirstPendingRowTimeNanos; + currentTableBuffer = null; + currentTableBufferSnapshotBytes = 0; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + retainedBatchRejection = null; + } + @Override public long rowsAppended() { return rowsAppended; @@ -4583,6 +4567,14 @@ private void checkTableSelected() { } } + private void clearSavepointAndRejection() { + savepointBatchMaxSymbolId = 0; + savepointPendingRowCount = 0; + savepointPendingBytes = 0; + savepointFirstPendingRowTimeNanos = 0; + retainedBatchRejection = null; + } + private synchronized Throwable closeRemainingResources(Throwable terminalError) { if (closeCleanupStarted) { return terminalError; @@ -4753,6 +4745,29 @@ private Endpoint currentEndpoint() { return idx < 0 ? null : endpoints.get(idx); } + private void discardPendingBatchState() { + // Drop the batch's symbol watermark along with the rows that raised it. A + // later flush encodes the delta section as + // [sentMaxSymbolId+1 .. currentBatchMaxSymbolId], so a watermark left behind + // by the discarded batch keeps re-encoding that batch's symbols: a single-row + // batch after reset() would still carry the whole abandoned range and hit the + // very cap rejection reset() is documented to clear, leaving the sender unable + // to flush anything at all. -1 is the same value resetTableBuffersAfterFlush + // leaves behind after a successful flush, and sendCommitMessage already reads + // it as an empty delta. + currentBatchMaxSymbolId = -1; + reclaimUnsentSymbolIds(); + pendingBytes = 0; + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + clearSavepointAndRejection(); + currentTableBuffer = null; + currentTableBufferSnapshotBytes = 0; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + /** * Build and offer a connection event to the dispatcher. No-op when the * dispatcher has not been allocated yet (e.g. very early in connect() @@ -5163,10 +5178,7 @@ private void flushPendingRows(boolean deferCommit) { ? 0 : currentTableBuffer.getBufferedBytes(); pendingRowCount = 0; firstPendingRowTimeNanos = 0; - savepointPendingRowCount = 0; - savepointPendingBytes = 0; - savepointFirstPendingRowTimeNanos = 0; - retainedBatchRejection = null; + clearSavepointAndRejection(); return; } @@ -5524,10 +5536,7 @@ private void resetTableBuffersAfterFlush() { currentTableBufferSnapshotBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; - savepointPendingRowCount = 0; - savepointPendingBytes = 0; - savepointFirstPendingRowTimeNanos = 0; - retainedBatchRejection = null; + clearSavepointAndRejection(); armIfEligible(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index e305210b1..0eba803d2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; @@ -487,7 +488,7 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_segment_bytes=64;"; + + ";sf_max_segment_bytes=64;auto_flush_bytes=off;"; String pad = TestUtils.repeat("x", 60); Sender sender = Sender.fromConfig(config); try { @@ -1040,7 +1041,14 @@ public void testSingleSymbolLargerThanTheCapThrowsWithNothingPublished() throws Assert.assertTrue(e.getMessage(), e.getMessage().contains("a single symbol value is too large for the server batch cap")); } + try { + sender.checkError(); + Assert.fail("the probe must report the retained batch"); + } catch (BatchTooLargeForCapException expected) { + // latched by the dictionary pre-flight + } sender.reset(); + sender.checkError(); } // Nothing may have been published -- not even a dictionary chunk. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 38c4866b0..881143f2c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -171,6 +171,41 @@ public void testHealthySenderReportsBufferState() throws Exception { } } + @Test + public void testResetToSavepointAfterAnOverCapFlushClearsTheProbeAndShipsTheEarlierRows() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.table("t").longColumn("v", 1L).atNow(); + sender.markSavepoint(); + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 6; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + try { + sender.flush(); + Assert.fail("the lease's rows do not fit a 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // retained for retry + } + try { + sender.checkError(); + Assert.fail("the probe must report the retained batch"); + } catch (BatchTooLargeForCapException expected) { + // the pool retires a slot on this + } + sender.resetToSavepoint(); + sender.checkError(); + Assert.assertEquals(1, sender.pendingRowCount()); + sender.flush(); + log.awaitDataFrames(1, 5_000); + } + } + } + @Test public void testResetToSavepointKeepsRowsBufferedBeforeTheMark() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); From e43a6d876ed84e2fd0c5433780ded17644dbf9df Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:48:24 +0100 Subject: [PATCH 32/57] fix(qwp): resetToSavepoint reclaims the symbol ids the dropped rows registered Restoring the batch watermark kept the retained rows from re-encoding the dropped rows' dictionary entries, but the entries stayed in the dictionary, so the next row with a new symbol re-shipped the whole orphaned range and an over-cap dictionary section came back on a later lease with the probe clean. In delta mode the non-empty branch now reclaims ids above the restored watermark, the sent watermark and the persisted dictionary, through the existing reclaim helper. A test drops six wide symbols, flushes the retained row and then adds a new symbol. Co-Authored-By: Claude Fable 5.1 --- .../main/java/io/questdb/client/Sender.java | 6 ++-- .../qwp/client/QwpWebSocketSender.java | 23 ++++++++++---- .../qwp/client/SenderProbeMethodsTest.java | 31 +++++++++++++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index d5e0a28c8..8b2ecba58 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -764,7 +764,7 @@ default Sender long256Column(CharSequence name, long l0, long l1, long l2, long * rolls back to: rows appended after this call are discarded by that * method, rows appended before it are kept. Any flush that ships the * pending rows empties the savepoint, so a later {@code resetToSavepoint()} - * behaves like {@link #reset()}. The pool marks a savepoint when it hands a + * discards every pending row like {@link #reset()}. The pool marks a savepoint when it hands a * slot to a borrower. Pure bookkeeping: it runs no error check. Default * no-op. */ @@ -842,8 +842,8 @@ default void resetSymbolDictionary() { * Discards the rows appended since the last {@link #markSavepoint()} and * keeps the rows buffered before it. Without a savepoint, or after a flush * emptied it, this discards every pending row like {@link #reset()}, and - * also removes the column definitions the server has not seen yet. A pooled sender's {@code reset()} - * calls this, so a borrower can drop its own bad batch without discarding + * also removes the column definitions the server has not seen yet. A pooled + * sender's {@code reset()} calls this, so a borrower can drop its own bad batch without discarding * rows earlier borrowers left in the shared slot buffer. Default calls * {@code reset()}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index ccd23b398..c6d965ff9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -476,8 +476,10 @@ public class QwpWebSocketSender implements Sender { private int pendingRowCount; // Monotonic for the sender's life; reset() does not clear it. Counts rows whether or not they shipped. private long rowsAppended; - // Pending counters at the last markSavepoint(); zero once a flush ships the - // pending rows, at which point resetToSavepoint() is reset(). + // Pending counters and batch symbol watermark at the last markSavepoint(); + // zero once a flush ships the pending rows, at which point resetToSavepoint() + // discards every pending row like reset() and also removes the column + // definitions the server has not seen yet. private int savepointBatchMaxSymbolId; private long savepointFirstPendingRowTimeNanos; private long savepointPendingBytes; @@ -3330,8 +3332,15 @@ public void resetToSavepoint() { } // Rows before the mark reference only symbol ids at or below the // watermark recorded at the mark, and the watermark only grows within a - // batch, so restoring it drops the ids the discarded rows registered. + // batch, so the batch range is restored to it. In delta mode the ids the + // discarded rows registered above it are reclaimed too (the floor keeps + // the ring's and the durable dictionary's ids), so the next new symbol + // does not re-ship the orphaned range. Full-dict mode cannot reclaim: + // its frames already on the ring define those ids. Its orphans stay in + // the dictionary, which reset() cannot shrink there either; they are + // only re-shipped once a later row's new symbol id lies above them. currentBatchMaxSymbolId = savepointBatchMaxSymbolId; + reclaimUnsentSymbolIds(savepointBatchMaxSymbolId + 1); pendingRowCount = savepointPendingRowCount; pendingBytes = savepointPendingBytes; firstPendingRowTimeNanos = savepointFirstPendingRowTimeNanos; @@ -4756,7 +4765,7 @@ private void discardPendingBatchState() { // leaves behind after a successful flush, and sendCommitMessage already reads // it as an empty delta. currentBatchMaxSymbolId = -1; - reclaimUnsentSymbolIds(); + reclaimUnsentSymbolIds(0); pendingBytes = 0; pendingRowCount = 0; firstPendingRowTimeNanos = 0; @@ -6857,17 +6866,19 @@ private void publishDictionaryChunk(int startId, int endId) { * * Handing either to a different string is the silent misattribution the dense * id space exists to prevent, so the floor is the max of the two. + * {@code minFloor} raises it further: resetToSavepoint() passes the ids the + * retained rows still reference. Pass 0 to reclaim everything unsent. *

* Full-dict mode is excluded. Its sections always start at id 0, so there is * no lifetime anchor to shrink and nothing to gain -- while reclaiming would * let a later frame define a different string at an id a frame already on the * ring defines, which is precisely the hazard above. */ - private void reclaimUnsentSymbolIds() { + private void reclaimUnsentSymbolIds(int minFloor) { if (!deltaDictEnabled) { return; } - int floor = sentMaxSymbolId + 1; + int floor = Math.max(sentMaxSymbolId + 1, minFloor); PersistedSymbolDict pd = cursorEngine == null ? null : cursorEngine.getPersistedSymbolDict(); if (pd != null) { int durable = pd.size(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 881143f2c..13d2f9023 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -234,6 +234,37 @@ public void testResetToSavepointKeepsRowsBufferedBeforeTheMark() throws Exceptio } } + @Test + public void testResetToSavepointReclaimsTheSymbolIdsOfTheDroppedRows() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.table("t").symbol("s", "a").atNow(); + sender.markSavepoint(); + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 6; i++) { + sender.table("t").symbol("s", wide + i).atNow(); + } + try { + sender.flush(); + Assert.fail("six wide symbols do not fit a 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // retained for retry + } + sender.resetToSavepoint(); + sender.flush(); + log.awaitDataFrames(1, 5_000); + sender.table("t").symbol("s", "b").atNow(); + sender.flush(); + log.awaitDataFrames(2, 5_000); + sender.checkError(); + } + } + } + @Test public void testResetToSavepointWithoutMarkIsReset() throws Exception { try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { From 23f4f0bf550d1eb073e4547c05afe78f755dfe57 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:56:00 +0100 Subject: [PATCH 33/57] fix(pool): reset() on a lease discards only that lease's rows With flush_on_close=off a slot buffer holds rows from earlier leases whose close() already returned. The delegate's reset() cleared every table buffer, so a borrower following the BatchTooLargeForCapException remedy silently lost those rows. borrow() now marks a savepoint on the delegate and the pooled reset() rolls back to it. Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 13 +++++- .../io/questdb/client/impl/SenderPool.java | 4 ++ .../client/test/QuestDBFlushOnCloseTest.java | 25 +++++++++++ .../client/test/impl/RecordingSender.java | 15 +++++++ .../impl/SenderPoolCloseSemanticsTest.java | 41 +++++++++++++++++++ 5 files changed, 96 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 69252039e..630431eb5 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -146,7 +146,8 @@ public void checkError() { * the rows stay in the slot's buffer and go out when an auto-flush * threshold trips on a later append or when the pool's housekeeper settles * the idle slot. With {@code flush_on_close=on} the rows are flushed first, - * as before 1.4. Does not close the underlying {@link Sender}; that only + * as before 1.4. {@link #reset()} on a lease discards only that lease's rows. + * Does not close the underlying {@link Sender}; that only * happens when the owning {@code QuestDB} is closed. If the pool is already * shutting down when this lease comes home, the underlying Sender is torn * down here, on the calling thread, which ships whatever it still buffers. @@ -412,9 +413,17 @@ public int pendingRowCount() { return slot.live(generation).pendingRowCount(); } + /** + * Discards the rows appended through this lease and keeps the rows earlier + * leases left in the slot's buffer: the pool marks a savepoint at borrow and + * this rolls back to it ({@link Sender#resetToSavepoint()}). After a + * {@code flush()} inside the lease every pending row is this lease's own and + * all of them go. Use it to drop a batch the server cannot accept, for + * example after a {@code BatchTooLargeForCapException}. + */ @Override public void reset() { - slot.live(generation).reset(); + slot.live(generation).resetToSavepoint(); } @Override diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 41466cc0d..2a322ef78 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1249,6 +1249,10 @@ public PooledSender borrow() { // wrapper handed out can be told apart from any prior, // now-stale borrow of the same slot. s.bumpGeneration(); + // Lease-scoped reset: the borrower's reset() rolls back to here, so rows + // earlier leases left in this buffer survive it. Pure bookkeeping on the + // delegate, no error check, no throw path while the lock is held. + s.delegate().markSavepoint(); return new PooledSender(s, s.generation()); } if (all.size() + inFlightCreations + closingSlots + leakedSlots + recoveringSlots < maxSize) { diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index ff296d1d4..58bc12e87 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -94,6 +94,31 @@ public void testHandleCloseShipsRowsLeftInSlot() throws Exception { } } + @Test + public void testLaterLeaseResetKeepsEarlierLeasesRows() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + try (Sender a = db.borrowSender()) { + a.table("t1").longColumn("v", 1L).atNow(); + } + try (Sender b = db.borrowSender()) { + b.table("t2").longColumn("v", 2L).atNow(); + b.reset(); // B drops its own bad batch + } + try (Sender c = db.borrowSender()) { + Assert.assertEquals("A's row is still buffered", 1, c.pendingRowCount()); + c.flush(); + } + log.awaitDataFrames(1, 5_000); + Assert.assertEquals("one frame, one table: t1 only", 1, log.tableCount(0)); + Assert.assertEquals(1, log.dataFrames()); + } + } + } + @Test public void testSettleShipsOnDefaultCadenceWhenIntervalUnset() throws Exception { FrameLog log = new FrameLog(); diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index 968a607ef..8b5325674 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -55,6 +55,7 @@ class RecordingSender implements Sender { volatile int pendingRows; volatile boolean rowInProgress; volatile long rowsAppended; + volatile int savepointRows; @Override public void at(long timestamp, ChronoUnit unit) { @@ -150,6 +151,7 @@ public void flush() { flushCount.incrementAndGet(); } pendingRows = 0; + savepointRows = 0; deferredFramesOpen = false; } @@ -165,6 +167,7 @@ public long flushDeferred() { return -1L; // nothing pending: the real sender publishes nothing } pendingRows = 0; + savepointRows = 0; deferredFramesOpen = true; return deferredFlushCount.incrementAndGet(); } @@ -204,6 +207,11 @@ public Sender longColumn(CharSequence name, long value) { return this; } + @Override + public void markSavepoint() { + savepointRows = pendingRows; + } + @Override public int pendingRowCount() { return pendingRows; @@ -212,6 +220,13 @@ public int pendingRowCount() { @Override public void reset() { pendingRows = 0; + savepointRows = 0; + rowInProgress = false; + } + + @Override + public void resetToSavepoint() { + pendingRows = savepointRows; rowInProgress = false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index e3d009bfe..2d41edf98 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -219,6 +219,28 @@ public void testLatchedServerErrorAtCloseKeepsItsType() { } } + @Test + public void testResetDiscardsOnlyTheResettingLeasesRows() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + try (Sender a = pool.borrow()) { + a.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + Assert.assertEquals("close() shipped nothing", 1, delegate.pendingRowCount()); + try (Sender b = pool.borrow()) { + Assert.assertEquals("same slot", 1, built.size()); + b.table("t").longColumn("v", 2L).atNow(); + Assert.assertEquals(2, delegate.pendingRowCount()); + b.reset(); + Assert.assertEquals("A's row survives B's reset", 1, delegate.pendingRowCount()); + } + Assert.assertEquals(1, pool.settleIdle()); + Assert.assertEquals(1, delegate.flushCount.get()); + Assert.assertEquals(0, delegate.pendingRowCount()); + } + } + @Test public void testStaleCloseDoesNotTouchReborrowedSlot() { List built = new ArrayList<>(); @@ -241,6 +263,25 @@ public void testStaleCloseDoesNotTouchReborrowedSlot() { } } + @Test + public void testStaleResetDoesNotTouchReborrowedSlot() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + first.close(); + try (Sender second = pool.borrow()) { + second.table("t").longColumn("v", 1L).atNow(); + try { + first.reset(); + Assert.fail("a stale lease must not reset the slot"); + } catch (IllegalStateException expected) { + // "sender handle is closed (returned to the pool)" + } + Assert.assertEquals(1, built.get(0).pendingRowCount()); + } + } + } + static SenderPool newPool(int min, int max, List built) { return new SenderPool(CONFIG, min, max, 150L, Long.MAX_VALUE, Long.MAX_VALUE, idx -> { RecordingSender s = new RecordingSender(); From 13838bfa0db630d1844b5c1b07f09bfcfcd8eac6 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:04:07 +0100 Subject: [PATCH 34/57] fix(pool): retire a slot whose batch the server cap rejects A settle flush that throws BatchTooLargeForCapException used to requeue the slot: the probe did not see the rejection, so the housekeeper retried every tick and later borrowers failed on their own appends. The settle now retires the slot and logs the dropped row count; at close the probe reports the retained batch, so the release retires the slot and the wrapper names the rows it dropped. borrow() now also marks a savepoint explicitly on a freshly created delegate, so every lease handed to a caller starts at a mark. Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 12 ++- .../io/questdb/client/impl/SenderPool.java | 41 ++++++--- .../client/test/QuestDBFlushOnCloseTest.java | 88 +++++++++++++++++++ .../test/impl/SenderPoolSettleTest.java | 21 +++++ 4 files changed, 146 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 630431eb5..79f2c4d89 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -156,7 +156,8 @@ public void checkError() { * {@link LineSenderException} is thrown so the mistake is visible. A * delegate that has latched a terminal error is retired instead of being * returned, and a new {@link LineSenderException} is thrown here whose - * cause is the delegate's error. A lease body that already threw the typed + * cause is the delegate's error and whose message carries the number of pending rows the + * retirement dropped. A lease body that already threw the typed * error keeps it as the primary exception, with close's wrapper * suppressed. The typed error and its classification live on the cause. *

@@ -187,6 +188,7 @@ public void close() { boolean cancelledRow = false; boolean recycle = false; RuntimeException failure = null; + int pendingRowsAtRetire = 0; try { // live(), not delegate(): re-validate the generation right before // touching the delegate, narrowing the duplicate-close race. @@ -218,6 +220,11 @@ public void close() { // latched an error, the flush failed, or the generation went // stale. Retire it; a wrapper around the original propagates // once this finally returns. + try { + pendingRowsAtRetire = slot.delegate().pendingRowCount(); + } catch (Throwable ignored) { + // best effort: the count is only for the message + } pool.discardBroken(this); } } @@ -229,7 +236,8 @@ public void close() { } String msg = failure.getMessage(); boolean retryable = failure instanceof LineSenderException && ((LineSenderException) failure).isRetryable(); - LineSenderException wrapper = new LineSenderException("pooled sender close failed: " + msg, retryable); + LineSenderException wrapper = new LineSenderException("pooled sender close failed, slot retired with " + + pendingRowsAtRetire + " pending row(s) dropped: " + msg, retryable); wrapper.initCause(failure); throw wrapper; } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 2a322ef78..fbed7e19e 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -30,6 +30,7 @@ import io.questdb.client.SenderError; import io.questdb.client.SenderErrorHandler; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; @@ -1323,6 +1324,8 @@ public PooledSender borrow() { created.bumpGeneration(); inFlightCreations--; creationFinished.signalAll(); + // Every lease handed to a caller starts at a mark. + created.delegate().markSavepoint(); return new PooledSender(created, created.generation()); } // Capacity-starved: re-probe retired slots BEFORE the terminal @@ -1730,8 +1733,10 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna * delegate is probed with {@link Sender#checkError()}: a latched error * retires the slot, anything else returns it to the free list, and rows a * failed flush left behind wait for the next settle. An {@link Error} - * thrown by the flush is not transient and retires the slot. The slot's idle clock is not - * touched, so reaping is unaffected. + * thrown by the flush is not transient and retires the slot, and so does a + * {@code BatchTooLargeForCapException}: that batch belongs to no lease and no retry can + * ship it, so the slot is retired and the dropped row count logged. The slot's idle clock + * is not touched, so reaping is unaffected. * * @return the number of slots whose flush completed */ @@ -1768,11 +1773,16 @@ public int settleIdle() { PooledSender lease = leases.get(i); boolean flushed = false; Throwable fatal = null; + BatchTooLargeForCapException overCap = null; try { // PooledSender.flush() already marks the commit on the slot. lease.flush(); lease.slot().settleFailureLogged(false); flushed = true; + } catch (BatchTooLargeForCapException e) { + // Deterministic and not latched: no retry ships this batch and no + // lease owns it any more. Retire the slot below. + overCap = e; } catch (Exception e) { // Transient (ring full, recycle in flight) or terminal: the // requeue below tells them apart with the probe. @@ -1785,6 +1795,10 @@ public int settleIdle() { if (fatal != null) { LOG.warn("settleIdle: flush threw an Error, retiring slot {}", lease.slot().slotIndex(), fatal); retireLease(lease, " during settle"); + } else if (overCap != null) { + LOG.warn("settleIdle: slot {} holds a batch the server cap rejects, retiring it and dropping {} pending row(s): {}", + lease.slot().slotIndex(), pendingRowsOf(lease.slot()), overCap.toString()); + retireLease(lease, " during settle"); } else { requeueSettled(lease); } @@ -1801,12 +1815,7 @@ public int settleIdle() { private void logSettleFailure(SenderSlot slot, Exception e) { int idx = slot.slotIndex(); - int pending = 0; - try { - pending = slot.delegate().pendingRowCount(); - } catch (Throwable ignored) { - // best effort: the count is only for the log line - } + int pending = pendingRowsOf(slot); String error = e.toString(); if (!slot.settleFailureLogged()) { slot.settleFailureLogged(true); @@ -1889,6 +1898,15 @@ private void waitForStartupRecoveryRetry() { * What {@link #settleIdle()} looks for in an idle slot. Phase A: buffered * rows and deferred frames awaiting their commit. */ + /** Best-effort pending row count of a slot's delegate, for log lines only. */ + private static int pendingRowsOf(SenderSlot slot) { + try { + return slot.delegate().pendingRowCount(); + } catch (Throwable ignored) { + return 0; + } + } + private static boolean hasUnshippedWork(Sender delegate) { return delegate.pendingRowCount() > 0 || delegate.hasDeferredFrames(); } @@ -1909,12 +1927,7 @@ private void requeueSettled(PooledSender lease) { // latched: retire below } if (!healthy) { - long pendingRows = 0; - try { - pendingRows = s.delegate().pendingRowCount(); - } catch (Throwable ignored) { - // best effort: the count is only for the log line - } + int pendingRows = pendingRowsOf(s); LOG.warn("settleIdle: delegate of slot {} latched an error, retiring it with {} pending row(s)", s.slotIndex(), pendingRows); retireLease(lease, " during settle"); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 58bc12e87..b857ea924 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -26,6 +26,9 @@ import io.questdb.client.QuestDB; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.impl.PooledSender; import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; @@ -119,6 +122,91 @@ public void testLaterLeaseResetKeepsEarlierLeasesRows() throws Exception { } } + @Test + public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off on purpose: defeat the cap-bound byte trigger so the batch outgrows the cap + try (QuestDB db = QuestDB.connect(config(server.getPort(), + "auto_flush_bytes=off;auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); + String wide = wideValue(1000); + for (int i = 0; i < 6; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").stringColumn("s", wide).atNow(); + } + } + Sender poisoned = null; + boolean closeThrew = false; + try (Sender s = db.borrowSender()) { + poisoned = ((PooledSender) s).getDelegateForTesting(); + try { + s.flush(); + Assert.fail("six 1 KB rows do not fit a 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // the retained batch belongs to nobody: close() must retire the slot + } + } catch (LineSenderException e) { + closeThrew = true; + Assert.assertTrue("close() wraps the retained rejection", e.getCause() instanceof BatchTooLargeForCapException); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("6 pending row(s) dropped")); + } + Assert.assertTrue("close() must report the retirement", closeThrew); + try (Sender s = db.borrowSender()) { + Assert.assertNotSame("the retired slot is replaced", poisoned, ((PooledSender) s).getDelegateForTesting()); + s.table("t").stringColumn("s", wide).atNow(); + s.flush(); + } + log.awaitDataFrames(2, 5_000); + } + } + } + + @Test + public void testRowOverTheCapFailsOnlyThatRowAndKeepsTheSlot() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); + Sender delegate; + try (Sender s = db.borrowSender()) { + delegate = ((PooledSender) s).getDelegateForTesting(); + try { + s.table("t").stringColumn("s", wideValue(5000)).atNow(); + Assert.fail("a 5 KB row cannot fit a 4 KiB cap"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains("row too large")); + } + s.table("t").longColumn("v", 1L).atNow(); // the lease carries on + } + try (Sender s = db.borrowSender()) { + Assert.assertSame("the slot stayed in service", delegate, ((PooledSender) s).getDelegateForTesting()); + Assert.assertEquals(1, s.pendingRowCount()); + s.flush(); + } + log.awaitDataFrames(2, 5_000); + } + } + } + @Test public void testSettleShipsOnDefaultCadenceWhenIntervalUnset() throws Exception { FrameLog log = new FrameLog(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index d44dda42d..79f867fd6 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -26,6 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import org.junit.Assert; import org.junit.Test; @@ -46,6 +48,25 @@ */ public class SenderPoolSettleTest { + @Test + public void testSettleRetiresSlotHoldingAnOverCapBatch() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + delegate.flushError = new BatchTooLargeForCapException("single table batch too large for server batch cap"); + Assert.assertEquals(0, pool.settleIdle()); + Assert.assertEquals("a batch the cap rejects never ships: the slot is retired", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + try (Sender next = pool.borrow()) { + Assert.assertEquals("the next borrow builds a fresh delegate", 2, built.size()); + Assert.assertNotSame(delegate, ((PooledSender) next).getDelegateForTesting()); + } + } + } + @Test public void testSettleShipsRowsLeftInIdleSlot() throws Exception { List built = new ArrayList<>(); From 66b8739b42a8969b86bec36d808a9bdda8c23d3a Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:14:02 +0100 Subject: [PATCH 35/57] fix(pool): place pendingRowsOf after hasUnshippedWork; say unshipped, not dropped, in the close wrapper The helper landed between hasUnshippedWork's Javadoc and its declaration. The close wrapper's count is read before the delegate closes, and a transient failure can still ship those rows from the delegate's close, so the message now says unshipped; the settle WARN keeps "dropping" because the over-cap path discards for real. Comments and Javadoc no longer claim the rejection is unlatched or that no node could ship it. Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/impl/PooledSender.java | 7 ++-- .../io/questdb/client/impl/SenderPool.java | 17 +++++---- .../client/test/QuestDBFlushOnCloseTest.java | 37 ++++++++++--------- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 79f2c4d89..07631a546 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -156,8 +156,9 @@ public void checkError() { * {@link LineSenderException} is thrown so the mistake is visible. A * delegate that has latched a terminal error is retired instead of being * returned, and a new {@link LineSenderException} is thrown here whose - * cause is the delegate's error and whose message carries the number of pending rows the - * retirement dropped. A lease body that already threw the typed + * cause is the delegate's error and whose message carries the number of unshipped rows at + * the retirement. The count lives on the wrapper of a non-server failure; the typed server + * branch below carries none. A lease body that already threw the typed * error keeps it as the primary exception, with close's wrapper * suppressed. The typed error and its classification live on the cause. *

@@ -237,7 +238,7 @@ public void close() { String msg = failure.getMessage(); boolean retryable = failure instanceof LineSenderException && ((LineSenderException) failure).isRetryable(); LineSenderException wrapper = new LineSenderException("pooled sender close failed, slot retired with " - + pendingRowsAtRetire + " pending row(s) dropped: " + msg, retryable); + + pendingRowsAtRetire + " unshipped row(s): " + msg, retryable); wrapper.initCause(failure); throw wrapper; } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index fbed7e19e..5cc78bfc1 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1734,8 +1734,8 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna * retires the slot, anything else returns it to the free list, and rows a * failed flush left behind wait for the next settle. An {@link Error} * thrown by the flush is not transient and retires the slot, and so does a - * {@code BatchTooLargeForCapException}: that batch belongs to no lease and no retry can - * ship it, so the slot is retired and the dropped row count logged. The slot's idle clock + * {@code BatchTooLargeForCapException}: that batch belongs to no lease, a settle retry + * cannot ship it, so the slot is retired and the dropped row count logged. The slot's idle clock * is not touched, so reaping is unaffected. * * @return the number of slots whose flush completed @@ -1780,8 +1780,9 @@ public int settleIdle() { lease.slot().settleFailureLogged(false); flushed = true; } catch (BatchTooLargeForCapException e) { - // Deterministic and not latched: no retry ships this batch and no - // lease owns it any more. Retire the slot below. + // Deterministic: the probe would report it too, but no settle retry + // can ship this batch and no lease owns it any more. Retire the + // slot below. overCap = e; } catch (Exception e) { // Transient (ring full, recycle in flight) or terminal: the @@ -1898,6 +1899,10 @@ private void waitForStartupRecoveryRetry() { * What {@link #settleIdle()} looks for in an idle slot. Phase A: buffered * rows and deferred frames awaiting their commit. */ + private static boolean hasUnshippedWork(Sender delegate) { + return delegate.pendingRowCount() > 0 || delegate.hasDeferredFrames(); + } + /** Best-effort pending row count of a slot's delegate, for log lines only. */ private static int pendingRowsOf(SenderSlot slot) { try { @@ -1907,10 +1912,6 @@ private static int pendingRowsOf(SenderSlot slot) { } } - private static boolean hasUnshippedWork(Sender delegate) { - return delegate.pendingRowCount() > 0 || delegate.hasDeferredFrames(); - } - /** * Returns a slot held by {@link #settleIdle()} to the free list without * resetting its idle clock, or retires it when the delegate has latched an diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index b857ea924..86d71feb2 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -132,13 +132,13 @@ public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exceptio // auto_flush_bytes=off on purpose: defeat the cap-bound byte trigger so the batch outgrows the cap try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_bytes=off;auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { - // warm-up lease: proves the connection and the advertised cap are in place - // (initial_connect_retry=async learns the cap on the I/O thread) - try (Sender s = db.borrowSender()) { - s.table("w").longColumn("v", 1L).atNow(); - s.flush(); - } - log.awaitDataFrames(1, 5_000); + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); String wide = wideValue(1000); for (int i = 0; i < 6; i++) { try (Sender s = db.borrowSender()) { @@ -146,7 +146,7 @@ public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exceptio } } Sender poisoned = null; - boolean closeThrew = false; + boolean hasCloseThrown = false; try (Sender s = db.borrowSender()) { poisoned = ((PooledSender) s).getDelegateForTesting(); try { @@ -156,17 +156,18 @@ public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exceptio // the retained batch belongs to nobody: close() must retire the slot } } catch (LineSenderException e) { - closeThrew = true; + hasCloseThrown = true; Assert.assertTrue("close() wraps the retained rejection", e.getCause() instanceof BatchTooLargeForCapException); - Assert.assertTrue(e.getMessage(), e.getMessage().contains("6 pending row(s) dropped")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("6 unshipped row(s)")); } - Assert.assertTrue("close() must report the retirement", closeThrew); + Assert.assertTrue("close() must report the retirement", hasCloseThrown); try (Sender s = db.borrowSender()) { Assert.assertNotSame("the retired slot is replaced", poisoned, ((PooledSender) s).getDelegateForTesting()); s.table("t").stringColumn("s", wide).atNow(); s.flush(); } log.awaitDataFrames(2, 5_000); + Assert.assertEquals("the over-cap batch never ships", 2, log.dataFrames()); } } } @@ -179,13 +180,13 @@ public void testRowOverTheCapFailsOnlyThatRowAndKeepsTheSlot() throws Exception server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { - // warm-up lease: proves the connection and the advertised cap are in place - // (initial_connect_retry=async learns the cap on the I/O thread) - try (Sender s = db.borrowSender()) { - s.table("w").longColumn("v", 1L).atNow(); - s.flush(); - } - log.awaitDataFrames(1, 5_000); + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); Sender delegate; try (Sender s = db.borrowSender()) { delegate = ((PooledSender) s).getDelegateForTesting(); From 41ab789273665faacddbfadf041291bd78f61066 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:16:55 +0100 Subject: [PATCH 36/57] fix(pool): retry failed startup recovery once a second, saturate housekeeper deadlines The two-deadline loop retried a failed recovery step on the settle cadence (about 100 ms) instead of the 5 s reap interval, and computed now + interval without overflow protection, so housekeeper_interval_ms=Long.MAX_VALUE made the loop wait forever and the settle never ran. Recovery retries now have their own one-second deadline, matching the direct pool's driver, and every deadline saturates. Drops the unused parameter of resolveSettleIntervalMillis and corrects the constructor Javadoc. Co-Authored-By: Claude Fable 5.1 --- .../io/questdb/client/QuestDBBuilder.java | 6 +- .../questdb/client/impl/PoolHousekeeper.java | 52 ++++++--- .../io/questdb/client/impl/SenderPool.java | 7 ++ .../client/test/QuestDBBuilderTest.java | 16 +-- .../test/QuestDBHousekeeperCadenceTest.java | 100 ++++++++++++++++++ 5 files changed, 157 insertions(+), 24 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index c35850b18..34105d1e4 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -325,7 +325,7 @@ public QuestDB build() { resolvedCommitRows = windowRows; resolvedCommitIntervalMillis = windowIntervalMillis; resolvedIngestConfig = ingestConfig; - long settleIntervalMillis = resolveSettleIntervalMillis(view, housekeeperIntervalMillis, windowIntervalMillis); + long settleIntervalMillis = resolveSettleIntervalMillis(view, windowIntervalMillis); return new QuestDBImpl( ingestConfig, @@ -599,11 +599,11 @@ private static String withDefaultAsyncConnect(String config) { * transport rejects both {@code auto_flush=off} and * {@code auto_flush_interval=off}, so the interval is a number or unset. * With a commit window, the shorter of that and {@code commitIntervalMillis} - * (0 = no window). {@code housekeeperIntervalMillis} is currently unused. + * (0 = no window). * Public only for the unit test, which lives in another package. */ @TestOnly - public static long resolveSettleIntervalMillis(ConfigView view, long housekeeperIntervalMillis, long commitIntervalMillis) { + public static long resolveSettleIntervalMillis(ConfigView view, long commitIntervalMillis) { // getLong uses the connect-string number syntax (underscore separators // included), the same one the Sender parse already accepted. long fromAutoFlush = Math.max(1L, view.getLong("auto_flush_interval", DEFAULT_SETTLE_INTERVAL_MILLIS)); diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 956af353a..8676d3553 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -46,6 +46,11 @@ final class PoolHousekeeper { // no application-level connect timeout); see the residual-window note on // SenderPool.recoverOneSlotStep. static final long STOP_TIMEOUT_MILLIS = 2_000; + // Wait between startup-recovery steps that did not find more work to do at + // once: a finished scan, an empty one, or a transient failure such as an + // unreachable server. The same cadence SenderPool's private driver uses, and + // deliberately not the settle cadence, which can be 50 ms. + static final long STARTUP_RECOVERY_RETRY_MILLIS = 1_000L; private final long intervalMillis; private final QueryClientPool queryPool; @@ -58,9 +63,9 @@ final class PoolHousekeeper { /** * @param intervalMillis reap cadence: idle and over-age slots * @param settleIntervalMillis settle cadence: ship rows left in idle slots; - * the shorter of the configured auto-flush and - * commit intervals, or {@code intervalMillis} - * when both are off + * {@code auto_flush_interval} (100 ms when + * unset), or the commit interval when that is + * shorter */ PoolHousekeeper(SenderPool senderPool, QueryClientPool queryPool, long intervalMillis, long settleIntervalMillis) { this.senderPool = senderPool; @@ -140,6 +145,16 @@ void stop() { * primitive. Keeping the two stop paths identical matters when an interrupt arrives during the first * join: it must be remembered without skipping the target interrupt and second join that follow. */ + /** + * {@code nowMillis + intervalMillis}, saturating at {@link Long#MAX_VALUE} + * so an interval of {@code Long.MAX_VALUE} ("never") does not wrap negative + * and either stall every other deadline or fire at once. + */ + static long deadline(long nowMillis, long intervalMillis) { + long d = nowMillis + intervalMillis; + return d < nowMillis ? Long.MAX_VALUE : d; + } + static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { final long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); final long deadlineNanos = System.nanoTime() + timeoutNanos; @@ -161,8 +176,10 @@ static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { } private void runLoop() { - long nextSettleAt = System.currentTimeMillis() + this.settleIntervalMillis; - long nextReapAt = System.currentTimeMillis() + intervalMillis; + long startMillis = System.currentTimeMillis(); + long nextSettleAt = deadline(startMillis, settleIntervalMillis); + long nextReapAt = deadline(startMillis, intervalMillis); + long nextRecoveryAt = startMillis; while (!stop) { // Per-slot startup SF recovery, driven on THIS (the reap-loop) // thread, one stranded slot per iteration. Doing it here rather than @@ -182,18 +199,27 @@ private void runLoop() { // No-op once recovery completes or the pool is closing. Best-effort: // a recovery failure (including an Error) must never kill this // daemon, so swallow Throwable -- exactly as the reap guards below. - boolean recovering; - try { - recovering = senderPool.runStartupRecoveryStep(); - } catch (Throwable ignored) { - recovering = false; + boolean recovering = false; + if (System.currentTimeMillis() >= nextRecoveryAt) { + try { + recovering = senderPool.runStartupRecoveryStep(); + } catch (Throwable ignored) { + recovering = false; + } + // A step with more work runs again at once. A step that finished, + // found nothing or hit a transient failure is retried on its own + // cadence, never on the settle cadence: a stranded slot against an + // unreachable server must not become a hot connect loop. + long afterStep = System.currentTimeMillis(); + nextRecoveryAt = recovering ? afterStep : deadline(afterStep, STARTUP_RECOVERY_RETRY_MILLIS); } synchronized (signalLock) { if (stop) { return; } if (!recovering) { - long waitMillis = Math.min(nextSettleAt, nextReapAt) - System.currentTimeMillis(); + long nextAt = Math.min(Math.min(nextSettleAt, nextReapAt), nextRecoveryAt); + long waitMillis = nextAt - System.currentTimeMillis(); if (waitMillis > 0) { try { signalLock.wait(waitMillis); @@ -215,7 +241,7 @@ private void runLoop() { // Best-effort: settleIdle() already guards per-slot // failures; a Throwable here must never kill the daemon. } - nextSettleAt = now + settleIntervalMillis; + nextSettleAt = deadline(now, settleIntervalMillis); } if (now >= nextReapAt) { try { @@ -236,7 +262,7 @@ private void runLoop() { // must never propagate, and Throwable (not RuntimeException) so // an Error from query-client teardown cannot kill the daemon. } - nextReapAt = now + intervalMillis; + nextReapAt = deadline(now, intervalMillis); } } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 5cc78bfc1..8669e6e77 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -363,6 +363,7 @@ public final class SenderPool implements AutoCloseable { private int recoveryFailStreak; private int recoveryFailStreakSlot = -1; private final IntList recoveryWarnedSlots = new IntList(); + private volatile int startupRecoveryBuildAttempts; public SenderPool( String configurationString, @@ -1719,6 +1720,11 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna this.afterStartupRecoveryJoinHook = afterJoinHook; } + @TestOnly + public int startupRecoveryBuildAttemptsForTesting() { + return startupRecoveryBuildAttempts; + } + /** * Ships rows left behind in idle slots. A pooled {@code close()} does not * flush by default, and auto-flush thresholds are only evaluated on append, @@ -2250,6 +2256,7 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { + startupRecoveryBuildAttempts++; return createSlot(recoverySenderFactory, slotIndex); } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java index 21d346a50..c830cb770 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBBuilderTest.java @@ -321,12 +321,12 @@ public void testCommitWindowBuilderCallsOverrideString() { @Test public void testSettleIntervalTakesShorterOfAutoFlushAndCommitIntervals() { ConfigView v = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); - Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 40L)); - Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 900L)); - Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 5_000L, 0L)); + Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(v, 40L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 900L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(v, 0L)); ConfigView unset = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); - Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 5_000L, 40L)); - Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 5_000L, 900L)); + Assert.assertEquals(40L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 40L)); + Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(unset, 900L)); } @Test @@ -396,11 +396,11 @@ public void testSettleIntervalFollowsAutoFlushInterval() { // auto_flush_interval=off, so the interval is unset (the sender's // 100 ms default) or a number (underscore separators allowed). ConfigView dflt = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;")); - Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 5_000L, 0L)); + Assert.assertEquals(100L, QuestDBBuilder.resolveSettleIntervalMillis(dflt, 0L)); ConfigView explicit = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=250;")); - Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 5_000L, 0L)); + Assert.assertEquals(250L, QuestDBBuilder.resolveSettleIntervalMillis(explicit, 0L)); ConfigView underscored = new ConfigView(ConfigString.parse("ws::addr=127.0.0.1:1;auto_flush_interval=1_000;")); - Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 5_000L, 0L)); + Assert.assertEquals(1_000L, QuestDBBuilder.resolveSettleIntervalMillis(underscored, 0L)); } @Test diff --git a/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java b/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java new file mode 100644 index 000000000..fa6144970 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java @@ -0,0 +1,100 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ +package io.questdb.client.test; + +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.concurrent.TimeUnit; + +/** + * The housekeeper's two cadences under extreme settings: a startup + * store-and-forward recovery that keeps failing, and a reap interval of + * {@code Long.MAX_VALUE}. + */ +public class QuestDBHousekeeperCadenceTest { + + @Rule + public final TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void testFailedStartupRecoveryRetriesOnItsOwnCadenceNotTheSettleCadence() throws Exception { + File sfDir = temp.newFolder("sf"); + int port; + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + port = silent.getPort(); + // Strand one slot: the server never acks, so the frame stays on disk after close. + try (QuestDB db = QuestDB.connect("ws::addr=localhost:" + port + ";sf_dir=" + sfDir.getAbsolutePath() + + ";sender_pool_min=1;sender_pool_max=1;query_pool_min=0;query_pool_max=1" + + ";close_flush_timeout_millis=200;housekeeper_interval_ms=60000;auto_flush_interval=60000;")) { + try (Sender s = db.borrowSender()) { + s.table("recover").longColumn("v", 1L).atNow(); + s.flush(); + } + } + } + // The port is closed now: every recovery step fails fast on a refused connect. + // Settle every 50 ms; recovery must not follow that cadence. + try (QuestDB db = QuestDB.connect("ws::addr=localhost:" + port + ";sf_dir=" + sfDir.getAbsolutePath() + + ";sender_pool_min=0;sender_pool_max=1;query_pool_min=0;query_pool_max=1" + + ";housekeeper_interval_ms=1000;auto_flush_interval=50;")) { + Thread.sleep(2_600); + int attempts = ((QuestDBImpl) db).getSenderPoolForTesting().startupRecoveryBuildAttemptsForTesting(); + Assert.assertTrue("about one attempt per second expected, got " + attempts + + "; sf_dir=" + java.util.Arrays.toString(sfDir.list()), attempts >= 1 && attempts <= 5); + } + } + + @Test + public void testHousekeeperIntervalAtLongMaxStillSettles() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(QuestDBFlushOnCloseTest.config(server.getPort(), + "housekeeper_interval_ms=" + Long.MAX_VALUE + ";"))) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + log.awaitDataFrames(1, 3_000); // the 100 ms settle must still run + } + } + } + + private static final class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // no ack: the frame stays unacked on disk + } + } +} From 09155329f2013534ef15fbebc1a3f6c34fc98fff Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:06 +0100 Subject: [PATCH 37/57] fix(pool): place deadline() and the recovery constant at their alphabetical positions deadline() had landed between joinIgnoringCallerInterrupts and its Javadoc, leaving that method undocumented and a dangling comment above the helper. Also moves the retry constant and the test accessor into order and corrects two comments. Co-Authored-By: Claude Fable 5.1 --- .../questdb/client/impl/PoolHousekeeper.java | 36 ++++++++++--------- .../io/questdb/client/impl/SenderPool.java | 10 +++--- .../test/QuestDBHousekeeperCadenceTest.java | 3 +- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 8676d3553..08f44d08e 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -33,6 +33,11 @@ */ final class PoolHousekeeper { + // Wait between startup-recovery steps that did not find more work to do at + // once: a finished scan, an empty one, or a transient failure such as an + // unreachable server. The same cadence SenderPool's private driver uses, and + // deliberately not the settle cadence, which can be 50 ms. + static final long STARTUP_RECOVERY_RETRY_MILLIS = 1_000L; // How long stop() waits for the daemon to exit. Kept ABOVE // SenderPool.RECOVERY_DRAIN_BUDGET_MILLIS so a startup-recovery drain still // in flight when close() arrives finishes well within this join (C1 fix). @@ -46,11 +51,6 @@ final class PoolHousekeeper { // no application-level connect timeout); see the residual-window note on // SenderPool.recoverOneSlotStep. static final long STOP_TIMEOUT_MILLIS = 2_000; - // Wait between startup-recovery steps that did not find more work to do at - // once: a finished scan, an empty one, or a transient failure such as an - // unreachable server. The same cadence SenderPool's private driver uses, and - // deliberately not the settle cadence, which can be 50 ms. - static final long STARTUP_RECOVERY_RETRY_MILLIS = 1_000L; private final long intervalMillis; private final QueryClientPool queryPool; @@ -62,7 +62,7 @@ final class PoolHousekeeper { /** * @param intervalMillis reap cadence: idle and over-age slots - * @param settleIntervalMillis settle cadence: ship rows left in idle slots; + * @param settleIntervalMillis settle cadence: ships rows left in idle slots; * {@code auto_flush_interval} (100 ms when * unset), or the commit interval when that is * shorter @@ -136,15 +136,6 @@ void stop() { } } - /** - * Waits up to the supplied budget without letting cancellation skip the caller's remaining shutdown - * work. Every {@link InterruptedException} clears the caller's flag, so remember it and spend only the - * remainder of the original budget before handing the information back to {@link #stop()}. - *

- * Package-private so SenderPool's direct recovery driver can use the same deadline-preserving shutdown - * primitive. Keeping the two stop paths identical matters when an interrupt arrives during the first - * join: it must be remembered without skipping the target interrupt and second join that follow. - */ /** * {@code nowMillis + intervalMillis}, saturating at {@link Long#MAX_VALUE} * so an interval of {@code Long.MAX_VALUE} ("never") does not wrap negative @@ -155,6 +146,15 @@ static long deadline(long nowMillis, long intervalMillis) { return d < nowMillis ? Long.MAX_VALUE : d; } + /** + * Waits up to the supplied budget without letting cancellation skip the caller's remaining shutdown + * work. Every {@link InterruptedException} clears the caller's flag, so remember it and spend only the + * remainder of the original budget before handing the information back to {@link #stop()}. + *

+ * Package-private so SenderPool's direct recovery driver can use the same deadline-preserving shutdown + * primitive. Keeping the two stop paths identical matters when an interrupt arrives during the first + * join: it must be remembered without skipping the target interrupt and second join that follow. + */ static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { final long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); final long deadlineNanos = System.nanoTime() + timeoutNanos; @@ -195,8 +195,10 @@ private void runLoop() { // sole residual overrun is an in-flight connect to a black-holed // host, which no interrupt breaks. See SenderPool.recoverOneSlotStep. // While recovery still has work we skip the idle wait so the backlog - // drains promptly; once done we fall back to the normal interval. - // No-op once recovery completes or the pool is closing. Best-effort: + // drains promptly; otherwise the step is retried every + // STARTUP_RECOVERY_RETRY_MILLIS until the pool reports it done, and + // stays cheap after that. A no-op once recovery completes or the + // pool is closing. Best-effort: // a recovery failure (including an Error) must never kill this // daemon, so swallow Throwable -- exactly as the reap guards below. boolean recovering = false; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 8669e6e77..12932a07b 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1502,6 +1502,11 @@ public void setCreationWaitRetryHookForTesting(Runnable hook) { this.creationWaitRetryHook = hook; } + @TestOnly + public int startupRecoveryBuildAttemptsForTesting() { + return startupRecoveryBuildAttempts; + } + /** * Raises the shutdown signal early -- without tearing down live delegates -- * so an in-flight startup-recovery step stops promptly between slots. Direct @@ -1720,11 +1725,6 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna this.afterStartupRecoveryJoinHook = afterJoinHook; } - @TestOnly - public int startupRecoveryBuildAttemptsForTesting() { - return startupRecoveryBuildAttempts; - } - /** * Ships rows left behind in idle slots. A pooled {@code close()} does not * flush by default, and auto-flush thresholds are only evaluated on append, diff --git a/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java b/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java index fa6144970..f117c4364 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBHousekeeperCadenceTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.util.Arrays; import java.util.concurrent.TimeUnit; /** @@ -71,7 +72,7 @@ public void testFailedStartupRecoveryRetriesOnItsOwnCadenceNotTheSettleCadence() Thread.sleep(2_600); int attempts = ((QuestDBImpl) db).getSenderPoolForTesting().startupRecoveryBuildAttemptsForTesting(); Assert.assertTrue("about one attempt per second expected, got " + attempts - + "; sf_dir=" + java.util.Arrays.toString(sfDir.list()), attempts >= 1 && attempts <= 5); + + "; sf_dir=" + Arrays.toString(sfDir.list()), attempts >= 1 && attempts <= 5); } } From 287f29d6d6d0613929cfa31a804b3206e301fd31 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:29:25 +0100 Subject: [PATCH 38/57] test(pool): pin the window age restart, flushAndGetSequence's mark, the retryable wrapper and the settle Error path Each of the four tests fails when its line is mutated: markCommitted without the age reset, flushAndGetSequence without the mark, a hard-coded retryable=false on the close wrapper, and a settle Error that requeues instead of retiring. testResetAfterAFlushInsideTheLeaseDropsEveryPendingRow additionally pins the savepointRows reset in RecordingSender.flush(). Co-Authored-By: Claude Fable 5.1 --- .../client/test/impl/RecordingSender.java | 5 +++ .../impl/SenderPoolCloseSemanticsTest.java | 42 +++++++++++++++++++ .../test/impl/SenderPoolCommitWindowTest.java | 42 +++++++++++++++++++ .../test/impl/SenderPoolSettleTest.java | 15 +++++++ 4 files changed, 104 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index 8b5325674..2635d5fcf 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -50,6 +50,7 @@ class RecordingSender implements Sender { volatile RuntimeException deferredFlushError; // thrown by flushDeferred() when set volatile boolean deferredFramesOpen; volatile RuntimeException flushError; // thrown by flush() when set + volatile Error flushFatal; // thrown by flush() when set, before flushError volatile CountDownLatch flushGate; // flush() blocks on it when set volatile RuntimeException latchedError; // thrown by checkError() when set volatile int pendingRows; @@ -141,6 +142,10 @@ public boolean drain(long timeoutMillis) { @Override public void flush() { awaitGate(); + Error fatal = flushFatal; + if (fatal != null) { + throw fatal; + } RuntimeException e = flushError; if (e != null) { throw e; diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index 2d41edf98..96109c575 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -86,6 +86,29 @@ public void testCloseDoesNotFlushByDefault() { } } + @Test + public void testCloseWrapperKeepsTheDelegatesRetryableFlag() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + Sender first = pool.borrow(); + built.get(0).latchedError = new LineSenderException("transient wire", true); + try { + first.close(); + Assert.fail("close() must rethrow the latched error"); + } catch (LineSenderException e) { + Assert.assertTrue("a retryable cause makes a retryable wrapper", e.isRetryable()); + } + Sender second = pool.borrow(); + built.get(1).latchedError = new LineSenderException("permanent", false); + try { + second.close(); + Assert.fail("close() must rethrow the latched error"); + } catch (LineSenderException e) { + Assert.assertFalse("a non-retryable cause makes a non-retryable wrapper", e.isRetryable()); + } + } + } + @Test public void testFlushOnCloseRestoresShippingAtClose() { List built = new ArrayList<>(); @@ -219,6 +242,25 @@ public void testLatchedServerErrorAtCloseKeepsItsType() { } } + @Test + public void testResetAfterAFlushInsideTheLeaseDropsEveryPendingRow() { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(1, 1, built)) { + try (Sender a = pool.borrow()) { + a.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + try (Sender b = pool.borrow()) { + b.table("t").longColumn("v", 2L).atNow(); + b.flush(); + Assert.assertEquals("the flush shipped both rows", 1, delegate.flushCount.get()); + b.table("t").longColumn("v", 3L).atNow(); + b.reset(); + Assert.assertEquals("the flush emptied the savepoint, so reset drops every pending row", 0, delegate.pendingRowCount()); + } + } + } + @Test public void testResetDiscardsOnlyTheResettingLeasesRows() { List built = new ArrayList<>(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java index 428a2f89e..dba77ffce 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCommitWindowTest.java @@ -39,6 +39,24 @@ */ public class SenderPoolCommitWindowTest { + @Test + public void testAgeClockRestartsAfterACommit() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(1_000, 30L); + borrowAppendClose(pool, 1L); // deferred: the window opens + Thread.sleep(60); + borrowAppendClose(pool, 2L); // older than 30 ms: commits + RecordingSender d = built.get(0); + Assert.assertEquals(1, d.flushCount.get()); + Assert.assertEquals(1, d.deferredFlushCount.get()); + borrowAppendClose(pool, 3L); // a fresh window: this close must defer + Assert.assertEquals("a clock that was not restarted at the commit would commit every close", 1, d.flushCount.get()); + Assert.assertEquals(2, d.deferredFlushCount.get()); + } + } + @Test public void testAgeWindowClosesAtNextRelease() throws Exception { List built = new ArrayList<>(); @@ -85,6 +103,30 @@ public void testDrainResetsTheWindow() { } } + @Test + public void testLeaseFlushAndGetSequenceResetsTheWindow() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + pool.setFlushOnClose(true); + pool.setCommitWindow(3, 60_000L); + borrowAppendClose(pool, 1L); // deferred: the window is open with 1 row + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 2L).atNow(); + s.flushAndGetSequence(); // app commits at 2 rows: the window restarts here + } + RecordingSender d = built.get(0); + Assert.assertEquals(1, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + borrowAppendClose(pool, 3L); // 3 rows in total: a counter that ignored the flush would commit here + borrowAppendClose(pool, 4L); + Assert.assertEquals(3, d.deferredFlushCount.get()); + Assert.assertEquals(1, d.flushCount.get()); + borrowAppendClose(pool, 5L); // 3 rows since the flush: the window closes + Assert.assertEquals(3, d.deferredFlushCount.get()); + Assert.assertEquals(2, d.flushCount.get()); + } + } + @Test public void testLeaseFlushResetsTheWindow() { List built = new ArrayList<>(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index 79f867fd6..97fac2471 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -67,6 +67,21 @@ public void testSettleRetiresSlotHoldingAnOverCapBatch() { } } + @Test + public void testSettleRetiresSlotWhenFlushThrowsAnError() { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(1, 1, built)) { + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + delegate.flushFatal = new AssertionError("buffer invariant broken"); + Assert.assertEquals(0, pool.settleIdle()); + Assert.assertEquals("an Error is not transient: the slot is retired", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + } + @Test public void testSettleShipsRowsLeftInIdleSlot() throws Exception { List built = new ArrayList<>(); From c42d7a5753fdb5b971720135e8cbaa0fd997427b Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:33:27 +0100 Subject: [PATCH 39/57] docs(pool): state frame-level fate sharing instead of promising replay; fix the examples; license headers The README said a rejected frame makes the client replay the healthy frames. It does not: a terminal halts the slot and a retriable replays the whole frame. The pool docs now say that one frame carries several leases' rows and name flush() and flush_on_close=on as the isolation tools. The examples no longer claim close() flushes. A test pins the documented behaviour on the wire. Co-Authored-By: Claude Fable 5.1 --- README.md | 14 ++- .../main/java/io/questdb/client/QuestDB.java | 6 + .../io/questdb/client/QuestDBBuilder.java | 3 + .../client/test/QuestDBCommitWindowTest.java | 24 ++++ .../client/test/QuestDBFlushOnCloseTest.java | 111 ++++++++++++++++++ .../qwp/client/SenderProbeMethodsTest.java | 24 ++++ .../main/java/com/example/QuestDBExample.java | 7 +- .../sender/WsConcurrentIngestExample.java | 7 +- .../java/com/example/sender/WsExample.java | 14 +-- 9 files changed, 192 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d5a5e71d5..ba524eb8a 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,16 @@ pool's next settle; call `flush()` before `close()` if you need it on the wire n If you borrow per message and want each message in the store-and-forward ring the moment `close()` returns, set `flush_on_close=on` together with a commit window (`commit_rows`, `commit_interval_ms`): every close ships one frame -flagged defer-commit, the server appends it, and the pool commits once per window instead of once per close. A -rejected frame rolls the open window back on the server and the client replays the healthy frames from the ring. +flagged defer-commit, the server appends it, and the pool commits once per window instead of once per close. + +A server rejection covers a whole frame. With the default `flush_on_close=off` one frame carries the rows of every +lease that closed since the last ship, so one rejected row discards the other leases' rows in that frame as well, +after their `close()` calls returned. The default error handler logs the rejection with the frame's sequence span, and +the error is thrown by the next call on that slot, possibly to a lease that wrote none of the rejected rows. A +retriable rejection (write error, internal error) replays the frame; a terminal one (schema mismatch, parse error, +security error) retires the slot, and with `sf_dir` its unacked frames stay in the slot directory for recovery. A +lease whose rows must not share a frame calls `flush()` before `close()`, or the handle sets `flush_on_close=on`. +`reset()` on a borrowed sender discards only the rows appended through that lease. ```java try (Sender sender = db.borrowSender()) { @@ -551,7 +559,7 @@ schema::key1=value1;key2=value2; | `acquire_timeout_ms` | `5000` | How long `borrowSender()`/`borrowQuery()` waits for a free slot | | `idle_timeout_ms` | `60000` | How long a pooled connection may stay idle before it is reaped | | `max_lifetime_ms` | `1800000` | Maximum lifetime of a pooled connection before it is recycled | -| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged. Close-time errors are wrapped; the delegate's error is the cause | +| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged. Close-time errors are wrapped; the delegate's error is the cause; with `on` each lease ships its own frame, so a server rejection cannot take other leases' rows with it | | `commit_rows` | _(unset)_ | Commit window: with `flush_on_close=on`, closes ship frames flagged defer-commit and the server commits once per window of this many rows. Acks arrive at the commit; `drain()` after `flush()` | | `commit_interval_ms` | _(unset)_ | Commit window age in milliseconds; setting either window key enables the window, the other defaults to 1000 rows / 100 ms. An idle slot's open group is committed at the settle cadence, so the interval only bounds groups on busy slots | diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 9c0efada5..310fc3f22 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -144,6 +144,12 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * later). Call {@link Sender#flush()} to ship now and * {@link Sender#drain(long)} to wait for the server. Set * {@code flush_on_close=on} to have {@code close()} flush, as before 1.4. + * With the default, one frame carries the rows of every lease that closed + * since the last ship, so a server rejection of that frame discards those + * leases' rows together; the error is logged by the error handler and thrown + * by a later call on that slot. Call {@link Sender#flush()} before close, or + * set {@code flush_on_close=on}, when a lease's rows must not share a frame. + * {@link Sender#reset()} on a pooled Sender discards only that lease's rows. * Note that in this client "flush" means published to the store-and-forward * ring, not acknowledged by the server. A real disconnect only happens at * {@link #close()} on this {@code QuestDB} handle. diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index 34105d1e4..f4eaed7d2 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -216,6 +216,9 @@ public QuestDBBuilder errorHandler(SenderErrorHandler handler) { * which is the slow path for applications that borrow per message. In this * client "flush" means published to the store-and-forward ring, not * acknowledged by the server; use {@link Sender#drain(long)} for that. + * With {@code on} every lease ships its own frame, so a server rejection + * cannot discard other leases' rows; with {@code off} leases share frames + * and a rejection covers all of them. * Overrides {@code flush_on_close} in the config string. */ public QuestDBBuilder flushOnClose(boolean flushOnClose) { diff --git a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java index c81366f42..ffcab234a 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBCommitWindowTest.java @@ -1,3 +1,27 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + package io.questdb.client.test; import io.questdb.client.LineSenderServerException; diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 86d71feb2..582b52c34 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -25,9 +25,11 @@ package io.questdb.client.test; import io.questdb.client.QuestDB; +import io.questdb.client.LineSenderServerException; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.impl.PooledSender; import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; @@ -241,6 +243,53 @@ public void testSettleShipsWithinAutoFlushInterval() throws Exception { } } + @Test + public void testTerminalRejectionDropsEveryLeaseRowInTheFrame() throws Exception { + RejectFirstFrameHandler handler = new RejectFirstFrameHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;") + + "reconnect_max_duration_millis=500;reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;")) { + try (Sender a = db.borrowSender()) { + a.table("t1").longColumn("v", 1L).atNow(); // a clean row + } + try (Sender b = db.borrowSender()) { + b.table("t2").longColumn("v", 2L).atNow(); // the row the server will reject + } + LineSenderServerException seen = null; + long deadline = System.currentTimeMillis() + 5_000; + for (int i = 0; seen == null && System.currentTimeMillis() < deadline; i++) { + try (Sender c = db.borrowSender()) { + if (i == 0) { + c.flush(); // ships A's and B's rows in ONE frame + } else { + c.table("t1").longColumn("v", 3L).atNow(); + } + } catch (LineSenderServerException e) { + seen = e; + } catch (LineSenderException e) { + if (!(e.getCause() instanceof LineSenderServerException)) { + throw e; + } + seen = (LineSenderServerException) e.getCause(); + } + Thread.sleep(20); + } + Assert.assertNotNull("the terminal surfaces on a later lease, not on A or B", seen); + Assert.assertEquals("both leases' rows shared the rejected frame", 1, handler.firstConnectionDataFrames()); + Assert.assertEquals(2, handler.firstConnectionTableCount()); + try (Sender d = db.borrowSender()) { + d.table("t1").longColumn("v", 4L).atNow(); + d.flush(); + } + handler.awaitLaterConnectionFrames(1, 5_000); + Assert.assertEquals("nothing is replayed after a terminal: only D's frame arrives on the fresh slot", + 1, handler.laterConnectionDataFrames()); + } + } + } + @Test public void testThresholdOnLaterLeaseShipsEarlierRows() throws Exception { FrameLog log = new FrameLog(); @@ -415,4 +464,66 @@ public synchronized int tableCount(int dataFrameIndex) { throw new AssertionError("no data frame at index " + dataFrameIndex); } } + + /** + * First connection: NACKs its first data frame with SCHEMA_MISMATCH (a + * terminal) and acks nothing else. Later connections: ack everything and + * count their data frames, so a replay would show up there. + */ + private static final class RejectFirstFrameHandler implements TestWebSocketServer.WebSocketServerHandler { + private static final int OFFSET_TABLE_COUNT = 6; + private int connections; + private TestWebSocketServer.ClientHandler currentClient; + private int firstConnectionDataFrames; + private int firstConnectionTableCount; + private int laterConnectionDataFrames; + private long nextSeq; + + void awaitLaterConnectionFrames(int atLeast, long timeoutMillis) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (laterConnectionDataFrames() < atLeast && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + } + + synchronized int firstConnectionDataFrames() { + return firstConnectionDataFrames; + } + + synchronized int firstConnectionTableCount() { + return firstConnectionTableCount; + } + + synchronized int laterConnectionDataFrames() { + return laterConnectionDataFrames; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + connections++; + nextSeq = 0; + } + long seq = nextSeq++; + int tableCount = data.length >= 8 ? ((data[OFFSET_TABLE_COUNT] & 0xFF) | ((data[OFFSET_TABLE_COUNT + 1] & 0xFF) << 8)) : 0; + try { + if (connections == 1) { + if (tableCount > 0) { + firstConnectionDataFrames++; + firstConnectionTableCount = tableCount; + client.sendBinary(QwpWireTestUtils.buildNack(seq, WebSocketResponse.STATUS_SCHEMA_MISMATCH)); + } + // rowless frames on the first connection: no ack, no nack + } else { + if (tableCount > 0) { + laterConnectionDataFrames++; + } + client.sendBinary(QwpWireTestUtils.buildAck(seq)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 13d2f9023..1df71d7d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -1,3 +1,27 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; diff --git a/examples/src/main/java/com/example/QuestDBExample.java b/examples/src/main/java/com/example/QuestDBExample.java index cc39e3e23..0f95f423e 100644 --- a/examples/src/main/java/com/example/QuestDBExample.java +++ b/examples/src/main/java/com/example/QuestDBExample.java @@ -33,9 +33,10 @@ public static void main(String[] args) throws InterruptedException { try (QuestDB db = QuestDB.connect("ws::addr=localhost:9000;"); Query q = db.borrowQuery()) { - // Ingest: borrow a Sender, write rows, close() flushes it and - // returns it to the pool. The WebSocket is NOT torn down here -- - // a real disconnect only happens at db.close(). + // Ingest: borrow a Sender, write rows, close() returns it to the pool. + // Close releases, flush ships, drain waits: the rows go out on the next + // auto-flush threshold or the pool's settle. The WebSocket is NOT torn + // down here -- a real disconnect only happens at db.close(). try (Sender sender = db.borrowSender()) { sender.table("trades") .symbol("symbol", "ETH-USD") diff --git a/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java b/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java index 2abcb6c87..e19f83b3b 100644 --- a/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java +++ b/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java @@ -13,8 +13,7 @@ * thread its own. *

* Each producer thread takes its own sender with {@link QuestDB#borrowSender()} - * and reuses it for the batch (see {@link WsExample}); close() flushes and - * returns it to the pool. We size the sender pool to the producer count so no + * and reuses it for the batch (see {@link WsExample}); flush() ships the batch and close() returns the sender to the pool. We size the sender pool to the producer count so no * thread ever waits on the acquire timeout. */ public class WsConcurrentIngestExample { @@ -45,8 +44,8 @@ public static void main(String[] args) throws InterruptedException { } private static void ingestBatch(QuestDB db, int producerId) { - // Each thread borrows its own sender and reuses it for the batch; - // close() (try-with-resources) flushes and returns it to the pool. + // Each thread borrows its own sender and reuses it for the batch; flush() + // ships it and close() (try-with-resources) returns it to the pool. try (Sender sender = db.borrowSender()) { String symbol = "SYM-" + producerId; for (int i = 0; i < ROWS_PER_PRODUCER; i++) { diff --git a/examples/src/main/java/com/example/sender/WsExample.java b/examples/src/main/java/com/example/sender/WsExample.java index 9ef0604dc..2ca6172e4 100644 --- a/examples/src/main/java/com/example/sender/WsExample.java +++ b/examples/src/main/java/com/example/sender/WsExample.java @@ -9,8 +9,7 @@ * The pooled {@link Sender} is the recommended way to ingest: one * {@code QuestDB} handle, created once and shared across threads, owns a pool * of senders that stay connected between uses. Take one with - * {@link QuestDB#borrowSender()} -- a lease per use: {@code close()} flushes - * pending rows and returns the sender to the pool (it does NOT disconnect). + * {@link QuestDB#borrowSender()} -- a lease per use: {@code close()} returns the sender to the pool (it does NOT disconnect or flush; rows ship on the next auto-flush threshold or the pool's settle, or on an explicit {@code flush()}). * Borrow one per producer thread, or once per batch, and close it when done. *

* A {@code Sender} is not thread-safe, so no two threads may share one -- the @@ -23,8 +22,8 @@ public class WsExample { public static void main(String[] args) { try (QuestDB db = QuestDB.connect("ws::addr=localhost:9000;")) { - // Borrowed: lease per use. try-with-resources close() flushes and - // returns the sender to the pool for reuse. + // Borrowed: lease per use. try-with-resources close() returns the + // sender to the pool for reuse; flush() below ships the batch. try (Sender sender = db.borrowSender()) { sender.table("trades") .symbol("symbol", "ETH-USD") @@ -46,10 +45,9 @@ public static void main(String[] args) { .doubleColumn("amount", 0.001) .atNow(); } - // Hand the buffered rows to the send engine. Auto-flush would - // also do this on the row/time thresholds; an explicit flush() - // makes the batch boundary deterministic. close() flushes too, - // so this call is optional here. + // Hand the buffered rows to the send engine. Auto-flush would also do + // this on the row/byte/time thresholds, and the pool settles an idle + // slot; an explicit flush() makes the batch boundary deterministic. sender.flush(); } From f586ac5a72bc134e702ea1cd039c7c004d35e4a3 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:36:51 +0100 Subject: [PATCH 40/57] docs(pool): remove the last example comment that says close() flushes; wrap two long lines WsExample still said that close() flushes pending rows, which is false with the default flush_on_close=off and contradicted the class Javadoc above it. The README's pooled-ingest example made the same claim in a comment. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- .../com/example/sender/WsConcurrentIngestExample.java | 5 +++-- examples/src/main/java/com/example/sender/WsExample.java | 9 ++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ba524eb8a..5792ceeaa 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ try (Sender sender = db.borrowSender()) { sender.flush(); } } - // Remaining rows are flushed by close(). + // Remaining rows ship on the pool's next settle; flush() ships them now. } ``` diff --git a/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java b/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java index e19f83b3b..baaa3ff9c 100644 --- a/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java +++ b/examples/src/main/java/com/example/sender/WsConcurrentIngestExample.java @@ -13,8 +13,9 @@ * thread its own. *

* Each producer thread takes its own sender with {@link QuestDB#borrowSender()} - * and reuses it for the batch (see {@link WsExample}); flush() ships the batch and close() returns the sender to the pool. We size the sender pool to the producer count so no - * thread ever waits on the acquire timeout. + * and reuses it for the batch (see {@link WsExample}); flush() ships the + * batch and close() returns the sender to the pool. We size the sender pool to + * the producer count so no thread ever waits on the acquire timeout. */ public class WsConcurrentIngestExample { diff --git a/examples/src/main/java/com/example/sender/WsExample.java b/examples/src/main/java/com/example/sender/WsExample.java index 2ca6172e4..60bb745d5 100644 --- a/examples/src/main/java/com/example/sender/WsExample.java +++ b/examples/src/main/java/com/example/sender/WsExample.java @@ -9,7 +9,10 @@ * The pooled {@link Sender} is the recommended way to ingest: one * {@code QuestDB} handle, created once and shared across threads, owns a pool * of senders that stay connected between uses. Take one with - * {@link QuestDB#borrowSender()} -- a lease per use: {@code close()} returns the sender to the pool (it does NOT disconnect or flush; rows ship on the next auto-flush threshold or the pool's settle, or on an explicit {@code flush()}). + * {@link QuestDB#borrowSender()} -- a lease per use: {@code close()} returns + * the sender to the pool (it does NOT disconnect or flush; rows ship on the + * next auto-flush threshold or the pool's settle, or on an explicit + * {@code flush()}). * Borrow one per producer thread, or once per batch, and close it when done. *

* A {@code Sender} is not thread-safe, so no two threads may share one -- the @@ -34,8 +37,8 @@ public static void main(String[] args) { } // Borrow one sender and reuse it for the whole batch. close() - // (try-with-resources) flushes pending rows and returns the sender - // to the pool -- the WebSocket is NOT torn down here. + // (try-with-resources) returns the sender to the pool -- the + // WebSocket is NOT torn down here; flush() below ships the batch. try (Sender sender = db.borrowSender()) { for (int i = 0; i < 1_000; i++) { sender.table("trades") From 219144ba3e862016d71800d70bc07db53bcb765f Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:05:33 +0100 Subject: [PATCH 41/57] fix(pool): a lease whose row crosses the cap margin no longer takes earlier leases' rows down with it The byte trigger fires after the crossing row is committed, so a row larger than the 10 percent margin makes the auto-flush reject a batch that is mostly other leases' rows. The release path used to retire the slot on the retained rejection and drop them all. It now rolls the lease back to its borrow mark, retries the flush, keeps the slot when the inherited rows ship, and retires only a batch the cap still rejects. Co-Authored-By: Claude Opus 5.5 --- .../io/questdb/client/impl/PooledSender.java | 44 ++++++++++++++++- .../client/test/QuestDBFlushOnCloseTest.java | 48 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 07631a546..74ccf7706 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -162,6 +163,16 @@ public void checkError() { * error keeps it as the primary exception, with close's wrapper * suppressed. The typed error and its classification live on the cause. *

+ * A server batch cap rejection ({@link BatchTooLargeForCapException}, retained + * by an auto-flush or thrown by close's own flush) does not retire the slot at + * once. The byte trigger fires after the crossing row is committed, so the + * rejected batch can hold mostly rows earlier leases left. The lease is rolled + * back to its borrow mark, which drops only this lease's rows, and the flush is + * retried: when the rows left by earlier leases ship, the slot is returned and a + * {@link LineSenderException} whose cause is the rejection names the number of + * this lease's rows dropped. Only an inherited batch the cap still rejects + * retires the slot, with the wrapper described above. + *

* Idempotent for sequential reuse: a stale generation (the lease was * already returned and the slot possibly re-borrowed) is a no-op. Concurrent * double-close of ONE lease from two threads is outside the Sender @@ -189,6 +200,8 @@ public void close() { boolean cancelledRow = false; boolean recycle = false; RuntimeException failure = null; + BatchTooLargeForCapException capRejection = null; + int droppedLeaseRows = 0; int pendingRowsAtRetire = 0; try { // live(), not delegate(): re-validate the generation right before @@ -213,6 +226,25 @@ public void close() { // slot is retired. failure = e; } + if (failure instanceof BatchTooLargeForCapException) { + // The byte trigger fires after the crossing row is committed, so + // the rejected batch can hold mostly rows earlier leases left. + // Drop only this lease's rows and retry: the slot stays in service + // when the rest ships, and an inherited batch the cap still + // rejects falls through to the retirement below. + try { + int pendingBeforeRollback = delegate.pendingRowCount(); + delegate.resetToSavepoint(); + droppedLeaseRows = pendingBeforeRollback - delegate.pendingRowCount(); + delegate.flush(); + slot.markCommitted(delegate.rowsAppended()); + capRejection = (BatchTooLargeForCapException) failure; + failure = null; + recycle = true; + } catch (RuntimeException e) { + failure = e; + } + } } finally { if (recycle) { pool.giveBack(this); @@ -220,9 +252,10 @@ public void close() { // The probe, the cancel or the ship threw: the delegate has // latched an error, the flush failed, or the generation went // stale. Retire it; a wrapper around the original propagates - // once this finally returns. + // once this finally returns. Rows a rollback above dropped are + // lost with the slot as well. try { - pendingRowsAtRetire = slot.delegate().pendingRowCount(); + pendingRowsAtRetire = droppedLeaseRows + slot.delegate().pendingRowCount(); } catch (Throwable ignored) { // best effort: the count is only for the message } @@ -242,6 +275,13 @@ public void close() { wrapper.initCause(failure); throw wrapper; } + if (capRejection != null) { + LineSenderException wrapper = new LineSenderException("pooled sender close dropped " + droppedLeaseRows + + " row(s) of this lease after the server batch cap rejected the batch; rows left by earlier leases shipped", + capRejection.isRetryable()); + wrapper.initCause(capRejection); + throw wrapper; + } if (cancelledRow) { throw new LineSenderException("row in progress at close(): the row was cancelled and the sender returned to the pool; " + "finish rows with at(), atNow() or cancelRow() before close()"); diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 582b52c34..9fccb0f4e 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -68,6 +68,54 @@ public void testCloseShipsNothingUntilSettleOrFlush() throws Exception { } } + @Test + public void testCrossingRowOverTheMarginDropsOnlyThatLeasesRowsAndShipsTheRest() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes unset: the byte trigger sits at 90 percent of the 4 KiB cap (3686 bytes) + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); + String wide = wideValue(1000); + for (int i = 0; i < 3; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").stringColumn("s", wide).atNow(); // 3 KB in all, below the trigger + } + } + Sender delegate = null; + boolean hasCloseThrown = false; + try (Sender s = db.borrowSender()) { + delegate = ((PooledSender) s).getDelegateForTesting(); + try { + s.table("t").stringColumn("s", wideValue(1500)).atNow(); + Assert.fail("the crossing row makes the auto-flushed batch 4.5 KB, over the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // the row stays committed and the rejection is retained + } + } catch (LineSenderException e) { + hasCloseThrown = true; + Assert.assertTrue("close() wraps the rejection", e.getCause() instanceof BatchTooLargeForCapException); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("dropped 1 row(s) of this lease")); + } + Assert.assertTrue("close() must report the dropped row", hasCloseThrown); + log.awaitDataFrames(2, 5_000); + Assert.assertEquals("the three earlier rows ship as one frame with table t", 1, log.tableCount(1)); + try (Sender s = db.borrowSender()) { + Assert.assertSame("the slot stayed in service", delegate, ((PooledSender) s).getDelegateForTesting()); + Assert.assertEquals("only the crossing row was dropped, the rest shipped", 0, s.pendingRowCount()); + } + } + } + } + @Test public void testFlushOnCloseShipsOneFramePerClose() throws Exception { FrameLog log = new FrameLog(); From 0b23f312202abee686a099501b66e44022a58193 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:08:53 +0100 Subject: [PATCH 42/57] fix(pool): pool messages drop the standalone remedy, client-side drops log at ERROR, savepoint Javadoc for pooled senders BatchTooLargeForCapException now separates its detail from the standalone remedy ("call reset() and keep this sender"), and the pool's close wrapper and settle log use the detail only. Slot retirements that drop rows log at ERROR, the level the default error handler uses for data loss. The Sender Javadoc states that a pooled sender ignores markSavepoint() and that resetToSavepoint() on it is the lease-scoped reset(). The retained rejection is cleared once a later flush attempt passes the pre-flight, the housekeeper reuses the pool's one-second recovery constant, and the cap-bound default documents the pre-handshake window. Co-Authored-By: Claude Opus 5.5 --- .../main/java/io/questdb/client/Sender.java | 29 ++++++++++++------- .../client/BatchTooLargeForCapException.java | 29 +++++++++++++++++++ .../qwp/client/QwpWebSocketSender.java | 25 ++++++++++++---- .../questdb/client/impl/PoolHousekeeper.java | 2 +- .../io/questdb/client/impl/PooledSender.java | 14 +++++---- .../io/questdb/client/impl/SenderPool.java | 15 +++++----- .../qwp/client/SenderProbeMethodsTest.java | 13 ++++++--- 7 files changed, 93 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 8b2ecba58..7ebd1704e 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -762,11 +762,15 @@ default Sender long256Column(CharSequence name, long l0, long l1, long l2, long /** * Marks the current buffer state as the point {@link #resetToSavepoint()} * rolls back to: rows appended after this call are discarded by that - * method, rows appended before it are kept. Any flush that ships the - * pending rows empties the savepoint, so a later {@code resetToSavepoint()} - * discards every pending row like {@link #reset()}. The pool marks a savepoint when it hands a - * slot to a borrower. Pure bookkeeping: it runs no error check. Default - * no-op. + * method, rows appended before it are kept. Take it between rows, not + * while a row is in progress. Any flush that ships the pending rows + * empties the savepoint, so a later {@code resetToSavepoint()} discards + * every pending row like {@link #reset()}. Pure bookkeeping: it runs no + * error check. Default no-op. + *

+ * A pooled sender ignores this call: the pool marks a savepoint when it + * hands a slot to a borrower, and {@code resetToSavepoint()} on the pooled + * sender rolls back to that mark. */ default void markSavepoint() { } @@ -830,8 +834,8 @@ default void resetSymbolDictionary() { * data format) and want to continue sending new data without retrying the problematic data. * After calling this method, you can start building a new row by calling {@link #table(CharSequence)}. *
- * Supported by the HTTP and WebSocket transports; TCP does not support it. On a pooled - * sender this discards only the rows appended through the current lease, see + * Supported by the HTTP, WebSocket and UDP transports; TCP does not support it. On a + * pooled sender this discards only the rows appended through the current lease, see * {@link #resetToSavepoint()}. * * @see #flush() @@ -842,10 +846,13 @@ default void resetSymbolDictionary() { * Discards the rows appended since the last {@link #markSavepoint()} and * keeps the rows buffered before it. Without a savepoint, or after a flush * emptied it, this discards every pending row like {@link #reset()}, and - * also removes the column definitions the server has not seen yet. A pooled - * sender's {@code reset()} calls this, so a borrower can drop its own bad batch without discarding - * rows earlier borrowers left in the shared slot buffer. Default calls - * {@code reset()}. + * also removes the column definitions the server has not seen yet. Default + * calls {@code reset()}. + *

+ * On a pooled sender this is the lease-scoped {@code reset()}: it rolls + * back to the savepoint the pool marked at borrow, so a borrower can drop + * its own bad batch without discarding rows earlier borrowers left in the + * shared slot buffer. */ default void resetToSavepoint() { reset(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java index b9ef97e8e..cd363c3ce 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java @@ -38,7 +38,36 @@ */ public class BatchTooLargeForCapException extends LineSenderException { + // Length of getMessage() when endDetail() ran: the detail ends there and the + // standalone remedy follows. -1 until then, when the whole message is the detail. + private int detailLength = -1; + public BatchTooLargeForCapException(CharSequence message) { super(message); } + + /** + * Marks the end of the detail: the text appended so far says what was rejected + * and the sizes involved, and the text appended after this call is the remedy for + * a standalone sender, which {@link #getDetail()} leaves out. + * + * @return this exception, to append the remedy + */ + public BatchTooLargeForCapException endDetail() { + detailLength = getMessage().length(); + return this; + } + + /** + * The message without the standalone remedy ("call reset() and keep this sender"), + * for a caller that handles the rejection itself, such as the connection pool, + * which retires the slot. Equals {@link #getMessage()} when the message carries no + * such remedy. + * + * @return what was rejected and the sizes involved + */ + public String getDetail() { + String message = getMessage(); + return detailLength < 0 || detailLength > message.length() ? message : message.substring(0, detailLength); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c6d965ff9..2b6c394fb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -135,7 +135,9 @@ public class QwpWebSocketSender implements Sender { * 90 percent of whatever batch cap the server advertises, and at no byte * threshold while it advertises none. The config-string default for the * WebSocket transport. An explicit {@code auto_flush_bytes=off} is {@code 0} - * and keeps byte flushing off whatever the server advertises. + * and keeps byte flushing off whatever the server advertises. Rows appended + * before the first handshake carry no byte trigger; the first append after it + * flushes the backlog. */ public static final int AUTO_FLUSH_BYTES_SERVER_CAP = -1; public static final long DEFAULT_AUTH_TIMEOUT_MS = 15_000L; @@ -462,8 +464,10 @@ public class QwpWebSocketSender implements Sender { // Optional owning-pool notification, relayed after the engine confirms the // flock release and slotLockReleased is visible to pool re-probes. private volatile Runnable slotLockReleaseListener; - // Pre-flight batch-cap rejection of the last flush attempt, kept until reset(), - // resetToSavepoint() or a successful flush clears it. Not a terminal: the + // Batch-cap rejection from a flush attempt's pre-flight. Cleared by reset(), + // resetToSavepoint() and by the next flush attempt whose batch passes the + // pre-flight, even when that attempt then fails for another reason, so an old + // rejection never outlives a batch the cap accepts. Not a terminal: the // standalone contract lets the caller reset() and carry on. checkError() // reports it so a pool can retire a slot whose batch no flush will ship. private BatchTooLargeForCapException retainedBatchRejection; @@ -5272,6 +5276,8 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { flushPendingRowsSplit(deferCommit, combinedBodyStart, cap, deltaBaseline); return; } + // The batch fits the cap: an earlier rejection no longer describes it. + retainedBatchRejection = null; // Write-ahead: durably persist this frame's new symbols BEFORE it is // published, so a recovered/orphan-drained slot can always rebuild the @@ -5444,14 +5450,18 @@ private void flushPendingRowsSplit( + "symbol for this data" : "call reset() to discard the retained batch and keep this sender, close the " + "sender to discard everything, or produce smaller batches"; - throw new BatchTooLargeForCapException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) + BatchTooLargeForCapException rejection = new BatchTooLargeForCapException( + "single table batch too large for server batch cap"); + rejection.put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) .put(", dictionaryFrameBytes=").put(frameOverhead) - .put(", serverMaxBatchSize=").put(cap).put(']') + .put(", serverMaxBatchSize=").put(cap).put(']'); + // The pool recovers on its own and reports only the detail. + rejection.endDetail() .put("; the batch is retained for retry and every flush() will " + "reject it again until a larger-cap node is reached -- ") .put(remedy); + throw rejection; } // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's // new ids, the remaining frames carry an empty delta above the baseline. @@ -5459,6 +5469,9 @@ private void flushPendingRowsSplit( simBaseline = currentBatchMaxSymbolId; } } + // Every split frame fits the cap: an earlier rejection no longer describes + // the batch. + retainedBatchRejection = null; int tableBodyOffset = combinedBodyStart; // Mirrors simBaseline above and advanceSentMaxSymbolId below, so the frame diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 08f44d08e..7ad2dc284 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -37,7 +37,7 @@ final class PoolHousekeeper { // once: a finished scan, an empty one, or a transient failure such as an // unreachable server. The same cadence SenderPool's private driver uses, and // deliberately not the settle cadence, which can be 50 ms. - static final long STARTUP_RECOVERY_RETRY_MILLIS = 1_000L; + static final long STARTUP_RECOVERY_RETRY_MILLIS = SenderPool.RECOVERY_RETRY_INTERVAL_MILLIS; // How long stop() waits for the daemon to exit. Kept ABOVE // SenderPool.RECOVERY_DRAIN_BUDGET_MILLIS so a startup-recovery drain still // in flight when close() arrives finishes well within this join (C1 fix). diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 74ccf7706..36950948d 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -268,7 +268,11 @@ public void close() { typed.initCause(failure); throw typed; } - String msg = failure.getMessage(); + // A cap rejection's message ends with the standalone remedy, which does + // not apply to a retired slot: report only its detail. + String msg = failure instanceof BatchTooLargeForCapException + ? ((BatchTooLargeForCapException) failure).getDetail() + : failure.getMessage(); boolean retryable = failure instanceof LineSenderException && ((LineSenderException) failure).isRetryable(); LineSenderException wrapper = new LineSenderException("pooled sender close failed, slot retired with " + pendingRowsAtRetire + " unshipped row(s): " + msg, retryable); @@ -465,10 +469,10 @@ public int pendingRowCount() { /** * Discards the rows appended through this lease and keeps the rows earlier * leases left in the slot's buffer: the pool marks a savepoint at borrow and - * this rolls back to it ({@link Sender#resetToSavepoint()}). After a - * {@code flush()} inside the lease every pending row is this lease's own and - * all of them go. Use it to drop a batch the server cannot accept, for - * example after a {@code BatchTooLargeForCapException}. + * this rolls back to it ({@link Sender#resetToSavepoint()}). After a flush + * inside the lease, explicit or by auto-flush, every pending row is this + * lease's own and all of them go. Use it to drop a batch the server cannot + * accept, for example after a {@code BatchTooLargeForCapException}. */ @Override public void reset() { diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 12932a07b..d2c2f6993 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -130,10 +130,6 @@ public final class SenderPool implements AutoCloseable { // the dominant server-wide transient failure keeps its retry-the-same- // candidate behavior for the first attempts. private static final int RECOVERY_MAX_SLOT_FAILURE_STREAK = 3; - // A direct SenderPool has no PoolHousekeeper to retry a transient startup - // recovery failure. Its private driver waits this long between failed - // attempts so an unavailable server does not cause a hot retry loop. - private static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000; // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, // TLS, or WebSocket setup. The creator retains ownership after this budget: // once construction returns, borrow() observes closed and tears down the @@ -146,6 +142,11 @@ public final class SenderPool implements AutoCloseable { // shutdown join is capped at QueryWorker.SHUTDOWN_JOIN_MILLIS (same value) // regardless of user config. static final long MAX_CLOSE_LEASE_WAIT_MILLIS = 5_000; + // A direct SenderPool has no PoolHousekeeper to retry a transient startup + // recovery failure. Its private driver waits this long between failed + // attempts so an unavailable server does not cause a hot retry loop. The + // housekeeper waits the same between its own startup-recovery steps. + static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000; private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; @@ -1803,8 +1804,8 @@ public int settleIdle() { LOG.warn("settleIdle: flush threw an Error, retiring slot {}", lease.slot().slotIndex(), fatal); retireLease(lease, " during settle"); } else if (overCap != null) { - LOG.warn("settleIdle: slot {} holds a batch the server cap rejects, retiring it and dropping {} pending row(s): {}", - lease.slot().slotIndex(), pendingRowsOf(lease.slot()), overCap.toString()); + LOG.error("settleIdle: slot {} holds a batch the server cap rejects, retiring it and dropping {} pending row(s): {}", + lease.slot().slotIndex(), pendingRowsOf(lease.slot()), overCap.getDetail()); retireLease(lease, " during settle"); } else { requeueSettled(lease); @@ -2024,7 +2025,7 @@ private void retireLease(PooledSender ps, String context) { } catch (Throwable t) { if (pendingAtRetire > 0) { String error = t.toString(); - LOG.warn("SenderPool: slot {} retired with {} unshipped row(s); delegate close failed: {}", + LOG.error("SenderPool: slot {} retired with {} unshipped row(s); delegate close failed: {}", s.slotIndex(), pendingAtRetire, error); } // Best-effort teardown: a delegate close() can throw an Error diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 1df71d7d9..00d62bceb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -37,10 +37,12 @@ import java.util.concurrent.TimeUnit; /** - * The three probe methods the pool uses at release: {@code checkError()}, - * {@code hasRowInProgress()} and {@code pendingRowCount()}. Exercised on a - * real WebSocket sender against the in-repo test server, which acks every - * frame. + * The sender methods the pool relies on, exercised on a real WebSocket sender + * against the in-repo test server, which acks every frame: the probes it runs + * at release ({@code checkError()}, {@code hasRowInProgress()} and + * {@code pendingRowCount()}), the savepoint pair ({@code markSavepoint()} and + * {@code resetToSavepoint()}) behind a lease's {@code reset()}, the deferred + * flush of the commit window and the cap-bound byte trigger. */ public class SenderProbeMethodsTest { @@ -62,6 +64,9 @@ public void testCheckErrorReportsARetainedOverCapBatchUntilReset() throws Except Assert.fail("six 1 KB rows do not fit a 4 KiB cap"); } catch (BatchTooLargeForCapException expected) { // retained for retry, as the message says + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains("call reset()")); + Assert.assertFalse("the detail leaves out the standalone remedy", expected.getDetail().contains("reset()")); + Assert.assertTrue(expected.getDetail(), expected.getDetail().endsWith("serverMaxBatchSize=4096]")); } try { sender.checkError(); From 89ee6d91dd687802c98851d29d71eff223dcaf85 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:16:49 +0100 Subject: [PATCH 43/57] fix(pool): mark the borrow savepoint after releasing the pool lock The mark walks every table the delegate has buffered; under the lock it serialized all borrowers (review round 2, critical 1). Co-Authored-By: Claude Opus 5.5 --- .../io/questdb/client/impl/SenderPool.java | 16 +++++--- .../client/test/impl/RecordingSender.java | 11 ++--- .../impl/SenderPoolCloseSemanticsTest.java | 41 +++++++++++++++++++ 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index d2c2f6993..159160778 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1252,11 +1252,14 @@ public PooledSender borrow() { // wrapper handed out can be told apart from any prior, // now-stale borrow of the same slot. s.bumpGeneration(); + PooledSender lease = new PooledSender(s, s.generation()); + lock.unlock(); // Lease-scoped reset: the borrower's reset() rolls back to here, so rows - // earlier leases left in this buffer survive it. Pure bookkeeping on the - // delegate, no error check, no throw path while the lock is held. + // earlier leases left in this buffer survive it. The slot is off the free + // list under a fresh generation, so no other thread touches the delegate, + // and the mark's walk over every buffered table no longer serializes borrowers. s.delegate().markSavepoint(); - return new PooledSender(s, s.generation()); + return lease; } if (all.size() + inFlightCreations + closingSlots + leakedSlots + recoveringSlots < maxSize) { inFlightCreations++; @@ -1326,9 +1329,12 @@ public PooledSender borrow() { created.bumpGeneration(); inFlightCreations--; creationFinished.signalAll(); - // Every lease handed to a caller starts at a mark. + PooledSender lease = new PooledSender(created, created.generation()); + lock.unlock(); + // Every lease handed to a caller starts at a mark. The new slot is not + // yet visible to other threads, so the mark runs outside the pool lock. created.delegate().markSavepoint(); - return new PooledSender(created, created.generation()); + return lease; } // Capacity-starved: re-probe retired slots BEFORE the terminal // timeout check — a deferred engine cleanup may have released a diff --git a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java index 2635d5fcf..ef71caef5 100644 --- a/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java +++ b/core/src/test/java/io/questdb/client/test/impl/RecordingSender.java @@ -53,6 +53,7 @@ class RecordingSender implements Sender { volatile Error flushFatal; // thrown by flush() when set, before flushError volatile CountDownLatch flushGate; // flush() blocks on it when set volatile RuntimeException latchedError; // thrown by checkError() when set + volatile CountDownLatch markSavepointGate; // markSavepoint() blocks on it when set volatile int pendingRows; volatile boolean rowInProgress; volatile long rowsAppended; @@ -141,7 +142,7 @@ public boolean drain(long timeoutMillis) { @Override public void flush() { - awaitGate(); + awaitGate(flushGate, "flushGate"); Error fatal = flushFatal; if (fatal != null) { throw fatal; @@ -162,7 +163,7 @@ public void flush() { @Override public long flushDeferred() { - awaitGate(); + awaitGate(flushGate, "flushGate"); RuntimeException deferredError = deferredFlushError; if (deferredError != null) { throw deferredError; @@ -214,6 +215,7 @@ public Sender longColumn(CharSequence name, long value) { @Override public void markSavepoint() { + awaitGate(markSavepointGate, "markSavepointGate"); savepointRows = pendingRows; } @@ -266,12 +268,11 @@ public Sender timestampColumn(CharSequence name, Instant value) { return this; } - private void awaitGate() { - CountDownLatch gate = flushGate; + private static void awaitGate(CountDownLatch gate, String name) { if (gate != null) { try { if (!gate.await(10, TimeUnit.SECONDS)) { - throw new AssertionError("flushGate was never released"); + throw new AssertionError(name + " was never released"); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index 96109c575..dde00b3ae 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -37,6 +37,8 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; /** * The pooled close() contract after the release-mode change: close releases, @@ -305,6 +307,38 @@ public void testStaleCloseDoesNotTouchReborrowedSlot() { } } + @Test + public void testMarkSavepointRunsOutsideThePoolLock() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = newPool(2, 2, built)) { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + a.close(); + b.close(); + CountDownLatch gate = new CountDownLatch(1); + for (RecordingSender rs : built) { + rs.markSavepointGate = gate; + } + AtomicReference first = new AtomicReference<>(); + AtomicReference second = new AtomicReference<>(); + Thread t1 = new Thread(() -> first.set(pool.borrow()), "first"); + Thread t2 = new Thread(() -> second.set(pool.borrow()), "second"); + try { + t1.start(); + awaitAvailable(pool, 1); + t2.start(); + awaitAvailable(pool, 0); + Assert.assertEquals("a second borrower takes the other slot while the first is still marking", 0, pool.availableSize()); + } finally { + gate.countDown(); + t1.join(2_000); + t2.join(2_000); + } + first.get().close(); + second.get().close(); + } + } + @Test public void testStaleResetDoesNotTouchReborrowedSlot() { List built = new ArrayList<>(); @@ -341,4 +375,11 @@ static Object slotOf(Sender pooledWrapper) { throw new AssertionError(e); } } + + private static void awaitAvailable(SenderPool pool, int target) throws InterruptedException { + long deadline = System.nanoTime() + 2_000_000_000L; + while (pool.availableSize() != target && System.nanoTime() < deadline) { + Thread.sleep(5); + } + } } From 24be1a0499120a0e8ecb87a8501cccfb76e6ae18 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:19:48 +0100 Subject: [PATCH 44/57] fix(pool): a release ships rows older than auto_flush_interval Settle cannot reach a leased slot, so a lease that held the slot without appending kept earlier leases' rows indefinitely (review round 2, critical 2). The release now runs the age check every append runs; the documented bound names the holding lease. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++-- .../main/java/io/questdb/client/QuestDB.java | 8 ++-- .../io/questdb/client/QuestDBBuilder.java | 3 +- .../main/java/io/questdb/client/Sender.java | 9 ++++ .../qwp/client/QwpWebSocketSender.java | 11 +++++ .../io/questdb/client/impl/PooledSender.java | 10 +++- .../client/test/QuestDBFlushOnCloseTest.java | 48 +++++++++++++++++++ .../qwp/client/SenderProbeMethodsTest.java | 20 ++++++++ 8 files changed, 109 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5792ceeaa..b571c7d0f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ try (QuestDB db = QuestDB.connect("ws::addr=localhost:9000;")) { .doubleColumn("price", 2615.54) .doubleColumn("amount", 0.00044) .atNow(); - // close() returns the Sender to the pool; rows ship by threshold or within auto_flush_interval. + // close() returns the Sender to the pool; rows ship by threshold, at a later close once older than auto_flush_interval, or when the idle slot is settled. } } ``` @@ -99,8 +99,10 @@ snippets below borrow from its pools. ### Ingest Data Borrow a `Sender`, write rows, and close it. Close releases, flush ships, drain waits: `close()` returns the sender to -the pool and ships nothing by itself; rows go out when an auto-flush threshold trips on a later append or when the -pool's housekeeper settles the idle slot, typically within one `auto_flush_interval` (100 ms by default). Borrowing per +the pool and ships nothing by itself; rows go out when an auto-flush threshold trips on a later append, at a later `close()` once they are older than +`auto_flush_interval` (100 ms by default), or when the pool's housekeeper settles the idle slot, typically within one +`auto_flush_interval`. A lease that holds the slot without appending or closing, for example one that waits for input +inside the `try` block, holds those rows back until it does: borrow once the data is in hand. Borrowing per message is the intended pattern and batches like a sender held for the whole run. Call `flush()` to ship now and `drain()` to wait for the server. The underlying connection is only torn down when the `QuestDB` handle itself is closed. diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 310fc3f22..664b808a1 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -139,9 +139,11 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * {@link Sender#close()} on the returned instance to release it back to * the pool. Close releases, flush ships, drain waits: {@code close()} on a * pooled Sender returns the slot and by default ships nothing, the rows go - * out when an auto-flush threshold trips on a later append or when the - * housekeeper settles the idle slot (at most one {@code auto_flush_interval} - * later). Call {@link Sender#flush()} to ship now and + * out when an auto-flush threshold trips on a later append, at a later + * release once they are older than {@code auto_flush_interval}, or when the + * housekeeper settles the idle slot, within about one + * {@code auto_flush_interval} of it going idle. A lease that holds the slot + * without appending or closing holds those rows back until it does. Call {@link Sender#flush()} to ship now and * {@link Sender#drain(long)} to wait for the server. Set * {@code flush_on_close=on} to have {@code close()} flush, as before 1.4. * With the default, one frame carries the rows of every lease that closed diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index f4eaed7d2..7a5eaf166 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -211,7 +211,8 @@ public QuestDBBuilder errorHandler(SenderErrorHandler handler) { /** * Whether {@code close()} on a borrowed sender flushes its rows before * returning the slot. Off by default: close releases, flush ships, drain - * waits, and rows left in a slot ship within one auto-flush interval. On + * waits, and rows left in a slot ship within one auto-flush interval of the slot going + * idle, or at the next append or close on it after that interval. On * restores the pre-1.4 behaviour, one frame and one WAL commit per close, * which is the slow path for applications that borrow per message. In this * client "flush" means published to the store-and-forward ring, not diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 7ebd1704e..a8e05901f 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -276,6 +276,15 @@ static Sender fromEnv() { */ void atNow(); + /** + * Runs the auto-flush check that every append runs and ships the pending rows when it trips. + * With no row in progress, only the age threshold can trip here, which lets a pool ship rows + * a lease left behind once they are older than {@code auto_flush_interval}. + * Only the WebSocket QWP transport implements it. + */ + default void autoFlushIfDue() { + } + /** * Block until the server has acknowledged every frame up to {@code targetFsn}, * or until {@code timeoutMillis} elapses. Pair with {@link #flushAndGetSequence()} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2b6c394fb..8f9a3b1b9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1475,6 +1475,17 @@ public void atNow() { } } + @Override + public void autoFlushIfDue() { + checkNotClosed(); + if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { + return; + } + if (shouldAutoFlush()) { + flushPendingRows(transactional); + } + } + /** * Blocks until {@link #getAckedFsn()} {@code >= targetFsn}, or until * {@code timeoutMillis} elapses. Polls on a 50us park; surfaces I/O loop diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 36950948d..d0b9975f7 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -85,6 +85,11 @@ public void atNow() { slot.live(generation).atNow(); } + @Override + public void autoFlushIfDue() { + slot.live(generation).autoFlushIfDue(); + } + @Override public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { return slot.live(generation).awaitAckedFsn(targetFsn, timeoutMillis); @@ -561,7 +566,8 @@ SenderSlot slot() { * pending rows unflagged and commits the group; *

  • otherwise {@code flush_on_close=on}: {@code flushDeferred()} with a * window, {@code flush()} without one;
    • - *
  • otherwise nothing: rows wait for a threshold or the settle.
    • + *
  • otherwise {@code autoFlushIfDue()}: ships only rows older than + * {@code auto_flush_interval}.
    • * */ private void ship(SenderPool pool, Sender delegate) { @@ -580,6 +586,8 @@ private void ship(SenderPool pool, Sender delegate) { } if (pool.isFlushOnClose()) { delegate.flush(); + } else { + delegate.autoFlushIfDue(); } } } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 9fccb0f4e..b206f8e24 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -222,6 +222,54 @@ public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exceptio } } + @Test + public void testPartialRowReleaseAfterTheIntervalStillShipsEarlierRows() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=200;housekeeper_interval_ms=60000;"))) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + Sender delegate; + Sender s = db.borrowSender(); + delegate = ((PooledSender) s).getDelegateForTesting(); + s.table("t").longColumn("v", 2L); + Thread.sleep(400); + try { + s.close(); + Assert.fail("expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("row in progress at close()")); + } + Assert.assertEquals("the earlier lease's aged row shipped", 0, delegate.pendingRowCount()); + log.awaitDataFrames(1, 5_000); + } + } + } + + @Test + public void testReleaseShipsAnEarlierLeasesRowOnceOlderThanTheInterval() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=200;housekeeper_interval_ms=60000;"))) { + try (Sender s = db.borrowSender()) { + s.table("t").longColumn("v", 1L).atNow(); + } + Sender delegate; + try (Sender s = db.borrowSender()) { // the same slot; settle cannot reach a leased slot + delegate = ((PooledSender) s).getDelegateForTesting(); + Thread.sleep(400); + } + Assert.assertEquals("the release shipped the earlier lease's aged row", 0, delegate.pendingRowCount()); + log.awaitDataFrames(1, 5_000); + } + } + } + @Test public void testRowOverTheCapFailsOnlyThatRowAndKeepsTheSlot() throws Exception { FrameLog log = new FrameLog(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 00d62bceb..75ffb6e6f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -46,6 +46,25 @@ */ public class SenderProbeMethodsTest { + @Test + public void testAutoFlushIfDueShipsOnlyRowsOlderThanTheInterval() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=300;close_flush_timeout_millis=0;")) { + sender.table("t").longColumn("v", 1L).atNow(); + sender.autoFlushIfDue(); + Assert.assertEquals("younger than the interval: kept", 1, sender.pendingRowCount()); + Thread.sleep(400); + sender.autoFlushIfDue(); + Assert.assertEquals("older than the interval: shipped", 0, sender.pendingRowCount()); + log.awaitDataFrames(1, 5_000); + } + } + } + @Test public void testCheckErrorReportsARetainedOverCapBatchUntilReset() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); @@ -112,6 +131,7 @@ public void testCheckErrorThrowsLatchedTerminal() throws Exception { @Test public void testDefaultsOnNonWebSocketTransport() { try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { + sender.autoFlushIfDue(); // default: no-op sender.checkError(); // default: no-op Assert.assertFalse(sender.hasRowInProgress()); // default: false Assert.assertEquals(0, sender.pendingRowCount()); // default: 0 From f26fa8d3669c40645f84dbf014aaedb4d9df894e Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:28:49 +0100 Subject: [PATCH 45/57] feat(qwp): a table buffer can hold back its last committed row The column writer encodes the rows before it and the buffer then keeps the held row alone, as retainInProgressRow does for the UDP sender. Co-Authored-By: Claude Opus 5.5 --- .../cutlass/qwp/client/QwpColumnWriter.java | 15 +- .../cutlass/qwp/protocol/QwpTableBuffer.java | 184 +++++++++++++++++- .../qwp/protocol/QwpTableBufferTest.java | 169 ++++++++++++++++ 3 files changed, 363 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index f8cc669fb..ae9215baf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -316,7 +316,20 @@ private void writeTimestampColumn(long addr, int count, boolean useGorilla) { } void encodeTable(QwpTableBuffer tableBuffer, boolean useGlobalSymbols, boolean useGorilla) { - encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useGlobalSymbols, useGorilla); + if (tableBuffer.hasHeldRow()) { + // the table buffer keeps the held row in its columns; this writer encodes only the rows before it + encodeTable( + tableBuffer, + tableBuffer.getRowCount(), + tableBuffer.getHeldValueCounts(), + tableBuffer.getHeldStringDataSizes(), + tableBuffer.getHeldSymbolDictionarySizes(), + useGlobalSymbols, + useGorilla + ); + } else { + encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useGlobalSymbols, useGorilla); + } } void encodeTable( diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 9622e4379..e1c6c0c74 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -70,12 +70,21 @@ public class QwpTableBuffer implements QuietCloseable { private boolean columnDefsCacheValid; private int committedColumnCount; // columns that existed at last nextRow() private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access + // held* arrays: holdBackLastRow() captures each column's state before the + // last committed row here; while isRowHeld is set, the column writer encodes + // only the rows before the held one. + private int[] heldArrayDataOffsets; + private int[] heldArrayShapeOffsets; + private long[] heldStringDataSizes; + private int[] heldSymbolDictionarySizes; + private int[] heldValueCounts; private int inProgressColumnCount; + private boolean isRowHeld; private int rowCount; // Savepoint for rollbackToSavepoint(): the row count and column count at the - // last markSavepoint(). reset() and retainInProgressRow() move it to "no rows, - // all current columns", since a flushed buffer's columns are known to the - // server; clear() zeroes it. + // last markSavepoint(). reset(), retainInProgressRow() and retainHeldRow() + // move it to "no rows, all current columns", since a flushed buffer's columns + // are known to the server; clear() zeroes it. private int savepointColumnCount; private int savepointRowCount; @@ -135,6 +144,7 @@ public void clear() { columnAccessCursor = 0; committedColumnCount = 0; inProgressColumnCount = 0; + isRowHeld = false; rowCount = 0; savepointRowCount = 0; savepointColumnCount = 0; @@ -198,6 +208,30 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { return lookupColumn(name, type); } + /** + * Per-column string data size before the held row, indexed by column + * index. Valid only while {@link #hasHeldRow()}. + */ + public long[] getHeldStringDataSizes() { + return heldStringDataSizes; + } + + /** + * Per-column symbol dictionary size before the held row, indexed by column + * index. Valid only while {@link #hasHeldRow()}. + */ + public int[] getHeldSymbolDictionarySizes() { + return heldSymbolDictionarySizes; + } + + /** + * Per-column value count before the held row, indexed by column index. + * Valid only while {@link #hasHeldRow()}. + */ + public int[] getHeldValueCounts() { + return heldValueCounts; + } + /** * Gets or creates a column with the given name and type. *

    @@ -260,7 +294,7 @@ public int getRowCount() { /** * Row count recorded by the last {@link #markSavepoint()}; {@code 0} when * none was taken since the buffer was last reset, cleared or - * retained a row in progress. + * retained a row in progress or a held row. */ public int getSavepointRowCount() { return savepointRowCount; @@ -273,6 +307,14 @@ public String getTableName() { return tableName; } + /** + * Returns true while {@link #holdBackLastRow()} holds the last committed + * row back, until {@link #restoreHeldRow()} or {@link #retainHeldRow()}. + */ + public boolean hasHeldRow() { + return isRowHeld; + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -281,6 +323,41 @@ public boolean hasInProgressRow() { return inProgressColumnCount > 0; } + /** + * Holds the last committed row back: records each column's state before + * that row and decrements the row count, so the column writer encodes only + * the rows before it. The row's values stay in the column buffers. + * {@link #restoreHeldRow()} puts the row back; {@link #retainHeldRow()} + * drops the rows before it and keeps it as the only row. The caller + * resolves the hold with one of the two before it writes, cancels or rolls + * back rows. + * + * @return false, changing nothing, when the buffer has no committed row, + * a row is in progress, or a row is already held + */ + public boolean holdBackLastRow() { + if (rowCount == 0 || isRowHeld || hasInProgressRow()) { + return false; + } + int columnCount = columns.size(); + ensureHeldColumnCapacity(columnCount); + int lastRow = rowCount - 1; + for (int i = 0; i < columnCount; i++) { + fastColumns[i].captureStateBeforeLastRow( + lastRow, + i, + heldValueCounts, + heldStringDataSizes, + heldSymbolDictionarySizes, + heldArrayShapeOffsets, + heldArrayDataOffsets + ); + } + rowCount = lastRow; + isRowHeld = true; + return true; + } + /** * Remembers the current row count and column count. A later * {@link #rollbackToSavepoint()} truncates the buffer back to this point. @@ -371,11 +448,55 @@ public void reset() { columnAccessCursor = 0; committedColumnCount = columns.size(); inProgressColumnCount = 0; + isRowHeld = false; savepointRowCount = 0; savepointColumnCount = columns.size(); rowCount = 0; } + /** + * Undoes {@link #holdBackLastRow()}: the held row counts as committed + * again. A no-op when no row is held. + */ + public void restoreHeldRow() { + if (isRowHeld) { + rowCount++; + isRowHeld = false; + } + } + + /** + * Drops the rows before the held row and keeps the held row as the only + * committed row, as {@link #retainInProgressRow} does for a row in + * progress. The savepoint moves to "no rows, all current columns", since + * the caller has shipped the dropped rows and the server knows every + * column. A no-op when no row is held. + */ + public void retainHeldRow() { + if (!isRowHeld) { + return; + } + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // retainTailRow clears the geohash precision of a column whose held + // value is null; the server has locked it with the shipped rows. + int geohashPrecisionBefore = col.geohashPrecision; + col.retainTailRow( + rowCount, + heldValueCounts[i], + heldArrayShapeOffsets[i], + heldArrayDataOffsets[i] + ); + col.geohashPrecision = geohashPrecisionBefore; + } + rowCount = 1; + savepointRowCount = 0; + savepointColumnCount = columns.size(); + committedColumnCount = columns.size(); + columnAccessCursor = 0; + isRowHeld = false; + } + public void retainInProgressRow( int[] sizeBefore, int[] valueCountBefore, @@ -500,6 +621,23 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullB return col; } + private void ensureHeldColumnCapacity(int required) { + if (heldValueCounts != null && required <= heldValueCounts.length) { + return; + } + + int newCapacity = heldValueCounts != null ? heldValueCounts.length : 8; + while (newCapacity < required) { + newCapacity *= 2; + } + + heldValueCounts = new int[newCapacity]; + heldStringDataSizes = new long[newCapacity]; + heldSymbolDictionarySizes = new int[newCapacity]; + heldArrayShapeOffsets = new int[newCapacity]; + heldArrayDataOffsets = new int[newCapacity]; + } + private ColumnBuffer lookupColumn(CharSequence name, byte type) { // Fast path: predict next column in sequence int n = columns.size(); @@ -2009,5 +2147,43 @@ private void retainSymbolValue(int valueIndex) { symbolDict.put(symbol, 0); Unsafe.getUnsafe().putInt(dataBuffer.pageAddress(), 0); } + + /** + * Writes this column's state before {@code lastRow}, its last committed + * row, into slot {@code columnIndex} of each array. The symbol dictionary + * size stays the current local one: WebSocket frames encode global ids, + * so a local entry that only the last row uses rides along unused. + */ + void captureStateBeforeLastRow( + int lastRow, + int columnIndex, + int[] valueCounts, + long[] stringDataSizes, + int[] symbolDictionarySizes, + int[] arrayShapeOffsets, + int[] arrayDataOffsets + ) { + boolean isLastNull = useNullBitmap && isNull(lastRow); + int valueCountBefore = isLastNull ? valueCount : valueCount - 1; + valueCounts[columnIndex] = valueCountBefore; + stringDataSizes[columnIndex] = stringOffsets != null + ? Unsafe.getUnsafe().getInt(stringOffsets.pageAddress() + (long) valueCountBefore * Integer.BYTES) + : 0; + symbolDictionarySizes[columnIndex] = getSymbolDictionarySize(); + + int shapeOffsetBefore = arrayShapeOffset; + int dataOffsetBefore = arrayDataOffset; + if (!isLastNull && arrayDims != null) { + int nDims = arrayDims[valueCountBefore] & 0xFF; + shapeOffsetBefore -= nDims; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= arrayShapes[shapeOffsetBefore + d]; + } + dataOffsetBefore -= elemCount; + } + arrayShapeOffsets[columnIndex] = shapeOffsetBefore; + arrayDataOffsets[columnIndex] = dataOffsetBefore; + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 18769d250..eab2e49bb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; @@ -1150,6 +1151,81 @@ public void testGetOrCreateColumnThrowsWhenExceedingMaxColumnCount() throws Exce }); } + @Test + public void testHeldRowEncodesThePrefixAndRetainsTheTailRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer both = new QwpTableBuffer("t"); + QwpTableBuffer first = new QwpTableBuffer("t"); + QwpTableBuffer second = new QwpTableBuffer("t")) { + appendMixedRow(both, 1L, 1.5, true, "alpha", "n1", "sym", new double[]{1.0, 2.0}); + appendMixedRow(both, 2L, 2.5, false, "beta", null, "sym", new double[]{3.0, 4.0, 5.0}); + appendMixedRow(first, 1L, 1.5, true, "alpha", "n1", "sym", new double[]{1.0, 2.0}); + appendMixedRow(second, 2L, 2.5, false, "beta", null, "sym", new double[]{3.0, 4.0, 5.0}); + + assertTrue(both.holdBackLastRow()); + assertEquals(1, both.getRowCount()); + assertArrayEquals("the prefix encodes as the first row alone", encodeForTest(first), encodeForTest(both)); + + both.retainHeldRow(); + assertFalse(both.hasHeldRow()); + assertEquals(1, both.getRowCount()); + assertArrayEquals("the retained row encodes as the second row alone", encodeForTest(second), encodeForTest(both)); + } + }); + } + + @Test + public void testHoldBackLastRowGrowsPastTheInitialColumnCapacity() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer both = new QwpTableBuffer("t"); + QwpTableBuffer first = new QwpTableBuffer("t"); + QwpTableBuffer second = new QwpTableBuffer("t")) { + int columnCount = 20; + for (int i = 0; i < columnCount; i++) { + both.getOrCreateColumn("c" + i, QwpConstants.TYPE_LONG, false).addLong(i); + first.getOrCreateColumn("c" + i, QwpConstants.TYPE_LONG, false).addLong(i); + } + both.nextRow(); + first.nextRow(); + for (int i = 0; i < columnCount; i++) { + both.getOrCreateColumn("c" + i, QwpConstants.TYPE_LONG, false).addLong(100 + i); + second.getOrCreateColumn("c" + i, QwpConstants.TYPE_LONG, false).addLong(100 + i); + } + both.nextRow(); + second.nextRow(); + + assertTrue(both.holdBackLastRow()); + assertTrue(both.getHeldValueCounts().length >= columnCount); + assertArrayEquals(encodeForTest(first), encodeForTest(both)); + both.retainHeldRow(); + assertArrayEquals(encodeForTest(second), encodeForTest(both)); + } + }); + } + + @Test + public void testHoldBackLastRowRefusesWithoutACommittedRowOrMidRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + assertFalse("an empty buffer has no row to hold", table.holdBackLastRow()); + assertFalse(table.hasHeldRow()); + + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + assertTrue(table.hasInProgressRow()); + assertFalse("a row in progress blocks the hold", table.holdBackLastRow()); + assertFalse(table.hasHeldRow()); + assertEquals(1, table.getRowCount()); + + table.nextRow(); + assertTrue(table.holdBackLastRow()); + assertFalse("a held row blocks a second hold", table.holdBackLastRow()); + assertEquals(1, table.getRowCount()); + } + }); + } + @Test public void testLongArrayMultipleRows() throws Exception { assertMemoryLeak(() -> { @@ -1388,6 +1464,26 @@ public void testNonAsciiColumnNameCaseInsensitive() throws Exception { }); } + @Test + public void testResetAndClearDropTheHeldRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + appendMixedRow(table, 1L, 1.5, true, "alpha", "n1", "sym", new double[]{1.0, 2.0}); + appendMixedRow(table, 2L, 2.5, false, "beta", null, "sym", new double[]{3.0, 4.0, 5.0}); + assertTrue(table.holdBackLastRow()); + table.reset(); + assertFalse("reset() drops the hold", table.hasHeldRow()); + assertEquals(0, table.getRowCount()); + + appendMixedRow(table, 3L, 3.5, true, "gamma", "n3", "sym", new double[]{6.0}); + assertTrue(table.holdBackLastRow()); + table.clear(); + assertFalse("clear() drops the hold", table.hasHeldRow()); + assertEquals(0, table.getRowCount()); + } + }); + } + @Test public void testResetMovesTheSavepointToTheEmptyBuffer() throws Exception { assertMemoryLeak(() -> { @@ -1405,6 +1501,42 @@ public void testResetMovesTheSavepointToTheEmptyBuffer() throws Exception { }); } + @Test + public void testRestoreHeldRowPutsTheRowBack() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + appendMixedRow(table, 1L, 1.5, true, "alpha", "n1", "sym", new double[]{1.0, 2.0}); + appendMixedRow(table, 2L, 2.5, false, "beta", null, "sym", new double[]{3.0, 4.0, 5.0}); + byte[] before = encodeForTest(table); + + assertTrue(table.holdBackLastRow()); + table.restoreHeldRow(); + assertFalse(table.hasHeldRow()); + assertArrayEquals(before, encodeForTest(table)); + assertEquals(2, table.getRowCount()); + } + }); + } + + @Test + public void testRetainHeldRowKeepsGeoHashPrecisionOfANullHeldValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer g = table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true); + g.addGeoHash(123L, 20); + table.nextRow(); + table.getOrCreateColumn("g", QwpConstants.TYPE_GEOHASH, true).addNull(); + table.nextRow(); + + assertTrue(table.holdBackLastRow()); + table.retainHeldRow(); + assertEquals(1, table.getRowCount()); + assertTrue(g.isNull(0)); + assertEquals("the server locked the precision with the shipped rows", 20, g.getGeoHashPrecision()); + } + }); + } + @Test public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { assertMemoryLeak(() -> { @@ -1617,6 +1749,31 @@ private static void addSymbolUtf8(QwpTableBuffer.ColumnBuffer col, String value) } } + private static void appendMixedRow( + QwpTableBuffer table, + long a, + double d, + boolean b, + String s, + String n, + String y, + double[] arr + ) { + table.getOrCreateColumn("a", QwpConstants.TYPE_LONG, false).addLong(a); + table.getOrCreateColumn("d", QwpConstants.TYPE_DOUBLE, true).addDouble(d); + table.getOrCreateColumn("b", QwpConstants.TYPE_BOOLEAN, false).addBoolean(b); + table.getOrCreateColumn("s", QwpConstants.TYPE_VARCHAR, true).addString(s); + QwpTableBuffer.ColumnBuffer nColumn = table.getOrCreateColumn("n", QwpConstants.TYPE_VARCHAR, true); + if (n == null) { + nColumn.addNull(); + } else { + nColumn.addString(n); + } + table.getOrCreateColumn("y", QwpConstants.TYPE_SYMBOL, true).addSymbol(y); + table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, true).addDoubleArray(arr); + table.nextRow(); + } + private static long copyToNative(byte[] bytes) { long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); for (int i = 0; i < bytes.length; i++) { @@ -1625,6 +1782,18 @@ private static long copyToNative(byte[] bytes) { return ptr; } + private static byte[] encodeForTest(QwpTableBuffer table) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + int length = encoder.encode(table); + long ptr = encoder.getBuffer().getBufferPtr(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(ptr + i); + } + return bytes; + } + } + /** * Simulates the encoder's walk over array data — the same logic as * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat From 77473192f502a6152640da7cbad0d8d1428fecee Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:52:58 +0100 Subject: [PATCH 46/57] fix(qwp): a row that pushes the batch over the cap ships the rows before it The byte trigger fires after the crossing row is committed; with rows inherited from earlier leases a row above a tenth of the cap made a batch the pre-PR pool shipped unshippable (review round 2, moderate 3). The sender now holds that row back, ships the rest and keeps it buffered; only the path that used to throw changes. QuestDBFlushOnCloseTest's crossing-row test asserted the old rejection and close()'s dropped-row report; it now asserts the hold-back. SelfSufficientFramesTest pins full-dict mode: the frame that ships the held row must carry the dictionary again, which the delta-mode test cannot observe. Co-Authored-By: Claude Opus 5.5 --- .../qwp/client/QwpWebSocketSender.java | 70 ++++++++++-- .../client/test/QuestDBFlushOnCloseTest.java | 94 +++++++++++----- .../qwp/client/SelfSufficientFramesTest.java | 57 ++++++++++ .../qwp/client/SenderProbeMethodsTest.java | 106 +++++++++++++++++- 4 files changed, 289 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8f9a3b1b9..8a25c220b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4004,6 +4004,44 @@ private void atNanos(long timestampNanos) { sendRow(); } + /** + * Runs the auto-flush {@link #sendRow()} starts once a row trips a + * threshold. The byte trigger fires after the crossing row is committed, so + * that row alone can push the batch over the server batch cap, for example + * when earlier pooled leases left rows behind. On that rejection this method + * holds the row back and ships the rows before it, and + * {@link #resetTableBuffersAfterFlush()} keeps the row buffered alone. When + * the rows before it do not fit either, the row goes back into the batch, + * and the method retains and throws the first rejection, which describes the + * whole batch. + */ + private void autoFlushAfterRow() { + try { + flushPendingRows(transactional); + } catch (BatchTooLargeForCapException rejection) { + // The hold-back runs only where the sender used to throw. Every cap + // rejection precedes the first publish, so the whole batch is still + // buffered and the retry below sees exactly the rows before this one. + if (pendingRowCount < 2 || !currentTableBuffer.holdBackLastRow()) { + throw rejection; + } + try { + // Ships every row but the held one; resetTableBuffersAfterFlush + // resolves the hold by retaining the held row. + flushPendingRows(transactional); + } catch (Throwable t) { + // restoreHeldRow() does nothing when resetTableBuffersAfterFlush + // already retained the row. + currentTableBuffer.restoreHeldRow(); + if (t instanceof BatchTooLargeForCapException) { + retainedBatchRejection = rejection; + throw rejection; + } + throw t; + } + } + } + /** * Resolves the connect timeout for one {@code buildAndConnect} walk. * Foreground connects honour the configured value verbatim: 0 (the @@ -5559,16 +5597,34 @@ private void onSlotLockReleased() { private void resetTableBuffersAfterFlush() { for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { - flushTableBuffers.getQuick(i).reset(); + QwpTableBuffer tableBuffer = flushTableBuffers.getQuick(i); + if (!tableBuffer.hasHeldRow()) { + tableBuffer.reset(); + } } // Drop the references; the next flush re-collects. flushTableNames.clear(); flushTableBuffers.clear(); - currentBatchMaxSymbolId = -1; - pendingBytes = 0; - currentTableBufferSnapshotBytes = 0; - pendingRowCount = 0; - firstPendingRowTimeNanos = 0; + if (currentTableBuffer != null && currentTableBuffer.hasHeldRow()) { + // autoFlushAfterRow shipped every row but the one it held back, which + // stays buffered as the only pending row. currentBatchMaxSymbolId keeps + // its value: the held row's symbol ids are at or below it, and a + // full-dictionary frame must still carry them. In delta mode the frame + // that just shipped carried them, and advanceSentMaxSymbolId moved the + // baseline past them. + currentTableBuffer.retainHeldRow(); + long heldRowBytes = currentTableBuffer.getBufferedBytes(); + pendingRowCount = 1; + pendingBytes = heldRowBytes; + currentTableBufferSnapshotBytes = heldRowBytes; + firstPendingRowTimeNanos = System.nanoTime(); + } else { + currentBatchMaxSymbolId = -1; + pendingBytes = 0; + currentTableBufferSnapshotBytes = 0; + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + } clearSavepointAndRejection(); armIfEligible(); } @@ -7187,7 +7243,7 @@ private void sendRow() { currentTableBufferSnapshotBytes = bufferedNow; if (shouldAutoFlush()) { - flushPendingRows(transactional); + autoFlushAfterRow(); } } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index b206f8e24..d1c8ce256 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -69,7 +70,7 @@ public void testCloseShipsNothingUntilSettleOrFlush() throws Exception { } @Test - public void testCrossingRowOverTheMarginDropsOnlyThatLeasesRowsAndShipsTheRest() throws Exception { + public void testCrossingRowShipsTheEarlierRowsAndStaysBuffered() throws Exception { FrameLog log = new FrameLog(); try (TestWebSocketServer server = new TestWebSocketServer(log)) { server.setAdvertisedMaxBatchSize(4 * 1024); @@ -77,41 +78,23 @@ public void testCrossingRowOverTheMarginDropsOnlyThatLeasesRowsAndShipsTheRest() Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); // auto_flush_bytes unset: the byte trigger sits at 90 percent of the 4 KiB cap (3686 bytes) try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { - // warm-up lease: proves the connection and the advertised cap are in place - // (initial_connect_retry=async learns the cap on the I/O thread) - try (Sender s = db.borrowSender()) { - s.table("w").longColumn("v", 1L).atNow(); - s.flush(); - } - log.awaitDataFrames(1, 5_000); - String wide = wideValue(1000); - for (int i = 0; i < 3; i++) { - try (Sender s = db.borrowSender()) { - s.table("t").stringColumn("s", wide).atNow(); // 3 KB in all, below the trigger - } - } - Sender delegate = null; - boolean hasCloseThrown = false; + appendThreeWideRowsAfterAWarmUp(db, log); + Sender delegate; try (Sender s = db.borrowSender()) { delegate = ((PooledSender) s).getDelegateForTesting(); - try { - s.table("t").stringColumn("s", wideValue(1500)).atNow(); - Assert.fail("the crossing row makes the auto-flushed batch 4.5 KB, over the 4 KiB cap"); - } catch (BatchTooLargeForCapException expected) { - // the row stays committed and the rejection is retained - } - } catch (LineSenderException e) { - hasCloseThrown = true; - Assert.assertTrue("close() wraps the rejection", e.getCause() instanceof BatchTooLargeForCapException); - Assert.assertTrue(e.getMessage(), e.getMessage().contains("dropped 1 row(s) of this lease")); + // the auto-flushed batch would be 4.5 KB, over the 4 KiB cap: the sender + // ships the three rows before the crossing row and keeps that row + s.table("t").stringColumn("s", wideValue(1500)).atNow(); + Assert.assertEquals("only the crossing row stays buffered", 1, s.pendingRowCount()); } - Assert.assertTrue("close() must report the dropped row", hasCloseThrown); log.awaitDataFrames(2, 5_000); Assert.assertEquals("the three earlier rows ship as one frame with table t", 1, log.tableCount(1)); try (Sender s = db.borrowSender()) { Assert.assertSame("the slot stayed in service", delegate, ((PooledSender) s).getDelegateForTesting()); - Assert.assertEquals("only the crossing row was dropped, the rest shipped", 0, s.pendingRowCount()); + Assert.assertEquals(1, s.pendingRowCount()); + s.flush(); } + log.awaitDataFrames(3, 5_000); } } } @@ -270,6 +253,31 @@ public void testReleaseShipsAnEarlierLeasesRowOnceOlderThanTheInterval() throws } } + @Test + public void testResetAfterACrossingRowDropsOnlyThatRow() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes unset: the byte trigger sits at 90 percent of the 4 KiB cap (3686 bytes) + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + appendThreeWideRowsAfterAWarmUp(db, log); + try (Sender s = db.borrowSender()) { + s.table("t").stringColumn("s", wideValue(1500)).atNow(); + // the three earlier rows shipped with the auto-flush, so the + // crossing row is the only row left and it is this lease's own + s.reset(); + Assert.assertEquals("reset() drops the crossing row", 0, s.pendingRowCount()); + } + log.awaitDataFrames(2, 5_000); + try (Sender s = db.borrowSender()) { + Assert.assertEquals(0, s.pendingRowCount()); + } + } + } + } + @Test public void testRowOverTheCapFailsOnlyThatRowAndKeepsTheSlot() throws Exception { FrameLog log = new FrameLog(); @@ -432,6 +440,22 @@ static void appendThreeRowsInThreeLeases(QuestDB db) { } } + static void appendThreeWideRowsAfterAWarmUp(QuestDB db, FrameLog log) throws InterruptedException { + // warm-up lease: proves the connection and the advertised cap are in place + // (initial_connect_retry=async learns the cap on the I/O thread) + try (Sender s = db.borrowSender()) { + s.table("w").longColumn("v", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); + String wide = wideValue(1000); + for (int i = 0; i < 3; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").stringColumn("s", wide).atNow(); // 3 KB in all, below the trigger + } + } + } + static String config(int port, String extra) { return "ws::addr=localhost:" + port + ";sender_pool_min=1;sender_pool_max=1" @@ -443,13 +467,13 @@ static String config(int port, String extra) { public static String wideValue(int length) { char[] chars = new char[length]; - java.util.Arrays.fill(chars, 'x'); + Arrays.fill(chars, 'x'); return new String(chars); } /** * Parses every QWP frame in each WebSocket message (a read can carry more - * than one on replay), records its flags byte and table count, and acks + * than one on replay), records its flags byte, table count and bytes, and acks * cumulatively so the sender never backs up. Public: the sender-level * probe test in a sub-package and the benchmark reuse it. */ @@ -459,6 +483,7 @@ public static final class FrameLog implements TestWebSocketServer.WebSocketServe private static final int OFFSET_PAYLOAD_LENGTH = 8; private static final int OFFSET_TABLE_COUNT = 6; private final List flags = new ArrayList<>(); + private final List frames = new ArrayList<>(); private final List tableCounts = new ArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; private long nextSeq; @@ -517,6 +542,14 @@ public synchronized boolean deferFlag(int dataFrameIndex) { throw new AssertionError("no data frame at index " + dataFrameIndex); } + /** + * A copy of every frame {@link #onBinaryMessage} parsed, in arrival order, + * each frame's header and payload as one array. + */ + public synchronized List frames() { + return new ArrayList<>(frames); + } + @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { if (currentClient != client) { @@ -535,6 +568,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien break; } flags.add(data[pos + OFFSET_FLAGS] & 0xFF); + frames.add(Arrays.copyOfRange(data, pos, pos + HEADER_SIZE + (int) payloadLength)); tableCounts.add(tableCount); framesInMessage++; pos += HEADER_SIZE + (int) payloadLength; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 0eba803d2..5d66217e5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -45,6 +45,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -1009,6 +1010,62 @@ public void testFullDictNearCapOversizedBodyStrandsNoChunks() throws Exception { }); } + @Test + public void testHeldCrossingRowFrameCarriesTheDictionaryInFullDictMode() throws Exception { + // A row that pushes an auto-flushed batch over the cap stays buffered while + // the rows before it ship. The sender registered the row's symbol before it + // encoded that frame, so the frame defines the id. In full-dict mode every + // frame must define every id its rows reference, so the frame that later + // ships the held row must carry the dictionary again: the sender keeps the + // batch's symbol watermark when it retains the row. + Path sfDir = temporaryFolder.newFolder("qwp-sf-held-row").toPath(); + String slot = sfDir.resolve("default").toString(); + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // openCleanRW fails against nothing on disk, so the slot runs full-dict. + UnopenableDictFacade dictFf = new UnopenableDictFacade(); + CursorSendEngine engine = new CursorSendEngine( + slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, dictFf); + // the default auto_flush_bytes binds the byte trigger to 90% of the 4 KiB cap + try (Sender sender = QwpWebSocketSender.connect( + "localhost", port, null, 1_000_000, QwpWebSocketSender.DEFAULT_AUTO_FLUSH_BYTES, + 0L, null, false, engine)) { + Assert.assertFalse("the slot runs full-dict", + ((QwpWebSocketSender) sender).isDeltaDictEnabledForTest()); + String wide = TestUtils.repeat("x", 1000); + for (int i = 0; i < 3; i++) { + sender.table("t").symbol("y", "a").stringColumn("s", wide).atNow(); + } + // the auto-flushed batch would be 4.5 KB: the three rows before this + // one ship, and this row, which registered "b", stays buffered + sender.table("t").symbol("y", "b").stringColumn("s", TestUtils.repeat("x", 1500)).atNow(); + Assert.assertEquals(1, sender.pendingRowCount()); + sender.flush(); + sender.table("t").symbol("y", "b").atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 3, 10_000); + } + + Assert.assertEquals(3, handler.batches.size()); + for (int i = 0; i < 3; i++) { + byte[] frame = handler.batches.get(i); + Assert.assertEquals("frame " + i + " carries table t", 1, QwpWireTestUtils.tableCount(frame)); + List dictionary = new ArrayList<>(); + QwpWireTestUtils.accumulateDeltaDictionary(frame, dictionary); + Assert.assertEquals("frame " + i + " defines every id its rows reference", + Arrays.asList("a", "b"), dictionary); + } + } + }); + } + @Test public void testSingleSymbolLargerThanTheCapThrowsWithNothingPublished() throws Exception { // A symbol value cannot be split across frames, so one larger than the cap is a diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 75ffb6e6f..cd45536ad 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -34,6 +34,9 @@ import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; /** @@ -42,7 +45,8 @@ * at release ({@code checkError()}, {@code hasRowInProgress()} and * {@code pendingRowCount()}), the savepoint pair ({@code markSavepoint()} and * {@code resetToSavepoint()}) behind a lease's {@code reset()}, the deferred - * flush of the commit window and the cap-bound byte trigger. + * flush of the commit window, the cap-bound byte trigger and the crossing row + * the sender holds back when the cap rejects the auto-flushed batch. */ public class SenderProbeMethodsTest { @@ -128,6 +132,72 @@ public void testCheckErrorThrowsLatchedTerminal() throws Exception { } } + @Test + public void testCrossingRowShipsTheRowsBeforeIt() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // no auto_flush_bytes key: the byte trigger sits at 90% of the 4 KiB cap (3686 bytes) + try (Sender sender = Sender.fromConfig(configWithoutAutoFlushBytes(server.getPort()))) { + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 3; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + // the auto-flushed batch would be 4.5 KB, over the cap: the sender ships + // the three rows before the crossing row and keeps that row + sender.table("t").stringColumn("s", QuestDBFlushOnCloseTest.wideValue(1500)).atNow(); + Assert.assertEquals("only the crossing row stays buffered", 1, sender.pendingRowCount()); + log.awaitDataFrames(1, 5_000); + sender.flush(); + log.awaitDataFrames(2, 5_000); + } + } + } + + @Test + public void testCrossingRowWhoseEarlierRowsStillExceedTheCapThrowsAndKeepsEveryRow() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: only the row trigger fires, at the sixth row, when the + // five rows before it already exceed the cap + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_bytes=off;auto_flush_rows=6;auto_flush_interval=60000;close_flush_timeout_millis=0;")) { + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 5; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + BatchTooLargeForCapException rejection = null; + try { + sender.table("t").stringColumn("s", wide).atNow(); + Assert.fail("the five rows before the crossing row still exceed the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + rejection = expected; + } + Assert.assertEquals("the crossing row goes back into the batch", 6, sender.pendingRowCount()); + Thread.sleep(300); + Assert.assertEquals("nothing ships", 0, log.dataFrames()); + try { + sender.checkError(); + Assert.fail("the probe must report the retained batch"); + } catch (BatchTooLargeForCapException expected) { + Assert.assertSame("the probe reports the rejection of the whole batch", rejection, expected); + } + try { + sender.flush(); + Assert.fail("the retained batch still exceeds the cap"); + } catch (BatchTooLargeForCapException expected) { + Assert.assertEquals("flush() encodes the same six rows", rejection.getMessage(), expected.getMessage()); + } + sender.reset(); + } + } + } + @Test public void testDefaultsOnNonWebSocketTransport() { try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { @@ -220,6 +290,36 @@ public void testHealthySenderReportsBufferState() throws Exception { } } + @Test + public void testHeldCrossingRowWithANewSymbolKeepsTheDictionaryGapFree() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(configWithoutAutoFlushBytes(server.getPort()))) { + Assert.assertTrue("memory mode selects delta mode", ((QwpWebSocketSender) sender).isDeltaDictEnabledForTest()); + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 3; i++) { + sender.table("t").symbol("y", "a").stringColumn("s", wide).atNow(); + } + // the held crossing row registers "b" before the frame of the three + // rows ahead of it ships + sender.table("t").symbol("y", "b").stringColumn("s", QuestDBFlushOnCloseTest.wideValue(1500)).atNow(); + sender.flush(); + sender.table("t").symbol("y", "b").atNow(); + sender.flush(); + log.awaitDataFrames(3, 5_000); + List dict = new ArrayList<>(); + for (byte[] frame : log.frames()) { + QwpWireTestUtils.accumulateDeltaDictionary(frame, dict, false); + } + Assert.assertEquals(Arrays.asList("a", "b"), dict); + sender.checkError(); + } + } + } + @Test public void testResetToSavepointAfterAnOverCapFlushClearsTheProbeAndShipsTheEarlierRows() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); @@ -353,6 +453,10 @@ private static String config(int port) { return "ws::addr=localhost:" + port + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;"; } + private static String configWithoutAutoFlushBytes(int port) { + return "ws::addr=localhost:" + port + ";auto_flush_rows=1000000;auto_flush_interval=60000;close_flush_timeout_millis=0;"; + } + private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { private TestWebSocketServer.ClientHandler currentClient; private long nextSeq; From c9259f8eaf45d082f23380993431477b4047b509 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:15:05 +0100 Subject: [PATCH 47/57] fix(qwp): a held row alone in its table keeps the savepoint columns When the crossing row is the only row of its table in the batch, the prefix frame leaves that table out, so the server has seen none of the columns the row added. retainHeldRow() still moved the savepoint to every current column, and a lease reset() after the hold-back kept those columns: a later lease padded them with nulls and shipped them. retainHeldRow() now moves the savepoint columns only when rows of the table shipped before the held row, and otherwise keeps the count the last mark or flush recorded. QuestDBFlushOnCloseTest also regains a wire test of PooledSender close()'s rollback-and-retry success branch, which the replaced crossing-row test used to cover: an explicit over-cap flush() with auto_flush_bytes=off, then close() drops only the lease's rows and ships the rest. Co-Authored-By: Claude Opus 5.5 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 19 +++- .../client/test/QuestDBFlushOnCloseTest.java | 104 ++++++++++++++++++ .../qwp/protocol/QwpTableBufferTest.java | 51 +++++++++ 3 files changed, 169 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index e1c6c0c74..337673c9f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -84,7 +84,8 @@ public class QwpTableBuffer implements QuietCloseable { // Savepoint for rollbackToSavepoint(): the row count and column count at the // last markSavepoint(). reset(), retainInProgressRow() and retainHeldRow() // move it to "no rows, all current columns", since a flushed buffer's columns - // are known to the server; clear() zeroes it. + // are known to the server; retainHeldRow() keeps the column count when no + // row of this table shipped. clear() zeroes it. private int savepointColumnCount; private int savepointRowCount; @@ -468,14 +469,20 @@ public void restoreHeldRow() { /** * Drops the rows before the held row and keeps the held row as the only * committed row, as {@link #retainInProgressRow} does for a row in - * progress. The savepoint moves to "no rows, all current columns", since - * the caller has shipped the dropped rows and the server knows every - * column. A no-op when no row is held. + * progress. The savepoint moves to no rows. When rows preceded the held + * row, the caller has shipped them with every current column, so the + * savepoint also takes every current column. When the held row is the + * table's only row, nothing of the table shipped, so the savepoint keeps + * its column count, which still names only columns the server knows: none + * for a new buffer, otherwise those at the last mark or flush. A later + * {@link #rollbackToSavepoint()} then drops the columns the held row + * added. A no-op when no row is held. */ public void retainHeldRow() { if (!isRowHeld) { return; } + boolean hasShippedRows = rowCount > 0; for (int i = 0, n = columns.size(); i < n; i++) { ColumnBuffer col = fastColumns[i]; // retainTailRow clears the geohash precision of a column whose held @@ -491,7 +498,9 @@ public void retainHeldRow() { } rowCount = 1; savepointRowCount = 0; - savepointColumnCount = columns.size(); + if (hasShippedRows) { + savepointColumnCount = columns.size(); + } committedColumnCount = columns.size(); columnAccessCursor = 0; isRowHeld = false; diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index d1c8ce256..40c74b20d 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -29,6 +29,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.BatchTooLargeForCapException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.impl.PooledSender; import io.questdb.client.test.cutlass.qwp.client.QwpWireTestUtils; @@ -37,6 +38,7 @@ import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -205,6 +207,47 @@ public void testOverCapBatchRetiresTheSlotInsteadOfPoisoningIt() throws Exceptio } } + @Test + public void testOverCapFlushDropsOnlyThatLeasesRowsAndShipsTheRest() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes=off: no byte trigger, so the sender never holds a row back + // and the lease's explicit flush() meets the whole over-cap batch + try (QuestDB db = QuestDB.connect(config(server.getPort(), + "auto_flush_bytes=off;auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + appendThreeWideRowsAfterAWarmUp(db, log); + Sender delegate = null; + boolean hasCloseThrown = false; + try (Sender s = db.borrowSender()) { + delegate = ((PooledSender) s).getDelegateForTesting(); + String wide = wideValue(1000); + s.table("t").stringColumn("s", wide).atNow(); + s.table("t").stringColumn("s", wide).atNow(); + try { + s.flush(); + Assert.fail("the lease's two rows make the batch 5 KB, over the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // the batch is retained: close() rolls the lease back and retries + } + } catch (LineSenderException e) { + hasCloseThrown = true; + Assert.assertTrue("close() wraps the rejection", e.getCause() instanceof BatchTooLargeForCapException); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("dropped 2 row(s) of this lease")); + } + Assert.assertTrue("close() must report the dropped rows", hasCloseThrown); + log.awaitDataFrames(2, 5_000); + Assert.assertEquals("the three earlier rows ship as one frame with table t", 1, log.tableCount(1)); + try (Sender s = db.borrowSender()) { + Assert.assertSame("the slot stayed in service", delegate, ((PooledSender) s).getDelegateForTesting()); + Assert.assertEquals("only the lease's rows were dropped, the rest shipped", 0, s.pendingRowCount()); + } + } + } + } + @Test public void testPartialRowReleaseAfterTheIntervalStillShipsEarlierRows() throws Exception { FrameLog log = new FrameLog(); @@ -253,6 +296,53 @@ public void testReleaseShipsAnEarlierLeasesRowOnceOlderThanTheInterval() throws } } + @Test + public void testResetAfterACrossingRowAloneInItsTableDropsTheColumnItAdded() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes unset: the byte trigger sits at 90 percent of the 4 KiB cap (3686 bytes) + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + // warm-up lease: the server learns late_table with kept_column, and late_table + // precedes t in the sender's tables, so its frame carries the batch's new + // symbols when the batch splits + try (Sender s = db.borrowSender()) { + s.table("late_table").longColumn("kept_column", 1L).atNow(); + s.flush(); + } + log.awaitDataFrames(1, 5_000); + for (int i = 0; i < 3; i++) { + try (Sender s = db.borrowSender()) { + s.table("t").symbol("y", i + wideValue(800)).atNow(); // 2.4 KB of new symbols + } + } + try (Sender s = db.borrowSender()) { + Assert.assertTrue("memory mode selects delta mode", + ((QwpWebSocketSender) ((PooledSender) s).getDelegateForTesting()).isDeltaDictEnabledForTest()); + // late_table's frame would carry this 3.7 KB row and the 2.4 KB of new symbols: + // the sender ships t's rows with the symbols and holds the row, the only row + // of late_table in the batch + s.table("late_table").longColumn("kept_column", 2L).stringColumn("discarded_column", wideValue(3700)).atNow(); + Assert.assertEquals("only the crossing row stays buffered", 1, s.pendingRowCount()); + s.reset(); + } + log.awaitDataFrames(2, 5_000); + try (Sender s = db.borrowSender()) { + s.table("late_table").longColumn("kept_column", 3L).atNow(); + s.flush(); + } + log.awaitDataFrames(3, 5_000); + List frames = log.frames(); + Assert.assertTrue("late_table ships kept_column", containsAscii(frames.get(frames.size() - 1), "kept_column")); + for (byte[] frame : frames) { + Assert.assertFalse("reset() dropped the column the server never saw", containsAscii(frame, "discarded_column")); + } + } + } + } + @Test public void testResetAfterACrossingRowDropsOnlyThatRow() throws Exception { FrameLog log = new FrameLog(); @@ -465,6 +555,20 @@ static String config(int port, String extra) { + extra; } + private static boolean containsAscii(byte[] frame, String text) { + byte[] needle = text.getBytes(StandardCharsets.US_ASCII); + for (int i = 0, n = frame.length - needle.length; i <= n; i++) { + int j = 0; + while (j < needle.length && frame[i + j] == needle[j]) { + j++; + } + if (j == needle.length) { + return true; + } + } + return false; + } + public static String wideValue(int length) { char[] chars = new char[length]; Arrays.fill(chars, 'x'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index eab2e49bb..ba68eb7aa 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1537,6 +1537,57 @@ public void testRetainHeldRowKeepsGeoHashPrecisionOfANullHeldValue() throws Exce }); } + @Test + public void testRetainHeldRowMovesTheSavepointColumnsOnlyWhenEarlierRowsShipped() throws Exception { + assertMemoryLeak(() -> { + // rows before the held row: the caller shipped them with every column, + // including the one the held row added, so the savepoint takes it + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.getOrCreateColumn("z", QwpConstants.TYPE_VARCHAR, true).addString("held"); + table.nextRow(); + assertTrue(table.holdBackLastRow()); + table.retainHeldRow(); + assertEquals(1, table.getRowCount()); + assertEquals(0, table.getSavepointRowCount()); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals("the server knows both columns", 2, table.getColumnCount()); + } + // the held row is the only row of a new buffer: nothing of the table + // shipped, so the savepoint keeps no column + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("z", QwpConstants.TYPE_VARCHAR, true).addString("held"); + table.nextRow(); + assertTrue(table.holdBackLastRow()); + table.retainHeldRow(); + assertEquals(1, table.getRowCount()); + assertEquals(0, table.getSavepointRowCount()); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals("the server has seen no column", 0, table.getColumnCount()); + } + // the held row is the only row after a flush: the savepoint keeps the + // column the flush shipped, not the one the held row added + try (QwpTableBuffer table = new QwpTableBuffer("t")) { + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(1L); + table.nextRow(); + table.reset(); + table.getOrCreateColumn("v", QwpConstants.TYPE_LONG, false).addLong(2L); + table.getOrCreateColumn("z", QwpConstants.TYPE_VARCHAR, true).addString("held"); + table.nextRow(); + assertTrue(table.holdBackLastRow()); + table.retainHeldRow(); + assertEquals(1, table.getRowCount()); + table.rollbackToSavepoint(); + assertEquals(0, table.getRowCount()); + assertEquals("only the column the flush shipped stays", 1, table.getColumnCount()); + } + }); + } + @Test public void testRetainInProgressRowFastClearsUnstagedNullableColumn() throws Exception { assertMemoryLeak(() -> { From d2bba3bf845630299ee7d70af282e5f0f2441fa1 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:21:17 +0100 Subject: [PATCH 48/57] fix(qwp): a held row over the byte budget flushes on its own at once nextRow()'s per-row guard measures raw column bytes, so a row just under the cap can still have a frame over it once the table name and schema are added. The hold-back shipped the rows before such a row and kept it buffered, so it outlived its lease: the next lease's flush was rejected, and the slot was retired with the rows of that batch. After the hold-back ships the rows before it, autoFlushAfterRow() now runs the normal auto-flush check once for the held row, with no further hold-back. A row at or above the byte budget would flush with the next append anyway; flushing it now puts the rejection of its own frame on its own at() call, so a pooled close() rolls back only that row and the slot stays in service, as before the hold-back. A held row below the budget keeps batching. The column test from the previous commit now holds a 2.6 KB row, below the budget, so its held row still stays buffered. Co-Authored-By: Claude Opus 5.5 --- .../qwp/client/QwpWebSocketSender.java | 17 +++++-- .../client/test/QuestDBFlushOnCloseTest.java | 50 +++++++++++++++++-- .../qwp/client/SenderProbeMethodsTest.java | 39 +++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8a25c220b..3d96cf960 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4010,10 +4010,11 @@ private void atNanos(long timestampNanos) { * that row alone can push the batch over the server batch cap, for example * when earlier pooled leases left rows behind. On that rejection this method * holds the row back and ships the rows before it, and - * {@link #resetTableBuffersAfterFlush()} keeps the row buffered alone. When - * the rows before it do not fit either, the row goes back into the batch, - * and the method retains and throws the first rejection, which describes the - * whole batch. + * {@link #resetTableBuffersAfterFlush()} keeps the row buffered alone. A + * held row that still trips a threshold on its own is flushed at once, so a + * rejection of its own frame reaches this caller. When the rows before it do + * not fit either, the row goes back into the batch, and the method retains + * and throws the first rejection, which describes the whole batch. */ private void autoFlushAfterRow() { try { @@ -4022,6 +4023,11 @@ private void autoFlushAfterRow() { // The hold-back runs only where the sender used to throw. Every cap // rejection precedes the first publish, so the whole batch is still // buffered and the retry below sees exactly the rows before this one. + // Once the rows before it ship, the held row gets the normal auto-flush + // check once, with no further hold-back: at or above the byte budget the + // next append would flush it anyway, and flushing it now puts a rejection + // of its own frame on this row's at() call, which a pooled close() rolls + // back while the slot stays in service. if (pendingRowCount < 2 || !currentTableBuffer.holdBackLastRow()) { throw rejection; } @@ -4039,6 +4045,9 @@ private void autoFlushAfterRow() { } throw t; } + if (shouldAutoFlush()) { + flushPendingRows(transactional); + } } } diff --git a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java index 40c74b20d..3a9573c22 100644 --- a/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java +++ b/core/src/test/java/io/questdb/client/test/QuestDBFlushOnCloseTest.java @@ -101,6 +101,47 @@ public void testCrossingRowShipsTheEarlierRowsAndStaysBuffered() throws Exceptio } } + @Test + public void testCrossingRowWhoseOwnFrameExceedsTheCapDropsOnlyThatRow() throws Exception { + FrameLog log = new FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // auto_flush_bytes unset: the byte trigger sits at 90 percent of the 4 KiB cap (3686 bytes) + try (QuestDB db = QuestDB.connect(config(server.getPort(), "auto_flush_interval=60000;housekeeper_interval_ms=60000;"))) { + appendThreeWideRowsAfterAWarmUp(db, log); + Sender delegate = null; + boolean hasCloseThrown = false; + try (Sender s = db.borrowSender()) { + delegate = ((PooledSender) s).getDelegateForTesting(); + try { + // 4070 bytes pass the per-row guard, but the table name and schema + // take the row's own frame over the cap + s.table("u").stringColumn("z", wideValue(4070)).atNow(); + Assert.fail("the row's own frame exceeds the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // the rows before it shipped; the row stays and the rejection is retained + } + } catch (LineSenderException e) { + hasCloseThrown = true; + Assert.assertTrue("close() wraps the rejection", e.getCause() instanceof BatchTooLargeForCapException); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("dropped 1 row(s) of this lease")); + } + Assert.assertTrue("close() must report the dropped row", hasCloseThrown); + log.awaitDataFrames(2, 5_000); + Assert.assertEquals("the three earlier rows ship as one frame with table t", 1, log.tableCount(1)); + try (Sender s = db.borrowSender()) { + Assert.assertSame("the slot stayed in service", delegate, ((PooledSender) s).getDelegateForTesting()); + Assert.assertEquals(0, s.pendingRowCount()); + s.table("w").longColumn("v", 2L).atNow(); + s.flush(); + } + log.awaitDataFrames(3, 5_000); + } + } + } + @Test public void testFlushOnCloseShipsOneFramePerClose() throws Exception { FrameLog log = new FrameLog(); @@ -315,16 +356,17 @@ public void testResetAfterACrossingRowAloneInItsTableDropsTheColumnItAdded() thr log.awaitDataFrames(1, 5_000); for (int i = 0; i < 3; i++) { try (Sender s = db.borrowSender()) { - s.table("t").symbol("y", i + wideValue(800)).atNow(); // 2.4 KB of new symbols + // 1.8 KB of new symbols and 1.5 KB of strings in all, below the trigger + s.table("t").symbol("y", i + wideValue(600)).stringColumn("s", wideValue(500)).atNow(); } } try (Sender s = db.borrowSender()) { Assert.assertTrue("memory mode selects delta mode", ((QwpWebSocketSender) ((PooledSender) s).getDelegateForTesting()).isDeltaDictEnabledForTest()); - // late_table's frame would carry this 3.7 KB row and the 2.4 KB of new symbols: + // late_table's frame would carry this 2.6 KB row and the 1.8 KB of new symbols: // the sender ships t's rows with the symbols and holds the row, the only row - // of late_table in the batch - s.table("late_table").longColumn("kept_column", 2L).stringColumn("discarded_column", wideValue(3700)).atNow(); + // of late_table in the batch and below the byte trigger on its own + s.table("late_table").longColumn("kept_column", 2L).stringColumn("discarded_column", wideValue(2600)).atNow(); Assert.assertEquals("only the crossing row stays buffered", 1, s.pendingRowCount()); s.reset(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index cd45536ad..aa0323508 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -198,6 +198,45 @@ public void testCrossingRowWhoseEarlierRowsStillExceedTheCapThrowsAndKeepsEveryR } } + @Test + public void testCrossingRowWhoseOwnFrameExceedsTheCapThrowsAfterTheRowsBeforeItShip() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // no auto_flush_bytes key: the byte trigger sits at 90% of the 4 KiB cap (3686 bytes) + try (Sender sender = Sender.fromConfig(configWithoutAutoFlushBytes(server.getPort()))) { + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 3; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + // 4070 bytes pass the per-row guard, but the table name and schema take the + // row's own frame over the cap: the rows before it ship, and the row, alone + // over the byte trigger, fails its own flush + BatchTooLargeForCapException rejection = null; + try { + sender.table("u").stringColumn("z", QuestDBFlushOnCloseTest.wideValue(4070)).atNow(); + Assert.fail("the row's own frame exceeds the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + rejection = expected; + } + log.awaitDataFrames(1, 5_000); + Assert.assertEquals("the three t rows ship as one frame with table t", 1, log.tableCount(0)); + try { + sender.checkError(); + Assert.fail("the probe must report the retained row"); + } catch (BatchTooLargeForCapException expected) { + Assert.assertSame(rejection, expected); + } + Assert.assertEquals("only the crossing row stays buffered", 1, sender.pendingRowCount()); + sender.reset(); + Assert.assertEquals(0, sender.pendingRowCount()); + sender.checkError(); + } + } + } + @Test public void testDefaultsOnNonWebSocketTransport() { try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { From 49bf8ae7b572418141ad2d341b185869c5290090 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:23:50 +0100 Subject: [PATCH 49/57] fix(pool): housekeeper deadlines follow a monotonic clock A backward wall-clock step delayed settling idle slots by the size of the step (review round 2, moderate 5). Co-Authored-By: Claude Opus 5.5 --- .../io/questdb/client/impl/PoolHousekeeper.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java index 7ad2dc284..78ce4ed2f 100644 --- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java +++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java @@ -53,6 +53,7 @@ final class PoolHousekeeper { static final long STOP_TIMEOUT_MILLIS = 2_000; private final long intervalMillis; + private final long originNanos = System.nanoTime(); private final QueryClientPool queryPool; private final SenderPool senderPool; private final long settleIntervalMillis; @@ -175,8 +176,12 @@ static boolean joinIgnoringCallerInterrupts(Thread target, long timeoutMillis) { return callerWasInterrupted; } + private long monotonicMillis() { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - originNanos); + } + private void runLoop() { - long startMillis = System.currentTimeMillis(); + long startMillis = monotonicMillis(); long nextSettleAt = deadline(startMillis, settleIntervalMillis); long nextReapAt = deadline(startMillis, intervalMillis); long nextRecoveryAt = startMillis; @@ -202,7 +207,7 @@ private void runLoop() { // a recovery failure (including an Error) must never kill this // daemon, so swallow Throwable -- exactly as the reap guards below. boolean recovering = false; - if (System.currentTimeMillis() >= nextRecoveryAt) { + if (monotonicMillis() >= nextRecoveryAt) { try { recovering = senderPool.runStartupRecoveryStep(); } catch (Throwable ignored) { @@ -212,7 +217,7 @@ private void runLoop() { // found nothing or hit a transient failure is retried on its own // cadence, never on the settle cadence: a stranded slot against an // unreachable server must not become a hot connect loop. - long afterStep = System.currentTimeMillis(); + long afterStep = monotonicMillis(); nextRecoveryAt = recovering ? afterStep : deadline(afterStep, STARTUP_RECOVERY_RETRY_MILLIS); } synchronized (signalLock) { @@ -221,7 +226,7 @@ private void runLoop() { } if (!recovering) { long nextAt = Math.min(Math.min(nextSettleAt, nextReapAt), nextRecoveryAt); - long waitMillis = nextAt - System.currentTimeMillis(); + long waitMillis = nextAt - monotonicMillis(); if (waitMillis > 0) { try { signalLock.wait(waitMillis); @@ -235,7 +240,7 @@ private void runLoop() { if (stop) { return; } - long now = System.currentTimeMillis(); + long now = monotonicMillis(); if (now >= nextSettleAt) { try { senderPool.settleIdle(); From 95d540f0851dc3486abdbad80cc50a3f705759cd Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:25:50 +0100 Subject: [PATCH 50/57] fix(pool): settle takes one idle slot at a time Taking every dirty slot up front kept them all out of the free list while one flush blocked on a full ring, so borrowSender() timed out with idle slots (review round 2, moderate 6). Co-Authored-By: Claude Opus 5.5 --- .../io/questdb/client/impl/SenderPool.java | 114 ++++++++++-------- .../test/impl/SenderPoolSettleTest.java | 88 ++++++++++++++ 2 files changed, 154 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 159160778..640f45a40 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1738,8 +1738,9 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna * so a slot nobody borrows again would keep its last rows until the pool * closes. The housekeeper calls this on its settle cadence. *

    - * Each candidate slot is taken off the free list under the pool lock and - * stamped with a fresh generation, which makes it an internal lease: a + * The settle lists the dirty slots under the pool lock, then takes one slot at + * a time off the free list under that lock and stamps it with a fresh + * generation, which makes it an internal lease: a * borrower arriving meanwhile takes another slot or waits, and * {@link #close()} waits for it like any outstanding lease. The flush runs * outside the lock. After the flush, whether it succeeded or not, the @@ -1757,76 +1758,93 @@ public int settleIdle() { if (closed) { return 0; } - ArrayList leases = null; + ArrayList candidates = null; lock.lock(); try { if (closed) { return 0; } - Iterator it = available.iterator(); - while (it.hasNext()) { - SenderSlot s = it.next(); + for (SenderSlot s : available) { if (hasUnshippedWork(s.delegate())) { - it.remove(); - s.bumpGeneration(); - if (leases == null) { - leases = new ArrayList<>(); + if (candidates == null) { + candidates = new ArrayList<>(); } - leases.add(new PooledSender(s, s.generation())); + candidates.add(s); } } } finally { lock.unlock(); } - if (leases == null) { + if (candidates == null) { return 0; } int settled = 0; - for (int i = 0, n = leases.size(); i < n; i++) { - PooledSender lease = leases.get(i); - boolean flushed = false; - Throwable fatal = null; - BatchTooLargeForCapException overCap = null; - try { - // PooledSender.flush() already marks the commit on the slot. - lease.flush(); - lease.slot().settleFailureLogged(false); - flushed = true; - } catch (BatchTooLargeForCapException e) { - // Deterministic: the probe would report it too, but no settle retry - // can ship this batch and no lease owns it any more. Retire the - // slot below. - overCap = e; - } catch (Exception e) { - // Transient (ring full, recycle in flight) or terminal: the - // requeue below tells them apart with the probe. - logSettleFailure(lease.slot(), e); - } catch (Throwable t) { - // An Error (AssertionError, OutOfMemoryError) is not transient. - fatal = t; - } + for (int i = 0, n = candidates.size(); i < n; i++) { + SenderSlot s = candidates.get(i); + PooledSender lease; + lock.lock(); try { - if (fatal != null) { - LOG.warn("settleIdle: flush threw an Error, retiring slot {}", lease.slot().slotIndex(), fatal); - retireLease(lease, " during settle"); - } else if (overCap != null) { - LOG.error("settleIdle: slot {} holds a batch the server cap rejects, retiring it and dropping {} pending row(s): {}", - lease.slot().slotIndex(), pendingRowsOf(lease.slot()), overCap.getDetail()); - retireLease(lease, " during settle"); - } else { - requeueSettled(lease); + if (closed) { + break; + } + // A borrower may have taken the slot since the listing: check + // membership before reading the delegate of a leased slot. + if (!available.contains(s) || !hasUnshippedWork(s.delegate())) { + continue; } - } catch (Throwable t) { - // Keep going: the remaining leases must not stay out of `available`. - LOG.warn("settleIdle: could not requeue or retire slot {}", lease.slot().slotIndex(), t); + available.remove(s); + s.bumpGeneration(); + lease = new PooledSender(s, s.generation()); + } finally { + lock.unlock(); } - if (flushed) { + if (settleOne(lease)) { settled++; } } return settled; } + private boolean settleOne(PooledSender lease) { + boolean flushed = false; + Throwable fatal = null; + BatchTooLargeForCapException overCap = null; + try { + // PooledSender.flush() already marks the commit on the slot. + lease.flush(); + lease.slot().settleFailureLogged(false); + flushed = true; + } catch (BatchTooLargeForCapException e) { + // Deterministic: the probe would report it too, but no settle retry + // can ship this batch and no lease owns it any more. Retire the + // slot below. + overCap = e; + } catch (Exception e) { + // Transient (ring full, recycle in flight) or terminal: the + // requeue below tells them apart with the probe. + logSettleFailure(lease.slot(), e); + } catch (Throwable t) { + // An Error (AssertionError, OutOfMemoryError) is not transient. + fatal = t; + } + try { + if (fatal != null) { + LOG.warn("settleIdle: flush threw an Error, retiring slot {}", lease.slot().slotIndex(), fatal); + retireLease(lease, " during settle"); + } else if (overCap != null) { + LOG.error("settleIdle: slot {} holds a batch the server cap rejects, retiring it and dropping {} pending row(s): {}", + lease.slot().slotIndex(), pendingRowsOf(lease.slot()), overCap.getDetail()); + retireLease(lease, " during settle"); + } else { + requeueSettled(lease); + } + } catch (Throwable t) { + // Keep going: the remaining leases must not stay out of `available`. + LOG.warn("settleIdle: could not requeue or retire slot {}", lease.slot().slotIndex(), t); + } + return flushed; + } + private void logSettleFailure(SenderSlot slot, Exception e) { int idx = slot.slotIndex(); int pending = pendingRowsOf(slot); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index 97fac2471..edc5338a8 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -243,6 +243,94 @@ public void testPoolCloseDuringSettleRetiresSlotOnSettleThread() throws Exceptio } } + @Test + public void testSettleKeepsOtherSlotsBorrowableWhileOneFlushBlocks() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPool(2, 2, built)) { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + a.table("t").longColumn("v", 1L).atNow(); + b.table("t").longColumn("v", 2L).atNow(); + a.close(); + b.close(); + CountDownLatch gate = new CountDownLatch(1); + for (RecordingSender rs : built) { + rs.flushGate = gate; + } + Thread settler = new Thread(pool::settleIdle, "settler"); + settler.start(); + try { + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() == 2 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals("settle holds one slot at a time", 1, pool.availableSize()); + try (Sender borrowed = pool.borrow()) { + RecordingSender other = (RecordingSender) ((PooledSender) borrowed).getDelegateForTesting(); + gate.countDown(); + settler.join(2_000); + Assert.assertFalse(settler.isAlive()); + Assert.assertEquals("settle never flushes a slot a borrower took mid-pass", 0, other.flushCount.get()); + } + } finally { + gate.countDown(); + } + } + } + + @Test + public void testSettleStopsTakingSlotsOnceThePoolCloses() throws Exception { + List built = new ArrayList<>(); + SenderPool pool = SenderPoolCloseSemanticsTest.newPool(2, 2, built); + try { + Sender a = pool.borrow(); + Sender b = pool.borrow(); + a.table("t").longColumn("v", 1L).atNow(); + b.table("t").longColumn("v", 2L).atNow(); + a.close(); + b.close(); + CountDownLatch gate = new CountDownLatch(1); + for (RecordingSender rs : built) { + rs.flushGate = gate; + } + AtomicReference settleFailure = new AtomicReference<>(); + Thread settler = new Thread(() -> { + try { + pool.settleIdle(); + } catch (Throwable t) { + settleFailure.set(t); + } + }, "settler"); + Thread closer = new Thread(pool::close, "closer"); + settler.start(); + try { + long deadline = System.currentTimeMillis() + 2_000; + while (pool.availableSize() == 2 && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertTrue(pool.availableSize() < 2); + closer.start(); + deadline = System.currentTimeMillis() + 2_000; + while (closer.getState() != Thread.State.TIMED_WAITING && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + Assert.assertEquals("close() must be waiting for the internal lease", + Thread.State.TIMED_WAITING, closer.getState()); + } finally { + gate.countDown(); + } + settler.join(2_000); + closer.join(2_000); + Assert.assertFalse(settler.isAlive()); + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(settleFailure.get()); + Assert.assertEquals("a closing pool stops the settle after the slot in flight", 1, + built.get(0).flushCount.get() + built.get(1).flushCount.get()); + } finally { + pool.close(); + } + } + @Test public void testSettleHandlesSeveralCandidatesIndependently() { List built = new ArrayList<>(); From 425561340301fea28675f94615ccb355cd412927 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:27:50 +0100 Subject: [PATCH 51/57] fix(qwp): appendBlocking parks instead of spinning when interrupted parkNanos returns at once while the interrupt flag is set, so an interrupted producer, such as the housekeeper after stop(), spun at full CPU until the append deadline (review round 2, moderate 6). The flag is restored on exit. Co-Authored-By: Claude Opus 5.5 --- .../client/sf/cursor/CursorSendEngine.java | 48 ++++++++++++------- .../sf/cursor/CursorSendEngineTest.java | 39 +++++++++++++++ 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index daf38b77f..22966536f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -942,25 +942,37 @@ public long appendBlocking(long payloadAddr, int payloadLen) { // deadline clock. counters.backpressureStalls.incrementAndGet(); long deadlineNs = System.nanoTime() + appendDeadlineNanos; - while (true) { - long now = System.nanoTime(); - if (now >= deadlineNs) { - throw new io.questdb.client.cutlass.line.LineSenderException( - "cursor ring backpressured for ").put(appendDeadlineNanos / 1_000_000L) - .put(" ms - wire path is not draining (server slow / disconnected, " - + "periodic disk sync is slow, or sf_max_total_bytes is too small)"); - } - if (now - lastBackpressureLogNs >= BACKPRESSURE_LOG_THROTTLE_NANOS) { - lastBackpressureLogNs = now; - LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O or periodic disk sync; " - + "will throw after {} ms", - counters.backpressureStalls.get(), appendDeadlineNanos / 1_000_000L); + boolean wasInterrupted = false; + try { + // parkNanos returns at once while the interrupt flag is set, so the wait + // clears the flag before each park and restores it on exit. + while (true) { + long now = System.nanoTime(); + if (now >= deadlineNs) { + throw new io.questdb.client.cutlass.line.LineSenderException( + "cursor ring backpressured for ").put(appendDeadlineNanos / 1_000_000L) + .put(" ms - wire path is not draining (server slow / disconnected, " + + "periodic disk sync is slow, or sf_max_total_bytes is too small)"); + } + if (now - lastBackpressureLogNs >= BACKPRESSURE_LOG_THROTTLE_NANOS) { + lastBackpressureLogNs = now; + LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O or periodic disk sync; " + + "will throw after {} ms", + counters.backpressureStalls.get(), appendDeadlineNanos / 1_000_000L); + } + if (Thread.interrupted()) { + wasInterrupted = true; + } + LockSupport.parkNanos(50_000L); // 50 µs + fsn = ring.appendOrFsn(payloadAddr, payloadLen); + if (fsn >= 0) return fsn; + if (fsn == SegmentRing.PAYLOAD_TOO_LARGE) { + throw new MmapSegmentException("payload too large for segment"); + } } - LockSupport.parkNanos(50_000L); // 50 µs - fsn = ring.appendOrFsn(payloadAddr, payloadLen); - if (fsn >= 0) return fsn; - if (fsn == SegmentRing.PAYLOAD_TOO_LARGE) { - throw new MmapSegmentException("payload too large for segment"); + } finally { + if (wasInterrupted) { + Thread.currentThread().interrupt(); } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 7f802535a..405ca8fc6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -44,9 +44,12 @@ import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; @@ -344,6 +347,42 @@ public void testCheckDurabilitySurfacesLatchedFailureToSenderEntryPoints() throw }); } + @Test + public void testAppendBlockingParksInsteadOfSpinningWhenInterrupted() throws Exception { + ThreadMXBean mx = ManagementFactory.getThreadMXBean(); + Assume.assumeTrue(mx.isCurrentThreadCpuTimeSupported()); + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096, 2 * 4096, TimeUnit.MILLISECONDS.toNanos(300))) { + while (true) { // fill the ring; nothing drains it + try { + engine.appendBlocking(buf, 16); + } catch (LineSenderException full) { + break; + } + } + Thread.currentThread().interrupt(); + long cpuStart = mx.getCurrentThreadCpuTime(); + long wallStart = System.nanoTime(); + try { + engine.appendBlocking(buf, 16); + fail("a full ring rejects after the deadline"); + } catch (LineSenderException expected) { + // backpressure deadline + } + long wall = System.nanoTime() - wallStart; + long cpu = mx.getCurrentThreadCpuTime() - cpuStart; + assertTrue("the interrupt flag survives the wait", Thread.interrupted()); + assertTrue("waited out the deadline", wall >= TimeUnit.MILLISECONDS.toNanos(300)); + System.out.println("TASK7 cpu=" + cpu + " wall=" + wall); + assertTrue("parked, not spinning: cpu=" + cpu + " wall=" + wall, cpu < wall / 2); + } finally { + Thread.interrupted(); + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testAppendChecksLatchedDurabilityFailureBeforePublishing() throws Exception { TestUtils.assertMemoryLeak(() -> { From 8b29691fcfc72e4c70b5342bf61fd47a54f8a3c8 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:28:04 +0100 Subject: [PATCH 52/57] test(qwp): drop debug print from appendBlocking interrupt test Co-Authored-By: Claude Opus 5.5 --- .../test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 405ca8fc6..3536c518d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -374,7 +374,6 @@ public void testAppendBlockingParksInsteadOfSpinningWhenInterrupted() throws Exc long cpu = mx.getCurrentThreadCpuTime() - cpuStart; assertTrue("the interrupt flag survives the wait", Thread.interrupted()); assertTrue("waited out the deadline", wall >= TimeUnit.MILLISECONDS.toNanos(300)); - System.out.println("TASK7 cpu=" + cpu + " wall=" + wall); assertTrue("parked, not spinning: cpu=" + cpu + " wall=" + wall, cpu < wall / 2); } finally { Thread.interrupted(); From df6b84190c516410503df46feda92a1468649153 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:30:37 +0100 Subject: [PATCH 53/57] fix(qwp): an empty savepoint stores symbol watermark -1 Only the empty-savepoint branch of resetToSavepoint() kept watermark 0 from reaching the next flush, and no test covered it (review round 2, moderate 7). Co-Authored-By: Claude Opus 5.5 --- .../qwp/client/QwpWebSocketSender.java | 4 ++-- .../qwp/client/SenderProbeMethodsTest.java | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3d96cf960..d7793c6ab 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -484,7 +484,7 @@ public class QwpWebSocketSender implements Sender { // zero once a flush ships the pending rows, at which point resetToSavepoint() // discards every pending row like reset() and also removes the column // definitions the server has not seen yet. - private int savepointBatchMaxSymbolId; + private int savepointBatchMaxSymbolId = -1; private long savepointFirstPendingRowTimeNanos; private long savepointPendingBytes; private int savepointPendingRowCount; @@ -4639,7 +4639,7 @@ private void checkTableSelected() { } private void clearSavepointAndRejection() { - savepointBatchMaxSymbolId = 0; + savepointBatchMaxSymbolId = -1; savepointPendingRowCount = 0; savepointPendingBytes = 0; savepointFirstPendingRowTimeNanos = 0; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index aa0323508..5320290a1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -284,6 +284,26 @@ public void testDeferredFlushShipsFlaggedFrameAndFlushCommits() throws Exception } } + @Test + public void testFlushAfterAnEmptySavepointResetShipsTheNextRow() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(server.getPort()))) { + sender.markSavepoint(); + sender.table("t").longColumn("v", 1L).atNow(); + sender.flush(); // empties the savepoint + sender.table("t").longColumn("v", 2L).atNow(); + sender.resetToSavepoint(); // drops row 2 + sender.table("t").longColumn("v", 3L).atNow(); + sender.flush(); + log.awaitDataFrames(2, 5_000); + sender.checkError(); + } + } + } + @Test public void testFlushEmptiesTheSavepoint() throws Exception { try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) { From b5d5fb4fb0ccea74e3f7cfaad92f7ead5cf2a8b2 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:32:38 +0100 Subject: [PATCH 54/57] docs(pool): scope rejection isolation to no commit window; state the byte-trigger default Also corrects stale comments on the byte trigger, the savepoint counters, discardPendingBatchState and the new-slot borrow path. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 5 +++-- .../main/java/io/questdb/client/QuestDBBuilder.java | 8 +++++--- core/src/main/java/io/questdb/client/Sender.java | 2 +- .../cutlass/qwp/client/QwpWebSocketSender.java | 13 +++++++------ .../java/io/questdb/client/impl/PooledSender.java | 12 +++++++++--- .../java/io/questdb/client/impl/SenderPool.java | 4 ++-- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b571c7d0f..e8e808cfb 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,8 @@ pool's next settle; call `flush()` before `close()` if you need it on the wire n If you borrow per message and want each message in the store-and-forward ring the moment `close()` returns, set `flush_on_close=on` together with a commit window (`commit_rows`, `commit_interval_ms`): every close ships one frame -flagged defer-commit, the server appends it, and the pool commits once per window instead of once per close. +flagged defer-commit, the server appends it, and the pool commits once per window instead of once per close. A rejected +frame then rolls back the whole open group, so every lease since the last commit loses its rows. A server rejection covers a whole frame. With the default `flush_on_close=off` one frame carries the rows of every lease that closed since the last ship, so one rejected row discards the other leases' rows in that frame as well, @@ -561,7 +562,7 @@ schema::key1=value1;key2=value2; | `acquire_timeout_ms` | `5000` | How long `borrowSender()`/`borrowQuery()` waits for a free slot | | `idle_timeout_ms` | `60000` | How long a pooled connection may stay idle before it is reaped | | `max_lifetime_ms` | `1800000` | Maximum lifetime of a pooled connection before it is recycled | -| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged. Close-time errors are wrapped; the delegate's error is the cause; with `on` each lease ships its own frame, so a server rejection cannot take other leases' rows with it | +| `flush_on_close` | `off` | `on` makes a pooled `close()` flush before releasing (one frame and one commit per close, the pre-1.4 behaviour). Note: "flush" means published to the store-and-forward ring, not acknowledged. Close-time errors are wrapped; the delegate's error is the cause; with `on` and no commit window each lease ships its own frame, so a server rejection cannot discard other leases' rows; with `off` leases share frames and a rejection covers all of them; with a commit window a rejection rolls back the whole open group, every lease since the last commit | | `commit_rows` | _(unset)_ | Commit window: with `flush_on_close=on`, closes ship frames flagged defer-commit and the server commits once per window of this many rows. Acks arrive at the commit; `drain()` after `flush()` | | `commit_interval_ms` | _(unset)_ | Commit window age in milliseconds; setting either window key enables the window, the other defaults to 1000 rows / 100 ms. An idle slot's open group is committed at the settle cadence, so the interval only bounds groups on busy slots | diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java index 7a5eaf166..a2ed24b7e 100644 --- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java +++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java @@ -217,9 +217,11 @@ public QuestDBBuilder errorHandler(SenderErrorHandler handler) { * which is the slow path for applications that borrow per message. In this * client "flush" means published to the store-and-forward ring, not * acknowledged by the server; use {@link Sender#drain(long)} for that. - * With {@code on} every lease ships its own frame, so a server rejection - * cannot discard other leases' rows; with {@code off} leases share frames - * and a rejection covers all of them. + * With {@code on} and no commit window every lease ships its own frame, so + * a server rejection cannot discard other leases' rows. With {@code off} + * leases share frames and a rejection covers all of them. With a commit + * window a rejection rolls back the whole open group, every lease since the + * last commit. * Overrides {@code flush_on_close} in the config string. */ public QuestDBBuilder flushOnClose(boolean flushOnClose) { diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index a8e05901f..071ea84df 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1473,7 +1473,7 @@ public LineSenderBuilder authTimeoutMillis(long millis) { *
    * This is only used when communicating over WebSocket transport. *
    - * Default value is 0, which disables byte-based auto-flush. + * Unset by default, which flushes at 90 percent of the batch cap the server advertises, and not by bytes when it advertises none. 0 disables byte-based auto-flush. * * @param bytes maximum bytes per batch * @return this instance for method chaining diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d7793c6ab..6291056c7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -481,9 +481,9 @@ public class QwpWebSocketSender implements Sender { // Monotonic for the sender's life; reset() does not clear it. Counts rows whether or not they shipped. private long rowsAppended; // Pending counters and batch symbol watermark at the last markSavepoint(); - // zero once a flush ships the pending rows, at which point resetToSavepoint() - // discards every pending row like reset() and also removes the column - // definitions the server has not seen yet. + // the counters are zero and the watermark is -1 once a flush ships the pending + // rows, at which point resetToSavepoint() discards every pending row like + // reset() and also removes the column definitions the server has not seen yet. private int savepointBatchMaxSymbolId = -1; private long savepointFirstPendingRowTimeNanos; private long savepointPendingBytes; @@ -4823,9 +4823,10 @@ private void discardPendingBatchState() { // by the discarded batch keeps re-encoding that batch's symbols: a single-row // batch after reset() would still carry the whole abandoned range and hit the // very cap rejection reset() is documented to clear, leaving the sender unable - // to flush anything at all. -1 is the same value resetTableBuffersAfterFlush - // leaves behind after a successful flush, and sendCommitMessage already reads - // it as an empty delta. + // to flush anything at all. -1 is what resetTableBuffersAfterFlush leaves + // behind after a flush that ships every pending row (after a hold-back flush + // the watermark stays at the batch max while the held row is pending), and + // sendCommitMessage already reads -1 as an empty delta. currentBatchMaxSymbolId = -1; reclaimUnsentSymbolIds(0); pendingBytes = 0; diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index d0b9975f7..8e2525298 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -170,7 +170,10 @@ public void checkError() { *

    * A server batch cap rejection ({@link BatchTooLargeForCapException}, retained * by an auto-flush or thrown by close's own flush) does not retire the slot at - * once. The byte trigger fires after the crossing row is committed, so the + * once. QwpWebSocketSender handles a byte-trigger crossing row itself, so a + * rejection that reaches close comes from a lease whose own rows push the + * batch over the cap through an explicit flush() with the byte trigger off, + * or from a held row whose own frame exceeds the cap; either way the * rejected batch can hold mostly rows earlier leases left. The lease is rolled * back to its borrow mark, which drops only this lease's rows, and the flush is * retried: when the rows left by earlier leases ship, the slot is returned and a @@ -232,8 +235,11 @@ public void close() { failure = e; } if (failure instanceof BatchTooLargeForCapException) { - // The byte trigger fires after the crossing row is committed, so - // the rejected batch can hold mostly rows earlier leases left. + // QwpWebSocketSender handles a byte-trigger crossing row itself, so + // this rejection comes from the lease's own rows pushing the batch + // over the cap through an explicit flush() with the byte trigger + // off, or from a held row whose own frame exceeds the cap. The + // rejected batch can hold mostly rows earlier leases left. // Drop only this lease's rows and retry: the slot stays in service // when the rest ships, and an inherited batch the cap still // rejects falls through to the retirement below. diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 640f45a40..6472925c9 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -1331,8 +1331,8 @@ public PooledSender borrow() { creationFinished.signalAll(); PooledSender lease = new PooledSender(created, created.generation()); lock.unlock(); - // Every lease handed to a caller starts at a mark. The new slot is not - // yet visible to other threads, so the mark runs outside the pool lock. + // Every lease handed to a caller starts at a mark. The new slot is off + // the free list and owned by this borrower, so the mark runs outside the pool lock. created.delegate().markSavepoint(); return lease; } From b2de8e4f4cb3ee5eb5edd394abf85e6d8731fd84 Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:58:18 +0100 Subject: [PATCH 55/57] fix(qwp): final review round-2 fix-ups A. autoFlushAfterRow post-retain recheck also flushes a held row at 90 percent of the advertised cap with auto_flush_bytes=off B. README and QuestDB.borrowSender Javadoc: close() ships rows older than auto_flush_interval C. Sender.autoFlushIfDue Javadoc names the row, byte and age thresholds D. PooledSender.close Javadoc and catch comment give cap rejection sources as examples E. testAutoFlushIfDueShipsOnlyRowsOlderThanTheInterval uses a 2000 ms interval F. QwpTableBuffer.holdBackLastRow Javadoc covers prefix columns and symbols G. rename wasInterrupted, wrap long Javadoc lines Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- .../main/java/io/questdb/client/QuestDB.java | 8 ++-- .../main/java/io/questdb/client/Sender.java | 10 +++-- .../qwp/client/QwpWebSocketSender.java | 13 ++++++- .../client/sf/cursor/CursorSendEngine.java | 6 +-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 6 +++ .../io/questdb/client/impl/PooledSender.java | 13 ++++--- .../qwp/client/SenderProbeMethodsTest.java | 38 ++++++++++++++++++- 8 files changed, 76 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e8e808cfb..92872a1b7 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,8 @@ snippets below borrow from its pools. ### Ingest Data Borrow a `Sender`, write rows, and close it. Close releases, flush ships, drain waits: `close()` returns the sender to -the pool and ships nothing by itself; rows go out when an auto-flush threshold trips on a later append, at a later `close()` once they are older than -`auto_flush_interval` (100 ms by default), or when the pool's housekeeper settles the idle slot, typically within one +the pool and ships only rows older than `auto_flush_interval` (100 ms by default), otherwise it only releases; the other rows go out when an auto-flush threshold trips on a later append, at a later `close()` once they are older than +that interval, or when the pool's housekeeper settles the idle slot, typically within one `auto_flush_interval`. A lease that holds the slot without appending or closing, for example one that waits for input inside the `try` block, holds those rows back until it does: borrow once the data is in hand. Borrowing per message is the intended pattern and batches like a sender held for the whole run. Call `flush()` to ship now and diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 664b808a1..f383fc9de 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -138,12 +138,14 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * Borrows a {@link Sender} from the pool. The caller MUST call * {@link Sender#close()} on the returned instance to release it back to * the pool. Close releases, flush ships, drain waits: {@code close()} on a - * pooled Sender returns the slot and by default ships nothing, the rows go + * pooled Sender returns the slot and by default ships only rows older than + * {@code auto_flush_interval}, otherwise it only releases; the other rows go * out when an auto-flush threshold trips on a later append, at a later - * release once they are older than {@code auto_flush_interval}, or when the + * release once they are older than that interval, or when the * housekeeper settles the idle slot, within about one * {@code auto_flush_interval} of it going idle. A lease that holds the slot - * without appending or closing holds those rows back until it does. Call {@link Sender#flush()} to ship now and + * without appending or closing holds those rows back until it does. Call + * {@link Sender#flush()} to ship now and * {@link Sender#drain(long)} to wait for the server. Set * {@code flush_on_close=on} to have {@code close()} flush, as before 1.4. * With the default, one frame carries the rows of every lease that closed diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 071ea84df..48c757094 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -277,9 +277,10 @@ static Sender fromEnv() { void atNow(); /** - * Runs the auto-flush check that every append runs and ships the pending rows when it trips. - * With no row in progress, only the age threshold can trip here, which lets a pool ship rows - * a lease left behind once they are older than {@code auto_flush_interval}. + * Runs the same row, byte and age thresholds every append runs and ships the pending rows + * when one trips. At a pooled release the age threshold is the one that usually trips, which + * lets a pool ship rows a lease left behind once they are older than + * {@code auto_flush_interval}. It does nothing while a row is in progress. * Only the WebSocket QWP transport implements it. */ default void autoFlushIfDue() { @@ -1473,7 +1474,8 @@ public LineSenderBuilder authTimeoutMillis(long millis) { *
    * This is only used when communicating over WebSocket transport. *
    - * Unset by default, which flushes at 90 percent of the batch cap the server advertises, and not by bytes when it advertises none. 0 disables byte-based auto-flush. + * Unset by default, which flushes at 90 percent of the batch cap the server advertises, + * and not by bytes when it advertises none. 0 disables byte-based auto-flush. * * @param bytes maximum bytes per batch * @return this instance for method chaining diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6291056c7..28d1b325a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3904,6 +3904,10 @@ public boolean wasEverConnected() { return hasLoopEverConnected || (l != null && l.hasEverConnected()); } + private static long byteBudgetForCap(int cap) { + return (long) cap * 9 / 10; + } + private static Throwable captureCloseError(Throwable terminalError, Throwable t) { if (terminalError == null) { return t; @@ -3976,7 +3980,7 @@ public void applyServerBatchSizeLimit(int advertisedMaxBatchSize) { // a conservative proxy for wire size (compression usually shrinks the // payload), but schema and dict-delta overhead can push the wire size // above the raw total in pathological cases. - long safeBudget = (long) advertisedMaxBatchSize * 9 / 10; + long safeBudget = byteBudgetForCap(advertisedMaxBatchSize); if (autoFlushBytes > 0 && autoFlushBytes < safeBudget) { effectiveAutoFlushBytes = autoFlushBytes; } else { @@ -4045,7 +4049,12 @@ private void autoFlushAfterRow() { } throw t; } - if (shouldAutoFlush()) { + // With auto_flush_bytes=off shouldAutoFlush() has no byte test, so a held row + // whose own frame exceeds the advertised cap would stay in the slot after its + // lease. Flush it at once when it reaches the same 90 percent budget the + // default byte trigger uses. + int cap = serverMaxBatchSize; + if (shouldAutoFlush() || (cap > 0 && getPendingBytes() >= byteBudgetForCap(cap))) { flushPendingRows(transactional); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 22966536f..5ae526cbe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -942,7 +942,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { // deadline clock. counters.backpressureStalls.incrementAndGet(); long deadlineNs = System.nanoTime() + appendDeadlineNanos; - boolean wasInterrupted = false; + boolean hasClearedInterrupt = false; try { // parkNanos returns at once while the interrupt flag is set, so the wait // clears the flag before each park and restores it on exit. @@ -961,7 +961,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { counters.backpressureStalls.get(), appendDeadlineNanos / 1_000_000L); } if (Thread.interrupted()) { - wasInterrupted = true; + hasClearedInterrupt = true; } LockSupport.parkNanos(50_000L); // 50 µs fsn = ring.appendOrFsn(payloadAddr, payloadLen); @@ -971,7 +971,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { } } } finally { - if (wasInterrupted) { + if (hasClearedInterrupt) { Thread.currentThread().interrupt(); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 337673c9f..3965bfec4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -332,6 +332,12 @@ public boolean hasInProgressRow() { * drops the rows before it and keeps it as the only row. The caller * resolves the hold with one of the two before it writes, cancels or rolls * back rows. + *

    + * Columns first used by the held row are part of the prefix frame, all-null + * for the prefix rows, so the server learns them, including their type and + * any decimal scale or geohash precision, even if the held row is later + * rolled back. The local symbol dictionary of the prefix may carry entries + * only the held row uses. * * @return false, changing nothing, when the buffer has no committed row, * a row is in progress, or a row is already held diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 8e2525298..adc3d6b25 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -171,9 +171,9 @@ public void checkError() { * A server batch cap rejection ({@link BatchTooLargeForCapException}, retained * by an auto-flush or thrown by close's own flush) does not retire the slot at * once. QwpWebSocketSender handles a byte-trigger crossing row itself, so a - * rejection that reaches close comes from a lease whose own rows push the - * batch over the cap through an explicit flush() with the byte trigger off, - * or from a held row whose own frame exceeds the cap; either way the + * rejection that reaches close comes, for example, from a lease whose own + * rows push the batch over the cap through an explicit flush() with the byte + * trigger off, or from a held row whose own frame exceeds the cap. The * rejected batch can hold mostly rows earlier leases left. The lease is rolled * back to its borrow mark, which drops only this lease's rows, and the flush is * retried: when the rows left by earlier leases ship, the slot is returned and a @@ -236,9 +236,10 @@ public void close() { } if (failure instanceof BatchTooLargeForCapException) { // QwpWebSocketSender handles a byte-trigger crossing row itself, so - // this rejection comes from the lease's own rows pushing the batch - // over the cap through an explicit flush() with the byte trigger - // off, or from a held row whose own frame exceeds the cap. The + // this rejection comes, for example, from the lease's own rows + // pushing the batch over the cap through an explicit flush() with + // the byte trigger off, or from a held row whose own frame exceeds + // the cap. The // rejected batch can hold mostly rows earlier leases left. // Drop only this lease's rows and retry: the slot stays in service // when the rest ships, and an inherited batch the cap still diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 5320290a1..956f5adb6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -57,11 +57,11 @@ public void testAutoFlushIfDueShipsOnlyRowsOlderThanTheInterval() throws Excepti server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() - + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=300;close_flush_timeout_millis=0;")) { + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=2000;close_flush_timeout_millis=0;")) { sender.table("t").longColumn("v", 1L).atNow(); sender.autoFlushIfDue(); Assert.assertEquals("younger than the interval: kept", 1, sender.pendingRowCount()); - Thread.sleep(400); + Thread.sleep(2200); sender.autoFlushIfDue(); Assert.assertEquals("older than the interval: shipped", 0, sender.pendingRowCount()); log.awaitDataFrames(1, 5_000); @@ -237,6 +237,40 @@ public void testCrossingRowWhoseOwnFrameExceedsTheCapThrowsAfterTheRowsBeforeItS } } + @Test + public void testCrossingRowWhoseOwnFrameExceedsTheCapFlushesAtOnceWithTheByteTriggerOff() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_bytes=off;auto_flush_rows=4;auto_flush_interval=60000;close_flush_timeout_millis=0;")) { + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 3; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + try { + sender.table("u").stringColumn("z", QuestDBFlushOnCloseTest.wideValue(4070)).atNow(); + Assert.fail("the row's own frame exceeds the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + // the held row flushed at once and its own rejection reached this call + } + log.awaitDataFrames(1, 5_000); + Assert.assertEquals("only the crossing row stays buffered", 1, sender.pendingRowCount()); + try { + sender.checkError(); + Assert.fail("the probe must report the retained row"); + } catch (BatchTooLargeForCapException expected) { + // the pool retires a slot on this + } + sender.reset(); + Assert.assertEquals(0, sender.pendingRowCount()); + sender.checkError(); + } + } + } + @Test public void testDefaultsOnNonWebSocketTransport() { try (Sender sender = Sender.fromConfig("http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;")) { From d8ab9be7422125f080c9c3a7a37e8a0b1d4ae55a Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:46:11 +0100 Subject: [PATCH 56/57] fix(qwp): keep released rows visible at pool close and reap SenderPool.close() now logs ERROR with the row count when an idle slot's delegate close fails, as retireLease already did. Before, rows earlier leases released into the slot were dropped with no exception and no ERROR line. reapIdle() skips idle slots that still hold rows or an open commit group, so the settle keeps retrying instead of the reaper closing the slot and dropping them. A directly built SenderPool defaults flushOnClose to true again: it has no housekeeper to settle released rows. QuestDBImpl sets the value from flush_on_close, so the facade default stays off. The three older pool test classes go back to their base form, and the release-only test helper sets flushOnClose(false) explicitly. Adds tests for the reap skip, the shutdown ERROR line, the direct-pool default, a single over-cap row alone in the batch, and the defer flag on a transactional hold-back prefix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011PpCzTucC66DiMgFiSuduj --- .../io/questdb/client/impl/SenderPool.java | 37 ++++++--- .../qwp/client/SenderProbeMethodsTest.java | 61 +++++++++++++++ .../impl/SenderPoolCloseSemanticsTest.java | 75 ++++++++++++++++++- .../test/impl/SenderPoolErrorSafetyTest.java | 1 - .../test/impl/SenderPoolSettleTest.java | 30 ++++++++ .../client/test/impl/SenderPoolSfTest.java | 1 - .../client/test/impl/SenderPoolTest.java | 8 +- 7 files changed, 194 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 6472925c9..ab4ada32d 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -255,7 +255,9 @@ public final class SenderPool implements AutoCloseable { // production; lifecycle tests use it to acknowledge distinct retries // without inspecting transient Condition queue membership. private volatile Runnable creationWaitRetryHook; - private volatile boolean flushOnClose; + // On for a directly built pool, which has no housekeeper to settle rows a + // release leaves in an idle slot. QuestDBImpl sets it from flush_on_close. + private volatile boolean flushOnClose = true; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -1652,9 +1654,18 @@ public void close() { // Close each idle delegate outside the lock so a slow real-close() // doesn't keep the pool latched. for (int i = 0; i < idleSnapshot.length; i++) { + SenderSlot s = idleSnapshot[i]; + // Rows earlier leases released into the slot; a failed close drops + // them, and nothing else reports it. + int pendingRows = pendingRowsOf(s); try { - idleSnapshot[i].delegate().close(); - } catch (Throwable ignored) { + s.delegate().close(); + } catch (Throwable t) { + if (pendingRows > 0) { + String error = t.toString(); + LOG.error("SenderPool.close(): idle slot {} closed with {} unshipped row(s); delegate close failed: {}", + s.slotIndex(), pendingRows, error); + } // Best-effort: an Error from one delegate's teardown must not // abort the loop and strand the remaining delegates unclosed. } @@ -1733,8 +1744,9 @@ public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runna } /** - * Ships rows left behind in idle slots. A pooled {@code close()} does not - * flush by default, and auto-flush thresholds are only evaluated on append, + * Ships rows left behind in idle slots. With flush on close off, the + * {@code QuestDB} facade's default, a pooled {@code close()} does not flush, + * and auto-flush thresholds are only evaluated on append, * so a slot nobody borrows again would keep its last rows until the pool * closes. The housekeeper calls this on its settle cadence. *

    @@ -2080,7 +2092,10 @@ private void retireLease(PooledSender ps, String context) { /** * Closes idle slots that have exceeded {@code idleTimeoutMillis} or that * have aged past {@code maxLifetimeMillis}. Never shrinks below - * {@code minSize}. Called by the {@link PoolHousekeeper} on its tick. + * {@code minSize}. Skips a slot that still holds unshipped work: closing it + * would drop rows a release left there, so it waits until + * {@link #settleIdle()} ships them. Called by the {@link PoolHousekeeper} + * on its tick. */ public void reapIdle() { if (closed) { @@ -2105,7 +2120,7 @@ public void reapIdle() { && (now - s.idleSinceMillis()) >= idleTimeoutMillis; boolean overAge = maxLifetimeMillis < Long.MAX_VALUE && (now - s.createdAtMillis()) >= maxLifetimeMillis; - if (idleExpired || overAge) { + if ((idleExpired || overAge) && !hasUnshippedWork(s.delegate())) { it.remove(); all.remove(s); // Keep the SF slot reserved until its flock is released @@ -2197,9 +2212,11 @@ boolean isClosed() { /** * Whether a pooled {@code close()} ships the lease's rows before releasing - * the slot (the pre-1.4 behaviour) or only releases it. Off by default: - * rows then ship when an auto-flush threshold trips on a later append, or - * when the housekeeper settles the idle slot. + * the slot (the pre-1.4 behaviour) or only releases it. On by default for a + * directly built pool, since nothing settles its idle slots. The + * {@code QuestDB} facade sets it from {@code flush_on_close}, off by + * default: rows then ship when an auto-flush threshold trips on a later + * append, or when the housekeeper settles the idle slot. */ public boolean isFlushOnClose() { return flushOnClose; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java index 956f5adb6..6cd7e2e8e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SenderProbeMethodsTest.java @@ -132,6 +132,34 @@ public void testCheckErrorThrowsLatchedTerminal() throws Exception { } } + @Test + public void testCrossingRowInATransactionShipsTheRowsBeforeItDeferred() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + // transaction=on, which a commit window injects: the rows before the crossing + // row ship inside the open transaction, and only flush() commits it + try (Sender sender = Sender.fromConfig(configWithoutAutoFlushBytes(server.getPort()) + "transaction=on;")) { + String wide = QuestDBFlushOnCloseTest.wideValue(1000); + for (int i = 0; i < 3; i++) { + sender.table("t").stringColumn("s", wide).atNow(); + } + sender.table("t").stringColumn("s", QuestDBFlushOnCloseTest.wideValue(1500)).atNow(); + Assert.assertEquals("only the crossing row stays buffered", 1, sender.pendingRowCount()); + log.awaitDataFrames(1, 5_000); + Assert.assertTrue("the rows before the crossing row ship without committing", log.deferFlag(0)); + Assert.assertTrue(sender.hasDeferredFrames()); + sender.flush(); + log.awaitDataFrames(2, 5_000); + Assert.assertFalse("flush() commits the transaction with the held row", log.deferFlag(1)); + Assert.assertEquals(0, log.commitFrames()); + sender.checkError(); + } + } + } + @Test public void testCrossingRowShipsTheRowsBeforeIt() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); @@ -521,6 +549,39 @@ public void testResetToSavepointWithoutMarkIsReset() throws Exception { } } + @Test + public void testSingleRowWhoseOwnFrameExceedsTheCapThrowsAndStaysBuffered() throws Exception { + QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); + try (TestWebSocketServer server = new TestWebSocketServer(log)) { + server.setAdvertisedMaxBatchSize(4 * 1024); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(configWithoutAutoFlushBytes(server.getPort()))) { + // the crossing row is the only pending row: no rows before it can ship, + // so the sender holds nothing back and the rejection reaches this call + BatchTooLargeForCapException rejection = null; + try { + sender.table("u").stringColumn("z", QuestDBFlushOnCloseTest.wideValue(4070)).atNow(); + Assert.fail("the only pending row's own frame exceeds the 4 KiB cap"); + } catch (BatchTooLargeForCapException expected) { + rejection = expected; + } + Assert.assertEquals("the row stays buffered", 1, sender.pendingRowCount()); + try { + sender.checkError(); + Assert.fail("the probe must report the retained row"); + } catch (BatchTooLargeForCapException expected) { + Assert.assertSame(rejection, expected); + } + sender.reset(); + sender.table("u").longColumn("v", 1L).atNow(); + sender.flush(); + log.awaitDataFrames(1, 5_000); + sender.checkError(); + } + } + } + @Test public void testUnsetAutoFlushBytesShipsAtTheAdvertisedCap() throws Exception { QuestDBFlushOnCloseTest.FrameLog log = new QuestDBFlushOnCloseTest.FrameLog(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java index dde00b3ae..9816b5649 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseSemanticsTest.java @@ -25,6 +25,10 @@ package io.questdb.client.test.impl; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import io.questdb.client.LineSenderServerException; import io.questdb.client.Sender; import io.questdb.client.SenderError; @@ -33,6 +37,7 @@ import io.questdb.client.impl.SenderPool; import org.junit.Assert; import org.junit.Test; +import org.slf4j.LoggerFactory; import java.lang.reflect.Field; import java.util.ArrayList; @@ -74,7 +79,7 @@ public void testCancelRowThrowingAtCloseRetiresSlot() { } @Test - public void testCloseDoesNotFlushByDefault() { + public void testCloseDoesNotFlushWithFlushOnCloseOff() { List built = new ArrayList<>(); try (SenderPool pool = newPool(1, 1, built)) { Assert.assertFalse(pool.isFlushOnClose()); @@ -111,6 +116,21 @@ public void testCloseWrapperKeepsTheDelegatesRetryableFlag() { } } + @Test + public void testDirectPoolFlushesOnCloseByDefault() { + List built = new ArrayList<>(); + try (SenderPool pool = newPoolWithDefaults(1, 1, Long.MAX_VALUE, built)) { + Assert.assertTrue("a direct pool has no housekeeper to settle released rows", pool.isFlushOnClose()); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + Assert.assertEquals(1, delegate.flushCount.get()); + Assert.assertEquals(0, delegate.pendingRowCount()); + Assert.assertEquals(1, pool.availableSize()); + } + } + @Test public void testFlushOnCloseRestoresShippingAtClose() { List built = new ArrayList<>(); @@ -244,6 +264,49 @@ public void testLatchedServerErrorAtCloseKeepsItsType() { } } + @Test + public void testPoolCloseLogsRowsAFailedIdleCloseDrops() { + Logger poolLogger = (Logger) LoggerFactory.getLogger(SenderPool.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + Level savedLevel = poolLogger.getLevel(); + poolLogger.setLevel(Level.ALL); + poolLogger.addAppender(appender); + try { + List built = new ArrayList<>(); + SenderPool pool = new SenderPool(CONFIG, 1, 1, 150L, Long.MAX_VALUE, Long.MAX_VALUE, idx -> { + RecordingSender s = new RecordingSender() { + @Override + public void close() { + super.close(); + throw new LineSenderException("cursor SF append failed"); + } + }; + built.add(s); + return s; + }); + pool.setFlushOnClose(false); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + s.table("t").longColumn("v", 2L).atNow(); + } + pool.close(); + Assert.assertEquals(1, built.get(0).closeCount.get()); + boolean logged = false; + for (ILoggingEvent e : appender.list) { + String msg = e.getFormattedMessage(); + if (e.getLevel() == Level.ERROR && msg.contains("2 unshipped row(s)") + && msg.contains("cursor SF append failed")) { + logged = true; + } + } + Assert.assertTrue("a failed idle close must log the rows it drops", logged); + } finally { + poolLogger.detachAppender(appender); + poolLogger.setLevel(savedLevel); + } + } + @Test public void testResetAfterAFlushInsideTheLeaseDropsEveryPendingRow() { List built = new ArrayList<>(); @@ -358,8 +421,16 @@ public void testStaleResetDoesNotTouchReborrowedSlot() { } } + // Release-only close, the QuestDB facade's default. A directly built pool + // flushes on close unless told otherwise. static SenderPool newPool(int min, int max, List built) { - return new SenderPool(CONFIG, min, max, 150L, Long.MAX_VALUE, Long.MAX_VALUE, idx -> { + SenderPool pool = newPoolWithDefaults(min, max, Long.MAX_VALUE, built); + pool.setFlushOnClose(false); + return pool; + } + + static SenderPool newPoolWithDefaults(int min, int max, long idleTimeoutMillis, List built) { + return new SenderPool(CONFIG, min, max, 150L, idleTimeoutMillis, Long.MAX_VALUE, idx -> { RecordingSender s = new RecordingSender(); built.add(s); return s; diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 941b5ba93..38561c46a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -171,7 +171,6 @@ public void flushErrorDiscardsBrokenSenderInsteadOfRecycling() throws Exception IntFunction factory = slotIndex -> flushThrowingSender(); try (SenderPool pool = newPool(CFG, 1, 1, 1_000, factory)) { - pool.setFlushOnClose(true); // this test is about the flushing close path Sender first = pool.borrow(); // Capture the underlying slot before close(): borrow() always hands // out a FRESH PooledSender wrapper, so assertNotSame(first, second) diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java index edc5338a8..24ee8eb90 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSettleTest.java @@ -48,6 +48,35 @@ */ public class SenderPoolSettleTest { + @Test + public void testReapSkipsSlotUntilSettleShipsItsRows() throws Exception { + List built = new ArrayList<>(); + try (SenderPool pool = SenderPoolCloseSemanticsTest.newPoolWithDefaults(0, 1, 1L, built)) { + pool.setFlushOnClose(false); + try (Sender s = pool.borrow()) { + s.table("t").longColumn("v", 1L).atNow(); + } + RecordingSender delegate = built.get(0); + Thread.sleep(20); + pool.reapIdle(); + Assert.assertEquals("an idle-expired slot holding rows is not reaped", 1, pool.totalSize()); + Assert.assertEquals(0, delegate.closeCount.get()); + Assert.assertEquals(1, delegate.pendingRowCount()); + + delegate.deferredFramesOpen = true; + delegate.pendingRows = 0; + pool.reapIdle(); + Assert.assertEquals("an open commit group is unshipped work too", 1, pool.totalSize()); + + delegate.deferredFramesOpen = false; + delegate.pendingRows = 1; + Assert.assertEquals(1, pool.settleIdle()); + pool.reapIdle(); + Assert.assertEquals("the reap closes the slot once the settle shipped it", 0, pool.totalSize()); + Assert.assertEquals(1, delegate.closeCount.get()); + } + } + @Test public void testSettleRetiresSlotHoldingAnOverCapBatch() { List built = new ArrayList<>(); @@ -194,6 +223,7 @@ public void testPoolCloseDuringSettleRetiresSlotOnSettleThread() throws Exceptio built.add(rs); return rs; }); + pool.setFlushOnClose(false); try { try (Sender s = pool.borrow()) { s.table("t").longColumn("v", 1L).atNow(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index ae07944ae..7744cd1a1 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -1410,7 +1410,6 @@ public void testBorrowerWokenWhenRecycleRetainedEngineReleasesSlot() throws Exce String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; Thread borrower = null; try (SenderPool pool = new SenderPool(config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - pool.setFlushOnClose(true); PooledSender lease = pool.borrow(); QwpWebSocketSender delegate = (QwpWebSocketSender) getDelegate(lease); lease.table("t").symbol("s", "a").longColumn("v", 1L).atNow(); diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java index 792f2622d..1aedb3ea1 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java @@ -44,9 +44,9 @@ *

    * Tests never call methods that would attempt a network round-trip * (no {@code flush}, no row builders that auto-flush). Pooled - * {@link io.questdb.client.impl.PooledSender#close()} only flushes when - * {@code setFlushOnClose(true)} is set; the two tests that need the flushing - * path set it. + * {@link io.questdb.client.impl.PooledSender#close()} does call + * {@code delegate.flush()}, but on an empty buffer that path is a no-op for + * HTTP transport. */ public class SenderPoolTest { @@ -82,7 +82,6 @@ public void testBrokenSenderIsNotReturnedToPool() throws Exception { // is also unusable. Either way, the next borrower must get a fresh // slot, not the broken one. try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - pool.setFlushOnClose(true); // this test is about the flushing close path Sender first = pool.borrow(); Object firstSlot = slotOf(first); first.table("t").longColumn("v", 1).atNow(); @@ -139,7 +138,6 @@ public void testLeaseReturnedAfterCloseTearsDownItsOwnDelegate() { // acquireTimeout is kept short: it doubles as close()'s bounded // lease-wait budget, which this test intentionally exhausts. try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 150, Long.MAX_VALUE, Long.MAX_VALUE)) { - pool.setFlushOnClose(true); // this test is about the flushing close path Sender s = pool.borrow(); // A row in the buffer forces flush() to attempt a real HTTP // request on close(); against the unreachable DEAD_HTTP target From 47603c712b22ae674bf54ea7bb7ef85e63d2956e Mon Sep 17 00:00:00 2001 From: Sergei Minaev <5072859+jovfer@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:46:11 +0100 Subject: [PATCH 57/57] docs(pool): describe what a release ships accurately autoFlushIfDue() measures the age of the oldest pending row and then ships every pending row, and the row and byte thresholds apply too. The README, QuestDB.borrowSender and PooledSender.ship Javadoc said a release ships only rows older than auto_flush_interval. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011PpCzTucC66DiMgFiSuduj --- README.md | 9 +++++---- core/src/main/java/io/questdb/client/QuestDB.java | 10 +++++----- .../main/java/io/questdb/client/impl/PooledSender.java | 5 +++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 92872a1b7..3052c9c6b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ try (QuestDB db = QuestDB.connect("ws::addr=localhost:9000;")) { .doubleColumn("price", 2615.54) .doubleColumn("amount", 0.00044) .atNow(); - // close() returns the Sender to the pool; rows ship by threshold, at a later close once older than auto_flush_interval, or when the idle slot is settled. + // close() returns the Sender to the pool; rows ship once an auto-flush threshold trips (age, rows or bytes), or when the idle slot is settled. } } ``` @@ -99,9 +99,10 @@ snippets below borrow from its pools. ### Ingest Data Borrow a `Sender`, write rows, and close it. Close releases, flush ships, drain waits: `close()` returns the sender to -the pool and ships only rows older than `auto_flush_interval` (100 ms by default), otherwise it only releases; the other rows go out when an auto-flush threshold trips on a later append, at a later `close()` once they are older than -that interval, or when the pool's housekeeper settles the idle slot, typically within one -`auto_flush_interval`. A lease that holds the slot without appending or closing, for example one that waits for input +the pool and ships the slot's pending rows once the oldest is older than `auto_flush_interval` (100 ms by default) or +another auto-flush threshold trips, otherwise it only releases; rows left in the slot go out when an auto-flush +threshold trips on a later append or `close()`, or when the pool's housekeeper settles the idle slot, typically within +one `auto_flush_interval`. A lease that holds the slot without appending or closing, for example one that waits for input inside the `try` block, holds those rows back until it does: borrow once the data is in hand. Borrowing per message is the intended pattern and batches like a sender held for the whole run. Call `flush()` to ship now and `drain()` to wait for the server. The underlying connection is only torn down when the `QuestDB` handle itself is diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index f383fc9de..e6dee7a82 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -138,11 +138,11 @@ static QuestDB connect(CharSequence configurationString, HttpTokenProvider token * Borrows a {@link Sender} from the pool. The caller MUST call * {@link Sender#close()} on the returned instance to release it back to * the pool. Close releases, flush ships, drain waits: {@code close()} on a - * pooled Sender returns the slot and by default ships only rows older than - * {@code auto_flush_interval}, otherwise it only releases; the other rows go - * out when an auto-flush threshold trips on a later append, at a later - * release once they are older than that interval, or when the - * housekeeper settles the idle slot, within about one + * pooled Sender returns the slot and by default ships the slot's pending + * rows once the oldest is older than {@code auto_flush_interval} or another + * auto-flush threshold trips, otherwise it only releases; rows left in the + * slot go out when an auto-flush threshold trips on a later append or + * release, or when the housekeeper settles the idle slot, within about one * {@code auto_flush_interval} of it going idle. A lease that holds the slot * without appending or closing holds those rows back until it does. Call * {@link Sender#flush()} to ship now and diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index adc3d6b25..a3f1183d5 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -573,8 +573,9 @@ SenderSlot slot() { * pending rows unflagged and commits the group; *

  • otherwise {@code flush_on_close=on}: {@code flushDeferred()} with a * window, {@code flush()} without one;
    • - *
  • otherwise {@code autoFlushIfDue()}: ships only rows older than - * {@code auto_flush_interval}.
    • + *
  • otherwise {@code autoFlushIfDue()}: ships every pending row once + * the oldest is older than {@code auto_flush_interval} or another + * auto-flush threshold trips.
    • * */ private void ship(SenderPool pool, Sender delegate) {