From 62376ee833123aa35b16360350f9e55636b04ce8 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 3 Oct 2026 17:03:49 +0530 Subject: [PATCH 01/13] fix(store): size the RocksDB memory budget from available memory The shipped application-pd.yml set rocksdb.total_memory_size to 32 GB. RaftRocksdbOptions splits that into the write buffer manager's cache and the block cache, both off-heap, so -Xmx and the container limit do not bound them. A Store under a 4 GiB limit was OOM-killed once it held about 1 GB of data (#3254). The heap-size fallback in AppConfig never ran, because the key was always set. total_memory_size now ships as 0. Unset or 0 sizes the budget as half of what is left after the heap, MaxDirectMemorySize (heap size when unset) and a 1.5 GiB reserve for the raft log cache and other native memory, clamped to 256 MiB..32 GB. Physical memory comes from the OperatingSystemMXBean, which on JDK 17 reports the cgroup v1/v2 limit in a container; older JDK 17 updates may not see a limit set only on a parent cgroup. A positive value is used as given, but values under 1 MiB are refused, since the write buffer manager's 2/3 share would round to 0 bytes and lift the memtable limit. A non-numeric value such as "4GB" fails with a clear message instead of a bare NumberFormatException. Startup warns when the floor applies, when an explicit value is under 256 MiB, or when it exceeds what is left after the heap, direct memory and the reserve. The Docker entrypoint, which builds SPRING_APPLICATION_JSON itself, had no HG_STORE_* variable for any rocksdb.* key. It now accepts HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE (bytes) and adds it only when set; the value outranks the config file and -Drocksdb.total_memory_size. AppConfig also stops computing a zero raft disruptor buffer for budgets under 1 GB, where log(0) rounded to a size of 0; the formula moved to RocksdbMemoryBudget.disruptorBufferSize() so it can be tested. The shipped application.yml pins disruptorBufferSize, so this only applies when that key is unset. The comment in helm/hugegraph/values-cluster.yaml that said container limits do not configure the RocksDB budget now describes the auto-sizing and HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE. Fixes #3254 --- .github/workflows/pd-store-ci.yml | 4 + helm/hugegraph/values-cluster.yaml | 8 +- .../travis/test-store-docker-entrypoint.sh | 107 ++++++++++++ hugegraph-store/README.md | 4 + hugegraph-store/docs/deployment-guide.md | 35 +++- .../store/options/RocksdbMemoryBudget.java | 160 ++++++++++++++++++ .../hg-store-dist/docker/docker-entrypoint.sh | 19 ++- .../assembly/static/conf/application-pd.yml | 14 +- .../hugegraph/store/node/AppConfig.java | 13 +- .../src/main/resources/application-pd.yml | 5 +- .../hugegraph/store/core/CoreSuiteTest.java | 4 +- .../options/RocksdbMemoryBudgetTest.java | 97 +++++++++++ 12 files changed, 441 insertions(+), 29 deletions(-) create mode 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh create mode 100644 hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java create mode 100644 hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 8dcc74b8f7..76aaab94f5 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -406,6 +406,10 @@ jobs: run: | mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp + - name: Run Store docker entrypoint rocksdb memory tests + run: | + $TRAVIS_DIR/test-store-docker-entrypoint.sh + - name: Check startup test prerequisites (Store) id: store-preflight run: | diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index c9b39bd3d9..8915b47aa9 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -57,9 +57,11 @@ store: -XX:MaxMetaspaceSize=256m -XX:MaxDirectMemorySize=512m -XX:+UseContainerSupport - # Container limits do not configure JVM heap or RocksDB native cache budgets. - # Size this preset for the workload and configure those application budgets - # separately; javaOpts accepts explicit JVM/system-property overrides. + # Container limits do not configure the JVM heap; javaOpts does. Store images + # with auto-sized RocksDB memory (apache/hugegraph#3254) give RocksDB half of + # the container limit left after heap, direct memory and a 1.5 GiB reserve; + # with this preset that is 2.5 GiB. Set store.extraEnv + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE (bytes) to use a different value. resources: requests: cpu: "1" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh new file mode 100755 index 0000000000..a16198f11b --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Checks that the Store Docker entrypoint passes HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE +# into SPRING_APPLICATION_JSON as rocksdb.total_memory_size, leaves the rocksdb key +# out when the variable is unset (so conf/application-pd.yml or auto-sizing applies), +# and refuses values that are not a plain number of bytes. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" +ENTRYPOINT="${REPO_ROOT}/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh" +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/store-entrypoint-test.XXXXXX") +trap 'rm -rf "${TMP_DIR}"' EXIT + +PASS=0 +FAIL=0 + +[[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; } + +mkdir -p "${TMP_DIR}/bin" +cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh" +# Stand in for the launcher: record the generated config instead of starting Store +cat > "${TMP_DIR}/bin/start-hugegraph-store.sh" <<'STUB' +#!/usr/bin/env bash +printf '%s' "${SPRING_APPLICATION_JSON}" > ./spring.json +STUB +chmod +x "${TMP_DIR}/bin/start-hugegraph-store.sh" "${TMP_DIR}/docker-entrypoint.sh" + +run_entrypoint() { + rm -f "${TMP_DIR}/spring.json" + # Drop any value from the caller's environment so "unset" really is unset + (cd "${TMP_DIR}" && env -u HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE \ + HG_STORE_PD_ADDRESS=pd0:8686 \ + HG_STORE_GRPC_HOST=store0 \ + HG_STORE_RAFT_ADDRESS=store0:8510 \ + "$@" \ + ./docker-entrypoint.sh 2>&1) +} + +expect_memory() { + # expect_memory [NAME=VALUE ...] + local name="$1" expected="$2" + shift 2 + local out + if ! out=$(run_entrypoint "$@"); then + echo " FAIL ${name}: entrypoint exited non-zero" + printf '%s\n' "${out}" | tail -3 + FAIL=$((FAIL + 1)) + return + fi + if ! EXPECTED="${expected}" python3 - "${TMP_DIR}/spring.json" <<'PY' +import json, os, sys +with open(sys.argv[1], encoding="utf-8") as fh: + doc = json.load(fh) +want = os.environ["EXPECTED"] +got = doc.get("rocksdb", {}).get("total_memory_size", "absent") +if got != want: + print(" rocksdb.total_memory_size is %r, expected %r" % (got, want)) + sys.exit(1) +if doc["app"]["data-path"] != "/hugegraph-store/storage": + print(" app.data-path was lost") + sys.exit(1) +PY + then + echo " FAIL ${name}: invalid JSON or wrong rocksdb.total_memory_size" + FAIL=$((FAIL + 1)) + return + fi + echo " PASS ${name}" + PASS=$((PASS + 1)) +} + +echo "Store docker-entrypoint rocksdb memory override" +expect_memory "unset leaves the key out" absent +expect_memory "empty leaves the key out" absent HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE= +expect_memory "bytes are passed through" 2147483648 \ + HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=2147483648 +expect_memory "0 is passed through for auto-sizing" 0 HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=0 + +for bad in '4GB' '-1' '1.5e9' '1 2' '1"}'; do + if run_entrypoint HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE="${bad}" >/dev/null; then + echo " FAIL '${bad}' was accepted" + FAIL=$((FAIL + 1)) + else + echo " PASS '${bad}' is refused" + PASS=$((PASS + 1)) + fi +done + +echo "${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" -eq 0 ]] diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index e742391348..f60fa453ff 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -389,6 +389,10 @@ docker run -d \ hugegraph/store:latest ``` +Add `-e HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=` to pin the off-heap RocksDB budget. +Without it the Store sizes the budget from the container memory limit; see +[Store Configuration](docs/deployment-guide.md#store-configuration) in the deployment guide. + **Exposed Ports**: - `8520`: REST API (management, metrics) - `8500`: gRPC (client connections) diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 3d4f798a59..2b98ac3766 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -347,7 +347,7 @@ File: `hugegraph-store/conf/application-pd.yml` (RocksDB tuning) ```yaml rocksdb: # Memory Configuration - total_memory_size: 32000000000 # Total memory for RocksDB (32GB) + total_memory_size: 0 # Off-heap RocksDB budget in bytes; 0 = auto-size write_buffer_size: 134217728 # Memtable size (128MB) max_write_buffer_number: 6 # Max memtables min_write_buffer_number_to_merge: 2 # Min memtables to merge @@ -356,14 +356,32 @@ rocksdb: level0_file_num_compaction_trigger: 4 max_background_jobs: 8 # Background compaction/flush threads - # Block Cache - block_cache_size: 16000000000 # Block cache (16GB) + # Budget split: this share goes to memtables, the rest to the block cache + write_buffer_ratio: 0.66 # SST File Size target_file_size_base: 134217728 # Target SST size (128MB) max_bytes_for_level_base: 1073741824 # L1 size (1GB) ``` +`total_memory_size` bounds the memtables (`write_buffer_ratio` of it) and the block cache +(the rest). Both live outside the Java heap, so `-Xmx` does not limit them. A fixed 1 GiB +cache for the Raft log comes on top of this budget. + +With `total_memory_size: 0`, or the key unset, the Store sizes the budget at startup as half +of what is left after the heap, the maximum direct memory and a 1.5 GiB reserve, clamped to +256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, so a Store +with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without +`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts +it that way, so set it explicitly in small containers or the budget falls to the 256 MiB floor. +The chosen value is logged by `RocksdbMemoryBudget` at startup, with a warning when the floor +applies, or when an explicit value is under 256 MiB or exceeds the memory left after the heap, +direct memory and the reserve. Set a number of bytes, at least 1048576 (1 MiB), to pin it; a +smaller value or a unit suffix such as `4GB` is rejected. Before this default existed the +shipped value was `32000000000`, which can exceed a container limit and get the Store +OOM-killed. Both caps are soft: RocksDB can briefly go over them under heavy writes, and a +small budget with many partitions means frequent memtable flushes. + ### Server Configuration File: `hugegraph-server/conf/graphs/hugegraph.properties` @@ -726,8 +744,15 @@ environment: HG_STORE_REST_PORT: "8520" # maps to server.port HG_STORE_RAFT_ADDRESS: store0:8510 # maps to raft.address HG_STORE_DATA_PATH: /hugegraph-store/storage # maps to app.data-path + # optional; maps to rocksdb.total_memory_size (bytes); unset = config file or auto-size + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE: "2147483648" ``` +`HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` must be a plain number of bytes; the entrypoint refuses +anything else. When set it outranks `rocksdb.total_memory_size` in a mounted +`conf/application-pd.yml` and a `-Drocksdb.total_memory_size` in `JAVA_OPTS`. Leave it unset to let the Store size the budget from the container +memory limit (see [Store Configuration](#store-configuration)). + **Server environment variables**: ```yaml @@ -826,6 +851,10 @@ spec: value: "$(POD_NAME).hugegraph-store" - name: HG_STORE_RAFT_ADDRESS value: "$(POD_NAME).hugegraph-store:8510" + # Optional: pin the off-heap RocksDB budget (bytes). Unset, it is sized + # from the container memory limit below. + # - name: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE + # value: "4294967296" volumeMounts: - name: data mountPath: /hugegraph-store/storage diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java new file mode 100644 index 0000000000..47b104f77b --- /dev/null +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.lang.management.ManagementFactory; + +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.OperatingSystemMXBean; + +import lombok.extern.slf4j.Slf4j; + +/** + * Resolves rocksdb.total_memory_size, the budget RaftRocksdbOptions splits into the + * write buffer manager's cache and the block cache. Both caches are off-heap, so an + * unset or "0" budget is sized from the memory left after the heap, direct memory and + * the fixed raft log cache, instead of a fixed number that can exceed a container limit. + */ +@Slf4j +public final class RocksdbMemoryBudget { + + public static final String KEY = "total_memory_size"; + + static final long MIB = 1024L * 1024L; + // The 1 GiB raft log block cache (RaftRocksdbOptions) plus thread stacks, metaspace + // and other native allocations that sit outside the budget + static final long RESERVED = 1024 * MIB + 512 * MIB; + static final long MIN_AUTO = 256 * MIB; + // RaftRocksdbOptions gives the write buffer manager 2/3 of the budget, and a 0-byte + // write buffer manager means no memtable limit at all + static final long MIN_EXPLICIT = MIB; + // The value shipped before auto-sizing; a large host never gets more than it had + static final long MAX_AUTO = 32_000_000_000L; + + private RocksdbMemoryBudget() { + } + + /** + * Returns the configured budget in bytes, or an automatic one when the value is + * missing, blank or 0. + */ + public static long resolve(String configured) { + long value = isAuto(configured) ? 0L : parse(configured); + long heap = Runtime.getRuntime().maxMemory(); + long physical = physicalMemory(); + if (value > 0) { + if (value < MIN_EXPLICIT) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be 0 (auto) or at least %s bytes, " + + "got %s", MIN_EXPLICIT, value)); + } + if (value < MIN_AUTO) { + log.warn("rocksdb.total_memory_size={} is below the {} bytes auto-sizing " + + "never goes under", value, MIN_AUTO); + } + long direct = maxDirectMemory(heap); + long left = physical - heap - direct - RESERVED; + if (physical > 0 && value > left) { + log.warn("rocksdb.total_memory_size={} exceeds the {} bytes left after the " + + "heap, direct memory and the {} bytes reserve (physical={}, heap={}, " + + "direct={}); the process may be OOM-killed", + value, left, RESERVED, physical, heap, direct); + } + return value; + } + long direct = maxDirectMemory(heap); + long budget = auto(physical, heap, direct); + log.info("rocksdb.total_memory_size is unset or 0, auto-sizing to {} bytes " + + "(physical={}, heap={}, direct={})", budget, physical, heap, direct); + if (physical > 0 && physical - heap - direct - RESERVED < 2 * MIN_AUTO) { + log.warn("Heap, direct memory and the {} bytes reserve leave too little of the " + + "{} bytes physical memory, rocksdb gets the {} bytes floor; lower -Xmx " + + "or -XX:MaxDirectMemorySize", RESERVED, physical, MIN_AUTO); + } + return budget; + } + + static boolean isAuto(String configured) { + return configured == null || configured.trim().isEmpty(); + } + + static long parse(String configured) { + try { + long value = Long.parseLong(configured.trim()); + if (value < 0) { + throw new NumberFormatException("negative"); + } + return value; + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be a number of bytes, got '%s'", + configured), e); + } + } + + /** + * Half of what is left after the heap, direct memory and the reserve, clamped to + * [MIN_AUTO, MAX_AUTO]. Falls back to the heap size, the previous default, when + * physical memory is unknown. + */ + static long auto(long physical, long heap, long direct) { + if (physical <= 0) { + return Math.min(Math.max(heap, MIN_AUTO), MAX_AUTO); + } + long available = physical - heap - direct - RESERVED; + return Math.min(Math.max(available / 2, MIN_AUTO), MAX_AUTO); + } + + /** + * The raft disruptor buffer size AppConfig derives from the budget when + * raft.disruptorBufferSize is 0: the budget in GB, at least 1, rounded to a power + * of two, times 32. + */ + public static int disruptorBufferSize(long totalMemory) { + // A budget under 1 GB would otherwise give log(0) and a zero-sized buffer + int size = (int) Math.max(1, totalMemory / 1000 / 1000 / 1000); + return (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; + } + + private static long physicalMemory() { + try { + // Reports the cgroup v1/v2 memory limit inside a container; older JDK 17 + // updates may not see a limit set only on a parent cgroup + OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean( + OperatingSystemMXBean.class); + return os == null ? -1L : os.getTotalMemorySize(); + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read physical memory size", e); + return -1L; + } + } + + private static long maxDirectMemory(long heap) { + try { + HotSpotDiagnosticMXBean hotspot = ManagementFactory.getPlatformMXBean( + HotSpotDiagnosticMXBean.class); + long direct = Long.parseLong( + hotspot.getVMOption("MaxDirectMemorySize").getValue()); + // 0 means unset, and the JVM then allows as much direct memory as heap + return direct > 0 ? direct : heap; + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read MaxDirectMemorySize, assuming the heap size", e); + return heap; + } + } +} diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..2e90b75ff9 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -54,6 +54,22 @@ require_env "HG_STORE_RAFT_ADDRESS" : "${HG_STORE_GRPC_PORT:=8500}" : "${HG_STORE_REST_PORT:=8520}" : "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE:=}" + +# Bytes only: the Store parses the value as a long and refuses to start otherwise +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" && + ! "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" =~ ^[0-9]+$ ]]; then + echo "ERROR: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE must be a number of bytes," \ + "got '${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}'" >&2 + exit 2 +fi + +# Only emitted when set: SPRING_APPLICATION_JSON outranks conf/application-pd.yml +ROCKSDB_JSON="" +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" ]]; then + ROCKSDB_JSON=", + \"rocksdb\": { \"total_memory_size\": \"${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}\" }" +fi # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat <}" ./bin/start-hugegraph-store.sh -d false -j "${JAVA_OPTS:-}" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 56cbdd8e63..6101ae0951 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -28,17 +28,9 @@ management: rocksdb: # Match the explicit launcher opt-in when enabling Topling. provider: rocksdb - # rocksdb total memory usage, force flush to disk when reaching this value - # TODO: default this lower, or size it from the container memory limit, or map it from an - # HG_STORE_* variable in the Docker entrypoint. A default of 0 only makes AppConfig fall back - # to the JVM max heap, which is another heap-sized native budget, not a safe container total. - # It is the capacity of the RocksDB block and write LRU caches (RaftRocksdbOptions), not an - # upfront allocation, but pinned at 32 GB those native caches can grow past a smaller - # container memory limit under load and get the Store OOM-killed. Until then the only - # override is the JVM system property -Drocksdb.total_memory_size= (JAVA_OPTS, the - # Helm chart's store.javaOpts; helm/hugegraph). - # https://github.com/apache/hugegraph/issues/3254 - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache, in bytes. 0 sizes it from + # the memory left after heap and direct memory (container limit aware, capped at 32 GB). + total_memory_size: 0 # memtable size used by rocksdb write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index c69f5fe89d..2ec5bb5acd 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -23,6 +23,7 @@ import javax.annotation.PostConstruct; import org.apache.hugegraph.store.options.JobOptions; +import org.apache.hugegraph.store.options.RocksdbMemoryBudget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -104,15 +105,11 @@ public void init() { } Map rocksdb = rocksdbConfig.rocksdb; - if (!rocksdb.containsKey("total_memory_size") - || "0".equals(rocksdb.get("total_memory_size"))) { - rocksdb.put("total_memory_size", Long.toString(rt.maxMemory())); - } - long totalMemory = Long.parseLong(rocksdbConfig.rocksdb.get("total_memory_size")); + long totalMemory = RocksdbMemoryBudget.resolve(rocksdb.get(RocksdbMemoryBudget.KEY)); + rocksdb.put(RocksdbMemoryBudget.KEY, Long.toString(totalMemory)); if (raft.getDisruptorBufferSize() == 0) { - int size = (int) (totalMemory / 1000 / 1000 / 1000); - size = (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; - raft.setDisruptorBufferSize(size); // Increase one buffer every 32M + // Increase one buffer every 32M + raft.setDisruptorBufferSize(RocksdbMemoryBudget.disruptorBufferSize(totalMemory)); } if (!rocksdb.containsKey("write_buffer_size") || diff --git a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml index 7985e67da1..337a61042a 100644 --- a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml +++ b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml @@ -33,6 +33,7 @@ logging: level: root: info rocksdb: - # total memory size used by rocksdb - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache, in bytes. 0 sizes it from + # the memory left after heap and direct memory (container limit aware, capped at 32 GB). + total_memory_size: 0 write_buffer_size: 32000000 diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java index e9b5be8c48..31258b1d5e 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java @@ -21,6 +21,7 @@ import org.apache.hugegraph.store.StoreIdChangeTest; import org.apache.hugegraph.store.core.snapshot.HgSnapshotHandlerTest; import org.apache.hugegraph.store.core.store.meta.PartitionMetaCodecTest; +import org.apache.hugegraph.store.options.RocksdbMemoryBudgetTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -51,7 +52,8 @@ MetricSessionTest.class, HgSnapshotHandlerTest.class, StoreIdChangeTest.class, - PartitionMetaCodecTest.class + PartitionMetaCodecTest.class, + RocksdbMemoryBudgetTest.class }) @Slf4j public class CoreSuiteTest { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java new file mode 100644 index 0000000000..4835a498cc --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MAX_AUTO; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIB; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIN_AUTO; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.Test; + +public class RocksdbMemoryBudgetTest { + + private static final long GIB = 1024 * MIB; + + @Test + public void testExplicitValueIsKept() { + assertEquals(32_000_000_000L, RocksdbMemoryBudget.resolve("32000000000")); + assertEquals(4_000_000_000L, RocksdbMemoryBudget.resolve(" 4000000000 ")); + assertEquals(MIB, RocksdbMemoryBudget.resolve(String.valueOf(MIB))); + } + + @Test + public void testExplicitValueBelowOneMibIsRefused() { + for (String value : new String[]{"1", "2", String.valueOf(MIB - 1)}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("at least")); + } + } + } + + @Test + public void testUnsetZeroOrBlankIsAuto() { + for (String value : new String[]{null, "", " ", "0", " 0 ", "00"}) { + long budget = RocksdbMemoryBudget.resolve(value); + assertTrue(String.valueOf(value), budget >= MIN_AUTO && budget <= MAX_AUTO); + } + } + + @Test + public void testNonNumericValueIsRefused() { + for (String value : new String[]{"4GB", "4g", "-1", "1.5e9", "99999999999999999999"}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("number of bytes")); + } + } + } + + @Test + public void testAutoFitsContainerLimit() { + // The setup from apache/hugegraph#3254: 4 GiB limit, 1 GiB heap, 512 MiB direct + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(4 * GIB, GIB, 512 * MIB)); + assertEquals(2560 * MIB, RocksdbMemoryBudget.auto(8 * GIB, GIB, 512 * MIB)); + } + + @Test + public void testAutoIsClamped() { + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(2 * GIB, GIB, GIB)); + assertEquals(MAX_AUTO, RocksdbMemoryBudget.auto(1024 * GIB, 8 * GIB, 8 * GIB)); + } + + @Test + public void testDisruptorBufferSizeIsNeverZero() { + assertEquals(32, RocksdbMemoryBudget.disruptorBufferSize(512 * MIB)); + assertEquals(32, RocksdbMemoryBudget.disruptorBufferSize(1_000_000_000L)); + assertEquals(1024, RocksdbMemoryBudget.disruptorBufferSize(32_000_000_000L)); + } + + @Test + public void testUnknownPhysicalMemoryFallsBackToHeap() { + assertEquals(2 * GIB, RocksdbMemoryBudget.auto(-1L, 2 * GIB, 2 * GIB)); + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(0L, 64 * MIB, 64 * MIB)); + } +} From 1927fbee87d434d194d3d6696bb910460d88680c Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 15 Oct 2026 10:41:33 +0530 Subject: [PATCH 02/13] fix(store): keep the derived write buffer above RocksDB's minimum With write_buffer_size unset or 0, AppConfig derives it as a thousandth of the RocksDB budget. An auto-sized budget can now be under 1 GiB: a 4 GiB container with -Xmx1g and -XX:MaxDirectMemorySize=512m gets 512 MiB, which derived 536870 bytes. rocksdb.write_buffer_size requires at least 1 MiB (RocksDBOptions.MEMTABLE_SIZE), so Store startup would have rejected the config in BusinessHandlerImpl.initRocksdb(). RocksdbMemoryBudget.writeBufferSize() clamps the derived value to 1 MiB, and a test runs it through the same OptionSpace/HugeConfig validation. The shipped write_buffer_size of 32000000 never took this path. --- .../store/options/RocksdbMemoryBudget.java | 10 +++++ .../hugegraph/store/node/AppConfig.java | 3 +- .../options/RocksdbMemoryBudgetTest.java | 37 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 47b104f77b..e63e9cf1b6 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -131,6 +131,16 @@ public static int disruptorBufferSize(long totalMemory) { return (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; } + /** + * The write_buffer_size AppConfig derives when the key is unset or 0: a thousandth + * of the budget, but never below the 1 MiB rocksdb.write_buffer_size accepts + * (RocksDBOptions.MEMTABLE_SIZE), which an auto-sized budget under 1 GiB would + * otherwise fall under. + */ + public static long writeBufferSize(long totalMemory) { + return Math.max(totalMemory / 1000, MIB); + } + private static long physicalMemory() { try { // Reports the cgroup v1/v2 memory limit inside a container; older JDK 17 diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index 2ec5bb5acd..3165e3cd62 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -114,7 +114,8 @@ public void init() { if (!rocksdb.containsKey("write_buffer_size") || "0".equals(rocksdb.get("write_buffer_size"))) { - rocksdb.put("write_buffer_size", Long.toString(totalMemory / 1000)); + rocksdb.put("write_buffer_size", + Long.toString(RocksdbMemoryBudget.writeBufferSize(totalMemory))); } } diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 4835a498cc..2ac9b999b2 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -24,6 +24,13 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; import org.junit.Test; public class RocksdbMemoryBudgetTest { @@ -89,6 +96,36 @@ public void testDisruptorBufferSizeIsNeverZero() { assertEquals(1024, RocksdbMemoryBudget.disruptorBufferSize(32_000_000_000L)); } + @Test + public void testDerivedWriteBufferSizePassesRocksdbValidation() { + // The 4 GiB container from apache/hugegraph#3254 auto-sizes to 512 MiB, whose + // thousandth is under the 1 MiB rocksdb.write_buffer_size minimum + long budget = RocksdbMemoryBudget.auto(4 * GIB, GIB, 512 * MIB); + assertEquals(MIB, RocksdbMemoryBudget.writeBufferSize(budget)); + assertEquals(MIB, RocksdbMemoryBudget.writeBufferSize(MIB)); + assertEquals(32_000_000L, RocksdbMemoryBudget.writeBufferSize(32_000_000_000L)); + + // The same validation BusinessHandlerImpl.initRocksdb() runs at Store startup + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + RocksDBOptions.instance(); + HugeConfig config = new HugeConfig(rocksdbConfig(budget, + RocksdbMemoryBudget.writeBufferSize(budget))); + assertEquals(MIB, (long) config.get(RocksDBOptions.MEMTABLE_SIZE)); + try { + new HugeConfig(rocksdbConfig(budget, budget / 1000)); + fail("accepted a write buffer under 1 MiB"); + } catch (ConfigException e) { + assertTrue(e.getMessage().contains("write_buffer_size")); + } + } + + private static Map rocksdbConfig(long totalMemory, long writeBufferSize) { + Map config = new HashMap<>(); + config.put("rocksdb.total_memory_size", String.valueOf(totalMemory)); + config.put("rocksdb.write_buffer_size", String.valueOf(writeBufferSize)); + return config; + } + @Test public void testUnknownPhysicalMemoryFallsBackToHeap() { assertEquals(2 * GIB, RocksdbMemoryBudget.auto(-1L, 2 * GIB, 2 * GIB)); From e691489e4556a6711f04ca43ba10e47c69e9498c Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 15 Oct 2026 18:43:20 +0530 Subject: [PATCH 03/13] ci: re-run checks From 84e8e27f3ee6d903249271a480651273a9a229bd Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 10:03:04 +0530 Subject: [PATCH 04/13] fix(store): count unset direct memory as at most 1 GiB in the budget Auto-sizing counted an unset -XX:MaxDirectMemorySize as a second full heap, because that is what the JVM allows. bin/start-hugegraph-store.sh never sets the flag, so a bare-metal Store with a raised heap fell to the 256 MiB floor on upgrade: -Xmx8g on a 16 GiB host, or -Xmx36g on a 64 GiB host, went from the old fixed 32000000000 bytes to 268435456. The Store's direct buffers (gRPC/Netty) do not grow with the heap, so an unset cap now counts as the heap size capped at 1 GiB. Those two hosts get 2.75 GiB and 12.75 GiB. An explicit -XX:MaxDirectMemorySize is still counted as given, so the container case from #3254 is unchanged. The deployment guide now says what bare-metal hosts get. --- hugegraph-store/docs/deployment-guide.md | 11 ++++++++-- .../store/options/RocksdbMemoryBudget.java | 22 ++++++++++++++----- .../options/RocksdbMemoryBudgetTest.java | 21 ++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 2b98ac3766..d1c50a4893 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -372,8 +372,15 @@ With `total_memory_size: 0`, or the key unset, the Store sizes the budget at sta of what is left after the heap, the maximum direct memory and a 1.5 GiB reserve, clamped to 256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, so a Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without -`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts -it that way, so set it explicitly in small containers or the budget falls to the 256 MiB floor. +`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap; the budget counts that +as at most 1 GiB, so set the flag when the Store needs more direct memory than that, and in +small containers, where an uncapped 1 GiB heap already leaves only the 256 MiB floor. + +On a host without a container limit the starting point is physical memory. The default +`bin/start-hugegraph-store.sh` caps the heap at 2 GiB, so a 16 GiB host gets about 5.75 GiB. +A Store whose heap was raised gets less: 2.75 GiB on a 16 GiB host with `-Xmx8g`, 12.75 GiB on +a 64 GiB host with `-Xmx36g`. Before this default every host got a fixed `32000000000`, so pin +the value after upgrading if a large node relied on it. The chosen value is logged by `RocksdbMemoryBudget` at startup, with a warning when the floor applies, or when an explicit value is under 256 MiB or exceeds the memory left after the heap, direct memory and the reserve. Set a number of bytes, at least 1048576 (1 MiB), to pin it; a diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index e63e9cf1b6..84947d703f 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -40,6 +40,10 @@ public final class RocksdbMemoryBudget { // and other native allocations that sit outside the budget static final long RESERVED = 1024 * MIB + 512 * MIB; static final long MIN_AUTO = 256 * MIB; + // Direct memory counted when -XX:MaxDirectMemorySize is unset. The JVM then allows + // as much as the heap, but the Store's gRPC/Netty buffers do not grow with the heap, + // and counting a 36 GB heap twice would leave rocksdb the floor on a large host + static final long UNSET_DIRECT = 1024 * MIB; // RaftRocksdbOptions gives the write buffer manager 2/3 of the budget, and a 0-byte // write buffer manager means no memtable limit at all static final long MIN_EXPLICIT = MIB; @@ -158,13 +162,19 @@ private static long maxDirectMemory(long heap) { try { HotSpotDiagnosticMXBean hotspot = ManagementFactory.getPlatformMXBean( HotSpotDiagnosticMXBean.class); - long direct = Long.parseLong( - hotspot.getVMOption("MaxDirectMemorySize").getValue()); - // 0 means unset, and the JVM then allows as much direct memory as heap - return direct > 0 ? direct : heap; + return directMemory(Long.parseLong( + hotspot.getVMOption("MaxDirectMemorySize").getValue()), heap); } catch (RuntimeException | LinkageError e) { - log.warn("Failed to read MaxDirectMemorySize, assuming the heap size", e); - return heap; + log.warn("Failed to read MaxDirectMemorySize, assuming it is unset", e); + return directMemory(0L, heap); } } + + /** + * The direct memory the budget leaves room for: an explicit -XX:MaxDirectMemorySize + * as given, otherwise (0, unset) the heap size capped at UNSET_DIRECT. + */ + static long directMemory(long configured, long heap) { + return configured > 0 ? configured : Math.min(heap, UNSET_DIRECT); + } } diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 2ac9b999b2..0209ddd10d 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -83,6 +83,27 @@ public void testAutoFitsContainerLimit() { assertEquals(2560 * MIB, RocksdbMemoryBudget.auto(8 * GIB, GIB, 512 * MIB)); } + @Test + public void testUnsetDirectMemoryIsCappedAtOneGib() { + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(512 * MIB, 8 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.directMemory(4 * GIB, 8 * GIB)); + assertEquals(GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB)); + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(0L, 512 * MIB)); + } + + @Test + public void testLargeHeapWithoutDirectCapKeepsAUsableBudget() { + // Bare-metal Stores with a raised heap and no -XX:MaxDirectMemorySize; counting + // the unset cap as a second heap gave all of these the 256 MiB floor + assertEquals(2816 * MIB, RocksdbMemoryBudget.auto( + 16 * GIB, 8 * GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB))); + assertEquals(13056 * MIB, RocksdbMemoryBudget.auto( + 64 * GIB, 36 * GIB, RocksdbMemoryBudget.directMemory(0L, 36 * GIB))); + // The default launcher caps the heap at 2 GiB + assertEquals(5888 * MIB, RocksdbMemoryBudget.auto( + 16 * GIB, 2 * GIB, RocksdbMemoryBudget.directMemory(0L, 2 * GIB))); + } + @Test public void testAutoIsClamped() { assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(2 * GIB, GIB, GIB)); From 3fb0da01f734b6a5ff1ba368aada937eeae3f582 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 19:55:47 +0530 Subject: [PATCH 05/13] ci: select the store suite for the Store entrypoint test test-store-docker-entrypoint.sh runs in the store job of pd-store-ci.yml, but the selector didn't map it, so a PR that changed only that script selected server, docker and macOS paths and skipped the test. Map it to {"store"} in CONTRACT_SUITES and extend the selector test so it checks the store job's condition and step as well. --- .github/scripts/ci-policy.py | 1 + .github/scripts/test_ci_policy.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 6241fffda4..e5a6d30872 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -101,6 +101,7 @@ TRAVIS + "test-check-jacoco-report.sh": {"struct"}, TRAVIS + "test-codecov-upload-config.sh": {"struct"}, TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, + TRAVIS + "test-store-docker-entrypoint.sh": {"store"}, } diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 65e7666324..14f9d89926 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -356,14 +356,15 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): ("test-check-java-runtime-logs.sh", {"struct"}), ("test-check-jacoco-report.sh", {"struct"}), ("test-codecov-upload-config.sh", {"struct"}), - ("test-riscv64-java-runtime.sh", {"server", "riscv64"})] + ("test-riscv64-java-runtime.sh", {"server", "riscv64"}), + ("test-store-docker-entrypoint.sh", {"store"})] for name, suites in cases: script = policy.TRAVIS + name with self.subTest(script=script): plan = self.plan_for([script]) self.assertEqual(suites, set(plan["selected"])) callers = self.coordinator_jobs(plan) - if "struct" in suites: + if "struct" in suites or "store" in suites: self.assertEqual({"pd_store"}, callers) root = (workflows / "server-ci.yml").read_text() caller = re.search(r"^ pd_store:\n(.*?)(?=^ [\w-]+:|\Z)", root, re.M | re.S).group(1) @@ -373,12 +374,13 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower())) inputs[key] = eval(binding, {"__builtins__": {}}) workflow = (workflows / "pd-store-ci.yml").read_text() - struct = workflow.split("\n struct:\n", 1)[1].split("\n pd:\n", 1)[0] - condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) + job = "struct" if "struct" in suites else "store" + body = re.search(r"^ " + job + r":\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + condition = re.search(r"^ if: (.+)$", body, re.M).group(1) condition = condition.replace("inputs.run-struct", str(inputs["struct"])) condition = condition.replace("inputs.run-store", str(inputs["store"])) self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}})) - step = next(step for step in struct.split("\n - ") if Path(script).name in step) + step = next(step for step in body.split("\n - ") if Path(script).name in step) self.assertNotRegex(step, r"(?m)^ if:") else: self.assertIn("server-rocksdb", callers) From 03a4d436754274d144d76101bf43c496bfb2a0d8 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 19:55:47 +0530 Subject: [PATCH 06/13] fix(store): honor ancestor cgroup limits and flag the old 32 GB value Older JDK 17 updates can miss a cgroup memory limit set only on a parent cgroup and report host memory, so auto-sizing could still give RocksDB most of host RAM inside a smaller pod or systemd slice. RocksdbMemoryBudget now also reads the cgroup itself: on v2 it walks memory.max from the process's cgroup up to the mount root, on v1 it reads hierarchical_memory_limit, and it uses the lower of that and the JVM value. A limit above a private cgroup namespace stays invisible, which the deployment guide now says. An existing deployment with a mounted or kept application-pd.yml still has total_memory_size: 32000000000, which is explicit and bypasses auto-sizing. The Store now warns when it sees that exact value, and the deployment guide has an upgrade step to set it to 0, remove it, or set HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE. --- hugegraph-store/docs/deployment-guide.md | 19 ++- .../store/options/RocksdbMemoryBudget.java | 122 +++++++++++++++++- .../options/RocksdbMemoryBudgetTest.java | 85 ++++++++++++ 3 files changed, 220 insertions(+), 6 deletions(-) diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index d1c50a4893..1b37669525 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -370,8 +370,10 @@ cache for the Raft log comes on top of this budget. With `total_memory_size: 0`, or the key unset, the Store sizes the budget at startup as half of what is left after the heap, the maximum direct memory and a 1.5 GiB reserve, clamped to -256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, so a Store -with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without +256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, including a +limit set only on a parent cgroup when the Store can see that cgroup; inside a private cgroup +namespace only the container's own limit is visible, so pin the value there if only the pod or +slice is limited. A Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without `-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap; the budget counts that as at most 1 GiB, so set the flag when the Store needs more direct memory than that, and in small containers, where an uncapped 1 GiB heap already leaves only the 256 MiB floor. @@ -381,6 +383,7 @@ On a host without a container limit the starting point is physical memory. The d A Store whose heap was raised gets less: 2.75 GiB on a 16 GiB host with `-Xmx8g`, 12.75 GiB on a 64 GiB host with `-Xmx36g`. Before this default every host got a fixed `32000000000`, so pin the value after upgrading if a large node relied on it. + The chosen value is logged by `RocksdbMemoryBudget` at startup, with a warning when the floor applies, or when an explicit value is under 256 MiB or exceeds the memory left after the heap, direct memory and the reserve. Set a number of bytes, at least 1048576 (1 MiB), to pin it; a @@ -389,6 +392,12 @@ shipped value was `32000000000`, which can exceed a container limit and get the OOM-killed. Both caps are soft: RocksDB can briefly go over them under heavy writes, and a small budget with many partitions means frequent memtable flushes. +**Upgrading an existing deployment.** Auto-sizing applies only when the loaded configuration +has `total_memory_size: 0` or no value. A Store that mounts its own `conf/application-pd.yml`, +or a tarball install that kept its `conf/` from an earlier release, still has `32000000000` and +keeps that fixed budget; the Store logs a warning naming the old value at startup. Set the key +to `0` or remove it, or set `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` in Docker. + ### Server Configuration File: `hugegraph-server/conf/graphs/hugegraph.properties` @@ -757,8 +766,10 @@ environment: `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` must be a plain number of bytes; the entrypoint refuses anything else. When set it outranks `rocksdb.total_memory_size` in a mounted -`conf/application-pd.yml` and a `-Drocksdb.total_memory_size` in `JAVA_OPTS`. Leave it unset to let the Store size the budget from the container -memory limit (see [Store Configuration](#store-configuration)). +`conf/application-pd.yml` and a `-Drocksdb.total_memory_size` in `JAVA_OPTS`. Leave it unset +to let the Store size the budget from the container memory limit, provided the loaded config +also has `total_memory_size: 0` or no value; a mounted `application-pd.yml` from an earlier +release still has `32000000000` (see [Store Configuration](#store-configuration)). **Server environment variables**: diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 84947d703f..9f9ade8150 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -17,7 +17,13 @@ package org.apache.hugegraph.store.options; +import java.io.IOException; import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import com.sun.management.HotSpotDiagnosticMXBean; import com.sun.management.OperatingSystemMXBean; @@ -71,6 +77,12 @@ public static long resolve(String configured) { log.warn("rocksdb.total_memory_size={} is below the {} bytes auto-sizing " + "never goes under", value, MIN_AUTO); } + if (value == MAX_AUTO) { + // Most likely an application-pd.yml kept from before auto-sizing + log.warn("rocksdb.total_memory_size={} is the fixed value shipped before " + + "auto-sizing; set it to 0 or remove it to size the budget from " + + "the available memory", value); + } long direct = maxDirectMemory(heap); long left = physical - heap - direct - RESERVED; if (physical > 0 && value > left) { @@ -146,16 +158,122 @@ public static long writeBufferSize(long totalMemory) { } private static long physicalMemory() { + long jvm; try { - // Reports the cgroup v1/v2 memory limit inside a container; older JDK 17 + // Reports the cgroup v1/v2 memory limit inside a container, but older JDK 17 // updates may not see a limit set only on a parent cgroup OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean( OperatingSystemMXBean.class); - return os == null ? -1L : os.getTotalMemorySize(); + jvm = os == null ? -1L : os.getTotalMemorySize(); } catch (RuntimeException | LinkageError e) { log.warn("Failed to read physical memory size", e); + jvm = -1L; + } + long cgroup = cgroupMemoryLimit(Paths.get("/proc/self/cgroup"), + Paths.get("/sys/fs/cgroup")); + return effectivePhysical(jvm, cgroup); + } + + /** + * The smaller of what the JVM reports and the cgroup limit, ignoring unknown (<= 0) + * values. + */ + static long effectivePhysical(long jvm, long cgroup) { + if (cgroup <= 0) { + return jvm; + } + return jvm <= 0 ? cgroup : Math.min(jvm, cgroup); + } + + /** + * The lowest memory limit on this process's cgroup or any visible ancestor, or -1 + * when there is none or it can't be read. cgroup v2 walks memory.max from the + * process's cgroup up to the mount root; cgroup v1 reads hierarchical_memory_limit, + * which already accounts for ancestors. Inside a private cgroup namespace the mount + * root is the container's own cgroup, so a limit above it stays invisible. + */ + static long cgroupMemoryLimit(Path procSelfCgroup, Path cgroupRoot) { + List lines; + try { + lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { return -1L; } + long limit = -1L; + for (String line : lines) { + String[] parts = line.split(":", 3); + if (parts.length < 3) { + continue; + } + String relative = parts[2].startsWith("/") ? parts[2].substring(1) : parts[2]; + if (parts[0].equals("0") && parts[1].isEmpty()) { + limit = lowest(limit, cgroupV2Limit(cgroupRoot, relative)); + } else if (("," + parts[1] + ",").contains(",memory,")) { + limit = lowest(limit, cgroupV1Limit(cgroupRoot.resolve("memory"), relative)); + } + } + return limit; + } + + private static long cgroupV2Limit(Path root, String relative) { + Path dir = root.resolve(relative).normalize(); + if (!dir.startsWith(root) || !Files.isDirectory(dir)) { + // The process path is not visible from this mount (a cgroup namespace) + dir = root; + } + long limit = -1L; + for (Path d = dir; d != null && d.startsWith(root); d = d.getParent()) { + limit = lowest(limit, readLimit(d.resolve("memory.max"))); + } + return limit; + } + + private static long cgroupV1Limit(Path base, String relative) { + Path dir = base.resolve(relative).normalize(); + if (!dir.startsWith(base) || !Files.isDirectory(dir)) { + dir = base; + } + long limit = readLimit(dir.resolve("memory.limit_in_bytes")); + try { + for (String line : Files.readAllLines(dir.resolve("memory.stat"), + StandardCharsets.UTF_8)) { + if (line.startsWith("hierarchical_memory_limit ")) { + limit = lowest(limit, parseLimit(line.substring(line.indexOf(' ') + 1))); + } + } + } catch (IOException | RuntimeException e) { + // memory.stat is optional here, memory.limit_in_bytes already counted + } + return limit; + } + + private static long readLimit(Path file) { + try { + return parseLimit(new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + } catch (IOException | RuntimeException e) { + return -1L; + } + } + + // "max" (v2) and the near-Long.MAX_VALUE page-aligned v1 default mean no limit + private static long parseLimit(String value) { + String trimmed = value.trim(); + if (trimmed.isEmpty() || trimmed.equals("max")) { + return -1L; + } + try { + long limit = Long.parseLong(trimmed); + return limit <= 0 || limit >= Long.MAX_VALUE / 2 ? -1L : limit; + } catch (NumberFormatException e) { + return -1L; + } + } + + private static long lowest(long a, long b) { + if (a <= 0) { + return b; + } + return b <= 0 ? a : Math.min(a, b); } private static long maxDirectMemory(long heap) { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 0209ddd10d..fbef17a1a7 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -24,9 +24,15 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import org.apache.commons.io.FileUtils; + import org.apache.hugegraph.config.ConfigException; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.OptionSpace; @@ -104,6 +110,85 @@ public void testLargeHeapWithoutDirectCapKeepsAUsableBudget() { 16 * GIB, 2 * GIB, RocksdbMemoryBudget.directMemory(0L, 2 * GIB))); } + @Test + public void testAncestorCgroupV2LimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // A 4 GiB limit on the pod, none on the container or above the pod + write(root, "kubepods/memory.max", "max"); + write(root, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(root, "kubepods/pod1/ctr/memory.max", "max\n"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, root); + assertEquals(4 * GIB, limit); + + // The case from review: a 64 GiB host reported by the JVM, -Xmx1g and + // -XX:MaxDirectMemorySize=512m get the 4 GiB limit's budget, not host RAM's + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + + // The container's own lower limit wins over the pod's + write(root, "kubepods/pod1/ctr/memory.max", String.valueOf(2 * GIB)); + assertEquals(2 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2NamespaceRootAndNoLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // Inside a cgroup namespace the process sees "/" and the mount is its cgroup + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + write(root, "memory.max", "max"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + write(root, "memory.max", String.valueOf(3 * GIB)); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + + // A path the mount doesn't show falls back to the mount root + Path hidden = write(root, "proc-hidden", "0::/system.slice/hidden.service\n"); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(hidden, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1HierarchicalLimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1"); + try { + Path proc = write(root, "proc-self-cgroup", + "12:cpu,cpuacct:/docker/abc\n4:memory:/docker/abc\n"); + // Docker mounts the container's own memory cgroup, so /docker/abc isn't there + write(root, "memory/memory.limit_in_bytes", "9223372036854771712"); + write(root, "memory/memory.stat", + "cache 0\nhierarchical_memory_limit " + (4 * GIB) + "\n"); + assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + + write(root, "memory/memory.stat", "hierarchical_memory_limit 9223372036854771712\n"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testUnreadableCgroupKeepsJvmMemory() { + Path missing = Path.of("/nonexistent/proc/self/cgroup"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(missing, Path.of("/nonexistent"))); + assertEquals(16 * GIB, RocksdbMemoryBudget.effectivePhysical(16 * GIB, -1L)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(-1L, 4 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(4 * GIB, 8 * GIB)); + } + + private static Path write(Path root, String relative, String content) throws IOException { + Path file = root.resolve(relative); + Files.createDirectories(file.getParent()); + Files.write(file, content.getBytes(StandardCharsets.UTF_8)); + return file; + } + @Test public void testAutoIsClamped() { assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(2 * GIB, GIB, GIB)); From 33e53ee18caf668669ff6619eac603e8dea2971e Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 20:12:50 +0530 Subject: [PATCH 07/13] fix(store): rebase the cgroup path onto its mount root /proc/self/cgroup gives the membership path from the hierarchy root, but the cgroupfs can be mounted from a subtree: mountinfo root /kubepods at /sys/fs/cgroup puts the pod's memory.max at /sys/fs/cgroup/pod1, not /sys/fs/cgroup/kubepods/pod1. The resolver looked for the latter, fell back to the unlimited mount root and missed the visible 4 GiB pod limit. Read the cgroup2 mount, or the cgroup v1 mount carrying memory, from /proc/self/mountinfo and strip its root from the membership path before walking ancestors. Without a mountinfo entry the default /sys/fs/cgroup layout is used as before. --- .../store/options/RocksdbMemoryBudget.java | 94 +++++++++++++++++-- .../options/RocksdbMemoryBudgetTest.java | 75 +++++++++++++-- 2 files changed, 153 insertions(+), 16 deletions(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 9f9ade8150..9478fa0296 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -170,6 +170,7 @@ private static long physicalMemory() { jvm = -1L; } long cgroup = cgroupMemoryLimit(Paths.get("/proc/self/cgroup"), + Paths.get("/proc/self/mountinfo"), Paths.get("/sys/fs/cgroup")); return effectivePhysical(jvm, cgroup); } @@ -187,12 +188,17 @@ static long effectivePhysical(long jvm, long cgroup) { /** * The lowest memory limit on this process's cgroup or any visible ancestor, or -1 - * when there is none or it can't be read. cgroup v2 walks memory.max from the - * process's cgroup up to the mount root; cgroup v1 reads hierarchical_memory_limit, - * which already accounts for ancestors. Inside a private cgroup namespace the mount - * root is the container's own cgroup, so a limit above it stays invisible. + * when there is none or it can't be read. The membership path from /proc/self/cgroup + * is rebased onto the controller's mount using its root in /proc/self/mountinfo, so a + * cgroupfs mounted from a subtree (root /kubepods at /sys/fs/cgroup) is walked from + * the right directory; without a mountinfo entry the default root is used. cgroup v2 + * walks memory.max from the process's cgroup up to the mount; cgroup v1 reads + * hierarchical_memory_limit, which already accounts for ancestors. A limit above the + * mount root, as inside a private cgroup namespace, stays invisible. */ - static long cgroupMemoryLimit(Path procSelfCgroup, Path cgroupRoot) { + static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { + String[] v2Mount = cgroupMount(mountinfo, true); + String[] v1Mount = cgroupMount(mountinfo, false); List lines; try { lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); @@ -205,16 +211,88 @@ static long cgroupMemoryLimit(Path procSelfCgroup, Path cgroupRoot) { if (parts.length < 3) { continue; } - String relative = parts[2].startsWith("/") ? parts[2].substring(1) : parts[2]; if (parts[0].equals("0") && parts[1].isEmpty()) { - limit = lowest(limit, cgroupV2Limit(cgroupRoot, relative)); + Path point = v2Mount == null ? defaultRoot : Paths.get(v2Mount[1]); + String root = v2Mount == null ? "/" : v2Mount[0]; + limit = lowest(limit, cgroupV2Limit(point, rebase(parts[2], root))); } else if (("," + parts[1] + ",").contains(",memory,")) { - limit = lowest(limit, cgroupV1Limit(cgroupRoot.resolve("memory"), relative)); + Path point = v1Mount == null ? defaultRoot.resolve("memory") : + Paths.get(v1Mount[1]); + String root = v1Mount == null ? "/" : v1Mount[0]; + limit = lowest(limit, cgroupV1Limit(point, rebase(parts[2], root))); } } return limit; } + /** + * The {root, mount point} of the cgroup v2 mount, or of the cgroup v1 mount carrying + * the memory controller, from /proc/self/mountinfo; null when there is none. + */ + private static String[] cgroupMount(Path mountinfo, boolean v2) { + List lines; + try { + lines = Files.readAllLines(mountinfo, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { + return null; + } + for (String line : lines) { + // id parent major:minor root mount-point options [optional...] - type source super + int separator = line.indexOf(" - "); + if (separator < 0) { + continue; + } + String[] fields = line.substring(0, separator).split(" "); + String[] tail = line.substring(separator + 3).split(" "); + if (fields.length < 5 || tail.length < 1) { + continue; + } + boolean match = v2 ? tail[0].equals("cgroup2") : + tail[0].equals("cgroup") && tail.length >= 3 && + ("," + tail[2] + ",").contains(",memory,"); + if (match) { + return new String[]{unescape(fields[3]), unescape(fields[4])}; + } + } + return null; + } + + /** + * The membership path relative to a mount whose root is mountRoot: /kubepods/pod1/ctr + * under root /kubepods is pod1/ctr. A path outside the mount root is kept as is and + * then falls back to the mount itself. + */ + static String rebase(String membership, String mountRoot) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = mountRoot.endsWith("/") && mountRoot.length() > 1 ? + mountRoot.substring(0, mountRoot.length() - 1) : mountRoot; + if (!root.equals("/")) { + if (path.equals(root)) { + return ""; + } + if (path.startsWith(root + "/")) { + path = path.substring(root.length()); + } + } + return path.substring(1); + } + + // mountinfo writes space, tab, newline and backslash as octal escapes such as \040 + private static String unescape(String field) { + StringBuilder out = new StringBuilder(field.length()); + for (int i = 0; i < field.length(); i++) { + char c = field.charAt(i); + if (c == '\\' && i + 4 <= field.length() && + field.substring(i + 1, i + 4).matches("[0-7]{3}")) { + out.append((char) Integer.parseInt(field.substring(i + 1, i + 4), 8)); + i += 3; + } else { + out.append(c); + } + } + return out.toString(); + } + private static long cgroupV2Limit(Path root, String relative) { Path dir = root.resolve(relative).normalize(); if (!dir.startsWith(root) || !Files.isDirectory(dir)) { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index fbef17a1a7..6aaadb933d 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -42,6 +42,7 @@ public class RocksdbMemoryBudgetTest { private static final long GIB = 1024 * MIB; + private static final Path NO_MOUNTINFO = Path.of("/nonexistent/proc/self/mountinfo"); @Test public void testExplicitValueIsKept() { @@ -119,7 +120,7 @@ public void testAncestorCgroupV2LimitIsUsed() throws IOException { write(root, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); write(root, "kubepods/pod1/ctr/memory.max", "max\n"); Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); - long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, root); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root); assertEquals(4 * GIB, limit); // The case from review: a 64 GiB host reported by the JVM, -Xmx1g and @@ -129,7 +130,7 @@ public void testAncestorCgroupV2LimitIsUsed() throws IOException { // The container's own lower limit wins over the pod's write(root, "kubepods/pod1/ctr/memory.max", String.valueOf(2 * GIB)); - assertEquals(2 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + assertEquals(2 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); } finally { FileUtils.deleteQuietly(root.toFile()); } @@ -142,13 +143,13 @@ public void testCgroupV2NamespaceRootAndNoLimit() throws IOException { // Inside a cgroup namespace the process sees "/" and the mount is its cgroup Path proc = write(root, "proc-self-cgroup", "0::/\n"); write(root, "memory.max", "max"); - assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); write(root, "memory.max", String.valueOf(3 * GIB)); - assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); // A path the mount doesn't show falls back to the mount root Path hidden = write(root, "proc-hidden", "0::/system.slice/hidden.service\n"); - assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(hidden, root)); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(hidden, NO_MOUNTINFO, root)); } finally { FileUtils.deleteQuietly(root.toFile()); } @@ -164,19 +165,77 @@ public void testCgroupV1HierarchicalLimitIsUsed() throws IOException { write(root, "memory/memory.limit_in_bytes", "9223372036854771712"); write(root, "memory/memory.stat", "cache 0\nhierarchical_memory_limit " + (4 * GIB) + "\n"); - assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); write(root, "memory/memory.stat", "hierarchical_memory_limit 9223372036854771712\n"); - assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, root)); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); } finally { FileUtils.deleteQuietly(root.toFile()); } } + @Test + public void testCgroupV2SubtreeMountIsRebased() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-subtree"); + try { + // cgroupfs mounted from /kubepods: the pod's limit is at /pod1, not at + // /kubepods/pod1, and the mount root itself is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "pod1/memory.max", String.valueOf(4 * GIB)); + write(mount, "pod1/ctr/memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + Path mountinfo = write(root, "mountinfo", + "25 1 8:1 / / rw - ext4 /dev/sda1 rw\n" + + "30 25 0:26 /kubepods " + mount + + " rw,nosuid shared:4 - cgroup2 cgroup2 rw\n"); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root); + assertEquals(4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + + // Without the mount root the path is looked up under kubepods/ and missed + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, mount)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1SubtreeMountAndEscapedMountPoint() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-subtree"); + try { + Path mount = root.resolve("memory dir"); + write(mount, "memory.limit_in_bytes", "9223372036854771712"); + write(mount, "memory.stat", "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/docker/abc\n"); + // mountinfo writes the space in the mount point as \040 + Path mountinfo = write(root, "mountinfo", + "40 25 0:30 /docker/abc " + + mount.toString().replace(" ", "\\040") + + " rw,nosuid - cgroup cgroup rw,memory\n"); + assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMembershipIsRebasedOntoMountRoot() { + assertEquals("pod1/ctr", RocksdbMemoryBudget.rebase("/kubepods/pod1/ctr", "/kubepods")); + assertEquals("pod1/ctr", RocksdbMemoryBudget.rebase("/kubepods/pod1/ctr", "/kubepods/")); + assertEquals("", RocksdbMemoryBudget.rebase("/kubepods", "/kubepods")); + assertEquals("a/b", RocksdbMemoryBudget.rebase("/a/b", "/")); + assertEquals("", RocksdbMemoryBudget.rebase("/", "/")); + // A path outside the mount root is kept and then falls back to the mount + assertEquals("other/x", RocksdbMemoryBudget.rebase("/other/x", "/kubepods")); + assertEquals("kubepodsx/y", RocksdbMemoryBudget.rebase("/kubepodsx/y", "/kubepods")); + } + @Test public void testUnreadableCgroupKeepsJvmMemory() { Path missing = Path.of("/nonexistent/proc/self/cgroup"); - assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(missing, Path.of("/nonexistent"))); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(missing, NO_MOUNTINFO, Path.of("/nonexistent"))); assertEquals(16 * GIB, RocksdbMemoryBudget.effectivePhysical(16 * GIB, -1L)); assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(-1L, 4 * GIB)); assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(4 * GIB, 8 * GIB)); From 535c2d01b3d1e95e68b94af3b6b469303d482e4b Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 16 Oct 2026 22:50:12 +0530 Subject: [PATCH 08/13] fix(store): take the lowest limit over every matching cgroup mount mountinfo can list several mounts of the same controller, and the resolver used the first one. An unrelated subtree or a leaf-only view listed before the full hierarchy hid the pod's visible limit, so the result depended on mount order. Read every cgroup2 mount, or every v1 mount carrying memory, skip those whose root doesn't contain the process's cgroup, walk each remaining one and keep the lowest limit. With no matching mount the default /sys/fs/cgroup layout is used as before. --- .../store/options/RocksdbMemoryBudget.java | 66 ++++++++++++------ .../options/RocksdbMemoryBudgetTest.java | 68 +++++++++++++++++++ 2 files changed, 112 insertions(+), 22 deletions(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 9478fa0296..267357ea98 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -23,6 +23,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import com.sun.management.HotSpotDiagnosticMXBean; @@ -189,16 +190,18 @@ static long effectivePhysical(long jvm, long cgroup) { /** * The lowest memory limit on this process's cgroup or any visible ancestor, or -1 * when there is none or it can't be read. The membership path from /proc/self/cgroup - * is rebased onto the controller's mount using its root in /proc/self/mountinfo, so a - * cgroupfs mounted from a subtree (root /kubepods at /sys/fs/cgroup) is walked from - * the right directory; without a mountinfo entry the default root is used. cgroup v2 + * is rebased onto every mount of the controller in /proc/self/mountinfo whose root + * contains it, so a cgroupfs mounted from a subtree (root /kubepods at /sys/fs/cgroup) + * is walked from the right directory, and the lowest limit over all those mounts is + * taken: a leaf-only or unrelated mount listed first can't hide a fuller view listed + * later. Without a matching mount the default root is used. cgroup v2 * walks memory.max from the process's cgroup up to the mount; cgroup v1 reads * hierarchical_memory_limit, which already accounts for ancestors. A limit above the * mount root, as inside a private cgroup namespace, stays invisible. */ static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { - String[] v2Mount = cgroupMount(mountinfo, true); - String[] v1Mount = cgroupMount(mountinfo, false); + List v2Mounts = cgroupMounts(mountinfo, true); + List v1Mounts = cgroupMounts(mountinfo, false); List lines; try { lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); @@ -211,30 +214,42 @@ static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultR if (parts.length < 3) { continue; } - if (parts[0].equals("0") && parts[1].isEmpty()) { - Path point = v2Mount == null ? defaultRoot : Paths.get(v2Mount[1]); - String root = v2Mount == null ? "/" : v2Mount[0]; - limit = lowest(limit, cgroupV2Limit(point, rebase(parts[2], root))); - } else if (("," + parts[1] + ",").contains(",memory,")) { - Path point = v1Mount == null ? defaultRoot.resolve("memory") : - Paths.get(v1Mount[1]); - String root = v1Mount == null ? "/" : v1Mount[0]; - limit = lowest(limit, cgroupV1Limit(point, rebase(parts[2], root))); + boolean v2 = parts[0].equals("0") && parts[1].isEmpty(); + if (!v2 && !("," + parts[1] + ",").contains(",memory,")) { + continue; + } + boolean matched = false; + for (String[] mount : v2 ? v2Mounts : v1Mounts) { + if (!contains(mount[0], parts[2])) { + // A mount of another subtree says nothing about this cgroup + continue; + } + matched = true; + Path point = Paths.get(mount[1]); + String relative = rebase(parts[2], mount[0]); + limit = lowest(limit, v2 ? cgroupV2Limit(point, relative) : + cgroupV1Limit(point, relative)); + } + if (!matched) { + String relative = rebase(parts[2], "/"); + limit = lowest(limit, v2 ? cgroupV2Limit(defaultRoot, relative) : + cgroupV1Limit(defaultRoot.resolve("memory"), relative)); } } return limit; } /** - * The {root, mount point} of the cgroup v2 mount, or of the cgroup v1 mount carrying - * the memory controller, from /proc/self/mountinfo; null when there is none. + * The {root, mount point} of every cgroup v2 mount, or of every cgroup v1 mount + * carrying the memory controller, in /proc/self/mountinfo order. */ - private static String[] cgroupMount(Path mountinfo, boolean v2) { + private static List cgroupMounts(Path mountinfo, boolean v2) { + List mounts = new ArrayList<>(); List lines; try { lines = Files.readAllLines(mountinfo, StandardCharsets.UTF_8); } catch (IOException | RuntimeException e) { - return null; + return mounts; } for (String line : lines) { // id parent major:minor root mount-point options [optional...] - type source super @@ -251,16 +266,23 @@ private static String[] cgroupMount(Path mountinfo, boolean v2) { tail[0].equals("cgroup") && tail.length >= 3 && ("," + tail[2] + ",").contains(",memory,"); if (match) { - return new String[]{unescape(fields[3]), unescape(fields[4])}; + mounts.add(new String[]{unescape(fields[3]), unescape(fields[4])}); } } - return null; + return mounts; + } + + // Whether a mount whose root is mountRoot shows the cgroup at membership + static boolean contains(String mountRoot, String membership) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = mountRoot.endsWith("/") && mountRoot.length() > 1 ? + mountRoot.substring(0, mountRoot.length() - 1) : mountRoot; + return root.equals("/") || path.equals(root) || path.startsWith(root + "/"); } /** * The membership path relative to a mount whose root is mountRoot: /kubepods/pod1/ctr - * under root /kubepods is pod1/ctr. A path outside the mount root is kept as is and - * then falls back to the mount itself. + * under root /kubepods is pod1/ctr. A path outside the mount root is kept as is. */ static String rebase(String membership, String mountRoot) { String path = membership.startsWith("/") ? membership : "/" + membership; diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 6aaadb933d..37ce1f4552 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -21,6 +21,7 @@ import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIB; import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIN_AUTO; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -220,6 +221,73 @@ public void testCgroupV1SubtreeMountAndEscapedMountPoint() throws IOException { } } + @Test + public void testCgroupV2MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-mounts"); + try { + // A full view of the hierarchy with the 4 GiB limit on the pod + Path full = root.resolve("full"); + write(full, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(full, "kubepods/pod1/ctr/memory.max", "max"); + // A leaf-only view of the container and an unrelated subtree, both unlimited + Path leaf = root.resolve("leaf"); + write(leaf, "memory.max", "max"); + Path other = root.resolve("other"); + write(other, "memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + String fullLine = "31 25 0:26 / " + full + " rw - cgroup2 cgroup2 rw\n"; + String leafLine = "32 25 0:26 /kubepods/pod1/ctr " + leaf + " rw - cgroup2 cgroup2 rw\n"; + String otherLine = "33 25 0:26 /system.slice " + other + " rw - cgroup2 cgroup2 rw\n"; + for (String order : new String[]{otherLine + leafLine + fullLine, + fullLine + leafLine + otherLine, + leafLine + otherLine + fullLine}) { + Path mountinfo = write(root, "mountinfo", order); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root); + assertEquals(order, 4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-mounts"); + try { + Path full = root.resolve("full"); + write(full, "kubepods/pod1/ctr/memory.limit_in_bytes", "9223372036854771712"); + write(full, "kubepods/pod1/ctr/memory.stat", + "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path other = root.resolve("other"); + write(other, "memory.limit_in_bytes", "9223372036854771712"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/kubepods/pod1/ctr\n"); + String fullLine = "41 25 0:30 / " + full + " rw - cgroup cgroup rw,memory\n"; + String otherLine = "42 25 0:30 /system.slice " + other + " rw - cgroup cgroup rw,memory\n"; + // A cpu-only mount is not the memory controller and is ignored + String cpuLine = "43 25 0:31 / " + other + " rw - cgroup cgroup rw,cpu,cpuacct\n"; + for (String order : new String[]{otherLine + cpuLine + fullLine, + fullLine + otherLine + cpuLine}) { + Path mountinfo = write(root, "mountinfo", order); + assertEquals(order, 4 * GIB, + RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root)); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMountRootContainsMembership() { + assertTrue(RocksdbMemoryBudget.contains("/", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods/", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods/pod1/ctr", "/kubepods/pod1/ctr")); + assertFalse(RocksdbMemoryBudget.contains("/system.slice", "/kubepods/pod1/ctr")); + assertFalse(RocksdbMemoryBudget.contains("/kubepods", "/kubepodsx/pod1")); + } + @Test public void testMembershipIsRebasedOntoMountRoot() { assertEquals("pod1/ctr", RocksdbMemoryBudget.rebase("/kubepods/pod1/ctr", "/kubepods")); From 0b287bccc7a5ff83c4ac98c6c39a543341466152 Mon Sep 17 00:00:00 2001 From: ayanx Date: Sat, 17 Oct 2026 12:06:43 +0530 Subject: [PATCH 09/13] fix(store): flag a cgroup mount that can't be matched to the process In a cgroup namespace /proc/self/cgroup says "/" while a cgroup2 mount made outside the namespace has root "/..". No mount contains the process's cgroup, and the child cgroup that is the namespace root can't be located from inside, so its tighter limit is invisible. The resolver used to fall back to /sys/fs/cgroup as if the layout were simply unlimited. Now it reads the limit at each such mount's own root, which still bounds the process, and reports the mapping as unmatched. When auto-sizing, the Store logs a warning saying a tighter limit may be missed and the budget should be set explicitly. With no mountinfo entry at all the default layout is used as before and nothing is flagged. --- .../store/options/RocksdbMemoryBudget.java | 63 +++++++++++++++---- .../options/RocksdbMemoryBudgetTest.java | 59 +++++++++++++++++ 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 267357ea98..432b8435a0 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -67,7 +67,10 @@ private RocksdbMemoryBudget() { public static long resolve(String configured) { long value = isAuto(configured) ? 0L : parse(configured); long heap = Runtime.getRuntime().maxMemory(); - long physical = physicalMemory(); + CgroupLimit cgroup = cgroupLimit(Paths.get("/proc/self/cgroup"), + Paths.get("/proc/self/mountinfo"), + Paths.get("/sys/fs/cgroup")); + long physical = effectivePhysical(jvmMemory(), cgroup.bytes); if (value > 0) { if (value < MIN_EXPLICIT) { throw new IllegalArgumentException(String.format( @@ -98,6 +101,13 @@ public static long resolve(String configured) { long budget = auto(physical, heap, direct); log.info("rocksdb.total_memory_size is unset or 0, auto-sizing to {} bytes " + "(physical={}, heap={}, direct={})", budget, physical, heap, direct); + if (cgroup.unmatched) { + log.warn("No cgroup mount in /proc/self/mountinfo contains this process's " + + "cgroup (an inherited mount in a cgroup namespace), so only the " + + "limits visible at the mount root are counted and a tighter limit on " + + "the process's own cgroup may be missed; set rocksdb.total_memory_size " + + "or HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE explicitly"); + } if (physical > 0 && physical - heap - direct - RESERVED < 2 * MIN_AUTO) { log.warn("Heap, direct memory and the {} bytes reserve leave too little of the " + "{} bytes physical memory, rocksdb gets the {} bytes floor; lower -Xmx " + @@ -158,22 +168,33 @@ public static long writeBufferSize(long totalMemory) { return Math.max(totalMemory / 1000, MIB); } - private static long physicalMemory() { - long jvm; + private static long jvmMemory() { try { // Reports the cgroup v1/v2 memory limit inside a container, but older JDK 17 // updates may not see a limit set only on a parent cgroup OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean( OperatingSystemMXBean.class); - jvm = os == null ? -1L : os.getTotalMemorySize(); + return os == null ? -1L : os.getTotalMemorySize(); } catch (RuntimeException | LinkageError e) { log.warn("Failed to read physical memory size", e); - jvm = -1L; + return -1L; + } + } + + /** + * A cgroup memory limit in bytes (-1 for none), and whether a cgroup mount existed + * but none of them contained this process's cgroup, in which case bytes only covers + * what is visible at those mounts' roots. + */ + static final class CgroupLimit { + + final long bytes; + final boolean unmatched; + + CgroupLimit(long bytes, boolean unmatched) { + this.bytes = bytes; + this.unmatched = unmatched; } - long cgroup = cgroupMemoryLimit(Paths.get("/proc/self/cgroup"), - Paths.get("/proc/self/mountinfo"), - Paths.get("/sys/fs/cgroup")); - return effectivePhysical(jvm, cgroup); } /** @@ -200,15 +221,20 @@ static long effectivePhysical(long jvm, long cgroup) { * mount root, as inside a private cgroup namespace, stays invisible. */ static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { + return cgroupLimit(procSelfCgroup, mountinfo, defaultRoot).bytes; + } + + static CgroupLimit cgroupLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { List v2Mounts = cgroupMounts(mountinfo, true); List v1Mounts = cgroupMounts(mountinfo, false); List lines; try { lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); } catch (IOException | RuntimeException e) { - return -1L; + return new CgroupLimit(-1L, false); } long limit = -1L; + boolean unmatched = false; for (String line : lines) { String[] parts = line.split(":", 3); if (parts.length < 3) { @@ -218,8 +244,9 @@ static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultR if (!v2 && !("," + parts[1] + ",").contains(",memory,")) { continue; } + List mounts = v2 ? v2Mounts : v1Mounts; boolean matched = false; - for (String[] mount : v2 ? v2Mounts : v1Mounts) { + for (String[] mount : mounts) { if (!contains(mount[0], parts[2])) { // A mount of another subtree says nothing about this cgroup continue; @@ -230,13 +257,23 @@ static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultR limit = lowest(limit, v2 ? cgroupV2Limit(point, relative) : cgroupV1Limit(point, relative)); } - if (!matched) { + if (!matched && !mounts.isEmpty()) { + // In a cgroup namespace /proc/self/cgroup says "/" while a mount made + // outside it has a root such as "/..": the process's own cgroup can't be + // located from here. The limits at each mount root still bound it. + unmatched = true; + for (String[] mount : mounts) { + Path point = Paths.get(mount[1]); + limit = lowest(limit, v2 ? readLimit(point.resolve("memory.max")) : + cgroupV1Limit(point, "")); + } + } else if (!matched) { String relative = rebase(parts[2], "/"); limit = lowest(limit, v2 ? cgroupV2Limit(defaultRoot, relative) : cgroupV1Limit(defaultRoot.resolve("memory"), relative)); } } - return limit; + return new CgroupLimit(limit, unmatched); } /** diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 37ce1f4552..4602bab518 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -278,6 +278,65 @@ public void testCgroupV1MountOrderDoesNotHideAncestorLimit() throws IOException } } + @Test + public void testInheritedNamespaceMountIsFlaggedNotTreatedAsUnlimited() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-namespace"); + try { + // cgroup_namespaces(7): /proc/self/cgroup says "/" while a cgroup2 mount made + // outside the namespace has root "/..". The namespace root is a child with a + // 2 GiB limit whose name can't be learned from inside; its parent is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "ns-child/memory.max", String.valueOf(2 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "35 25 0:26 /.. " + mount + " rw - cgroup2 cgroup2 rw\n"); + RocksdbMemoryBudget.CgroupLimit limit = + RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertTrue(limit.unmatched); + assertEquals(-1L, limit.bytes); + + // A limit at the visible mount root still bounds the result + write(mount, "memory.max", String.valueOf(8 * GIB)); + limit = RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertTrue(limit.unmatched); + assertEquals(8 * GIB, limit.bytes); + + // v1 with the same layout + Path v1 = root.resolve("memory"); + write(v1, "memory.limit_in_bytes", "9223372036854771712"); + write(v1, "memory.stat", "hierarchical_memory_limit " + (8 * GIB) + "\n"); + Path proc1 = write(root, "proc-v1", "4:memory:/\n"); + Path mountinfo1 = write(root, "mountinfo-v1", + "45 25 0:30 /.. " + v1 + " rw - cgroup cgroup rw,memory\n"); + limit = RocksdbMemoryBudget.cgroupLimit(proc1, mountinfo1, root); + assertTrue(limit.unmatched); + assertEquals(8 * GIB, limit.bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMatchedOrMissingMountsAreNotFlagged() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-flag"); + try { + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", String.valueOf(4 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "30 25 0:26 / " + mount + " rw - cgroup2 cgroup2 rw\n"); + RocksdbMemoryBudget.CgroupLimit limit = + RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertFalse(limit.unmatched); + assertEquals(4 * GIB, limit.bytes); + // Without any mountinfo entry the default layout is used and nothing is flagged + assertFalse(RocksdbMemoryBudget.cgroupLimit(proc, NO_MOUNTINFO, mount).unmatched); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + @Test public void testMountRootContainsMembership() { assertTrue(RocksdbMemoryBudget.contains("/", "/kubepods/pod1/ctr")); From 5139e1842b6bccee3967dcdf305f930bd623f68f Mon Sep 17 00:00:00 2001 From: ayanx Date: Sat, 17 Oct 2026 12:13:19 +0530 Subject: [PATCH 10/13] fix(store): budget the JVM's real direct-memory allowance With -XX:MaxDirectMemorySize unset the JVM allows direct buffers up to the heap size, but the budget counted at most 1 GiB. The Store accepts gRPC messages up to 1000 MB, so direct-buffer-heavy traffic could use more than was reserved while the RocksDB caches stayed sized. bin/start-hugegraph-store.sh now passes -XX:MaxDirectMemorySize equal to the heap it picks, unless -j sets one (a later flag wins), and the budget counts whatever the JVM reports: the configured cap, or the heap when unset. The launcher's default 2 GiB heap and cap give a 16 GiB host about 5.25 GiB. A raised heap should come with a smaller explicit cap, or direct memory counts as a second heap and the 256 MiB floor applies; the deployment guide says so. --- hugegraph-store/docs/deployment-guide.md | 17 ++++++++------ .../store/options/RocksdbMemoryBudget.java | 12 +++++----- .../static/bin/start-hugegraph-store.sh | 5 ++++- .../options/RocksdbMemoryBudgetTest.java | 22 +++++++++---------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 1b37669525..aac87e6d3f 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -374,15 +374,18 @@ of what is left after the heap, the maximum direct memory and a 1.5 GiB reserve, limit set only on a parent cgroup when the Store can see that cgroup; inside a private cgroup namespace only the container's own limit is visible, so pin the value there if only the pod or slice is limited. A Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without -`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap; the budget counts that -as at most 1 GiB, so set the flag when the Store needs more direct memory than that, and in -small containers, where an uncapped 1 GiB heap already leaves only the 256 MiB floor. +`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts +it that way. `bin/start-hugegraph-store.sh` sets the flag to the heap it picks unless `-j` +passes one, so the budget always counts a limit the JVM enforces. In small containers set a +cap below the heap, or the budget falls to the 256 MiB floor. On a host without a container limit the starting point is physical memory. The default -`bin/start-hugegraph-store.sh` caps the heap at 2 GiB, so a 16 GiB host gets about 5.75 GiB. -A Store whose heap was raised gets less: 2.75 GiB on a 16 GiB host with `-Xmx8g`, 12.75 GiB on -a 64 GiB host with `-Xmx36g`. Before this default every host got a fixed `32000000000`, so pin -the value after upgrading if a large node relied on it. +`bin/start-hugegraph-store.sh` caps the heap at 2 GiB and direct memory at the same 2 GiB, so a +16 GiB host gets about 5.25 GiB. When you raise the heap, pass a direct-memory cap with it, +for example `-j "-Xmx36g -XX:MaxDirectMemorySize=2g"`; otherwise direct memory counts as a +second heap. With a 2 GiB cap a 16 GiB host with `-Xmx8g` gets 2.25 GiB and a 64 GiB host with +`-Xmx36g` gets 12.25 GiB. Before this default every host got a fixed `32000000000`, so pin the +value after upgrading if a large node relied on it. The chosen value is logged by `RocksdbMemoryBudget` at startup, with a warning when the floor applies, or when an explicit value is under 256 MiB or exceeds the memory left after the heap, diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java index 432b8435a0..b96f6213f3 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -47,10 +47,6 @@ public final class RocksdbMemoryBudget { // and other native allocations that sit outside the budget static final long RESERVED = 1024 * MIB + 512 * MIB; static final long MIN_AUTO = 256 * MIB; - // Direct memory counted when -XX:MaxDirectMemorySize is unset. The JVM then allows - // as much as the heap, but the Store's gRPC/Netty buffers do not grow with the heap, - // and counting a 36 GB heap twice would leave rocksdb the floor on a large host - static final long UNSET_DIRECT = 1024 * MIB; // RaftRocksdbOptions gives the write buffer manager 2/3 of the budget, and a 0-byte // write buffer manager means no memtable limit at all static final long MIN_EXPLICIT = MIB; @@ -426,10 +422,12 @@ private static long maxDirectMemory(long heap) { } /** - * The direct memory the budget leaves room for: an explicit -XX:MaxDirectMemorySize - * as given, otherwise (0, unset) the heap size capped at UNSET_DIRECT. + * The direct memory the budget leaves room for: -XX:MaxDirectMemorySize as given, or, + * when it is unset (0), the heap size, which is what the JVM then allows. + * bin/start-hugegraph-store.sh sets the flag to the heap it picks unless -j overrides + * it, so a large heap with a smaller cap should pass that cap explicitly. */ static long directMemory(long configured, long heap) { - return configured > 0 ? configured : Math.min(heap, UNSET_DIRECT); + return configured > 0 ? configured : heap; } } diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index d1ccc651c7..edca9f57a0 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -169,7 +169,10 @@ if [ "$JAVA_OPTIONS" = "" ]; then >> ${OUTPUT} exit 1 fi - JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " + # Cap direct memory at the heap size picked here, the JVM's own unset default, so the + # RocksDB auto-sizing (RocksdbMemoryBudget) counts a limit the JVM actually enforces. + # A -XX:MaxDirectMemorySize in -j comes later on the command line and wins. + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:MaxDirectMemorySize=${XMX}m -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " # JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" # Rolling out detailed GC logs diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java index 4602bab518..a1d9daa87a 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -92,24 +92,24 @@ public void testAutoFitsContainerLimit() { } @Test - public void testUnsetDirectMemoryIsCappedAtOneGib() { + public void testUnsetDirectMemoryCountsAsTheHeap() { assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(512 * MIB, 8 * GIB)); assertEquals(4 * GIB, RocksdbMemoryBudget.directMemory(4 * GIB, 8 * GIB)); - assertEquals(GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB)); + // Unset: the JVM allows as much direct memory as heap, so the budget counts that + assertEquals(8 * GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB)); assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(0L, 512 * MIB)); } @Test - public void testLargeHeapWithoutDirectCapKeepsAUsableBudget() { - // Bare-metal Stores with a raised heap and no -XX:MaxDirectMemorySize; counting - // the unset cap as a second heap gave all of these the 256 MiB floor - assertEquals(2816 * MIB, RocksdbMemoryBudget.auto( - 16 * GIB, 8 * GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB))); - assertEquals(13056 * MIB, RocksdbMemoryBudget.auto( + public void testBareMetalBudgetsWithTheLauncherDirectCap() { + // The default launcher picks a heap of at most 2 GiB and caps direct memory at it + assertEquals(5376 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 2 * GIB, 2 * GIB)); + // A raised heap with a smaller explicit -XX:MaxDirectMemorySize keeps a usable budget + assertEquals(2304 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 8 * GIB, 2 * GIB)); + assertEquals(12544 * MIB, RocksdbMemoryBudget.auto(64 * GIB, 36 * GIB, 2 * GIB)); + // Without one, direct memory counts as a second heap and the floor applies + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto( 64 * GIB, 36 * GIB, RocksdbMemoryBudget.directMemory(0L, 36 * GIB))); - // The default launcher caps the heap at 2 GiB - assertEquals(5888 * MIB, RocksdbMemoryBudget.auto( - 16 * GIB, 2 * GIB, RocksdbMemoryBudget.directMemory(0L, 2 * GIB))); } @Test From 119f819a7ba0c221c013c239a25be5b209bdb6ad Mon Sep 17 00:00:00 2001 From: ayanx Date: Sat, 17 Oct 2026 12:20:33 +0530 Subject: [PATCH 11/13] fix(store): leave the direct-memory cap to a -j heap The launcher put -XX:MaxDirectMemorySize= before the -j options, so a -j that only set -Xmx kept the cap from the heap the launcher had computed: -j -Xmx512m still allowed 2 GiB of direct memory, and a raised heap kept the smaller cap. Leave the flag out when -j sets -Xmx or -XX:MaxDirectMemorySize. The JVM then defaults the cap to the -j heap, or uses the given cap, and RocksdbMemoryBudget counts whichever applies. test-start-hugegraph-store.sh covers no -j, a lower and a raised heap-only -j, and an explicit cap. --- .../travis/test-start-hugegraph-store.sh | 23 ++++++++++++++++++- hugegraph-store/docs/deployment-guide.md | 6 +++-- .../static/bin/start-hugegraph-store.sh | 9 ++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 6e1bc0394d..8e4fad58b3 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -172,7 +172,8 @@ JAVA printf '%s\n' '#!/bin/sh' 'echo launcher-test' > "$fixture/jdk/bin/uname" chmod +x "$fixture/jdk/bin/"* "$fixture/bin/start-hugegraph-store.sh" for option in default custom-java-options user-options default-serial default-zgc \ - g1 G1 zgc ZGC invalid missing-module-options; do + g1 G1 zgc ZGC direct-default direct-heap-lower direct-heap-raised \ + direct-explicit-cap invalid missing-module-options; do capture="$fixture/$option.args" local -a args=(-d false) jvm_options="-Xms64m -Xmx64m" @@ -180,6 +181,11 @@ JAVA default) jvm_options="" ;; custom-java-options) jvm_options="$jvm_options -Dlauncher.test.custom=true" ;; user-options) jvm_options=""; args+=(-j "-Xmx96m -Dlauncher.test=true") ;; + # The direct-memory cap follows the heap the JVM actually gets (calc_xmx: 1024) + direct-default) jvm_options="" ;; + direct-heap-lower) jvm_options=""; args+=(-j "-Xmx512m") ;; + direct-heap-raised) jvm_options=""; args+=(-j "-Dlauncher.test=true -Xmx4g") ;; + direct-explicit-cap) jvm_options=""; args+=(-j "-XX:MaxDirectMemorySize=256m") ;; missing-module-options) rm "$fixture/bin/jvm-module.options" ;; default-serial) jvm_options="$jvm_options -XX:+UseSerialGC" ;; default-zgc) jvm_options="$jvm_options -XX:+UseZGC" ;; @@ -217,6 +223,21 @@ JAVA elif [[ "$option" == user-options ]] && ! grep -Fxq -- '-Dlauncher.test=true' "$capture"; then fail "launcher dropped custom -j options" + elif [[ "$option" == direct-* ]]; then + local cap want + # The JVM uses the last -XX:MaxDirectMemorySize on the command line + cap=$(grep -E '^-XX:MaxDirectMemorySize=' "$capture" | tail -1) + case "$option" in + direct-default) want='-XX:MaxDirectMemorySize=1024m' ;; + direct-explicit-cap) want='-XX:MaxDirectMemorySize=256m' ;; + # A heap-only -j leaves the cap to the JVM, which defaults it to that heap + *) want='' ;; + esac + if [[ "$cap" == "$want" ]]; then + pass "launcher '$option' sets direct memory to '${want:-the JVM default (-j heap)}'" + else + fail "launcher '$option' set direct memory to '${cap:-none}', expected '${want:-none}'" + fi elif [[ "$option" == zgc || "$option" == ZGC ]]; then if grep -Fxq -- '-XX:+UseZGC' "$capture" && \ ! grep -Fxq -- '-XX:InitiatingHeapOccupancyPercent=50' "$capture"; then diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index aac87e6d3f..7f20841de8 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -375,8 +375,10 @@ limit set only on a parent cgroup when the Store can see that cgroup; inside a p namespace only the container's own limit is visible, so pin the value there if only the pod or slice is limited. A Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without `-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts -it that way. `bin/start-hugegraph-store.sh` sets the flag to the heap it picks unless `-j` -passes one, so the budget always counts a limit the JVM enforces. In small containers set a +it that way. `bin/start-hugegraph-store.sh` sets the flag to the heap it picks; when `-j` sets +`-Xmx` or `-XX:MaxDirectMemorySize` itself, the launcher leaves the flag out and the JVM's +default (that heap) or the given cap applies. Either way the budget counts a limit the JVM +enforces. In small containers set a cap below the heap, or the budget falls to the 256 MiB floor. On a host without a container limit the starting point is physical memory. The default diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index edca9f57a0..7bbf06bbca 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -171,8 +171,13 @@ if [ "$JAVA_OPTIONS" = "" ]; then fi # Cap direct memory at the heap size picked here, the JVM's own unset default, so the # RocksDB auto-sizing (RocksdbMemoryBudget) counts a limit the JVM actually enforces. - # A -XX:MaxDirectMemorySize in -j comes later on the command line and wins. - JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:MaxDirectMemorySize=${XMX}m -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " + # Leave it out when -j sets the heap or the cap itself: a fixed cap from the computed + # heap would no longer match a -j -Xmx, and the JVM then defaults it to that heap. + DIRECT_MEMORY_OPTION="-XX:MaxDirectMemorySize=${XMX}m" + case " ${USER_OPTION} " in + *" -Xmx"* | *"-XX:MaxDirectMemorySize="*) DIRECT_MEMORY_OPTION="" ;; + esac + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${DIRECT_MEMORY_OPTION} -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " # JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" # Rolling out detailed GC logs From 643db967cc5f47335fba07a606b7d68b4ae6bc5b Mon Sep 17 00:00:00 2001 From: ayanx Date: Sat, 17 Oct 2026 12:35:49 +0530 Subject: [PATCH 12/13] fix(store): detect a -j heap the way the JVM receives it The check for a heap or cap in -j matched a literal " -Xmx", so options separated by a tab or newline, or the -XX:MaxHeapSize alias, still got the computed direct-memory cap. Split -j the way it reaches the JVM (unquoted, on spaces, tabs and newlines) and match each option against -Xmx*, -XX:MaxHeapSize=* and -XX:MaxDirectMemorySize=*. Two launcher test cases cover the tab-separated and alias forms. --- .../src/assembly/travis/test-start-hugegraph-store.sh | 5 ++++- .../src/assembly/static/bin/start-hugegraph-store.sh | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 8e4fad58b3..27bf8fd88d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -173,7 +173,8 @@ JAVA chmod +x "$fixture/jdk/bin/"* "$fixture/bin/start-hugegraph-store.sh" for option in default custom-java-options user-options default-serial default-zgc \ g1 G1 zgc ZGC direct-default direct-heap-lower direct-heap-raised \ - direct-explicit-cap invalid missing-module-options; do + direct-heap-tab direct-heap-alias direct-explicit-cap \ + invalid missing-module-options; do capture="$fixture/$option.args" local -a args=(-d false) jvm_options="-Xms64m -Xmx64m" @@ -185,6 +186,8 @@ JAVA direct-default) jvm_options="" ;; direct-heap-lower) jvm_options=""; args+=(-j "-Xmx512m") ;; direct-heap-raised) jvm_options=""; args+=(-j "-Dlauncher.test=true -Xmx4g") ;; + direct-heap-tab) jvm_options=""; args+=(-j $'-Dlauncher.test=true\t-Xmx4g') ;; + direct-heap-alias) jvm_options=""; args+=(-j "-XX:MaxHeapSize=4g") ;; direct-explicit-cap) jvm_options=""; args+=(-j "-XX:MaxDirectMemorySize=256m") ;; missing-module-options) rm "$fixture/bin/jvm-module.options" ;; default-serial) jvm_options="$jvm_options -XX:+UseSerialGC" ;; diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 7bbf06bbca..8cb79d97fd 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -174,9 +174,12 @@ if [ "$JAVA_OPTIONS" = "" ]; then # Leave it out when -j sets the heap or the cap itself: a fixed cap from the computed # heap would no longer match a -j -Xmx, and the JVM then defaults it to that heap. DIRECT_MEMORY_OPTION="-XX:MaxDirectMemorySize=${XMX}m" - case " ${USER_OPTION} " in - *" -Xmx"* | *"-XX:MaxDirectMemorySize="*) DIRECT_MEMORY_OPTION="" ;; - esac + # Split -j the way it reaches the JVM below (unquoted: spaces, tabs and newlines) + for opt in ${USER_OPTION}; do + case "$opt" in + -Xmx* | -XX:MaxHeapSize=* | -XX:MaxDirectMemorySize=*) DIRECT_MEMORY_OPTION="" ;; + esac + done JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${DIRECT_MEMORY_OPTION} -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " # JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" From 9e6dc506b95cd06527813dcd12590aed4bd83464 Mon Sep 17 00:00:00 2001 From: ayanx Date: Sat, 17 Oct 2026 13:11:43 +0530 Subject: [PATCH 13/13] ci: keep start-server.sh config writes idempotent across restarts run-api-test.sh now starts the same Server directory twice: once anonymous, then with authentication. start-server.sh appended usePD, schema.sync_deletion and the HBase timeouts with >> on every start, so the second HStore start read usePD as '[true, true]' and init-store failed with "Invalid value for key 'usePD'". Add a set_property helper that deletes every existing line for the key (dots escaped) before appending it once, and use it for those four keys. test-start-server.sh runs the real script twice on one fixture with init-store and the launcher stubbed, for hstore and hbase, and checks each key appears exactly once. It runs in the Server rocksdb job and is mapped in CONTRACT_SUITES. --- .github/scripts/ci-policy.py | 1 + .github/scripts/test_ci_policy.py | 3 +- .github/workflows/server-tests.yml | 1 + .../src/assembly/travis/start-server.sh | 21 ++++- .../src/assembly/travis/test-start-server.sh | 78 +++++++++++++++++++ 5 files changed, 99 insertions(+), 5 deletions(-) create mode 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-server.sh diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index e5a6d30872..0220695df3 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -102,6 +102,7 @@ TRAVIS + "test-codecov-upload-config.sh": {"struct"}, TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, TRAVIS + "test-store-docker-entrypoint.sh": {"store"}, + TRAVIS + "test-start-server.sh": {"server"}, } diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 14f9d89926..0bcfc64c88 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -357,7 +357,8 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): ("test-check-jacoco-report.sh", {"struct"}), ("test-codecov-upload-config.sh", {"struct"}), ("test-riscv64-java-runtime.sh", {"server", "riscv64"}), - ("test-store-docker-entrypoint.sh", {"store"})] + ("test-store-docker-entrypoint.sh", {"store"}), + ("test-start-server.sh", {"server"})] for name, suites in cases: script = policy.TRAVIS + name with self.subTest(script=script): diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index f034a8c44e..8e7ccf9f6f 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -78,6 +78,7 @@ jobs: run: | "$TRAVIS_DIR/test-wait-storage.sh" "$TRAVIS_DIR/test-riscv64-java-runtime.sh" + "$TRAVIS_DIR/test-start-server.sh" bash "$TRAVIS_DIR/test-tinkerpop-reports.sh" - name: Validate Docker integration diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh index b7b70e8cc9..d68a43cc3a 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh @@ -42,6 +42,19 @@ function sed_in_place() { esac } +# Leave exactly one key=value line: run-api-test.sh starts the same directory +# more than once, and a key appended twice reaches HugeConfig as a list. +function set_property() { + local file=$1 + local key=$2 + local value=$3 + local pattern + + pattern=$(printf '%s' "$key" | sed 's/[][\.*^$/]/\\&/g') + sed_in_place "/^[[:space:]]*${pattern}[[:space:]]*[=:]/d" "$file" + echo "${key}=${value}" >> "$file" +} + case "$BACKEND" in memory) SERIALIZER=text @@ -64,18 +77,18 @@ write_property "$CONF" "serializer" "$SERIALIZER" # Set timeout for hbase if [ "$BACKEND" == "hbase" ]; then - echo "restserver.request_timeout=200" >> $REST_CONF - echo "gremlinserver.timeout=200" >> $REST_CONF + set_property "$REST_CONF" "restserver.request_timeout" "200" + set_property "$REST_CONF" "gremlinserver.timeout" "200" sed_in_place 's/evaluationTimeout.*/evaluationTimeout: 200000/' "$GREMLIN_CONF" fi # Set usePD=true for hstore if [ "$BACKEND" == "hstore" ]; then - echo "usePD=true" >> $REST_CONF + set_property "$REST_CONF" "usePD" "true" fi # Append schema.sync_deletion=true to config file -echo "schema.sync_deletion=true" >> $CONF +set_property "$CONF" "schema.sync_deletion" "true" JACOCO_OPTION="" if [ -n "$JACOCO_PORT" ]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-server.sh new file mode 100755 index 0000000000..c99d79d612 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-server.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# run-api-test.sh starts the same Server directory twice. Run start-server.sh +# twice on one fixture, with init-store and the launcher stubbed, and check +# that every key it writes appears exactly once: a duplicate usePD reached +# HugeConfig as '[true, true]' and failed the second init-store. +set -euo pipefail + +TRAVIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +STATIC_BIN="${TRAVIS_DIR}/../static/bin" +FIXTURE=$(mktemp -d "${TMPDIR:-/tmp}/start-server-test.XXXXXX") +trap 'rm -rf "${FIXTURE}"' EXIT + +PASS=0 +FAIL=0 + +expect_once() { + # expect_once + local lines + lines=$(grep -E "^[[:space:]]*${2//./\\.}[[:space:]]*[=:]" "$1" || true) + if [[ "${lines}" == "$2=$3" ]]; then + PASS=$((PASS + 1)) + else + echo " FAIL $(basename "$1"): expected one '$2=$3', got: ${lines:-nothing}" >&2 + FAIL=$((FAIL + 1)) + fi +} + +for backend in hstore hbase; do + base="${FIXTURE}/${backend}" + mkdir -p "${base}/bin" "${base}/conf/graphs" "${base}/travis" + cp "${TRAVIS_DIR}/start-server.sh" "${base}/travis/" + cp "${STATIC_BIN}/util.sh" "${base}/bin/" + printf '#!/bin/bash\ncat >/dev/null\n' > "${base}/bin/init-store.sh" + printf '#!/bin/bash\nexit 0\n' > "${base}/bin/start-hugegraph.sh" + chmod +x "${base}/bin/"*.sh "${base}/travis/start-server.sh" + printf 'backend=rocksdb\nserializer=binary\n' > "${base}/conf/graphs/hugegraph.properties" + printf 'restserver.url=http://127.0.0.1:8080\n' > "${base}/conf/rest-server.properties" + printf 'evaluationTimeout: 30000\n' > "${base}/conf/gremlin-server.yaml" + + for run in 1 2; do + if ! bash "${base}/travis/start-server.sh" "${base}" "${backend}" "" \ + > "${base}/start-${run}.log" 2>&1; then + echo " FAIL ${backend}: start-server.sh run ${run} failed" >&2 + tail -5 "${base}/start-${run}.log" >&2 + FAIL=$((FAIL + 1)) + fi + done + + graph="${base}/conf/graphs/hugegraph.properties" + rest="${base}/conf/rest-server.properties" + expect_once "${graph}" "backend" "${backend}" + expect_once "${graph}" "schema.sync_deletion" "true" + if [[ "${backend}" == hstore ]]; then + expect_once "${rest}" "usePD" "true" + else + expect_once "${rest}" "restserver.request_timeout" "200" + expect_once "${rest}" "gremlinserver.timeout" "200" + fi +done + +echo "start-server.sh twice on one directory: ${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" -eq 0 ]]