diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 6241fffda4..0220695df3 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -101,6 +101,8 @@ TRAVIS + "test-check-jacoco-report.sh": {"struct"}, TRAVIS + "test-codecov-upload-config.sh": {"struct"}, TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, + TRAVIS + "test-store-docker-entrypoint.sh": {"store"}, + TRAVIS + "test-start-server.sh": {"server"}, } diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 65e7666324..0bcfc64c88 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -356,14 +356,16 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): ("test-check-java-runtime-logs.sh", {"struct"}), ("test-check-jacoco-report.sh", {"struct"}), ("test-codecov-upload-config.sh", {"struct"}), - ("test-riscv64-java-runtime.sh", {"server", "riscv64"})] + ("test-riscv64-java-runtime.sh", {"server", "riscv64"}), + ("test-store-docker-entrypoint.sh", {"store"}), + ("test-start-server.sh", {"server"})] for name, suites in cases: script = policy.TRAVIS + name with self.subTest(script=script): plan = self.plan_for([script]) self.assertEqual(suites, set(plan["selected"])) callers = self.coordinator_jobs(plan) - if "struct" in suites: + if "struct" in suites or "store" in suites: self.assertEqual({"pd_store"}, callers) root = (workflows / "server-ci.yml").read_text() caller = re.search(r"^ pd_store:\n(.*?)(?=^ [\w-]+:|\Z)", root, re.M | re.S).group(1) @@ -373,12 +375,13 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower())) inputs[key] = eval(binding, {"__builtins__": {}}) workflow = (workflows / "pd-store-ci.yml").read_text() - struct = workflow.split("\n struct:\n", 1)[1].split("\n pd:\n", 1)[0] - condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) + job = "struct" if "struct" in suites else "store" + body = re.search(r"^ " + job + r":\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + condition = re.search(r"^ if: (.+)$", body, re.M).group(1) condition = condition.replace("inputs.run-struct", str(inputs["struct"])) condition = condition.replace("inputs.run-store", str(inputs["store"])) self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}})) - step = next(step for step in struct.split("\n - ") if Path(script).name in step) + step = next(step for step in body.split("\n - ") if Path(script).name in step) self.assertNotRegex(step, r"(?m)^ if:") else: self.assertIn("server-rocksdb", callers) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index ceb1d1121c..60bd3ea2e9 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -412,6 +412,10 @@ jobs: run: | mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp + - name: Run Store docker entrypoint rocksdb memory tests + run: | + $TRAVIS_DIR/test-store-docker-entrypoint.sh + - name: Check startup test prerequisites (Store) id: store-preflight run: | diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index c9b39bd3d9..8915b47aa9 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -57,9 +57,11 @@ store: -XX:MaxMetaspaceSize=256m -XX:MaxDirectMemorySize=512m -XX:+UseContainerSupport - # Container limits do not configure JVM heap or RocksDB native cache budgets. - # Size this preset for the workload and configure those application budgets - # separately; javaOpts accepts explicit JVM/system-property overrides. + # Container limits do not configure the JVM heap; javaOpts does. Store images + # with auto-sized RocksDB memory (apache/hugegraph#3254) give RocksDB half of + # the container limit left after heap, direct memory and a 1.5 GiB reserve; + # with this preset that is 2.5 GiB. Set store.extraEnv + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE (bytes) to use a different value. resources: requests: cpu: "1" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 6e1bc0394d..27bf8fd88d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -172,7 +172,9 @@ JAVA printf '%s\n' '#!/bin/sh' 'echo launcher-test' > "$fixture/jdk/bin/uname" chmod +x "$fixture/jdk/bin/"* "$fixture/bin/start-hugegraph-store.sh" for option in default custom-java-options user-options default-serial default-zgc \ - g1 G1 zgc ZGC invalid missing-module-options; do + g1 G1 zgc ZGC direct-default direct-heap-lower direct-heap-raised \ + direct-heap-tab direct-heap-alias direct-explicit-cap \ + invalid missing-module-options; do capture="$fixture/$option.args" local -a args=(-d false) jvm_options="-Xms64m -Xmx64m" @@ -180,6 +182,13 @@ JAVA default) jvm_options="" ;; custom-java-options) jvm_options="$jvm_options -Dlauncher.test.custom=true" ;; user-options) jvm_options=""; args+=(-j "-Xmx96m -Dlauncher.test=true") ;; + # The direct-memory cap follows the heap the JVM actually gets (calc_xmx: 1024) + direct-default) jvm_options="" ;; + direct-heap-lower) jvm_options=""; args+=(-j "-Xmx512m") ;; + direct-heap-raised) jvm_options=""; args+=(-j "-Dlauncher.test=true -Xmx4g") ;; + direct-heap-tab) jvm_options=""; args+=(-j $'-Dlauncher.test=true\t-Xmx4g') ;; + direct-heap-alias) jvm_options=""; args+=(-j "-XX:MaxHeapSize=4g") ;; + direct-explicit-cap) jvm_options=""; args+=(-j "-XX:MaxDirectMemorySize=256m") ;; missing-module-options) rm "$fixture/bin/jvm-module.options" ;; default-serial) jvm_options="$jvm_options -XX:+UseSerialGC" ;; default-zgc) jvm_options="$jvm_options -XX:+UseZGC" ;; @@ -217,6 +226,21 @@ JAVA elif [[ "$option" == user-options ]] && ! grep -Fxq -- '-Dlauncher.test=true' "$capture"; then fail "launcher dropped custom -j options" + elif [[ "$option" == direct-* ]]; then + local cap want + # The JVM uses the last -XX:MaxDirectMemorySize on the command line + cap=$(grep -E '^-XX:MaxDirectMemorySize=' "$capture" | tail -1) + case "$option" in + direct-default) want='-XX:MaxDirectMemorySize=1024m' ;; + direct-explicit-cap) want='-XX:MaxDirectMemorySize=256m' ;; + # A heap-only -j leaves the cap to the JVM, which defaults it to that heap + *) want='' ;; + esac + if [[ "$cap" == "$want" ]]; then + pass "launcher '$option' sets direct memory to '${want:-the JVM default (-j heap)}'" + else + fail "launcher '$option' set direct memory to '${cap:-none}', expected '${want:-none}'" + fi elif [[ "$option" == zgc || "$option" == ZGC ]]; then if grep -Fxq -- '-XX:+UseZGC' "$capture" && \ ! grep -Fxq -- '-XX:InitiatingHeapOccupancyPercent=50' "$capture"; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh new file mode 100755 index 0000000000..a16198f11b --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Checks that the Store Docker entrypoint passes HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE +# into SPRING_APPLICATION_JSON as rocksdb.total_memory_size, leaves the rocksdb key +# out when the variable is unset (so conf/application-pd.yml or auto-sizing applies), +# and refuses values that are not a plain number of bytes. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" +ENTRYPOINT="${REPO_ROOT}/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh" +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/store-entrypoint-test.XXXXXX") +trap 'rm -rf "${TMP_DIR}"' EXIT + +PASS=0 +FAIL=0 + +[[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; } + +mkdir -p "${TMP_DIR}/bin" +cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh" +# Stand in for the launcher: record the generated config instead of starting Store +cat > "${TMP_DIR}/bin/start-hugegraph-store.sh" <<'STUB' +#!/usr/bin/env bash +printf '%s' "${SPRING_APPLICATION_JSON}" > ./spring.json +STUB +chmod +x "${TMP_DIR}/bin/start-hugegraph-store.sh" "${TMP_DIR}/docker-entrypoint.sh" + +run_entrypoint() { + rm -f "${TMP_DIR}/spring.json" + # Drop any value from the caller's environment so "unset" really is unset + (cd "${TMP_DIR}" && env -u HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE \ + HG_STORE_PD_ADDRESS=pd0:8686 \ + HG_STORE_GRPC_HOST=store0 \ + HG_STORE_RAFT_ADDRESS=store0:8510 \ + "$@" \ + ./docker-entrypoint.sh 2>&1) +} + +expect_memory() { + # expect_memory [NAME=VALUE ...] + local name="$1" expected="$2" + shift 2 + local out + if ! out=$(run_entrypoint "$@"); then + echo " FAIL ${name}: entrypoint exited non-zero" + printf '%s\n' "${out}" | tail -3 + FAIL=$((FAIL + 1)) + return + fi + if ! EXPECTED="${expected}" python3 - "${TMP_DIR}/spring.json" <<'PY' +import json, os, sys +with open(sys.argv[1], encoding="utf-8") as fh: + doc = json.load(fh) +want = os.environ["EXPECTED"] +got = doc.get("rocksdb", {}).get("total_memory_size", "absent") +if got != want: + print(" rocksdb.total_memory_size is %r, expected %r" % (got, want)) + sys.exit(1) +if doc["app"]["data-path"] != "/hugegraph-store/storage": + print(" app.data-path was lost") + sys.exit(1) +PY + then + echo " FAIL ${name}: invalid JSON or wrong rocksdb.total_memory_size" + FAIL=$((FAIL + 1)) + return + fi + echo " PASS ${name}" + PASS=$((PASS + 1)) +} + +echo "Store docker-entrypoint rocksdb memory override" +expect_memory "unset leaves the key out" absent +expect_memory "empty leaves the key out" absent HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE= +expect_memory "bytes are passed through" 2147483648 \ + HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=2147483648 +expect_memory "0 is passed through for auto-sizing" 0 HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=0 + +for bad in '4GB' '-1' '1.5e9' '1 2' '1"}'; do + if run_entrypoint HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE="${bad}" >/dev/null; then + echo " FAIL '${bad}' was accepted" + FAIL=$((FAIL + 1)) + else + echo " PASS '${bad}' is refused" + PASS=$((PASS + 1)) + fi +done + +echo "${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" -eq 0 ]] diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index 3615649ff1..90ef3e3653 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -249,7 +249,9 @@ bin/start-hugegraph-store.sh [-g GC_TYPE] [-j "JVM_OPTIONS"] [-d DAEMON] set in `JAVA_OPTIONS` apply. Explicit `-g g1` selects G1 even when JVM ergonomics would choose Serial GC; unknown values are rejected. When using explicit `-g`, do not select another collector in `JAVA_OPTIONS` or `-j`; conflicting selections are rejected by the JVM. -- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g"`) +- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g -XX:MaxDirectMemorySize=2g"`). A `-j` heap replaces the + launcher's direct-memory cap, so pass one with it; otherwise the RocksDB budget counts a second heap + of direct memory (see the deployment guide's Store Configuration). - `-d`: Daemon mode (`true` = daemon, `false` = foreground; default: `true`). Set to `false` when running under Docker or a process supervisor so the container exits if Java dies. The launcher always loads `bin/jvm-module.options`, including with custom `JAVA_OPTIONS` or `-j`. @@ -389,6 +391,10 @@ docker run -d \ hugegraph/store:latest ``` +Add `-e HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=` to pin the off-heap RocksDB budget. +Without it the Store sizes the budget from the container memory limit; see +[Store Configuration](docs/deployment-guide.md#store-configuration) in the deployment guide. + **Exposed Ports**: - `8520`: REST API (management, metrics) - `8500`: gRPC (client connections) diff --git a/hugegraph-store/docs/best-practices.md b/hugegraph-store/docs/best-practices.md index 47d02521c3..74524943e7 100644 --- a/hugegraph-store/docs/best-practices.md +++ b/hugegraph-store/docs/best-practices.md @@ -85,7 +85,7 @@ app: **JVM Settings**: ```bash -JAVA_OPTS="-Xms32g -Xmx32g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20" +JAVA_OPTS="-Xms32g -Xmx32g -XX:MaxDirectMemorySize=2g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20" ``` ### PD Node Recommendations @@ -252,6 +252,7 @@ store.rpc_timeout=30000 # 30 seconds ```bash JAVA_OPTS=" -Xms32g -Xmx32g # Fixed heap size + -XX:MaxDirectMemorySize=2g # Direct-memory cap; RocksDB auto-sizing counts it -XX:+UseG1GC # G1 garbage collector -XX:MaxGCPauseMillis=200 # Target pause time -XX:G1ReservePercent=20 # Reserve for to-space @@ -267,6 +268,7 @@ JAVA_OPTS=" ```bash JAVA_OPTS=" -Xms32g -Xmx32g + -XX:MaxDirectMemorySize=2g # Direct-memory cap; RocksDB auto-sizing counts it -XX:+UseZGC # ZGC (sub-10ms pauses) -XX:ZCollectionInterval=120 # GC interval (seconds) -XX:+UnlockDiagnosticVMOptions diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 3d4f798a59..1394d56ec1 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -347,7 +347,7 @@ File: `hugegraph-store/conf/application-pd.yml` (RocksDB tuning) ```yaml rocksdb: # Memory Configuration - total_memory_size: 32000000000 # Total memory for RocksDB (32GB) + total_memory_size: 0 # Off-heap RocksDB budget in bytes; 0 = auto-size write_buffer_size: 134217728 # Memtable size (128MB) max_write_buffer_number: 6 # Max memtables min_write_buffer_number_to_merge: 2 # Min memtables to merge @@ -356,14 +356,53 @@ rocksdb: level0_file_num_compaction_trigger: 4 max_background_jobs: 8 # Background compaction/flush threads - # Block Cache - block_cache_size: 16000000000 # Block cache (16GB) + # Budget split: this share goes to memtables, the rest to the block cache + write_buffer_ratio: 0.66 # SST File Size target_file_size_base: 134217728 # Target SST size (128MB) max_bytes_for_level_base: 1073741824 # L1 size (1GB) ``` +`total_memory_size` bounds the memtables (`write_buffer_ratio` of it) and the block cache +(the rest). Both live outside the Java heap, so `-Xmx` does not limit them. A fixed 1 GiB +cache for the Raft log comes on top of this budget. + +With `total_memory_size: 0`, or the key unset, the Store sizes the budget at startup as half +of what is left after the heap, the maximum direct memory and a 1.5 GiB reserve, clamped to +256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, including a +limit set only on a parent cgroup when the Store can see that cgroup; inside a private cgroup +namespace only the container's own limit is visible, so pin the value there if only the pod or +slice is limited. A Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` gets 512 MiB. Without +`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts +it that way. `bin/start-hugegraph-store.sh` sets the flag to the heap it picks; when `-j` sets +`-Xmx`, `-XX:MaxHeapSize` or `-XX:MaxDirectMemorySize` itself, the launcher leaves the flag out and the JVM's +default (that heap) or the given cap applies. Either way the budget counts a limit the JVM +enforces. In small containers set a +cap below the heap, or the budget falls to the 256 MiB floor. + +On a host without a container limit the starting point is physical memory. The default +`bin/start-hugegraph-store.sh` caps the heap at 2 GiB and direct memory at the same 2 GiB, so a +16 GiB host gets about 5.25 GiB. When you raise the heap, pass a direct-memory cap with it, +for example `-j "-Xmx36g -XX:MaxDirectMemorySize=2g"`; otherwise direct memory counts as a +second heap. With a 2 GiB cap a 16 GiB host with `-Xmx8g` gets 2.25 GiB and a 64 GiB host with +`-Xmx36g` gets 12.25 GiB. Before this default every host got a fixed `32000000000`, so pin the +value after upgrading if a large node relied on it. + +The chosen value is logged by `RocksdbMemoryBudget` at startup, with a warning when the floor +applies, or when an explicit value is under 256 MiB or exceeds the memory left after the heap, +direct memory and the reserve. Set a number of bytes, at least 1048576 (1 MiB), to pin it; a +smaller value or a unit suffix such as `4GB` is rejected. Before this default existed the +shipped value was `32000000000`, which can exceed a container limit and get the Store +OOM-killed. Both caps are soft: RocksDB can briefly go over them under heavy writes, and a +small budget with many partitions means frequent memtable flushes. + +**Upgrading an existing deployment.** Auto-sizing applies only when the loaded configuration +has `total_memory_size: 0` or no value. A Store that mounts its own `conf/application-pd.yml`, +or a tarball install that kept its `conf/` from an earlier release, still has `32000000000` and +keeps that fixed budget; the Store logs a warning naming the old value at startup. Set the key +to `0` or remove it, or set `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` in Docker. + ### Server Configuration File: `hugegraph-server/conf/graphs/hugegraph.properties` @@ -726,8 +765,17 @@ environment: HG_STORE_REST_PORT: "8520" # maps to server.port HG_STORE_RAFT_ADDRESS: store0:8510 # maps to raft.address HG_STORE_DATA_PATH: /hugegraph-store/storage # maps to app.data-path + # optional; maps to rocksdb.total_memory_size (bytes); unset = config file or auto-size + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE: "2147483648" ``` +`HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` must be a plain number of bytes; the entrypoint refuses +anything else. When set it outranks `rocksdb.total_memory_size` in a mounted +`conf/application-pd.yml` and a `-Drocksdb.total_memory_size` in `JAVA_OPTS`. Leave it unset +to let the Store size the budget from the container memory limit, provided the loaded config +also has `total_memory_size: 0` or no value; a mounted `application-pd.yml` from an earlier +release still has `32000000000` (see [Store Configuration](#store-configuration)). + **Server environment variables**: ```yaml @@ -826,6 +874,10 @@ spec: value: "$(POD_NAME).hugegraph-store" - name: HG_STORE_RAFT_ADDRESS value: "$(POD_NAME).hugegraph-store:8510" + # Optional: pin the off-heap RocksDB budget (bytes). Unset, it is sized + # from the container memory limit below. + # - name: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE + # value: "4294967296" volumeMounts: - name: data mountPath: /hugegraph-store/storage diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java new file mode 100644 index 0000000000..b96f6213f3 --- /dev/null +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.OperatingSystemMXBean; + +import lombok.extern.slf4j.Slf4j; + +/** + * Resolves rocksdb.total_memory_size, the budget RaftRocksdbOptions splits into the + * write buffer manager's cache and the block cache. Both caches are off-heap, so an + * unset or "0" budget is sized from the memory left after the heap, direct memory and + * the fixed raft log cache, instead of a fixed number that can exceed a container limit. + */ +@Slf4j +public final class RocksdbMemoryBudget { + + public static final String KEY = "total_memory_size"; + + static final long MIB = 1024L * 1024L; + // The 1 GiB raft log block cache (RaftRocksdbOptions) plus thread stacks, metaspace + // and other native allocations that sit outside the budget + static final long RESERVED = 1024 * MIB + 512 * MIB; + static final long MIN_AUTO = 256 * MIB; + // RaftRocksdbOptions gives the write buffer manager 2/3 of the budget, and a 0-byte + // write buffer manager means no memtable limit at all + static final long MIN_EXPLICIT = MIB; + // The value shipped before auto-sizing; a large host never gets more than it had + static final long MAX_AUTO = 32_000_000_000L; + + private RocksdbMemoryBudget() { + } + + /** + * Returns the configured budget in bytes, or an automatic one when the value is + * missing, blank or 0. + */ + public static long resolve(String configured) { + long value = isAuto(configured) ? 0L : parse(configured); + long heap = Runtime.getRuntime().maxMemory(); + CgroupLimit cgroup = cgroupLimit(Paths.get("/proc/self/cgroup"), + Paths.get("/proc/self/mountinfo"), + Paths.get("/sys/fs/cgroup")); + long physical = effectivePhysical(jvmMemory(), cgroup.bytes); + if (value > 0) { + if (value < MIN_EXPLICIT) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be 0 (auto) or at least %s bytes, " + + "got %s", MIN_EXPLICIT, value)); + } + if (value < MIN_AUTO) { + log.warn("rocksdb.total_memory_size={} is below the {} bytes auto-sizing " + + "never goes under", value, MIN_AUTO); + } + if (value == MAX_AUTO) { + // Most likely an application-pd.yml kept from before auto-sizing + log.warn("rocksdb.total_memory_size={} is the fixed value shipped before " + + "auto-sizing; set it to 0 or remove it to size the budget from " + + "the available memory", value); + } + long direct = maxDirectMemory(heap); + long left = physical - heap - direct - RESERVED; + if (physical > 0 && value > left) { + log.warn("rocksdb.total_memory_size={} exceeds the {} bytes left after the " + + "heap, direct memory and the {} bytes reserve (physical={}, heap={}, " + + "direct={}); the process may be OOM-killed", + value, left, RESERVED, physical, heap, direct); + } + return value; + } + long direct = maxDirectMemory(heap); + long budget = auto(physical, heap, direct); + log.info("rocksdb.total_memory_size is unset or 0, auto-sizing to {} bytes " + + "(physical={}, heap={}, direct={})", budget, physical, heap, direct); + if (cgroup.unmatched) { + log.warn("No cgroup mount in /proc/self/mountinfo contains this process's " + + "cgroup (an inherited mount in a cgroup namespace), so only the " + + "limits visible at the mount root are counted and a tighter limit on " + + "the process's own cgroup may be missed; set rocksdb.total_memory_size " + + "or HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE explicitly"); + } + if (physical > 0 && physical - heap - direct - RESERVED < 2 * MIN_AUTO) { + log.warn("Heap, direct memory and the {} bytes reserve leave too little of the " + + "{} bytes physical memory, rocksdb gets the {} bytes floor; lower -Xmx " + + "or -XX:MaxDirectMemorySize", RESERVED, physical, MIN_AUTO); + } + return budget; + } + + static boolean isAuto(String configured) { + return configured == null || configured.trim().isEmpty(); + } + + static long parse(String configured) { + try { + long value = Long.parseLong(configured.trim()); + if (value < 0) { + throw new NumberFormatException("negative"); + } + return value; + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be a number of bytes, got '%s'", + configured), e); + } + } + + /** + * Half of what is left after the heap, direct memory and the reserve, clamped to + * [MIN_AUTO, MAX_AUTO]. Falls back to the heap size, the previous default, when + * physical memory is unknown. + */ + static long auto(long physical, long heap, long direct) { + if (physical <= 0) { + return Math.min(Math.max(heap, MIN_AUTO), MAX_AUTO); + } + long available = physical - heap - direct - RESERVED; + return Math.min(Math.max(available / 2, MIN_AUTO), MAX_AUTO); + } + + /** + * The raft disruptor buffer size AppConfig derives from the budget when + * raft.disruptorBufferSize is 0: the budget in GB, at least 1, rounded to a power + * of two, times 32. + */ + public static int disruptorBufferSize(long totalMemory) { + // A budget under 1 GB would otherwise give log(0) and a zero-sized buffer + int size = (int) Math.max(1, totalMemory / 1000 / 1000 / 1000); + return (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; + } + + /** + * The write_buffer_size AppConfig derives when the key is unset or 0: a thousandth + * of the budget, but never below the 1 MiB rocksdb.write_buffer_size accepts + * (RocksDBOptions.MEMTABLE_SIZE), which an auto-sized budget under 1 GiB would + * otherwise fall under. + */ + public static long writeBufferSize(long totalMemory) { + return Math.max(totalMemory / 1000, MIB); + } + + private static long jvmMemory() { + try { + // Reports the cgroup v1/v2 memory limit inside a container, but older JDK 17 + // updates may not see a limit set only on a parent cgroup + OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean( + OperatingSystemMXBean.class); + return os == null ? -1L : os.getTotalMemorySize(); + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read physical memory size", e); + return -1L; + } + } + + /** + * A cgroup memory limit in bytes (-1 for none), and whether a cgroup mount existed + * but none of them contained this process's cgroup, in which case bytes only covers + * what is visible at those mounts' roots. + */ + static final class CgroupLimit { + + final long bytes; + final boolean unmatched; + + CgroupLimit(long bytes, boolean unmatched) { + this.bytes = bytes; + this.unmatched = unmatched; + } + } + + /** + * The smaller of what the JVM reports and the cgroup limit, ignoring unknown (<= 0) + * values. + */ + static long effectivePhysical(long jvm, long cgroup) { + if (cgroup <= 0) { + return jvm; + } + return jvm <= 0 ? cgroup : Math.min(jvm, cgroup); + } + + /** + * The lowest memory limit on this process's cgroup or any visible ancestor, or -1 + * when there is none or it can't be read. The membership path from /proc/self/cgroup + * is rebased onto every mount of the controller in /proc/self/mountinfo whose root + * contains it, so a cgroupfs mounted from a subtree (root /kubepods at /sys/fs/cgroup) + * is walked from the right directory, and the lowest limit over all those mounts is + * taken: a leaf-only or unrelated mount listed first can't hide a fuller view listed + * later. Without a matching mount the default root is used. cgroup v2 + * walks memory.max from the process's cgroup up to the mount; cgroup v1 reads + * hierarchical_memory_limit, which already accounts for ancestors. A limit above the + * mount root, as inside a private cgroup namespace, stays invisible. + */ + static long cgroupMemoryLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { + return cgroupLimit(procSelfCgroup, mountinfo, defaultRoot).bytes; + } + + static CgroupLimit cgroupLimit(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { + List v2Mounts = cgroupMounts(mountinfo, true); + List v1Mounts = cgroupMounts(mountinfo, false); + List lines; + try { + lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { + return new CgroupLimit(-1L, false); + } + long limit = -1L; + boolean unmatched = false; + for (String line : lines) { + String[] parts = line.split(":", 3); + if (parts.length < 3) { + continue; + } + boolean v2 = parts[0].equals("0") && parts[1].isEmpty(); + if (!v2 && !("," + parts[1] + ",").contains(",memory,")) { + continue; + } + List mounts = v2 ? v2Mounts : v1Mounts; + boolean matched = false; + for (String[] mount : mounts) { + if (!contains(mount[0], parts[2])) { + // A mount of another subtree says nothing about this cgroup + continue; + } + matched = true; + Path point = Paths.get(mount[1]); + String relative = rebase(parts[2], mount[0]); + limit = lowest(limit, v2 ? cgroupV2Limit(point, relative) : + cgroupV1Limit(point, relative)); + } + if (!matched && !mounts.isEmpty()) { + // In a cgroup namespace /proc/self/cgroup says "/" while a mount made + // outside it has a root such as "/..": the process's own cgroup can't be + // located from here. The limits at each mount root still bound it. + unmatched = true; + for (String[] mount : mounts) { + Path point = Paths.get(mount[1]); + limit = lowest(limit, v2 ? readLimit(point.resolve("memory.max")) : + cgroupV1Limit(point, "")); + } + } else if (!matched) { + String relative = rebase(parts[2], "/"); + limit = lowest(limit, v2 ? cgroupV2Limit(defaultRoot, relative) : + cgroupV1Limit(defaultRoot.resolve("memory"), relative)); + } + } + return new CgroupLimit(limit, unmatched); + } + + /** + * The {root, mount point} of every cgroup v2 mount, or of every cgroup v1 mount + * carrying the memory controller, in /proc/self/mountinfo order. + */ + private static List cgroupMounts(Path mountinfo, boolean v2) { + List mounts = new ArrayList<>(); + List lines; + try { + lines = Files.readAllLines(mountinfo, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { + return mounts; + } + for (String line : lines) { + // id parent major:minor root mount-point options [optional...] - type source super + int separator = line.indexOf(" - "); + if (separator < 0) { + continue; + } + String[] fields = line.substring(0, separator).split(" "); + String[] tail = line.substring(separator + 3).split(" "); + if (fields.length < 5 || tail.length < 1) { + continue; + } + boolean match = v2 ? tail[0].equals("cgroup2") : + tail[0].equals("cgroup") && tail.length >= 3 && + ("," + tail[2] + ",").contains(",memory,"); + if (match) { + mounts.add(new String[]{unescape(fields[3]), unescape(fields[4])}); + } + } + return mounts; + } + + // Whether a mount whose root is mountRoot shows the cgroup at membership + static boolean contains(String mountRoot, String membership) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = mountRoot.endsWith("/") && mountRoot.length() > 1 ? + mountRoot.substring(0, mountRoot.length() - 1) : mountRoot; + return root.equals("/") || path.equals(root) || path.startsWith(root + "/"); + } + + /** + * The membership path relative to a mount whose root is mountRoot: /kubepods/pod1/ctr + * under root /kubepods is pod1/ctr. A path outside the mount root is kept as is. + */ + static String rebase(String membership, String mountRoot) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = mountRoot.endsWith("/") && mountRoot.length() > 1 ? + mountRoot.substring(0, mountRoot.length() - 1) : mountRoot; + if (!root.equals("/")) { + if (path.equals(root)) { + return ""; + } + if (path.startsWith(root + "/")) { + path = path.substring(root.length()); + } + } + return path.substring(1); + } + + // mountinfo writes space, tab, newline and backslash as octal escapes such as \040 + private static String unescape(String field) { + StringBuilder out = new StringBuilder(field.length()); + for (int i = 0; i < field.length(); i++) { + char c = field.charAt(i); + if (c == '\\' && i + 4 <= field.length() && + field.substring(i + 1, i + 4).matches("[0-7]{3}")) { + out.append((char) Integer.parseInt(field.substring(i + 1, i + 4), 8)); + i += 3; + } else { + out.append(c); + } + } + return out.toString(); + } + + private static long cgroupV2Limit(Path root, String relative) { + Path dir = root.resolve(relative).normalize(); + if (!dir.startsWith(root) || !Files.isDirectory(dir)) { + // The process path is not visible from this mount (a cgroup namespace) + dir = root; + } + long limit = -1L; + for (Path d = dir; d != null && d.startsWith(root); d = d.getParent()) { + limit = lowest(limit, readLimit(d.resolve("memory.max"))); + } + return limit; + } + + private static long cgroupV1Limit(Path base, String relative) { + Path dir = base.resolve(relative).normalize(); + if (!dir.startsWith(base) || !Files.isDirectory(dir)) { + dir = base; + } + long limit = readLimit(dir.resolve("memory.limit_in_bytes")); + try { + for (String line : Files.readAllLines(dir.resolve("memory.stat"), + StandardCharsets.UTF_8)) { + if (line.startsWith("hierarchical_memory_limit ")) { + limit = lowest(limit, parseLimit(line.substring(line.indexOf(' ') + 1))); + } + } + } catch (IOException | RuntimeException e) { + // memory.stat is optional here, memory.limit_in_bytes already counted + } + return limit; + } + + private static long readLimit(Path file) { + try { + return parseLimit(new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + } catch (IOException | RuntimeException e) { + return -1L; + } + } + + // "max" (v2) and the near-Long.MAX_VALUE page-aligned v1 default mean no limit + private static long parseLimit(String value) { + String trimmed = value.trim(); + if (trimmed.isEmpty() || trimmed.equals("max")) { + return -1L; + } + try { + long limit = Long.parseLong(trimmed); + return limit <= 0 || limit >= Long.MAX_VALUE / 2 ? -1L : limit; + } catch (NumberFormatException e) { + return -1L; + } + } + + private static long lowest(long a, long b) { + if (a <= 0) { + return b; + } + return b <= 0 ? a : Math.min(a, b); + } + + private static long maxDirectMemory(long heap) { + try { + HotSpotDiagnosticMXBean hotspot = ManagementFactory.getPlatformMXBean( + HotSpotDiagnosticMXBean.class); + return directMemory(Long.parseLong( + hotspot.getVMOption("MaxDirectMemorySize").getValue()), heap); + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read MaxDirectMemorySize, assuming it is unset", e); + return directMemory(0L, heap); + } + } + + /** + * The direct memory the budget leaves room for: -XX:MaxDirectMemorySize as given, or, + * when it is unset (0), the heap size, which is what the JVM then allows. + * bin/start-hugegraph-store.sh sets the flag to the heap it picks unless -j overrides + * it, so a large heap with a smaller cap should pass that cap explicitly. + */ + static long directMemory(long configured, long heap) { + return configured > 0 ? configured : heap; + } +} diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..2e90b75ff9 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -54,6 +54,22 @@ require_env "HG_STORE_RAFT_ADDRESS" : "${HG_STORE_GRPC_PORT:=8500}" : "${HG_STORE_REST_PORT:=8520}" : "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE:=}" + +# Bytes only: the Store parses the value as a long and refuses to start otherwise +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" && + ! "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" =~ ^[0-9]+$ ]]; then + echo "ERROR: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE must be a number of bytes," \ + "got '${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}'" >&2 + exit 2 +fi + +# Only emitted when set: SPRING_APPLICATION_JSON outranks conf/application-pd.yml +ROCKSDB_JSON="" +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" ]]; then + ROCKSDB_JSON=", + \"rocksdb\": { \"total_memory_size\": \"${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}\" }" +fi # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat <}" ./bin/start-hugegraph-store.sh -d false -j "${JAVA_OPTS:-}" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index d1ccc651c7..8cb79d97fd 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -169,7 +169,18 @@ if [ "$JAVA_OPTIONS" = "" ]; then >> ${OUTPUT} exit 1 fi - JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " + # Cap direct memory at the heap size picked here, the JVM's own unset default, so the + # RocksDB auto-sizing (RocksdbMemoryBudget) counts a limit the JVM actually enforces. + # Leave it out when -j sets the heap or the cap itself: a fixed cap from the computed + # heap would no longer match a -j -Xmx, and the JVM then defaults it to that heap. + DIRECT_MEMORY_OPTION="-XX:MaxDirectMemorySize=${XMX}m" + # Split -j the way it reaches the JVM below (unquoted: spaces, tabs and newlines) + for opt in ${USER_OPTION}; do + case "$opt" in + -Xmx* | -XX:MaxHeapSize=* | -XX:MaxDirectMemorySize=*) DIRECT_MEMORY_OPTION="" ;; + esac + done + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${DIRECT_MEMORY_OPTION} -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " # JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" # Rolling out detailed GC logs diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 56cbdd8e63..6101ae0951 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -28,17 +28,9 @@ management: rocksdb: # Match the explicit launcher opt-in when enabling Topling. provider: rocksdb - # rocksdb total memory usage, force flush to disk when reaching this value - # TODO: default this lower, or size it from the container memory limit, or map it from an - # HG_STORE_* variable in the Docker entrypoint. A default of 0 only makes AppConfig fall back - # to the JVM max heap, which is another heap-sized native budget, not a safe container total. - # It is the capacity of the RocksDB block and write LRU caches (RaftRocksdbOptions), not an - # upfront allocation, but pinned at 32 GB those native caches can grow past a smaller - # container memory limit under load and get the Store OOM-killed. Until then the only - # override is the JVM system property -Drocksdb.total_memory_size= (JAVA_OPTS, the - # Helm chart's store.javaOpts; helm/hugegraph). - # https://github.com/apache/hugegraph/issues/3254 - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache, in bytes. 0 sizes it from + # the memory left after heap and direct memory (container limit aware, capped at 32 GB). + total_memory_size: 0 # memtable size used by rocksdb write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index 0bc57d22dd..4926379c32 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -23,6 +23,7 @@ import javax.annotation.PostConstruct; import org.apache.hugegraph.store.options.JobOptions; +import org.apache.hugegraph.store.options.RocksdbMemoryBudget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -104,20 +105,17 @@ public void init() { } Map rocksdb = rocksdbConfig.rocksdb; - if (!rocksdb.containsKey("total_memory_size") - || "0".equals(rocksdb.get("total_memory_size"))) { - rocksdb.put("total_memory_size", Long.toString(rt.maxMemory())); - } - long totalMemory = Long.parseLong(rocksdbConfig.rocksdb.get("total_memory_size")); + long totalMemory = RocksdbMemoryBudget.resolve(rocksdb.get(RocksdbMemoryBudget.KEY)); + rocksdb.put(RocksdbMemoryBudget.KEY, Long.toString(totalMemory)); if (raft.getDisruptorBufferSize() == 0) { - int size = (int) (totalMemory / 1000 / 1000 / 1000); - size = (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; - raft.setDisruptorBufferSize(size); // Increase one buffer every 32M + // Increase one buffer every 32M + raft.setDisruptorBufferSize(RocksdbMemoryBudget.disruptorBufferSize(totalMemory)); } if (!rocksdb.containsKey("write_buffer_size") || "0".equals(rocksdb.get("write_buffer_size"))) { - rocksdb.put("write_buffer_size", Long.toString(totalMemory / 1000)); + rocksdb.put("write_buffer_size", + Long.toString(RocksdbMemoryBudget.writeBufferSize(totalMemory))); } } diff --git a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml index 7985e67da1..337a61042a 100644 --- a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml +++ b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml @@ -33,6 +33,7 @@ logging: level: root: info rocksdb: - # total memory size used by rocksdb - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache, in bytes. 0 sizes it from + # the memory left after heap and direct memory (container limit aware, capped at 32 GB). + total_memory_size: 0 write_buffer_size: 32000000 diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java index e9b5be8c48..31258b1d5e 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java @@ -21,6 +21,7 @@ import org.apache.hugegraph.store.StoreIdChangeTest; import org.apache.hugegraph.store.core.snapshot.HgSnapshotHandlerTest; import org.apache.hugegraph.store.core.store.meta.PartitionMetaCodecTest; +import org.apache.hugegraph.store.options.RocksdbMemoryBudgetTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -51,7 +52,8 @@ MetricSessionTest.class, HgSnapshotHandlerTest.class, StoreIdChangeTest.class, - PartitionMetaCodecTest.class + PartitionMetaCodecTest.class, + RocksdbMemoryBudgetTest.class }) @Slf4j public class CoreSuiteTest { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java new file mode 100644 index 0000000000..a1d9daa87a --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MAX_AUTO; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIB; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIN_AUTO; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.junit.Test; + +public class RocksdbMemoryBudgetTest { + + private static final long GIB = 1024 * MIB; + private static final Path NO_MOUNTINFO = Path.of("/nonexistent/proc/self/mountinfo"); + + @Test + public void testExplicitValueIsKept() { + assertEquals(32_000_000_000L, RocksdbMemoryBudget.resolve("32000000000")); + assertEquals(4_000_000_000L, RocksdbMemoryBudget.resolve(" 4000000000 ")); + assertEquals(MIB, RocksdbMemoryBudget.resolve(String.valueOf(MIB))); + } + + @Test + public void testExplicitValueBelowOneMibIsRefused() { + for (String value : new String[]{"1", "2", String.valueOf(MIB - 1)}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("at least")); + } + } + } + + @Test + public void testUnsetZeroOrBlankIsAuto() { + for (String value : new String[]{null, "", " ", "0", " 0 ", "00"}) { + long budget = RocksdbMemoryBudget.resolve(value); + assertTrue(String.valueOf(value), budget >= MIN_AUTO && budget <= MAX_AUTO); + } + } + + @Test + public void testNonNumericValueIsRefused() { + for (String value : new String[]{"4GB", "4g", "-1", "1.5e9", "99999999999999999999"}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("number of bytes")); + } + } + } + + @Test + public void testAutoFitsContainerLimit() { + // The setup from apache/hugegraph#3254: 4 GiB limit, 1 GiB heap, 512 MiB direct + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(4 * GIB, GIB, 512 * MIB)); + assertEquals(2560 * MIB, RocksdbMemoryBudget.auto(8 * GIB, GIB, 512 * MIB)); + } + + @Test + public void testUnsetDirectMemoryCountsAsTheHeap() { + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(512 * MIB, 8 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.directMemory(4 * GIB, 8 * GIB)); + // Unset: the JVM allows as much direct memory as heap, so the budget counts that + assertEquals(8 * GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB)); + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(0L, 512 * MIB)); + } + + @Test + public void testBareMetalBudgetsWithTheLauncherDirectCap() { + // The default launcher picks a heap of at most 2 GiB and caps direct memory at it + assertEquals(5376 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 2 * GIB, 2 * GIB)); + // A raised heap with a smaller explicit -XX:MaxDirectMemorySize keeps a usable budget + assertEquals(2304 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 8 * GIB, 2 * GIB)); + assertEquals(12544 * MIB, RocksdbMemoryBudget.auto(64 * GIB, 36 * GIB, 2 * GIB)); + // Without one, direct memory counts as a second heap and the floor applies + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto( + 64 * GIB, 36 * GIB, RocksdbMemoryBudget.directMemory(0L, 36 * GIB))); + } + + @Test + public void testAncestorCgroupV2LimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // A 4 GiB limit on the pod, none on the container or above the pod + write(root, "kubepods/memory.max", "max"); + write(root, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(root, "kubepods/pod1/ctr/memory.max", "max\n"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root); + assertEquals(4 * GIB, limit); + + // The case from review: a 64 GiB host reported by the JVM, -Xmx1g and + // -XX:MaxDirectMemorySize=512m get the 4 GiB limit's budget, not host RAM's + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + + // The container's own lower limit wins over the pod's + write(root, "kubepods/pod1/ctr/memory.max", String.valueOf(2 * GIB)); + assertEquals(2 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2NamespaceRootAndNoLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // Inside a cgroup namespace the process sees "/" and the mount is its cgroup + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + write(root, "memory.max", "max"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); + write(root, "memory.max", String.valueOf(3 * GIB)); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); + + // A path the mount doesn't show falls back to the mount root + Path hidden = write(root, "proc-hidden", "0::/system.slice/hidden.service\n"); + assertEquals(3 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(hidden, NO_MOUNTINFO, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1HierarchicalLimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1"); + try { + Path proc = write(root, "proc-self-cgroup", + "12:cpu,cpuacct:/docker/abc\n4:memory:/docker/abc\n"); + // Docker mounts the container's own memory cgroup, so /docker/abc isn't there + write(root, "memory/memory.limit_in_bytes", "9223372036854771712"); + write(root, "memory/memory.stat", + "cache 0\nhierarchical_memory_limit " + (4 * GIB) + "\n"); + assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); + + write(root, "memory/memory.stat", "hierarchical_memory_limit 9223372036854771712\n"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2SubtreeMountIsRebased() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-subtree"); + try { + // cgroupfs mounted from /kubepods: the pod's limit is at /pod1, not at + // /kubepods/pod1, and the mount root itself is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "pod1/memory.max", String.valueOf(4 * GIB)); + write(mount, "pod1/ctr/memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + Path mountinfo = write(root, "mountinfo", + "25 1 8:1 / / rw - ext4 /dev/sda1 rw\n" + + "30 25 0:26 /kubepods " + mount + + " rw,nosuid shared:4 - cgroup2 cgroup2 rw\n"); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root); + assertEquals(4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + + // Without the mount root the path is looked up under kubepods/ and missed + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(proc, NO_MOUNTINFO, mount)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1SubtreeMountAndEscapedMountPoint() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-subtree"); + try { + Path mount = root.resolve("memory dir"); + write(mount, "memory.limit_in_bytes", "9223372036854771712"); + write(mount, "memory.stat", "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/docker/abc\n"); + // mountinfo writes the space in the mount point as \040 + Path mountinfo = write(root, "mountinfo", + "40 25 0:30 /docker/abc " + + mount.toString().replace(" ", "\\040") + + " rw,nosuid - cgroup cgroup rw,memory\n"); + assertEquals(4 * GIB, RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root)); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-mounts"); + try { + // A full view of the hierarchy with the 4 GiB limit on the pod + Path full = root.resolve("full"); + write(full, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(full, "kubepods/pod1/ctr/memory.max", "max"); + // A leaf-only view of the container and an unrelated subtree, both unlimited + Path leaf = root.resolve("leaf"); + write(leaf, "memory.max", "max"); + Path other = root.resolve("other"); + write(other, "memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + String fullLine = "31 25 0:26 / " + full + " rw - cgroup2 cgroup2 rw\n"; + String leafLine = "32 25 0:26 /kubepods/pod1/ctr " + leaf + " rw - cgroup2 cgroup2 rw\n"; + String otherLine = "33 25 0:26 /system.slice " + other + " rw - cgroup2 cgroup2 rw\n"; + for (String order : new String[]{otherLine + leafLine + fullLine, + fullLine + leafLine + otherLine, + leafLine + otherLine + fullLine}) { + Path mountinfo = write(root, "mountinfo", order); + long limit = RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root); + assertEquals(order, 4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-mounts"); + try { + Path full = root.resolve("full"); + write(full, "kubepods/pod1/ctr/memory.limit_in_bytes", "9223372036854771712"); + write(full, "kubepods/pod1/ctr/memory.stat", + "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path other = root.resolve("other"); + write(other, "memory.limit_in_bytes", "9223372036854771712"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/kubepods/pod1/ctr\n"); + String fullLine = "41 25 0:30 / " + full + " rw - cgroup cgroup rw,memory\n"; + String otherLine = "42 25 0:30 /system.slice " + other + " rw - cgroup cgroup rw,memory\n"; + // A cpu-only mount is not the memory controller and is ignored + String cpuLine = "43 25 0:31 / " + other + " rw - cgroup cgroup rw,cpu,cpuacct\n"; + for (String order : new String[]{otherLine + cpuLine + fullLine, + fullLine + otherLine + cpuLine}) { + Path mountinfo = write(root, "mountinfo", order); + assertEquals(order, 4 * GIB, + RocksdbMemoryBudget.cgroupMemoryLimit(proc, mountinfo, root)); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testInheritedNamespaceMountIsFlaggedNotTreatedAsUnlimited() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-namespace"); + try { + // cgroup_namespaces(7): /proc/self/cgroup says "/" while a cgroup2 mount made + // outside the namespace has root "/..". The namespace root is a child with a + // 2 GiB limit whose name can't be learned from inside; its parent is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "ns-child/memory.max", String.valueOf(2 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "35 25 0:26 /.. " + mount + " rw - cgroup2 cgroup2 rw\n"); + RocksdbMemoryBudget.CgroupLimit limit = + RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertTrue(limit.unmatched); + assertEquals(-1L, limit.bytes); + + // A limit at the visible mount root still bounds the result + write(mount, "memory.max", String.valueOf(8 * GIB)); + limit = RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertTrue(limit.unmatched); + assertEquals(8 * GIB, limit.bytes); + + // v1 with the same layout + Path v1 = root.resolve("memory"); + write(v1, "memory.limit_in_bytes", "9223372036854771712"); + write(v1, "memory.stat", "hierarchical_memory_limit " + (8 * GIB) + "\n"); + Path proc1 = write(root, "proc-v1", "4:memory:/\n"); + Path mountinfo1 = write(root, "mountinfo-v1", + "45 25 0:30 /.. " + v1 + " rw - cgroup cgroup rw,memory\n"); + limit = RocksdbMemoryBudget.cgroupLimit(proc1, mountinfo1, root); + assertTrue(limit.unmatched); + assertEquals(8 * GIB, limit.bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMatchedOrMissingMountsAreNotFlagged() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-flag"); + try { + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", String.valueOf(4 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "30 25 0:26 / " + mount + " rw - cgroup2 cgroup2 rw\n"); + RocksdbMemoryBudget.CgroupLimit limit = + RocksdbMemoryBudget.cgroupLimit(proc, mountinfo, root); + assertFalse(limit.unmatched); + assertEquals(4 * GIB, limit.bytes); + // Without any mountinfo entry the default layout is used and nothing is flagged + assertFalse(RocksdbMemoryBudget.cgroupLimit(proc, NO_MOUNTINFO, mount).unmatched); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMountRootContainsMembership() { + assertTrue(RocksdbMemoryBudget.contains("/", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods/", "/kubepods/pod1/ctr")); + assertTrue(RocksdbMemoryBudget.contains("/kubepods/pod1/ctr", "/kubepods/pod1/ctr")); + assertFalse(RocksdbMemoryBudget.contains("/system.slice", "/kubepods/pod1/ctr")); + assertFalse(RocksdbMemoryBudget.contains("/kubepods", "/kubepodsx/pod1")); + } + + @Test + public void testMembershipIsRebasedOntoMountRoot() { + assertEquals("pod1/ctr", RocksdbMemoryBudget.rebase("/kubepods/pod1/ctr", "/kubepods")); + assertEquals("pod1/ctr", RocksdbMemoryBudget.rebase("/kubepods/pod1/ctr", "/kubepods/")); + assertEquals("", RocksdbMemoryBudget.rebase("/kubepods", "/kubepods")); + assertEquals("a/b", RocksdbMemoryBudget.rebase("/a/b", "/")); + assertEquals("", RocksdbMemoryBudget.rebase("/", "/")); + // A path outside the mount root is kept and then falls back to the mount + assertEquals("other/x", RocksdbMemoryBudget.rebase("/other/x", "/kubepods")); + assertEquals("kubepodsx/y", RocksdbMemoryBudget.rebase("/kubepodsx/y", "/kubepods")); + } + + @Test + public void testUnreadableCgroupKeepsJvmMemory() { + Path missing = Path.of("/nonexistent/proc/self/cgroup"); + assertEquals(-1L, RocksdbMemoryBudget.cgroupMemoryLimit(missing, NO_MOUNTINFO, Path.of("/nonexistent"))); + assertEquals(16 * GIB, RocksdbMemoryBudget.effectivePhysical(16 * GIB, -1L)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(-1L, 4 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(4 * GIB, 8 * GIB)); + } + + private static Path write(Path root, String relative, String content) throws IOException { + Path file = root.resolve(relative); + Files.createDirectories(file.getParent()); + Files.write(file, content.getBytes(StandardCharsets.UTF_8)); + return file; + } + + @Test + public void testAutoIsClamped() { + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(2 * GIB, GIB, GIB)); + assertEquals(MAX_AUTO, RocksdbMemoryBudget.auto(1024 * GIB, 8 * GIB, 8 * GIB)); + } + + @Test + public void testDisruptorBufferSizeIsNeverZero() { + assertEquals(32, RocksdbMemoryBudget.disruptorBufferSize(512 * MIB)); + assertEquals(32, RocksdbMemoryBudget.disruptorBufferSize(1_000_000_000L)); + assertEquals(1024, RocksdbMemoryBudget.disruptorBufferSize(32_000_000_000L)); + } + + @Test + public void testDerivedWriteBufferSizePassesRocksdbValidation() { + // The 4 GiB container from apache/hugegraph#3254 auto-sizes to 512 MiB, whose + // thousandth is under the 1 MiB rocksdb.write_buffer_size minimum + long budget = RocksdbMemoryBudget.auto(4 * GIB, GIB, 512 * MIB); + assertEquals(MIB, RocksdbMemoryBudget.writeBufferSize(budget)); + assertEquals(MIB, RocksdbMemoryBudget.writeBufferSize(MIB)); + assertEquals(32_000_000L, RocksdbMemoryBudget.writeBufferSize(32_000_000_000L)); + + // The same validation BusinessHandlerImpl.initRocksdb() runs at Store startup + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + RocksDBOptions.instance(); + HugeConfig config = new HugeConfig(rocksdbConfig(budget, + RocksdbMemoryBudget.writeBufferSize(budget))); + assertEquals(MIB, (long) config.get(RocksDBOptions.MEMTABLE_SIZE)); + try { + new HugeConfig(rocksdbConfig(budget, budget / 1000)); + fail("accepted a write buffer under 1 MiB"); + } catch (ConfigException e) { + assertTrue(e.getMessage().contains("write_buffer_size")); + } + } + + private static Map rocksdbConfig(long totalMemory, long writeBufferSize) { + Map config = new HashMap<>(); + config.put("rocksdb.total_memory_size", String.valueOf(totalMemory)); + config.put("rocksdb.write_buffer_size", String.valueOf(writeBufferSize)); + return config; + } + + @Test + public void testUnknownPhysicalMemoryFallsBackToHeap() { + assertEquals(2 * GIB, RocksdbMemoryBudget.auto(-1L, 2 * GIB, 2 * GIB)); + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(0L, 64 * MIB, 64 * MIB)); + } +}