From 079365befa9331230c3b019eb1628efb5433dd2f Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Mon, 5 Oct 2026 14:42:36 +0200 Subject: [PATCH] Added `cfengine test` i2, now reports a test-suite using the `assert`-promise module Runs test_*.cf files or a deployed policy's own assert: promises (or just a plain built-policy run) in disposable Docker container and reports a pytest-style pass/fail summary. Adds --dockerfile/--rebuild flags for testing against a custom base image. Ticket: ENT-14444 Signed-off-by: Simon Halvorsen Co-Authored-By: Claude Sonnet 5 --- CHEATSHEET.md | 25 + .../cfengine_wrapper/cfengine_commands.py | 54 +- src/cfengine_cli/commands.py | 12 +- src/cfengine_cli/container.py | 556 +++++++++++++++++- src/cfengine_cli/docker/test-agent/Dockerfile | 15 +- .../test-agent/assert-module/promise_agent.cf | 5 + src/cfengine_cli/main.py | 26 + src/cfengine_cli/report.py | 193 ++++++ tests/shell/009-test-assert-vocab.sh | 45 ++ tests/shell/010-test-suite-standalone.sh | 54 ++ tests/shell/011-test-suite-cfbs-project.sh | 82 +++ tests/shell/012-test-full-policy.sh | 40 ++ .../test_assert_vocab.cf | 90 +++ .../tests/fixtures/test_service_config.json | 3 + .../tests/test_computed_metrics.cf | 31 + .../tests/test_service_config.cf | 40 ++ .../fixtures/012-test-full-policy/cfbs.json | 27 + .../012-test-full-policy/overrides/def.json | 3 + .../overrides/services/init.cf | 8 + .../tests/fixtures/test_instance_a.json | 3 + .../tests/fixtures/test_instance_b.json | 3 + .../tests/test_instance_a.cf | 8 + .../tests/test_instance_b.cf | 8 + tests/unit/test_container.py | 194 ++++++ tests/unit/test_report.py | 109 ++++ 25 files changed, 1605 insertions(+), 29 deletions(-) create mode 100644 src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf create mode 100644 src/cfengine_cli/report.py create mode 100755 tests/shell/009-test-assert-vocab.sh create mode 100755 tests/shell/010-test-suite-standalone.sh create mode 100755 tests/shell/011-test-suite-cfbs-project.sh create mode 100755 tests/shell/012-test-full-policy.sh create mode 100644 tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/cfbs.json create mode 100644 tests/shell/fixtures/012-test-full-policy/overrides/def.json create mode 100644 tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf create mode 100644 tests/unit/test_container.py create mode 100644 tests/unit/test_report.py diff --git a/CHEATSHEET.md b/CHEATSHEET.md index f6bff581..eeef80f1 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -60,6 +60,31 @@ cfengine build --hub myhub --non-interactive cfengine deploy --hub myhub ``` +## Testing policy with `cfengine test` + +Runs `test_*.cf` files (bundles that contain their own `assert:` section) in disposable Docker containers and reports a pytest-style pass/fail summary. + +```bash +# Auto-discover and run tests/test_*.cf +cfengine test + +# Run specific files or directories instead +cfengine test tests/test_foo.cf +cfengine test tests/ + +# Inside a cfbs project: builds first, then runs tests in that project's built +# masterfiles (custom promise types, services/init.cf), against a minimal +# generated policy, not the project's real bundlesequence; see --full-policy below +cfengine test + +# Test against a custom base image (e.g. a different distro), bypassing the build cache +cfengine test --dockerfile ./my-ubuntu.Dockerfile --rebuild + + +# Inside a cfbs project: run the tests after a normal policy run +cfengine test --full-policy +``` + ## Running the CFEngine agent ```bash diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py index 008218ff..02b438f6 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py @@ -24,7 +24,12 @@ ) from cfengine_cli.utils import UserError -from cfengine_cli.container import run_in_container +from cfengine_cli.container import ( + run_in_container, + run_files_in_container, + discover_test_files, + discover_module_files, +) from cfengine_cli.cfengine_wrapper.cfengine_objects import ( Executable, ensure_default_agent_flags, @@ -287,12 +292,49 @@ def deploy( return error -def test() -> int: - rc = build_command() - if rc != 0: - return rc +def test( + files: list[str] | None = None, + dockerfile: str | None = None, + rebuild: bool = False, + full_policy: bool = False, +) -> int: + is_cfbs = is_cfbs_repo() + if is_cfbs: + # Build first so any custom promise types / modules this + # project uses are shipped + rc = build_command() + if rc != 0: + return rc + + if files: + return run_files_in_container( + files, + masterfiles_dir="out/masterfiles" if is_cfbs else None, + dockerfile=dockerfile, + rebuild=rebuild, + full_policy=full_policy, + ) + + discovered = discover_test_files() + if discovered: + return run_files_in_container( + discover_module_files() + discovered, + masterfiles_dir="out/masterfiles" if is_cfbs else None, + dockerfile=dockerfile, + rebuild=rebuild, + full_policy=full_policy, + ) + + if is_cfbs: + return run_in_container( + "out/masterfiles", dockerfile=dockerfile, rebuild=rebuild + ) - return run_in_container("out/masterfiles") + raise UserError( + "Nothing to test here -- no cfbs.json project and no test_*.cf files " + "found. Add a test_*.cf file, or run `cfengine init` to start a cfbs " + "project." + ) def show(target: list[str] | None = None) -> int: diff --git a/src/cfengine_cli/commands.py b/src/cfengine_cli/commands.py index 004de5f5..ecd4f1cd 100644 --- a/src/cfengine_cli/commands.py +++ b/src/cfengine_cli/commands.py @@ -10,6 +10,7 @@ from cfengine_cli.utils import UserError from cfengine_cli.up import validate_config, up_do, resolve_templates from cfengine_cli.initialize_project import init_policy_module, init_promise_type +from cfengine_cli.container import assert_module_declaration_path from cf_remote.paths import cf_remote_dir from cfbs.commands import init_command from cfengine_cli.cfengine_wrapper.cfengine_commands import test as _cfengine_test @@ -48,15 +49,20 @@ def lint(files, strict, syntax_path) -> int: return errors -def test(files, strict) -> int: - errors = _lint(files, strict, None) +def test(files, strict, dockerfile=None, rebuild=False, full_policy=False) -> int: + # Adds the assert module so linting passes + lint_targets = (list(files) if files else ["."]) + [ + assert_module_declaration_path() + ] + + errors = _lint(lint_targets, strict, None) if errors != 0: plural = "error" if errors == 1 else "errors" print(f"Lint failed, {errors} {plural} in total. Skipping build/deploy/run.") return errors print("Lint passed, no errors found.") - return _cfengine_test() + return _cfengine_test(files, dockerfile, rebuild, full_policy) def dev(subcommand, args) -> int: diff --git a/src/cfengine_cli/container.py b/src/cfengine_cli/container.py index 0f9a5293..745a93e2 100644 --- a/src/cfengine_cli/container.py +++ b/src/cfengine_cli/container.py @@ -1,11 +1,38 @@ +import glob +import hashlib +import json import os +import re import shutil import subprocess +import tempfile +import uuid +from typing import NamedTuple +from cfbs.utils import merge_json, read_json + +from cfengine_cli import paths +from cfengine_cli.report import ( + RunResults, + run_and_parse, + print_report, + run_and_scan_asserts, + print_assert_totals, +) from cfengine_cli.utils import UserError _DOCKERFILE_DIR = os.path.join(os.path.dirname(__file__), "docker", "test-agent") _IMAGE_TAG = "cfengine-cli-test-agent:latest" +_ASSERT_MODULE_DIR = "/var/cfengine/modules/promises" +_PROMISE_AGENT_DECLARATION_RE = re.compile(r"promise\s+agent\s+(\w+)") +_BUNDLE_AGENT_DECLARATION_RE = re.compile(r"bundle\s+agent\s+(\w+)") +_ASSERT_SECTION_RE = re.compile(r"^\s*assert\s*:", re.MULTILINE) + + +def assert_module_declaration_path() -> str: + """Path to the `promise agent assert {...}` declaration + so calling lint on files containing the assert-promise-type will pass""" + return os.path.join(_DOCKERFILE_DIR, "assert-module", "promise_agent.cf") def require_docker() -> None: @@ -15,38 +42,529 @@ def require_docker() -> None: ) -def _ensure_image_built() -> None: +def _resolve_dockerfile_dir(dockerfile: str | None) -> str: + """--dockerfile may point at a Dockerfile itself or at the directory + containing one; returns the directory `docker build` expects. + None => use the built-in Dockerfile.""" + if dockerfile is None: + return _DOCKERFILE_DIR + abs_path = os.path.abspath(dockerfile) + return os.path.dirname(abs_path) if os.path.isfile(abs_path) else abs_path + + +def _image_tag_for(dockerfile_dir: str) -> str: + """The built-in Dockerfile keeps the stable, human-readable tag; any + other --dockerfile gets its own tag derived from its path, as to not + conflict with the cached image.""" + if os.path.abspath(dockerfile_dir) == os.path.abspath(_DOCKERFILE_DIR): + return _IMAGE_TAG + digest = hashlib.sha256(os.path.abspath(dockerfile_dir).encode()).hexdigest()[:12] + return f"cfengine-cli-test-agent-custom-{digest}:latest" + + +def _ensure_image_built(dockerfile_dir: str, image_tag: str, rebuild: bool) -> None: + cmd = ["docker", "build", "-q", "-t", image_tag] + if rebuild: + cmd.append("--no-cache") + cmd.append(dockerfile_dir) + result = subprocess.run(cmd) + if result.returncode != 0: + raise UserError("Failed to build the cfengine-test Docker image.") + + +def _docker_cleanup(*args: str) -> None: + """Best-effort `docker ` for rm/rmi cleanup.""" + subprocess.run(["docker", *args], capture_output=True) + + +def _run_setup_and_commit( + mount_args: list, setup_commands: list, image_tag: str +) -> str: + """Runs setup_commands once in a throwaway container and commits the + result to an image, so every test starts pre-set-up.""" + container_name = f"cfengine-cli-test-prep-{uuid.uuid4().hex[:12]}" + script = " && ".join(setup_commands) result = subprocess.run( - ["docker", "build", "-q", "-t", _IMAGE_TAG, _DOCKERFILE_DIR] + [ + "docker", + "run", + "--name", + container_name, + *mount_args, + image_tag, + "sh", + "-c", + script, + ] ) if result.returncode != 0: - raise UserError("Failed to build the cfengine-test Docker image.") + _docker_cleanup("rm", "-f", container_name) + raise UserError( + "Failed to set up the test environment (deploy step) -- see output above." + ) + + image_tag = f"cfengine-cli-test-agent-prepped:{uuid.uuid4().hex[:12]}" + subprocess.run( + ["docker", "commit", container_name, image_tag], check=True, capture_output=True + ) + _docker_cleanup("rm", container_name) + return image_tag + +def _remove_image(image_tag: str) -> None: + _docker_cleanup("rmi", image_tag) -def run_in_container(masterfiles_dir: str) -> int: + +def run_in_container( + masterfiles_dir: str, dockerfile: str | None = None, rebuild: bool = False +) -> int: """ - Runs cf-agent against the given built masterfiles directory inside a container + Runs cf-agent against the given built masterfiles directory inside a + container. There's no test_*.cf harness here, so there's no per-test + breakdown -- but if the deployed policy's own promises happen to emit + [ASSERT] lines (e.g. assert: promises outside the usual test_*.cf + convention), those are still counted and reported like a test run; + otherwise this just shows the agent's own output and exit code. + + dockerfile overrides the built-in Dockerfile (e.g. to test against a + different base distro); rebuild forces a cache-busting rebuild instead + of reusing whatever's already cached for it. """ require_docker() - _ensure_image_built() + dockerfile_dir = _resolve_dockerfile_dir(dockerfile) + image_tag = _image_tag_for(dockerfile_dir) + _ensure_image_built(dockerfile_dir, image_tag, rebuild) - abs_masterfiles = os.path.abspath(masterfiles_dir) - result = subprocess.run( - [ + project = _project_setup(masterfiles_dir) + setup_commands = ( + ["rm -rf /var/cfengine/inputs"] + + project.deploy_commands + + project.post_deploy_commands + ) + prepped_image = _run_setup_and_commit(project.mount_args, setup_commands, image_tag) + try: + cmd = [ "docker", "run", "--rm", - "-v", - f"{abs_masterfiles}:/mnt/masterfiles:ro", - _IMAGE_TAG, + prepped_image, "sh", "-c", - "rm -rf /var/cfengine/inputs " - "&& rm -rf /var/cfengine/masterfiles " - "&& cp -r /mnt/masterfiles /var/cfengine/masterfiles " - "&& /var/cfengine/bin/cf-agent --bootstrap 127.0.0.1 " - "&& /var/cfengine/bin/cf-agent -KIf update.cf " - "&& /var/cfengine/bin/cf-agent -KI", + f"{paths.bin('cf-agent')} -KIf /var/cfengine/masterfiles/promises.cf", + ] + results = run_and_scan_asserts(cmd) + finally: + _remove_image(prepped_image) + + if results.passed == 0 and results.failed == 0: + return results.returncode + return print_assert_totals(results.passed, results.failed) + + +def _read_file(path: str) -> str: + with open(path) as file: + return file.read() + + +def _bundle_bodies_declared_in(content: str) -> list: + """[(name, body), ...] for every `bundle agent { ... }` in + content, in order, body being the brace-matched text between (and + including) its own braces.""" + bodies = [] + for match in _BUNDLE_AGENT_DECLARATION_RE.finditer(content): + name = match.group(1) + start = content.index("{", match.end()) + depth = 0 + end = start + for end in range(start, len(content)): + if content[end] == "{": + depth += 1 + elif content[end] == "}": + depth -= 1 + if depth == 0: + break + bodies.append((name, content[start : end + 1])) + return bodies + + +def _bundle_names_declared_in(path: str) -> list: + """Test (reportable) bundles in a file, in order -- a bundle only counts + as a test if it has its own `assert:` section. A bundle with no + `assert:` section (e.g. a shared bundle computing some vars) is just + available to be called via `methods:` from a test bundle, like any other + bundle in this file's inputs, without being reported as a test of its + own.""" + names = [ + name + for name, body in _bundle_bodies_declared_in(_read_file(path)) + if _ASSERT_SECTION_RE.search(body) + ] + if not names: + raise UserError(f"No bundle with an 'assert:' section found in {path}") + return names + + +def discover_test_files(tests_dir: str = "tests") -> list: + search_dir = tests_dir if os.path.isdir(tests_dir) else "." + return sorted( + os.path.join(search_dir, fname) + for fname in os.listdir(search_dir) + if fname.startswith("test_") and fname.endswith(".cf") + ) + + +def discover_module_files() -> list: + return sorted(glob.glob("*/main.cf")) + + +def _declared_promise_types(cf_file_path: str) -> set: + return set(_PROMISE_AGENT_DECLARATION_RE.findall(_read_file(cf_file_path))) + + +def _read_only_mount(host_path: str, container_path: str) -> list: + return ["-v", f"{host_path}:{container_path}:ro"] + + +def _expand_directories(paths: list) -> list: + expanded = [] + for path in paths: + if not os.path.isdir(path): + expanded.append(path) + continue + for root, dirs, files in os.walk(path, followlinks=True): + dirs[:] = [d for d in dirs if not d.startswith(".")] + expanded += sorted( + os.path.join(root, f) + for f in files + if f.endswith(".cf") and not f.startswith(".") + ) + return expanded + + +class ProjectSetup(NamedTuple): + mount_args: list + deploy_commands: list + post_deploy_commands: list + init_mount: str | None + declares_assert: bool + + +def _project_setup(masterfiles_dir: str | None) -> ProjectSetup: + """Everything needed to make a cfbs project's masterfiles, custom promise + types and services/init.cf available in the container; empty when + there's no masterfiles_dir (a bare, cfbs-less test_*.cf file).""" + if masterfiles_dir is None: + return ProjectSetup([], [], [], None, False) + + abs_masterfiles = os.path.abspath(masterfiles_dir) + deploy_commands = [ + "rm -rf /var/cfengine/masterfiles", + "cp -r /mnt/project_masterfiles /var/cfengine/masterfiles", + ] + + post_deploy_commands = [] + if os.path.isdir(os.path.join(abs_masterfiles, "modules")): + post_deploy_commands.append( + "cp -r /mnt/project_masterfiles/modules/. /var/cfengine/modules/" + ) + + # Bootstrapping to itself runs the project's own update.cf, + # This happens once, as part of the shared setup image every test + # reuses, so it doesn't add per-test cost. + post_deploy_commands += [ + paths.bin("cf-serverd"), + "sleep 1", + f"{paths.bin('cf-agent')} --bootstrap 127.0.0.1", + ] + + init_mount = None + declares_assert = False + project_init_file = os.path.join(abs_masterfiles, "services", "init.cf") + if os.path.isfile(project_init_file): + # The prep container's /mnt/project_masterfiles mount is gone by the + # time per-test containers start from its committed image; this path + # is what's still there. + init_mount = "/var/cfengine/masterfiles/services/init.cf" + declares_assert = "assert" in _declared_promise_types(project_init_file) + + return ProjectSetup( + mount_args=_read_only_mount(abs_masterfiles, "/mnt/project_masterfiles"), + deploy_commands=deploy_commands, + post_deploy_commands=post_deploy_commands, + init_mount=init_mount, + declares_assert=declares_assert, + ) + + +def _bundlesequence_and_defs(bundle_reports: list) -> tuple: + """bundle_reports is this file's [(bundle_name, report_id), ...], one + pair per bundle agent it declares (see _bundle_names_declared_in). + Returns (bundlesequence_names, bundle_def_lines): the cftest_start/ + marker bundle names in bundlesequence order (interleaved with the real + test bundle names), and their own `bundle agent` definitions. Each + bundle gets its own CFTEST-START/DONE pair, so several independent + check bundles in one file still get attributed their own report row.""" + bundlesequence = [] + bundle_defs = [] + for i, (bundle_name, report_id) in enumerate(bundle_reports): + start, marker = f"cftest_start_{i}", f"cftest_marker_{i}" + bundlesequence += [start, bundle_name, marker] + bundle_defs += [ + f"bundle agent {start}", + "{", + " reports:", + f' "[CFTEST-START] {report_id}";', + "}", + "", + f"bundle agent {marker}", + "{", + " reports:", + f' "[CFTEST-DONE] {report_id}";', + "}", + "", ] + return bundlesequence, bundle_defs + + +def _run_cf_content(bundle_reports: list, inputs: list) -> str: + """A standalone run.cf: its own `body common control` declaring exactly + this file's inputs and bundlesequence -- the test bundle(s) are all that + run, nothing from a real project's own policy. See _bundle_defs_content + for the --full-policy alternative.""" + inputs_str = ", ".join(f'"{i}"' for i in inputs) + bundlesequence, bundle_defs = _bundlesequence_and_defs(bundle_reports) + bundlesequence_str = ", ".join(f'"{name}"' for name in bundlesequence) + return "\n".join( + [ + "body common control", + "{", + f" inputs => {{ {inputs_str} }};", + f" bundlesequence => {{ {bundlesequence_str} }};", + "}", + "", + ] + + bundle_defs + ) + + +def _bundle_defs_content(bundle_reports: list) -> str: + """Just the cftest_start/marker `bundle agent` definitions, no `body + common control` of its own -- for --full-policy mode, where the real + project's promises.cf supplies the control body and bundlesequence.""" + _bundlesequence, bundle_defs = _bundlesequence_and_defs(bundle_reports) + return "\n".join(bundle_defs) + + +def _merge_def_json( + masterfiles_dir: str, + fixture_path: str | None, + extra_inputs: list, + extra_bundlesequence: list, +) -> str: + """--full-policy mode runs the project's own real promises.cf as the + entry point, instead of a from-scratch run.cf -- so the test's own + inputs and bundlesequence additions have to be merged into def.json + instead of declared directly.""" + merged = read_json(os.path.join(masterfiles_dir, "def.json")) or {} + + if fixture_path is not None: + merged = merge_json(merged, read_json(fixture_path) or {}) + + extra = { + "inputs": extra_inputs, + "vars": {"control_common_bundlesequence_end": extra_bundlesequence}, + } + merged = merge_json(merged, extra) + + return json.dumps(merged, indent=2) + + +def _inputs_for_test( + test_mount: str, project: ProjectSetup, library_mounts: list +) -> list: + inputs = ( + [] if project.declares_assert else [f"{_ASSERT_MODULE_DIR}/promise_agent.cf"] ) - return result.returncode + if project.init_mount is not None: + inputs.append(project.init_mount) + inputs += library_mounts + inputs.append(test_mount) + return inputs + + +def _prepare_test_run( + runners_dir: str, + test_file: str, + bundle_reports: list, + project: ProjectSetup, + library_mounts: list, + full_policy: bool = False, + masterfiles_dir: str | None = None, +) -> tuple: + """Writes this test's run.cf (and fixture, if any) under runners_dir, a + fresh per-test tempdir. bundle_reports is this file's [(bundle_name, + report_id), ...] (see _bundlesequence_and_defs). Returns (mount_args, + run_command). + + Run after the project's own real bundlesequence has converged for real, rather + than only ever seeing whatever state the one-time setup/bootstrap step left behind. + """ + test_mount = f"/mnt/tests/{os.path.basename(test_file)}" + mount_args = _read_only_mount(test_file, test_mount) + + inputs = _inputs_for_test(test_mount, project, library_mounts) + + # Fixtures are keyed by the test file's name, not its bundle name -- + # two files can share a bundle name. + test_stem = os.path.splitext(os.path.basename(test_file))[0] + fixture = os.path.join(os.path.dirname(test_file), "fixtures", f"{test_stem}.json") + fixture_path = fixture if os.path.isfile(fixture) else None + + if full_policy: + assert masterfiles_dir is not None # enforced by run_files_in_container + wrapper_mount = "/mnt/runners/cftest_wrapper.cf" + with open(os.path.join(runners_dir, "cftest_wrapper.cf"), "w") as wrapper_cf: + wrapper_cf.write(_bundle_defs_content(bundle_reports)) + + bundlesequence, _bundle_defs = _bundlesequence_and_defs(bundle_reports) + def_json = os.path.join(runners_dir, "def.json") + with open(def_json, "w") as def_json_file: + def_json_file.write( + _merge_def_json( + masterfiles_dir, + fixture_path, + inputs + [wrapper_mount], + bundlesequence, + ) + ) + # Augments (def.json) are loaded from next to the entry file being + # parsed -- /var/cfengine/masterfiles/promises.cf here -- so the + # merged def.json has to override that exact path, on top of the + # whole-directory /mnt/runners mount below that makes the wrapper + # file (and this same def.json) reachable at all. + mount_args += _read_only_mount(def_json, "/var/cfengine/masterfiles/def.json") + run_command = ( + f"{paths.bin('cf-agent')} -KIf /var/cfengine/masterfiles/promises.cf" + ) + return mount_args, run_command + + with open(os.path.join(runners_dir, "run.cf"), "w") as run_cf: + run_cf.write(_run_cf_content(bundle_reports, inputs)) + if fixture_path is not None: + shutil.copyfile(fixture_path, os.path.join(runners_dir, "def.json")) + + run_command = f"{paths.bin('cf-agent')} -KIf /mnt/runners/run.cf" + return mount_args, run_command + + +def run_files_in_container( + files: list, + masterfiles_dir: str | None = None, + dockerfile: str | None = None, + rebuild: bool = False, + full_policy: bool = False, +) -> int: + """Runs each test_*.cf file as its own isolated agent-run in its own + container, parses [ASSERT] PASS/FAIL lines out of the output, + and prints a pytest-style report, returning 0 only if everything passed. + A file may declare several bundles; each becomes its own "::" + report row within that one file's run. + + Every non-test_ file is a shared library added to every test's `inputs`. + A sibling fixtures/.json is auto-loaded as that run's + `def.json`. masterfiles_dir, if given, is a built cfbs policy set, + deployed once for every test to share. + + full_policy runs each test against the real project's own promises.cf + (extended via augments) instead of a minimal generated one -- see + _prepare_test_run -- and requires masterfiles_dir, since there's no + "real policy" to run without a cfbs project.""" + require_docker() + if full_policy and masterfiles_dir is None: + raise UserError( + "--full-policy needs a cfbs project to run the real policy " + "against -- there's no masterfiles_dir here." + ) + dockerfile_dir = _resolve_dockerfile_dir(dockerfile) + image_tag = _image_tag_for(dockerfile_dir) + _ensure_image_built(dockerfile_dir, image_tag, rebuild) + + abs_files = [os.path.abspath(f) for f in _expand_directories(files)] + test_files = [f for f in abs_files if os.path.basename(f).startswith("test_")] + library_files = [f for f in abs_files if f not in test_files] + if not test_files: + raise UserError( + "No test_*.cf files given to `cfengine test` -- nothing to run. " + "Pass at least one file whose name starts with 'test_', " + "e.g. `cfengine test tests/test_foo.cf`." + ) + + file_report_ids = [os.path.relpath(f).replace(" ", "_") for f in test_files] + bundle_reports_by_file = [ + [ + (bundle_name, f"{file_report_id}::{bundle_name}") + for bundle_name in _bundle_names_declared_in(test_file) + ] + for test_file, file_report_id in zip(test_files, file_report_ids) + ] + report_ids = [ + report_id + for bundle_reports in bundle_reports_by_file + for _bundle_name, report_id in bundle_reports + ] + + project = _project_setup(masterfiles_dir) + setup_commands = ( + ["rm -rf /var/cfengine/inputs"] + + project.deploy_commands + + project.post_deploy_commands + ) + + library_mount_args = [] + library_mounts = [] + for lib in library_files: + lib_mount = f"/mnt/lib/{os.path.basename(lib)}" + library_mount_args += _read_only_mount(lib, lib_mount) + library_mounts.append(lib_mount) + + prepped_image = _run_setup_and_commit(project.mount_args, setup_commands, image_tag) + try: + verdicts = {} + completed = set() + marks_by_test = {} + + for test_file, bundle_reports in zip(test_files, bundle_reports_by_file): + test_names = [report_id for _bundle_name, report_id in bundle_reports] + with tempfile.TemporaryDirectory( + prefix="cfengine-test-runners-" + ) as runners_dir: + test_mount_args, run_command = _prepare_test_run( + runners_dir, + test_file, + bundle_reports, + project, + library_mounts, + full_policy=full_policy, + masterfiles_dir=( + os.path.abspath(masterfiles_dir) + if masterfiles_dir is not None + else None + ), + ) + mount_args = ( + library_mount_args + + test_mount_args + + _read_only_mount(runners_dir, "/mnt/runners") + ) + + one_test_results = run_and_parse( + mount_args, run_command, test_names, image=prepped_image + ) + + verdicts.update(one_test_results.verdicts) + completed.update(one_test_results.completed) + marks_by_test.update(one_test_results.marks_by_test) + finally: + _remove_image(prepped_image) + + return print_report(report_ids, RunResults(verdicts, completed, marks_by_test)) diff --git a/src/cfengine_cli/docker/test-agent/Dockerfile b/src/cfengine_cli/docker/test-agent/Dockerfile index 2469584b..0999844b 100644 --- a/src/cfengine_cli/docker/test-agent/Dockerfile +++ b/src/cfengine_cli/docker/test-agent/Dockerfile @@ -3,11 +3,24 @@ ENV PATH="/root/.local/bin:${PATH}" RUN apt-get update \ && apt-get install -y --no-install-recommends \ - python3 pipx sudo procps + python3 pipx sudo procps git rsync RUN pipx install cfengine +RUN pipx install cfbs RUN cfengine install --clients localhost --edition community RUN mkdir -p /var/cfengine/bin/ \ && ln -sf "$(command -v python3)" /var/cfengine/bin/cfengine-selected-python + +RUN mkdir -p /var/cfengine/modules/promises /tmp/assert-module-fetch \ + && cd /tmp/assert-module-fetch \ + && cfbs --non-interactive --masterfiles no --git no init \ + && cfbs --non-interactive add library-for-promise-types-in-python promise-type-assert \ + && cfbs build \ + && cp out/masterfiles/modules/promises/assert_promise_type.py \ + out/masterfiles/modules/promises/cfengine_module_library.py \ + /var/cfengine/modules/promises/ \ + && cd / && rm -rf /tmp/assert-module-fetch + +COPY assert-module/promise_agent.cf /var/cfengine/modules/promises/promise_agent.cf diff --git a/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf b/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf new file mode 100644 index 00000000..d41638b4 --- /dev/null +++ b/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf @@ -0,0 +1,5 @@ +promise agent assert +{ + path => "$(sys.workdir)/modules/promises/assert_promise_type.py"; + interpreter => "/var/cfengine/bin/cfengine-selected-python"; +} diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index b794cbfa..e4951671 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -112,6 +112,29 @@ def _get_arg_parser(): nargs="*", help="Files/Folder to lint (default is . (the entire project) )", ) + tst.add_argument( + "--dockerfile", + type=str, + default=None, + help="Path to a custom Dockerfile (or its directory) to build the test " + "container from, instead of the built-in one -- e.g. to test against " + "a different base distro", + ) + tst.add_argument( + "--rebuild", + action="store_true", + help="Force a clean rebuild of the test container image, bypassing " + "Docker's build cache", + ) + tst.add_argument( + "--full-policy", + action="store_true", + help="Run each test against the real project's own promises.cf " + "(extended with the test bundle via augments), instead of a " + "minimal generated one -- so the project's real bundlesequence " + "converges for real before each test's assertions run. Requires a " + "cfbs project.", + ) dev_parser = subp.add_parser( "dev", help="Utilities intended for developers / maintainers of CFEngine" @@ -274,6 +297,9 @@ def run_command_with_args(args) -> int: return commands.test( args.files, (args.strict.lower() in ("y", "ye", "yes")), + args.dockerfile, + args.rebuild, + args.full_policy, ) if args.command == "report": return cfengine_commands.report( diff --git a/src/cfengine_cli/report.py b/src/cfengine_cli/report.py new file mode 100644 index 00000000..2e9b8c83 --- /dev/null +++ b/src/cfengine_cli/report.py @@ -0,0 +1,193 @@ +import re +import subprocess +import sys +from collections import Counter +from typing import NamedTuple + +_ASSERT_RE = re.compile(r"\[ASSERT\] (PASS|FAIL) (.+?)(?: # (.*))?$") +_START_RE = re.compile(r"\[CFTEST-START\] (\S+)") +_DONE_RE = re.compile(r"\[CFTEST-DONE\] (\S+)") +# An assert: promise that fails validation is skipped by cf-agent and never +# prints a verdict, so it has to be caught from the error itself +_INVALID_ASSERT_RE = re.compile( + r"error: (.*) for assert promise with promiser '([^']*)'" +) + +_GREEN = "\033[32m" +_RED = "\033[31m" +_RESET = "\033[0m" + + +def _color(text: str, code: str) -> str: + return f"{code}{text}{_RESET}" if sys.stdout.isatty() else text + + +class RunResults(NamedTuple): + verdicts: dict # (test, assertion) -> (outcome, reason) + completed: set # test (bundle) names that ran to completion + marks_by_test: dict # test name -> list of "." / "F", one per assert, in order + + +def run_and_parse( + mount_args: list, script: str, test_names: list, image: str +) -> RunResults: + proc = subprocess.Popen( + ["docker", "run", "--rm", *mount_args, image, "sh", "-c", script], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + assert proc.stdout is not None + + verdicts = {} + completed = set() + marks_by_test = {name: [] for name in test_names} + current = None # test whose cftest_start marker fired most recently + + for line in proc.stdout: + print(line, end="", flush=True) + + start_match = _START_RE.search(line) + if start_match: + current = start_match.group(1) + + assert_match = _ASSERT_RE.search(line) + if assert_match and current: + outcome = assert_match.group(1) + name = assert_match.group(2) + reason = assert_match.group(3) or "" + verdicts[(current, name)] = (outcome, reason) + marks_by_test[current].append("." if outcome == "PASS" else "F") + + invalid_match = _INVALID_ASSERT_RE.search(line) + if invalid_match and current: + reason = invalid_match.group(1) + name = invalid_match.group(2) + verdicts[(current, name)] = ("FAIL", f"invalid assert: {reason}") + marks_by_test[current].append("F") + + done_match = _DONE_RE.search(line) + if done_match: + completed.add(done_match.group(1)) + + proc.wait() + return RunResults(verdicts, completed, marks_by_test) + + +class DeployResults(NamedTuple): + returncode: ( + int # the deploy run's own exit code, used when it has no asserts at all + ) + passed: int + failed: int + + +def run_and_scan_asserts(cmd: list) -> DeployResults: + """Like run_and_parse, but for a single plain policy run with no + CFTEST-START/DONE wrapper -- there's only one implicit "test" (the whole + run), so asserts are just counted flat, not attributed to a test name.""" + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + assert proc.stdout is not None + + passed = failed = 0 + for line in proc.stdout: + print(line, end="", flush=True) + + assert_match = _ASSERT_RE.search(line) + if assert_match: + if assert_match.group(1) == "PASS": + passed += 1 + else: + failed += 1 + elif _INVALID_ASSERT_RE.search(line): + failed += 1 + + proc.wait() + return DeployResults(proc.returncode, passed, failed) + + +def print_assert_totals(passed: int, failed: int) -> int: + print() + print("=" * 40) + print( + f" cfengine test: {_color(f'{passed} passed', _GREEN)}, " + f"{_color(f'{failed} failed', _RED)}, 0 errors" + ) + print("=" * 40) + return 0 if failed == 0 else 1 + + +def _assert_dots(name: str, results: RunResults) -> str: + """One character per assert, in order; an incomplete test gets a + trailing "E", like pytest's "...E...." distinguishes error from fail.""" + marks = [ + _color(mark, _GREEN if mark == "." else _RED) + for mark in results.marks_by_test[name] + ] + if name not in results.completed: + marks.append(_color("E", _RED)) + return "".join(marks) + + +def _verdicts_for(name: str, results: RunResults) -> list: + """This test's own (assertion, outcome, reason) entries.""" + return [ + (assertion, outcome, reason) + for (test, assertion), (outcome, reason) in results.verdicts.items() + if test == name + ] + + +def _outcome_of(name: str, results: RunResults) -> str: + """cftest_marker only fires if the test bundle finished evaluating, so a + fatal abort (syntax error, missing bundle, ...) shows as ERROR rather + than FAIL.""" + if name not in results.completed: + return "ERROR" + if any( + outcome == "FAIL" + for _assertion, outcome, _reason in _verdicts_for(name, results) + ): + return "FAIL" + return "PASS" + + +def print_report(test_names: list, results: RunResults) -> int: + outcomes = [(name, _outcome_of(name, results)) for name in test_names] + + print() + for name in test_names: + print(f"{name} {_assert_dots(name, results)}") + print("=" * 40) + # Passed/failed count individual assert: checks, like pytest counts test + # functions; errors count per file since an aborted run never reports + # the rest of its checks. + check_counts = Counter(outcome for outcome, _reason in results.verdicts.values()) + errored_files = sum(1 for _, outcome in outcomes if outcome == "ERROR") + passed = check_counts["PASS"] + failed = check_counts["FAIL"] + print( + f" cfengine test: {_color(f'{passed} passed', _GREEN)}, " + f"{_color(f'{failed} failed', _RED)}, " + f"{errored_files} errors" + ) + print("=" * 40) + + if all(outcome == "PASS" for _, outcome in outcomes): + return 0 + + for name, outcome in outcomes: + if outcome == "FAIL": + for assertion, verdict, reason in sorted(_verdicts_for(name, results)): + if verdict == "FAIL": + suffix = f" -> {reason}" if reason else "" + print(f" {_color('FAIL', _RED)} {name}::{assertion}{suffix}") + elif outcome == "ERROR": + print( + f" {_color('ERROR', _RED)} {name} -> test did not run to completion -- check " + "output above for a cf-agent error (syntax error, missing bundle, ...)" + ) + print() + return 1 diff --git a/tests/shell/009-test-assert-vocab.sh b/tests/shell/009-test-assert-vocab.sh new file mode 100755 index 00000000..a36f76f1 --- /dev/null +++ b/tests/shell/009-test-assert-vocab.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf "$tmpdir/test_assert_vocab.cf" +cd "$tmpdir" + +if cfengine test test_assert_vocab.cf >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected test_assert_vocab.cf to fail (it has deliberately-failing assertions)" + exit 1 +fi +cat "$output_file" + +grep -qF "PASS: test_assert_vocab.cf" "$output_file" +grep -qF "Lint passed, no errors found." "$output_file" + +expected_fails=( + "FAIL test_assert_vocab.cf::test_assert_vocab::int_equals_fail -> expected 6, got 5" + "FAIL test_assert_vocab.cf::test_assert_vocab::int_less_than_fail -> 5 is not less than 1" + "FAIL test_assert_vocab.cf::test_assert_vocab::str_not_equals_fail -> expected not 'hello', but got it" + "FAIL test_assert_vocab.cf::test_assert_vocab::file_contents_fail -> /tmp/assert_vocab_fixture.txt contents did not match the expected contents" + "FAIL test_assert_vocab.cf::test_assert_vocab::dir_exists_fail -> expected exists=True, but exists=False for /this/path/should/not/exist/hopefully" + "FAIL test_assert_vocab.cf::test_assert_vocab::slist_contains_fail -> ['apple', 'banana', 'cherry'] does not contain 'durian'" +) +for line in "${expected_fails[@]}"; do + grep -qF "$line" "$output_file" || { + echo "FAIL: expected line not found in output: $line" + exit 1 + } +done + +# Exactly these six should fail +fail_count=$(grep -c "^ FAIL test_assert_vocab.cf::" "$output_file") +if [ "$fail_count" -ne 6 ]; then + echo "FAIL: expected exactly 6 failing assertions, got $fail_count" + exit 1 +fi + +grep -qF "cfengine test: 11 passed, 6 failed, 0 errors" "$output_file" diff --git a/tests/shell/010-test-suite-standalone.sh b/tests/shell/010-test-suite-standalone.sh new file mode 100755 index 00000000..264dfdc4 --- /dev/null +++ b/tests/shell/010-test-suite-standalone.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp -r tests/shell/fixtures/010-test-suite-standalone/. "$tmpdir/" +cd "$tmpdir" + +# --- Auto-discovery: bare `cfengine test`, no arguments --- +# Two files, each with two check bundles sharing the same acted-upon data +if cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail (it has deliberately-failing assertions)" + exit 1 +fi +cat "$output_file" + +grep -qF "cfengine test: 5 passed, 2 failed, 0 errors" "$output_file" + +expected_rows=( + "tests/test_service_config.cf::test_service_config_contents" + "tests/test_service_config.cf::test_service_config_permissions" + "tests/test_computed_metrics.cf::test_computed_metrics_equality" + "tests/test_computed_metrics.cf::test_computed_metrics_comparison" +) +for row in "${expected_rows[@]}"; do + grep -qF "$row" "$output_file" || { + echo "FAIL: expected report row not found: $row" + exit 1 + } +done + +expected_fails=( + "FAIL tests/test_service_config.cf::test_service_config_permissions::config file is not world-writable (deliberately wrong check) -> /tmp/cfengine_test_suite_service.conf has permissions 640, expected 777" + "FAIL tests/test_computed_metrics.cf::test_computed_metrics_comparison::half life doubled is less than five (deliberately wrong) -> 7.0 is not less than 5.0" +) +for line in "${expected_fails[@]}"; do + grep -qF "$line" "$output_file" || { + echo "FAIL: expected line not found in output: $line" + exit 1 + } +done + +# --- Explicit directory argument --- +if cfengine test tests/ >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail when passed as an explicit directory too" + exit 1 +fi +grep -qF "cfengine test: 5 passed, 2 failed, 0 errors" "$output_file" diff --git a/tests/shell/011-test-suite-cfbs-project.sh b/tests/shell/011-test-suite-cfbs-project.sh new file mode 100755 index 00000000..219eceaf --- /dev/null +++ b/tests/shell/011-test-suite-cfbs-project.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cd "$tmpdir" + +# A plain cfbs policy-set project built on the standard community +# masterfiles -- no assert-specific setup needed, `cfengine test` auto-injects +# the assert: promise type into every test run on its own. +cat >cfbs.json <<'EOF' +{ + "name": "Example project", + "description": "Example description", + "type": "policy-set", + "git": false, + "build": [ + { + "name": "masterfiles", + "description": "Official CFEngine Masterfiles Policy Framework (MPF)", + "url": "https://github.com/cfengine/masterfiles", + "commit": "1ac0cef8c592bfca1ce0e842744b354f90b5759b", + "branch": "master", + "added_by": "cfbs init", + "steps": ["run ./prepare.sh -y", "copy ./ ./"] + } + ] +} +EOF + +# A locally-authored module (discovered via */main.cf, same as a real cfbs +# project's own custom modules), and a single test showing the setup(act)/ +# assert flow: act by calling into the project's own bundle, then assert on +# the state it left behind. +mkdir -p app_module tests +cat >app_module/main.cf <<'EOF' +bundle agent configure_app +{ + files: + "/tmp/cfengine_test_suite_app.conf" + create => "true", + content => "listen_port=8080", + perms => app_perms; +} + +body perms app_perms +{ + mode => "0644"; +} +EOF + +cat >tests/test_app_config.cf <<'EOF' +bundle agent test_app_config +{ + methods: + "configure" usebundle => configure_app; + + assert: + "app config file exists" + file => "/tmp/cfengine_test_suite_app.conf", + exists => "true"; + + "app config file is not world-writable (deliberately wrong check)" + file => "/tmp/cfengine_test_suite_app.conf", + perms => "0777"; +} +EOF + +if cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the cfbs-project suite to fail (it has a deliberately-failing assertion)" + exit 1 +fi +cat "$output_file" + +grep -qF "cfengine test: 1 passed, 1 failed, 0 errors" "$output_file" +grep -qF "tests/test_app_config.cf::test_app_config" "$output_file" +grep -qF "FAIL tests/test_app_config.cf::test_app_config::app config file is not world-writable (deliberately wrong check) -> /tmp/cfengine_test_suite_app.conf has permissions 644, expected 777" "$output_file" diff --git a/tests/shell/012-test-full-policy.sh b/tests/shell/012-test-full-policy.sh new file mode 100755 index 00000000..467f41ef --- /dev/null +++ b/tests/shell/012-test-full-policy.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp -r tests/shell/fixtures/012-test-full-policy/. "$tmpdir/" +cd "$tmpdir" + +# The project's real policy (services/init.cf, wired into the default +# bundlesequence via def.json's control_common_bundlesequence_end) writes a +# file whose contents depend on a variable with no project-level default -- +# only a per-test fixture supplies it. Without --full-policy, that bundle +# only ever runs once, during the shared setup/bootstrap step, with no +# fixture in scope, so the variable is never resolved. With --full-policy, +# the real policy re-converges fresh inside each test's own container, using +# that test's own fixture. + +# --- Without --full-policy: the real bundle only ran once, with no +# per-test fixture in scope, so neither test's expected content matches. --- +if cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail without --full-policy" + exit 1 +fi +cat "$output_file" +grep -qF "cfengine test: 0 passed, 2 failed, 0 errors" "$output_file" + +# --- With --full-policy: the real bundle re-converges fresh per test file, +# using that test's own fixture, so both pass. --- +if ! cfengine test --full-policy >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to pass with --full-policy" + exit 1 +fi +cat "$output_file" +grep -qF "cfengine test: 2 passed, 0 failed, 0 errors" "$output_file" diff --git a/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf b/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf new file mode 100644 index 00000000..30a836e1 --- /dev/null +++ b/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf @@ -0,0 +1,90 @@ +bundle agent test_assert_vocab +{ + vars: + "fruits" slist => { "apple", "banana", "cherry" }; + "counts" ilist => { "1", "2", "3" }; + "prices" rlist => { "1.50", "2.000000", "3.75" }; + + files: + "/tmp/assert_vocab_fixture.txt" + create => "true", + content => "hello world"; + + assert: + # int + "int_equals_pass" + int => "5", + equals => "5"; + + "int_equals_fail" + int => "5", + equals => "6"; + + "int_greater_than_pass" + int => "5", + greater_than => "1"; + + "int_less_than_fail" + int => "5", + less_than => "1"; + + # real + "real_equals_pass" + real => "3.140000", + equals => "3.14"; + + # str + "str_equals_pass" + str => "hello", + equals => "hello"; + + "str_not_equals_fail" + str => "hello", + not_equals => "hello"; + + "str_contains_pass" + str => "hello world", + contains => "world"; + + # file + "file_exists_pass" + file => "/tmp/assert_vocab_fixture.txt", + exists => "true"; + + "file_contents_fail" + file => "/tmp/assert_vocab_fixture.txt", + contents => "wrong contents"; + + # dir + "dir_exists_pass" + dir => "/tmp", + exists => "true"; + + "dir_exists_fail" + dir => "/this/path/should/not/exist/hopefully", + exists => "true"; + + # command + "command_output_pass" + command => "echo hello", + output => "hello"; + + # slist + "slist_equals_pass" + slist => "@(fruits)", + equals => { "apple", "banana", "cherry" }; + + "slist_contains_fail" + slist => "@(fruits)", + contains => "durian"; + + # ilist (numeric-aware comparison) + "ilist_contains_pass" + ilist => "@(counts)", + contains => "2"; + + # rlist (numeric-aware: "2" should match "2.000000") + "rlist_contains_pass" + rlist => "@(prices)", + contains => "2"; +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json b/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json new file mode 100644 index 00000000..6902c69e --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json @@ -0,0 +1,3 @@ +{ + "vars": { "expected_contents": "mode=standalone\nretries=3\n" } +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf b/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf new file mode 100644 index 00000000..53953933 --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf @@ -0,0 +1,31 @@ +bundle agent variables +{ + vars: + "half_life" string => eval("3.5 * 2", "math", "infix"); +} + +bundle agent test_computed_metrics_equality +{ + methods: + "variables"; + + assert: + "half life doubled equals seven" + real => "$(variables.half_life)", + equals => "7"; +} + +bundle agent test_computed_metrics_comparison +{ + methods: + "variables"; + + assert: + "half life doubled is greater than five" + real => "$(variables.half_life)", + greater_than => "5"; + + "half life doubled is less than five (deliberately wrong)" + real => "$(variables.half_life)", + less_than => "5"; +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf b/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf new file mode 100644 index 00000000..16e3f892 --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf @@ -0,0 +1,40 @@ +bundle agent test_service_config_contents +{ + files: + "/tmp/cfengine_test_suite_service.conf" + create => "true", + content => "$(def.expected_contents)", + perms => demo_perms; + + assert: + "config file exists" + file => "/tmp/cfengine_test_suite_service.conf", + exists => "true"; + + "config file has the fixture's contents" + file => "/tmp/cfengine_test_suite_service.conf", + contents => "$(def.expected_contents)"; +} + +bundle agent test_service_config_permissions +{ + files: + "/tmp/cfengine_test_suite_service.conf" + create => "true", + content => "$(def.expected_contents)", + perms => demo_perms; + + assert: + "config file has mode 0640" + file => "/tmp/cfengine_test_suite_service.conf", + perms => "0640"; + + "config file is not world-writable (deliberately wrong check)" + file => "/tmp/cfengine_test_suite_service.conf", + perms => "0777"; +} + +body perms demo_perms +{ + mode => "0640"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/cfbs.json b/tests/shell/fixtures/012-test-full-policy/cfbs.json new file mode 100644 index 00000000..fcbb05da --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/cfbs.json @@ -0,0 +1,27 @@ +{ + "name": "Full policy example", + "description": "Example description", + "type": "policy-set", + "git": false, + "build": [ + { + "name": "masterfiles", + "description": "Official CFEngine Masterfiles Policy Framework (MPF)", + "url": "https://github.com/cfengine/masterfiles", + "commit": "1ac0cef8c592bfca1ce0e842744b354f90b5759b", + "branch": "master", + "added_by": "cfbs init", + "steps": ["run ./prepare.sh -y", "copy ./ ./"] + }, + { + "name": "./overrides/", + "description": "Local subdirectory added using cfbs command line", + "tags": ["local"], + "added_by": "cfbs add", + "steps": [ + "copy ./def.json def.json", + "copy ./services/init.cf services/init.cf" + ] + } + ] +} diff --git a/tests/shell/fixtures/012-test-full-policy/overrides/def.json b/tests/shell/fixtures/012-test-full-policy/overrides/def.json new file mode 100644 index 00000000..320ac1a7 --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/overrides/def.json @@ -0,0 +1,3 @@ +{ + "vars": { "control_common_bundlesequence_end": ["real_policy_work"] } +} diff --git a/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf b/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf new file mode 100644 index 00000000..7faff0dd --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf @@ -0,0 +1,8 @@ +bundle agent real_policy_work +{ + files: + "/tmp/cfengine_test_suite_full_policy.txt" + create => "true", + content => "ran for $(def.instance_id) +"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json new file mode 100644 index 00000000..c2cee3a8 --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json @@ -0,0 +1,3 @@ +{ + "vars": { "instance_id": "A" } +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json new file mode 100644 index 00000000..86ec6663 --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json @@ -0,0 +1,3 @@ +{ + "vars": { "instance_id": "B" } +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf new file mode 100644 index 00000000..bfef3fda --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf @@ -0,0 +1,8 @@ +bundle agent test_instance_a +{ + assert: + "real policy ran for instance A" + file => "/tmp/cfengine_test_suite_full_policy.txt", + contents => "ran for A +"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf new file mode 100644 index 00000000..21e5bcff --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf @@ -0,0 +1,8 @@ +bundle agent test_instance_b +{ + assert: + "real policy ran for instance B" + file => "/tmp/cfengine_test_suite_full_policy.txt", + contents => "ran for B +"; +} diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py new file mode 100644 index 00000000..36127220 --- /dev/null +++ b/tests/unit/test_container.py @@ -0,0 +1,194 @@ +import pytest + +import cfengine_cli.container as container +from cfengine_cli.report import DeployResults +from cfengine_cli.utils import UserError + + +@pytest.fixture +def no_real_docker(monkeypatch): + """run_files_in_container/run_in_container always start by checking for + and building the docker image, then setting up and committing a prepped + image -- stub all of that out so tests don't need a real docker daemon.""" + monkeypatch.setattr(container, "require_docker", lambda: None) + monkeypatch.setattr(container, "_ensure_image_built", lambda *a, **k: None) + monkeypatch.setattr( + container, "_run_setup_and_commit", lambda *a, **k: "prepped-image:latest" + ) + monkeypatch.setattr(container, "_remove_image", lambda *a, **k: None) + + +# --------------------------------------------------------------------------- +# require_docker +# --------------------------------------------------------------------------- + + +def test_require_docker_raises_when_missing(monkeypatch): + monkeypatch.setattr(container.shutil, "which", lambda name: None) + with pytest.raises(UserError): + container.require_docker() + + +def test_require_docker_passes_when_present(monkeypatch): + monkeypatch.setattr(container.shutil, "which", lambda name: "/usr/bin/docker") + container.require_docker() # doesn't raise + + +# --------------------------------------------------------------------------- +# _resolve_dockerfile_dir / _image_tag_for +# --------------------------------------------------------------------------- + + +def test_resolve_dockerfile_dir_defaults_to_builtin(): + assert container._resolve_dockerfile_dir(None) == container._DOCKERFILE_DIR + + +def test_resolve_dockerfile_dir_accepts_a_directory(tmp_path): + assert container._resolve_dockerfile_dir(str(tmp_path)) == str(tmp_path) + + +def test_resolve_dockerfile_dir_accepts_a_dockerfile_path(tmp_path): + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text("FROM debian:13-slim\n") + assert container._resolve_dockerfile_dir(str(dockerfile)) == str(tmp_path) + + +def test_image_tag_for_builtin_dockerfile_is_the_stable_tag(): + assert container._image_tag_for(container._DOCKERFILE_DIR) == container._IMAGE_TAG + + +def test_image_tag_for_custom_dockerfile_is_distinct_and_deterministic(tmp_path): + tag_a = container._image_tag_for(str(tmp_path)) + tag_b = container._image_tag_for(str(tmp_path)) + assert tag_a == tag_b + assert tag_a != container._IMAGE_TAG + assert tag_a.startswith("cfengine-cli-test-agent-custom-") + + +# --------------------------------------------------------------------------- +# _ensure_image_built +# --------------------------------------------------------------------------- + + +def test_ensure_image_built_passes_no_cache_only_when_rebuilding(monkeypatch): + seen_cmds = [] + monkeypatch.setattr( + container.subprocess, + "run", + lambda cmd: seen_cmds.append(cmd) + or container.subprocess.CompletedProcess(cmd, 0), + ) + + container._ensure_image_built("/some/dir", "some-tag:latest", rebuild=False) + container._ensure_image_built("/some/dir", "some-tag:latest", rebuild=True) + + assert "--no-cache" not in seen_cmds[0] + assert "--no-cache" in seen_cmds[1] + + +def test_ensure_image_built_raises_on_build_failure(monkeypatch): + monkeypatch.setattr( + container.subprocess, + "run", + lambda cmd: container.subprocess.CompletedProcess(cmd, 1), + ) + with pytest.raises(UserError): + container._ensure_image_built("/some/dir", "some-tag:latest", rebuild=False) + + +# --------------------------------------------------------------------------- +# _run_setup_and_commit +# --------------------------------------------------------------------------- + + +def test_run_setup_and_commit_raises_when_setup_fails(monkeypatch): + monkeypatch.setattr( + container.subprocess, + "run", + lambda *a, **k: container.subprocess.CompletedProcess(a, 1), + ) + monkeypatch.setattr(container, "_docker_cleanup", lambda *a: None) + + with pytest.raises(UserError): + container._run_setup_and_commit([], ["false"], "some-tag:latest") + + +# --------------------------------------------------------------------------- +# run_in_container +# --------------------------------------------------------------------------- + + +def test_run_in_container_reports_totals_when_asserts_are_present( + no_real_docker, monkeypatch, tmp_path, capsys +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults(returncode=1, passed=2, failed=1), + ) + + rc = container.run_in_container(str(tmp_path)) + out = capsys.readouterr().out + + assert rc == 1 + assert "2 passed" in out + assert "1 failed" in out + + +def test_run_in_container_returns_raw_exit_code_without_asserts( + no_real_docker, monkeypatch, tmp_path +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults(returncode=7, passed=0, failed=0), + ) + + assert container.run_in_container(str(tmp_path)) == 7 + + +# --------------------------------------------------------------------------- +# _bundle_names_declared_in +# --------------------------------------------------------------------------- + + +def test_bundle_names_declared_in_single_bundle(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text('bundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n') + assert container._bundle_names_declared_in(str(path)) == ["test_foo"] + + +def test_bundle_names_declared_in_excludes_bundles_without_assert_section(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text( + 'bundle agent variables\n{\n vars:\n "x" string => "1";\n}\n' + '\nbundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n' + ) + assert container._bundle_names_declared_in(str(path)) == ["test_foo"] + + +def test_bundle_names_declared_in_raises_without_declaration(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text("# no bundle here\n") + with pytest.raises(UserError): + container._bundle_names_declared_in(str(path)) + + +def test_bundle_names_declared_in_raises_when_no_bundle_has_assert_section(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text('bundle agent variables\n{\n vars:\n "x" string => "1";\n}\n') + with pytest.raises(UserError): + container._bundle_names_declared_in(str(path)) + + +# --------------------------------------------------------------------------- +# run_files_in_container +# --------------------------------------------------------------------------- + + +def test_run_files_in_container_raises_without_test_files(no_real_docker, tmp_path): + lib = tmp_path / "lib.cf" + lib.write_text("bundle agent lib {}\n") + + with pytest.raises(UserError): + container.run_files_in_container([str(lib)]) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py new file mode 100644 index 00000000..39444ad3 --- /dev/null +++ b/tests/unit/test_report.py @@ -0,0 +1,109 @@ +from cfengine_cli import report +from cfengine_cli.report import RunResults + + +class _FakeProc: + """Stands in for subprocess.Popen: stdout is pre-canned lines, no real + process behind it.""" + + def __init__(self, lines, returncode=0): + self.stdout = iter(lines) + self.returncode = returncode + + def wait(self): + return self.returncode + + +def test_run_and_parse_attributes_pass_and_fail_to_current_test(monkeypatch): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + " info: [ASSERT] PASS a check\n", + " error: [ASSERT] FAIL another check # it broke\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + assert results.verdicts[("file.cf::bundle_a", "a check")] == ("PASS", "") + assert results.verdicts[("file.cf::bundle_a", "another check")] == ( + "FAIL", + "it broke", + ) + assert results.completed == {"file.cf::bundle_a"} + assert results.marks_by_test["file.cf::bundle_a"] == [".", "F"] + + +def test_run_and_parse_records_invalid_assert_as_fail(monkeypatch): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + "error: wrong type for 'int' for assert promise with promiser 'oops'\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + outcome, reason = results.verdicts[("file.cf::bundle_a", "oops")] + assert outcome == "FAIL" + assert "invalid assert: wrong type for 'int'" == reason + assert results.marks_by_test["file.cf::bundle_a"] == ["F"] + + +def test_run_and_scan_asserts_counts_pass_fail_and_invalid(monkeypatch): + lines = [ + " info: [ASSERT] PASS one\n", + " error: [ASSERT] FAIL two # bad\n", + "error: wrong type for 'int' for assert promise with promiser 'three'\n", + ] + monkeypatch.setattr( + report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines, returncode=1) + ) + + results = report.run_and_scan_asserts(["docker", "run"]) + + assert results.passed == 1 + assert results.failed == 2 + assert results.returncode == 1 + + +def test_print_assert_totals_reports_zero_failures_as_success(capsys): + rc = report.print_assert_totals(passed=3, failed=0) + out = capsys.readouterr().out + assert rc == 0 + assert "3 passed" in out + assert "0 failed" in out + + +def test_print_assert_totals_reports_failures_as_failure(capsys): + rc = report.print_assert_totals(passed=1, failed=2) + assert rc == 1 + + +def test_assert_dots_marks_incomplete_test_with_trailing_e(): + results = RunResults(verdicts={}, completed=set(), marks_by_test={"t": [".", "."]}) + assert report._assert_dots("t", results) == "..E" + + +def test_outcome_of_is_error_when_test_never_completed(): + results = RunResults(verdicts={}, completed=set(), marks_by_test={"t": []}) + assert report._outcome_of("t", results) == "ERROR" + + +def test_print_report_returns_zero_when_everything_passes(capsys): + results = RunResults( + verdicts={("t", "a check"): ("PASS", "")}, + completed={"t"}, + marks_by_test={"t": ["."]}, + ) + rc = report.print_report(["t"], results) + assert rc == 0 + + +def test_print_report_prints_error_line_for_incomplete_test(capsys): + results = RunResults(verdicts={}, completed=set(), marks_by_test={"t": []}) + rc = report.print_report(["t"], results) + out = capsys.readouterr().out + assert rc == 1 + assert "ERROR" in out + assert "test did not run to completion" in out