PoC of a new in memory model for deployment artifact generations

tools/deployment-cli-tools/ch_cli_tools/ng/__init__.py

@@ -0,0 +1,8 @@
+# Imported for their @register_file side effect (wiring up project.skaffold/project.codefresh), not for re-export.
+from . import codefresh, skaffold  # noqa: F401
+from .model import CHDeployConfig, CHProject
+
+__all__ = [
+    "CHDeployConfig",
+    "CHProject",
+]

tools/deployment-cli-tools/ch_cli_tools/ng/api.py

@@ -0,0 +1,201 @@
+"""High-level, user/script-facing deployment generation entry points.
+
+This is the surface `harness-deployment` (and any other script) actually calls
+- one function per legacy top-level entry point in ch_cli_tools/{helm,
+dockercompose,configurationgenerator,skaffold,tilt,codefresh,preprocessing,
+migration,utils}.py. Signatures mirror the legacy ones so the existing tests
+for each (tests/test_helm.py, tests/test_skaffold.py, etc.) can be retargeted
+at these once implemented, instead of rewritten.
+
+First pass: signatures only, `pass` bodies. Implementation comes later, backed
+by CHProject/CHDeployConfig and the rest of the ng model instead of the legacy
+ConfigurationGenerator/HarnessMainConfig machinery.
+"""
+
+from dataclasses import replace
+
+from .model import CHDeployConfig, CHProject
+
+# --- ch_cli_tools/helm.py ---
+
+
+def create_helm_chart(
+    root_paths,
+    tag=None,
+    registry="",
+    local=True,
+    domain=None,
+    exclude=(),
+    secured=True,
+    output_path="./deployment",
+    include=None,
+    registry_secret_name=None,
+    tls=True,
+    env=None,
+    namespace=None,
+    name=None,
+    chart_version=None,
+    app_version=None,
+):
+    # secured/tls/name/chart_version/app_version: accepted for signature
+    # compat, not wired yet - see ng/api.py's module docstring and
+    # CHProject.build_final_helm_values (secured_gatekeepers is hardcoded True,
+    # no CHDeployConfig field for it yet; Chart.yaml metadata has no ng
+    # generator yet).
+    root_paths = list(root_paths)
+
+    config_kwargs = dict(
+        includes=list(include or []),
+        excludes=list(exclude or []),
+        registry=registry,
+        tag=tag,
+        local=local,
+        namespace=namespace,
+        registry_secret_name=registry_secret_name,
+        output_path=output_path,
+    )
+    if env:
+        config_kwargs["env"] = env
+    if domain:
+        config_kwargs["domain"] = domain
+
+    config = CHDeployConfig(**config_kwargs)
+    project = None
+    for path in root_paths:  # lowest priority first, most specific last
+        project = CHProject(path, base=project, config=config)
+    assert project, "Couldn't build the project root representation"
+    return project.build_final_helm_values()
+
+
+def deploy(namespace, output_path="./deployment"):
+    pass
+
+
+# --- ch_cli_tools/dockercompose.py ---
+
+
+def create_docker_compose_configuration(
+    root_paths,
+    tag=None,
+    registry="",
+    local=True,
+    domain=None,
+    exclude=(),
+    secured=True,
+    output_path="./deployment",
+    include=None,
+    registry_secret_name=None,
+    tls=True,
+    env=None,
+    namespace=None,
+):
+    pass
+
+
+# --- ch_cli_tools/configurationgenerator.py ---
+
+
+def hosts_info(values):
+    pass
+
+
+# --- ch_cli_tools/skaffold.py ---
+# create_skaffold_configuration's replacement already exists as
+# ng.skaffold.CHSkaffold.generate() - this stub is only about matching the
+# legacy call convention (root_paths/helm_values) so the old tests can drive it.
+
+
+def create_skaffold_configuration(
+    root_paths,
+    helm_values,
+    output_path=".",
+    manage_task_images=True,
+    backend_deploy=None,
+    env=None,
+):
+    base_config = helm_values._ch_project.config
+    config = replace(
+        base_config,
+        backend="compose" if backend_deploy == "docker-compose" else "helm",
+        env=env if env is not None else base_config.env,
+        manage_task_images=manage_task_images,
+    )
+
+    # builds the CHProject chain
+    project = None
+    for path in root_paths:  # lowest priority first, most specific last
+        project = CHProject(path, base=project, config=config)
+
+    assert project, "Couldn't build the project root representation"
+    return project.skaffold.generate(output_path=output_path)
+
+
+def create_vscode_debug_configuration(root_paths, helm_values):
+    pass
+
+
+# --- ch_cli_tools/tilt.py ---
+
+
+def create_tilt_configuration(
+    root_paths,
+    helm_values,
+    manage_task_images=True,
+    output_path=".",
+    name="",
+    namespace="",
+    domain="",
+):
+    pass
+
+
+# --- ch_cli_tools/codefresh.py ---
+
+
+def create_codefresh_deployment_scripts(
+    root_paths,
+    envs=(),
+    include=(),
+    exclude=(),
+    template_name=None,
+    base_image_name=None,
+    helm_values=None,
+    save=True,
+):
+    pass
+
+
+def write_env_file(helm_values, filename, image_cache_endpoint_url=None):
+    pass
+
+
+# --- ch_cli_tools/preprocessing.py ---
+# preprocess_build_overrides physically merges overlapping app/base-image
+# directories across root_paths into one filesystem tree, so a Docker build
+# context sees files from every root that touches that app, not just the one
+# that owns the Dockerfile. Whether this is still needed depends on whether
+# the ng model keeps allowing a root to contribute extra build-context files
+# alongside a Dockerfile it doesn't own itself (CHApp.base only merges
+# *values*, never the filesystem) - open question, not decided yet.
+
+
+def preprocess_build_overrides(root_paths, helm_values, merge_build_path=None):
+    pass
+
+
+def generate_hash_based_image_tags(root_paths, helm_values, merge_build_path=None):
+    pass
+
+
+# --- ch_cli_tools/utils.py ---
+
+
+def merge_app_directories(root_paths, destination) -> None:
+    pass
+
+
+# --- ch_cli_tools/migration.py ---
+
+
+def perform_migration(base_root, accept_all=False):
+    pass

tools/deployment-cli-tools/ch_cli_tools/ng/codefresh.py

@@ -0,0 +1,112 @@
+from functools import lru_cache
+
+from .model import CHValues, register_file
+from .utils import dict_merge  # type: ignore
+
+
+@register_file(
+    "codefresh_template",
+    lambda root: root / "deployment-configuration" / "codefresh-template.yaml",
+)
+class CHCodefreshTemplate(CHValues):
+    @lru_cache
+    def all_values(self):
+        layer = self.containing_project
+        own = self.merge_with_envs(self.project.config.envs)
+        if layer.base is None:
+            return own
+        return dict_merge(layer.base.codefresh_template.all_values(), own)
+
+
+@register_file(
+    "codefresh", lambda root: root / "deployment" / "codefresh.yaml", only_env=True
+)
+class CHCodefresh(CHValues):
+    def qualify(self, image_name):
+        registry = self.project.config.registry
+        if registry and not registry.endswith("/"):
+            registry = f"{registry}/"
+        return f"{registry}{image_name}"
+
+    def _collect_app_build_step(self, app):
+        step = {
+            "title": f"Building {app.name}",
+            "type": "build",
+            "image_name": self.qualify(app.image_name),
+            "working_directory": str(
+                app.path.resolve().relative_to(self.path.parent.resolve(), walk_up=True)
+            ),
+            "dockerfile": app.dockerfile.path.name,
+            "tag": "${{CF_SHORT_REVISION}}",
+        }
+        if self.project.config.local or self.project.config.debug:
+            step["build_arguments"] = ["DEBUG=true"]
+        # MISSING: env-specific dockerfile selection, the rest of build args beyond
+        # DEBUG, git-dependency clone steps, and a `stage:` assignment (no `stages`
+        # pipeline scaffolding is modeled here) - none of these are exposed by
+        # CHApp/CHDockerfile yet. `registry_secret_name`/`domain` are on
+        # CHDeployConfig now but have nothing to plug into yet either: registry auth
+        # for a push is a Codefresh registry integration reference, not a k8s secret
+        # name (that's a Helm-values concern, a generator we haven't built), and
+        # `domain` only matters for e2e test steps, which aren't generated at all.
+        return f"build_{app.name.replace('-', '_')}", step
+
+    def _collect_task_build_steps(self, app):
+        steps = {}
+        for task in app.tasks.values():
+            if not task.dockerfile.path.exists():
+                continue
+            steps[f"build_{task.name.replace('-', '_')}"] = {
+                "title": f"Building {task.name}",
+                "type": "build",
+                "image_name": self.qualify(task.image_name),
+                "working_directory": str(
+                    task.path.resolve().relative_to(
+                        self.path.parent.resolve(), walk_up=True
+                    )
+                ),
+                "dockerfile": task.dockerfile.path.name,
+                "tag": "${{CF_SHORT_REVISION}}",
+            }
+        return steps
+
+    def generate(self, write_on_disk=True, output_path="."):
+        project = self.project
+
+        base = project.codefresh_template.all_values()
+
+        build_steps = {}
+
+        for app in project.involved_apps:
+            if isinstance(app, str):
+                continue
+
+            if app.dockerfile.path.exists():
+                key, step = self._collect_app_build_step(app)
+                build_steps[key] = step
+
+            build_steps.update(self._collect_task_build_steps(app))
+
+        # MISSING: CHProject.base_images (infrastructure/base-images/ and
+        # infrastructure/common-images/) exists now (see skaffold.py, which uses it
+        # to resolve `requires`), but nothing here iterates it to emit a build step
+        # for the base/common image itself, and this file doesn't have a
+        # `requires`/build-ordering concept at all yet either - unlike the old
+        # codefresh_steps_from_base_path() passes over BASE_IMAGES_PATH/
+        # STATIC_IMAGES_PATH. Also, base images build from repo root context and
+        # common images from their own directory - CHBaseImage doesn't carry that
+        # distinction yet.
+
+        steps = base.setdefault("steps", {})
+        steps.update(build_steps)
+
+        # MISSING: `version`/`stages` pipeline scaffolding, the git-clone/
+        # prepare_deployment/deploy steps, e2e test environment wiring, and unit
+        # test steps (harness.test.unit) all came from the old
+        # codefresh-template.yaml + create_codefresh_deployment_scripts()'s own
+        # logic - none of that is reproduced here, only the per-app/task build steps.
+
+        if write_on_disk:
+            self.write(base, output_path=output_path)
+
+        return base