From cf88fd9ed58744e90af2c9ef37f0a728048da7f0 Mon Sep 17 00:00:00 2001 From: miki3421 <42601846+miki3421@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:00:31 +0200 Subject: [PATCH] Isolate shared monitoring endpoints from tenant action pods on 0.9.0 --- MONITORING_SECURITY.md | 99 +++++++++ .../alertmanager-00-networkpolicy.yaml | 52 +++++ deploy/alert-manager/alertmanager-03-dep.yaml | 3 +- deploy/monitoring/kubestate-03-dep.yaml | 6 +- .../monitoring-00-networkpolicy.yaml | 54 +++++ deploy/monitoring/prometheus-03-dep.yaml | 2 +- deploy/monitoring/prometheus-04-svc.yaml | 3 +- .../openserverless-monitoring-01-roles.yaml | 9 +- openserverless/main.py | 3 + openserverless/monitoring.py | 61 +++++- tests/monitoring_security_live.py | 194 ++++++++++++++++++ tests/test_monitoring_security.py | 181 ++++++++++++++++ 12 files changed, 645 insertions(+), 22 deletions(-) create mode 100644 MONITORING_SECURITY.md create mode 100644 deploy/alert-manager/alertmanager-00-networkpolicy.yaml create mode 100644 deploy/monitoring/monitoring-00-networkpolicy.yaml create mode 100644 tests/monitoring_security_live.py create mode 100644 tests/test_monitoring_security.py diff --git a/MONITORING_SECURITY.md b/MONITORING_SECURITY.md new file mode 100644 index 0000000..9dc2d15 --- /dev/null +++ b/MONITORING_SECURITY.md @@ -0,0 +1,99 @@ + + +# Monitoring access boundary + +Prometheus, kube-state-metrics and Alertmanager administer shared cluster state. +Tenant actions must not reach any of their listeners, including direct pod IPs. +NetworkPolicies deny ingress to Prometheus, allow only same-namespace Prometheus +pods to scrape kube-state-metrics and send alerts to Alertmanager, and allow +Alertmanager gossip only between its own peers. Allowed selectors explicitly +exclude every pod carrying the invoker-controlled `user-action-pod` label. +Copying trusted labels in another namespace does not grant access. + +Prometheus uses ClusterIP instead of the former NodePort 30299. Its destructive +admin API is disabled. Lifecycle reload remains enabled because the local config +reloader needs it; both processes communicate over loopback. The network boundary +blocks tenant access to lifecycle routes as well as to query, status and metrics +routes. No new application credentials or environment variables are required. + +## Operator access + +Use a Kubernetes identity authorized for pod port-forwarding: + +```sh +kubectl -n openserverless port-forward --address 127.0.0.1 \ + service/openserverless-prometheus-server 9090:9999 +``` + +Open `http://127.0.0.1:9090`. For Alertmanager, use a separate terminal: + +```sh +kubectl -n openserverless port-forward --address 127.0.0.1 \ + service/openserverless-prometheus-alertmanager 9093:9093 +``` + +Open `http://127.0.0.1:9093`. If kubectl runs on a remote server, forward its +loopback port over SSH. Keep the port-forward bound to loopback. The Kubernetes +API authorizes access; the native monitoring API does not authenticate callers. +Do not publish these listeners through an unauthenticated ingress or NodePort. + +## Installation and upgrade + +The cluster must enforce NetworkPolicy. K3s supports this when its network-policy +controller is enabled. A CNI without policy enforcement, K3s started with +`--disable-network-policy`, or another administrator-created broad allow policy +can leave the services accessible. Validate denial from a real action pod before +claiming tenant isolation. A pod with Kubernetes/host administration privileges, +or a compromised permitted Prometheus pod, is outside the ordinary tenant model. + +Install the matching task and operator changes together. The task removes the +collector's cluster-wide Secret list/watch permissions; the matching deployment +also stops collecting Secret metadata. During an operator update the admin client +reapplies the monitoring roles before changing the operator image. The operator +itself must not receive extra cluster permissions to perform that operation. +For an image-only upgrade, an administrator must first apply the matching +`setup/kubernetes/roles/openserverless-monitoring-roles.yaml` from the task repo. + +The operator reconciles monitoring on resume and configuration update, including +existing clusters. It applies isolation before workloads and propagates policy or +rollout errors. A pod-template annotation rolls legacy monitored pods once, +closing connections established before the policy. Component versions, PVC data, +receiver configuration and Alpha/Beta application source stay unchanged. +Monitoring also works when Alertmanager is disabled. Repeated reconciliation +does not roll unchanged monitored pod templates. + +Explicit component deletion retains policies until monitored pods terminate. +Whole-Whisk owner garbage collection has no policy-last ordering guarantee; this +is the same accepted uninstall limitation as the messaging boundary. Policies +are not a substitute for service authentication in threat models that distrust +node administrators or permitted control-plane pods. + +## Credential handling and validation + +Alertmanager 0.25 redacts its password and Slack URL fields in the status API. +The template still stores receiver credentials in a ConfigMap; moving them to +Secret-backed files is a separate hardening item for principals already allowed +to read cluster configuration. Do not describe status config output alone as +evidence of credential leakage. + +Run `python -m unittest discover -s tests -p test_monitoring_security.py` and +`python -m unittest discover -s tests -p test_alertmanager_config.py` in the operator +environment. Live acceptance must verify denied tenant connections by service +and pod IP, forged trusted labels, another namespace, trusted scraping and alert +delivery, reload, authenticated port-forward access, readiness, repeat resume, +RBAC reduction, and unchanged Alpha/Beta application behavior. diff --git a/deploy/alert-manager/alertmanager-00-networkpolicy.yaml b/deploy/alert-manager/alertmanager-00-networkpolicy.yaml new file mode 100644 index 0000000..a7e6f94 --- /dev/null +++ b/deploy/alert-manager/alertmanager-00-networkpolicy.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: openserverless-alertmanager-isolation + namespace: openserverless +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: openserverless-prometheus + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: openserverless-prometheus + matchExpressions: + - key: user-action-pod + operator: DoesNotExist + ports: + - port: 9093 + protocol: TCP + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: openserverless-prometheus + matchExpressions: + - key: user-action-pod + operator: DoesNotExist + ports: + - port: 9094 + protocol: TCP + - port: 9094 + protocol: UDP diff --git a/deploy/alert-manager/alertmanager-03-dep.yaml b/deploy/alert-manager/alertmanager-03-dep.yaml index 94c6532..b56f29d 100644 --- a/deploy/alert-manager/alertmanager-03-dep.yaml +++ b/deploy/alert-manager/alertmanager-03-dep.yaml @@ -41,6 +41,7 @@ spec: app.kubernetes.io/name: alertmanager name: alertmanager annotations: + openserverless.org/monitoring-boundary: "1" whisks.openserverless.org/annotate-version: "true" spec: serviceAccount: openserverless-prometheus-alertmanager @@ -97,4 +98,4 @@ spec: name: config - name: storage-volume persistentVolumeClaim: - claimName: storage-openserverless-prometheus-alertmanager \ No newline at end of file + claimName: storage-openserverless-prometheus-alertmanager diff --git a/deploy/monitoring/kubestate-03-dep.yaml b/deploy/monitoring/kubestate-03-dep.yaml index 10f9729..3f00cda 100644 --- a/deploy/monitoring/kubestate-03-dep.yaml +++ b/deploy/monitoring/kubestate-03-dep.yaml @@ -37,6 +37,8 @@ spec: type: RollingUpdate template: metadata: + annotations: + openserverless.org/monitoring-boundary: "1" labels: app.kubernetes.io/component: metrics app.kubernetes.io/instance: openserverless-prometheus @@ -80,7 +82,7 @@ spec: containers: - args: - --port=8080 - - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments + - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.8.0 imagePullPolicy: IfNotPresent livenessProbe: @@ -111,4 +113,4 @@ spec: resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File - terminationGracePeriodSeconds: 30 \ No newline at end of file + terminationGracePeriodSeconds: 30 diff --git a/deploy/monitoring/monitoring-00-networkpolicy.yaml b/deploy/monitoring/monitoring-00-networkpolicy.yaml new file mode 100644 index 0000000..cd4ba3c --- /dev/null +++ b/deploy/monitoring/monitoring-00-networkpolicy.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: openserverless-prometheus-isolation + namespace: openserverless +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: openserverless-prometheus + policyTypes: [Ingress] + # Administration uses Kubernetes-authenticated port-forwarding. + # Prometheus scrapes itself and the reloader over loopback. + ingress: [] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: openserverless-kube-state-metrics-isolation + namespace: openserverless +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: openserverless-prometheus + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: openserverless-prometheus + matchExpressions: + - key: user-action-pod + operator: DoesNotExist + ports: + - port: 8080 + protocol: TCP diff --git a/deploy/monitoring/prometheus-03-dep.yaml b/deploy/monitoring/prometheus-03-dep.yaml index 1fdd917..0a3ea8d 100644 --- a/deploy/monitoring/prometheus-03-dep.yaml +++ b/deploy/monitoring/prometheus-03-dep.yaml @@ -41,6 +41,7 @@ spec: app.kubernetes.io/name: prometheus name: prometheus annotations: + openserverless.org/monitoring-boundary: "1" whisks.openserverless.org/annotate-version: "true" spec: serviceAccount: openserverless-prometheus-server @@ -55,7 +56,6 @@ spec: - --web.console.libraries=/etc/prometheus/console_libraries - --web.console.templates=/etc/prometheus/consoles - --web.enable-lifecycle - - --web.enable-admin-api - --web.listen-address=:9999 image: quay.io/prometheus/prometheus:v2.43.1 imagePullPolicy: IfNotPresent diff --git a/deploy/monitoring/prometheus-04-svc.yaml b/deploy/monitoring/prometheus-04-svc.yaml index 136c689..b952f05 100644 --- a/deploy/monitoring/prometheus-04-svc.yaml +++ b/deploy/monitoring/prometheus-04-svc.yaml @@ -24,13 +24,12 @@ metadata: app.kubernetes.io/component: server app.kubernetes.io/instance: openserverless-prometheus spec: - type: NodePort + type: ClusterIP ports: - name: http port: 9999 protocol: TCP targetPort: 9999 - nodePort: 30299 selector: app.kubernetes.io/component: server app.kubernetes.io/instance: openserverless-prometheus diff --git a/deploy/openserverless-permissions/openserverless-monitoring-01-roles.yaml b/deploy/openserverless-permissions/openserverless-monitoring-01-roles.yaml index 6893617..c984540 100644 --- a/deploy/openserverless-permissions/openserverless-monitoring-01-roles.yaml +++ b/deploy/openserverless-permissions/openserverless-monitoring-01-roles.yaml @@ -223,13 +223,6 @@ rules: verbs: - list - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - list - - watch - apiGroups: - "" resources: @@ -292,4 +285,4 @@ roleRef: subjects: - kind: ServiceAccount name: openserverless-prometheus-kube-state-metrics - namespace: openserverless \ No newline at end of file + namespace: openserverless diff --git a/openserverless/main.py b/openserverless/main.py index ca02d9d..699a900 100644 --- a/openserverless/main.py +++ b/openserverless/main.py @@ -401,6 +401,7 @@ def whisk_update(spec, status, namespace, diff, name, **kwargs): owner = kube.get(f"wsk/{name}") messaging_security.reconcile(owner) + monitoring.reconcile(owner) patcher.patch(diff, status, owner, name) @kopf.on.resume('openserverless.org', 'v1', 'whisks') @@ -408,6 +409,8 @@ def whisk_resume(spec, status, name, **kwargs): operator_util.config_from_spec(spec, handler_type="on_resume") if cfg.get('components.zookeeper') or cfg.get('components.kafka'): messaging_security.reconcile(kube.get(f"wsk/{name}")) + if cfg.get('components.monitoring'): + monitoring.reconcile(kube.get(f"wsk/{name}")) operator_util.whisk_post_resume(name) def runtimes_filter(name, type, **kwargs): diff --git a/openserverless/monitoring.py b/openserverless/monitoring.py index 80b742e..6c18e68 100644 --- a/openserverless/monitoring.py +++ b/openserverless/monitoring.py @@ -21,6 +21,7 @@ import kopf import json, time +import yaml import openserverless.kube as kube import openserverless.kustomize as kus import openserverless.config as cfg @@ -28,6 +29,50 @@ import openserverless.util as util import logging +def reconcile(owner): + """Install the monitoring boundary on existing installations as well.""" + if not cfg.get('components.monitoring'): + return + if not owner or not owner.get('metadata', {}).get('uid'): + raise RuntimeError('Cannot reconcile monitoring without the Whisk owner') + return create(owner) + + +def _apply(spec, component): + expected = ({'openserverless-alertmanager-isolation'} if component == 'alert-manager' + else {'openserverless-prometheus-isolation', + 'openserverless-kube-state-metrics-isolation'}) + policies = [item for item in spec['items'] if item['kind'] == 'NetworkPolicy'] + if not expected.issubset({p['metadata']['name'] for p in policies}): + raise RuntimeError('Missing monitoring isolation policy') + # Apply the boundary first and fail closed if the cluster rejects it. + kube.apply({'apiVersion': 'v1', 'kind': 'List', 'items': policies}) + result = kube.apply(spec) + names = (['openserverless-prometheus-alertmanager'] if component == 'alert-manager' + else ['openserverless-prometheus-kube-state-metrics', + 'openserverless-prometheus-server']) + # Roll legacy pods once to close connections established before isolation. + for name in names: + kube.kubectl('rollout', 'status', 'deployment/' + name, '--timeout=300s') + return result + + +def _delete(spec): + """Keep isolation until every monitored pod has actually terminated.""" + policies = [item for item in spec['items'] if item['kind'] == 'NetworkPolicy'] + resources = [item for item in spec['items'] if item['kind'] != 'NetworkPolicy'] + result = kube.delete({'apiVersion': 'v1', 'kind': 'List', 'items': resources}) + for policy in policies: + labels = policy['spec']['podSelector']['matchLabels'] + selector = ','.join(k + '=' + v for k, v in labels.items()) + pods = kube.kubectl('get', 'pods', '-l', selector, '-o', 'name').split() + if pods: + kube.kubectl('wait', '--for=delete', *pods, '--timeout=300s') + if policies: + result += kube.delete({'apiVersion': 'v1', 'kind': 'List', 'items': policies}) + return result + + def create(owner=None): """ Configuring the operator monitoring via prometheus. @@ -52,7 +97,7 @@ def create(owner=None): else: cfg.put("state.alertmanager.spec", am_spec) - res = kube.apply(am_spec) + res = _apply(am_spec, 'alert-manager') # dynamically detect alert manager pod and wait for readiness util.wait_for_pod_ready(r"{.items[?(@.metadata.labels.app\.kubernetes\.io\/name == 'alertmanager')].metadata.name}") @@ -69,7 +114,7 @@ def create(owner=None): prometheus_data["run_as_user"]=65534 p_tplp.append("security-global-attach.yaml") - if(data['affinity'] or data['tolerations']): + if(prometheus_data['affinity'] or prometheus_data['tolerations']): p_tplp.append("affinity-tolerance-dep-core-attach.yaml") if len(p_tplp) > 0 : @@ -83,7 +128,7 @@ def create(owner=None): else: cfg.put("state.prometheus.spec", spec) - res = kube.apply(spec) + res = _apply(spec, 'monitoring') # dynamically detect prometheus pod and wait for readiness util.wait_for_pod_ready(r"{.items[?(@.metadata.labels.app\.kubernetes\.io\/name == 'prometheus')].metadata.name}") @@ -95,12 +140,12 @@ def delete_by_owner(): alert_manager = cfg.get("monitoring.alert-manager.enabled") or False spec = kus.build("monitoring") - res = kube.delete(spec) + res = _delete({'items': list(yaml.safe_load_all(spec))}) logging.info(f"delete prometheus: {res}") if alert_manager: spec = kus.build("alert-manager") - res = kube.delete(spec) + res = _delete({'items': list(yaml.safe_load_all(spec))}) logging.info(f"delete alert-manager: {res}") return res @@ -110,11 +155,11 @@ def delete_by_spec(): alert_manager_spec = cfg.get("state.alertmanager.spec") res = False if prometheus_spec: - res = kube.delete(prometheus_spec) + res = _delete(prometheus_spec) logging.info(f"delete prometheus: {res}") if alert_manager_spec: - res = kube.delete(alert_manager_spec) + res = _delete(alert_manager_spec) logging.info(f"delete alert manager: {res}") return res @@ -142,4 +187,4 @@ def patch(status, action, owner=None): logging.info(f"*** handled request to {action} monitoring") except Exception as e: logging.error('*** failed to update monitoring: %s' % e) - status['whisk_create']['monitoring']='error' \ No newline at end of file + status['whisk_create']['monitoring']='error' diff --git a/tests/monitoring_security_live.py b/tests/monitoring_security_live.py new file mode 100644 index 0000000..7bff91a --- /dev/null +++ b/tests/monitoring_security_live.py @@ -0,0 +1,194 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Explicit lab-only network boundary test; no application code is deployed.""" +import argparse +import datetime +import json +import socket +import subprocess +import time +import urllib.request + +NS = 'openserverless' +CHECKS = [] + + +def kubectl(*args, namespace=NS, data=None): + return subprocess.run(['kubectl', '-n', namespace, *args], input=data, + capture_output=True, text=True, timeout=180, check=True).stdout + + +def obj(*args, namespace=NS): + return json.loads(kubectl(*args, '-o', 'json', namespace=namespace)) + + +def check(name, passed): + CHECKS.append({'name': name, 'pass': bool(passed)}) + if not passed: + raise AssertionError(name) + + +def pod_for(name): + pods = obj('get', 'pods', '-l', 'app.kubernetes.io/name=' + name)['items'] + return next(p['metadata']['name'] for p in pods if p.get('status', {}).get('phase') == 'Running') + + +def fetch(url): + with urllib.request.urlopen(url, timeout=10) as response: + return response.read() + + +def wget(pod, url, payload=None): + args = ['exec', pod, '-c', 'prometheus-server', '--', '/bin/wget', '-q', '-T', '10', '-O', '-'] + if payload is not None: + args += ['--header=Content-Type: application/json', '--post-data=' + json.dumps(payload)] + return kubectl(*args, url) + + +def negative_matrix(pod, endpoints, namespace=NS, container=None): + code = '''import concurrent.futures,json,socket +targets=json.loads(TARGETS) +def probe(target): + try: + with socket.create_connection((target['host'],target['port']),1.5): return dict(target, denied=False) + except (TimeoutError,socket.timeout,ConnectionRefusedError,OSError): return dict(target,denied=True) +with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + print(json.dumps(list(executor.map(probe,targets)))) +'''.replace('TARGETS', repr(json.dumps(endpoints))) + args = ['exec', pod] + if container: + args += ['-c', container] + rows = json.loads(kubectl(*args, '--', 'python3', '-c', code, namespace=namespace)) + for row in rows: + check(namespace + '/' + pod + ':' + row['name'], row['denied']) + + +def run(): + parser = argparse.ArgumentParser() + parser.add_argument('--lab', action='store_true', required=True, + help='Authorize temporary synthetic pods and one resolved test alert in a lab.') + parser.parse_args() + pods = obj('get', 'pods')['items'] + tenants = [next(p for p in pods if ('-' + tenant + '-') in p['metadata']['name'] + and p['status']['phase'] == 'Running') for tenant in ['opsvalalpha', 'opsvalbeta']] + services = {} + endpoints = [] + for component, name, port in [ + ('prometheus', 'openserverless-prometheus-server', 9999), + ('kube-state-metrics', 'openserverless-prometheus-kube-state-metrics', 8080), + ('alertmanager', 'openserverless-prometheus-alertmanager', 9093)]: + service = obj('get', 'service', name) + services[component] = service + check(component + ':ClusterIP-only', service['spec']['type'] == 'ClusterIP') + p = obj('get', 'pod', pod_for(component)) + for label, host in [('service', name + '.openserverless.svc.cluster.local'), ('pod', p['status']['podIP'])]: + endpoints.append({'name': component + ':' + label, 'host': host, 'port': port}) + if component in ['kube-state-metrics', 'alertmanager']: + endpoints.append({'name': component + ':secondary-listener', + 'host': p['status']['podIP'], + 'port': 8081 if component == 'kube-state-metrics' else 9094}) + for tenant in tenants: + negative_matrix(tenant['metadata']['name'], endpoints, container='user-action') + + labels = {'app.kubernetes.io/name': 'prometheus', + 'app.kubernetes.io/instance': 'openserverless-prometheus'} + image = tenants[0]['spec']['containers'][0]['image'] + foreign = 'ops-monitoring-security-test' + fixture = 'ops-monitoring-forged-action' + # Abort instead of reusing/deleting an existing fixture owned by someone else. + existing = subprocess.run(['kubectl', 'get', 'namespace', foreign, '-o', 'name'], capture_output=True) + if existing.returncode == 0: + raise RuntimeError('Synthetic namespace already exists') + existing = subprocess.run(['kubectl', '-n', NS, 'get', 'job', fixture, '-o', 'name'], capture_output=True) + if existing.returncode == 0: + raise RuntimeError('Synthetic job already exists') + try: + kubectl('create', 'namespace', foreign) + def pod_spec(): + return {'restartPolicy': 'Never', 'automountServiceAccountToken': False, + 'containers': [{'name': 'probe', 'image': image, 'imagePullPolicy': 'IfNotPresent', + 'command': ['python3', '-c', 'import time;time.sleep(600)']}]} + foreign_pod = {'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': 'probe', 'namespace': foreign, 'labels': labels}, 'spec': pod_spec()} + kubectl('apply', '-f', '-', namespace=foreign, data=json.dumps(foreign_pod)) + job = {'apiVersion': 'batch/v1', 'kind': 'Job', 'metadata': {'name': fixture, 'namespace': NS}, + 'spec': {'backoffLimit': 0, 'template': {'metadata': {'labels': dict(labels, **{'user-action-pod': 'false'})}, 'spec': pod_spec()}}} + kubectl('apply', '-f', '-', data=json.dumps(job)) + kubectl('wait', '--for=condition=Ready', 'pod/probe', '--timeout=90s', namespace=foreign) + time.sleep(2) + fake = obj('get', 'pods', '-l', 'job-name=' + fixture)['items'][0]['metadata']['name'] + kubectl('wait', '--for=condition=Ready', 'pod/' + fake, '--timeout=90s') + negative_matrix('probe', endpoints, namespace=foreign) + negative_matrix(fake, endpoints) + finally: + kubectl('delete', 'job', fixture, '--ignore-not-found', '--wait=true') + kubectl('delete', 'namespace', foreign, '--ignore-not-found', '--wait=true') + + prom = pod_for('prometheus') + ksm = 'http://openserverless-prometheus-kube-state-metrics:8080/metrics' + metrics = wget(prom, ksm) + check('trusted:ksm-scrape', 'kube_pod_info' in metrics) + check('trusted:no-secret-series-from-collector', 'kube_secret_' not in metrics) + am = 'http://openserverless-prometheus-alertmanager:9093' + check('trusted:alertmanager-status', 'config' in json.loads(wget(prom, am + '/api/v2/status'))) + now = datetime.datetime.now(datetime.timezone.utc) + marker = 'OpsMonitoringBoundarySynthetic' + str(int(time.time())) + alert = {'labels': {'alertname': marker, 'service': 'synthetic-lab'}, + 'startsAt': (now - datetime.timedelta(seconds=1)).isoformat(), + 'endsAt': (now + datetime.timedelta(minutes=1)).isoformat()} + try: + wget(prom, am + '/api/v2/alerts', [alert]) + active = json.loads(wget(prom, am + '/api/v2/alerts')) + check('trusted:alert-delivery', any(a['labels'].get('alertname') == marker for a in active)) + finally: + alert['endsAt'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + wget(prom, am + '/api/v2/alerts', [alert]) + check('trusted:reload', wget(prom, 'http://127.0.0.1:9999/-/reload', {}) == '') + flags = json.loads(wget(prom, 'http://127.0.0.1:9999/api/v1/status/flags'))['data'] + check('trusted:admin-api-disabled', flags['web.enable-admin-api'] == 'false') + targets = json.loads(wget(prom, 'http://127.0.0.1:9999/api/v1/targets'))['data']['activeTargets'] + check('trusted:ksm-target-up', any(t['labels'].get('job') == 'kube-state-metrics' and t['health'] == 'up' for t in targets)) + rbac = subprocess.run(['kubectl', 'auth', 'can-i', '--as=system:serviceaccount:openserverless:openserverless-prometheus-kube-state-metrics', 'list', 'secrets', '--all-namespaces'], capture_output=True, text=True) + check('rbac:collector-cannot-list-secrets', rbac.stdout.strip() == 'no') + # Kubernetes-authenticated port-forward preserves administrator UI/query access. + forwarding = subprocess.Popen(['kubectl', '-n', NS, 'port-forward', '--address', '127.0.0.1', 'service/openserverless-prometheus-server', '19090:9999'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for attempt in range(50): + try: + response = fetch('http://127.0.0.1:19090/api/v1/query?query=up') + break + except OSError: + time.sleep(.1) + else: + raise RuntimeError('Port-forward did not become ready') + check('admin:authenticated-port-forward-query', json.loads(response)['status'] == 'success') + finally: + forwarding.terminate() + forwarding.wait(timeout=10) + try: + with socket.create_connection(('127.0.0.1', 30299), 1): + retired = False + except OSError: + retired = True + check('admin:legacy-nodeport-removed', retired) + + +if __name__ == '__main__': + try: + run() + finally: + print(json.dumps({'checks': CHECKS, 'count': len(CHECKS), 'failed': sum(not c['pass'] for c in CHECKS)})) diff --git a/tests/test_monitoring_security.py b/tests/test_monitoring_security.py new file mode 100644 index 0000000..d27805a --- /dev/null +++ b/tests/test_monitoring_security.py @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tenant isolation and upgrade/creation contracts for shared monitoring.""" +import copy +from pathlib import Path +import unittest +from unittest.mock import patch + +import yaml +from openserverless import config, main, monitoring + +ROOT = Path(__file__).resolve().parents[1] + + +def documents(component): + return list(yaml.safe_load_all(next((ROOT / 'deploy' / component).glob('*networkpolicy.yaml')).read_text())) + + +def accepts(policy, labels, port, namespace='openserverless'): + if namespace != 'openserverless': + return False + for rule in policy['spec']['ingress']: + if not any(item['port'] == port for item in rule['ports']): + continue + for peer in rule['from']: + selector = peer['podSelector'] + if any(labels.get(k) != v for k, v in selector.get('matchLabels', {}).items()): + continue + if any(e['key'] in labels for e in selector.get('matchExpressions', []) + if e['operator'] == 'DoesNotExist'): + continue + return True + return False + + +class MonitoringSecurityTests(unittest.TestCase): + def tearDown(self): + config.clean() + + def test_tenants_cannot_read_or_mutate_any_monitoring_listener(self): + for component in ['monitoring', 'alert-manager']: + for policy in documents(component): + for labels in [{}, {'user-action-pod': 'true'}, + {'name': 'controller'}, {'name': 'invoker'}]: + for port in [9999, 9093, 9094, 8080, 8081]: + self.assertFalse(accepts(policy, labels, port)) + + def test_trusted_labels_never_override_action_marker_or_namespace(self): + for policy in documents('monitoring') + documents('alert-manager'): + for name in ['prometheus', 'alertmanager']: + labels = {'app.kubernetes.io/name': name, + 'app.kubernetes.io/instance': 'openserverless-prometheus'} + for port in [9999, 9093, 9094, 8080, 8081]: + self.assertFalse(accepts(policy, labels, port, 'foreign')) + for marker in ['true', 'false', '']: + self.assertFalse(accepts(policy, dict(labels, **{'user-action-pod': marker}), port)) + + def test_prometheus_retains_scraping_and_alert_delivery_only(self): + labels = {'app.kubernetes.io/name': 'prometheus', + 'app.kubernetes.io/instance': 'openserverless-prometheus'} + server, metrics = documents('monitoring') + alertmanager = documents('alert-manager')[0] + self.assertFalse(accepts(server, labels, 9999)) + self.assertTrue(accepts(metrics, labels, 8080)) + self.assertFalse(accepts(metrics, labels, 8081)) + self.assertTrue(accepts(alertmanager, labels, 9093)) + self.assertFalse(accepts(alertmanager, labels, 9094)) + + def test_no_public_nodeport_or_destructive_admin_api(self): + service = yaml.safe_load((ROOT / 'deploy/monitoring/prometheus-04-svc.yaml').read_text()) + self.assertEqual(service['spec']['type'], 'ClusterIP') + self.assertTrue(all('nodePort' not in p for p in service['spec']['ports'])) + deployment = yaml.safe_load((ROOT / 'deploy/monitoring/prometheus-03-dep.yaml').read_text()) + containers = deployment['spec']['template']['spec']['containers'] + self.assertNotIn('--web.enable-admin-api', containers[0]['args']) + self.assertIn('--web.enable-lifecycle', containers[0]['args']) + self.assertIn('--reload-url=http://127.0.0.1:9999/-/reload', containers[1]['args']) + + def test_secret_collection_and_rbac_are_removed(self): + deployment = yaml.safe_load((ROOT / 'deploy/monitoring/kubestate-03-dep.yaml').read_text()) + args = deployment['spec']['template']['spec']['containers'][0]['args'] + resources = next(a.split('=', 1)[1] for a in args if a.startswith('--resources=')) + self.assertNotIn('secrets', resources.split(',')) + roles = list(yaml.safe_load_all((ROOT / 'deploy/openserverless-permissions/openserverless-monitoring-01-roles.yaml').read_text())) + self.assertTrue(all('secrets' not in r.get('resources', []) for role in roles for r in role.get('rules', []))) + + def test_policy_failure_stops_all_workloads(self): + for component in ['monitoring', 'alert-manager']: + spec = {'items': documents(component) + [{'kind': 'Deployment'}]} + with patch.object(monitoring.kube, 'apply', side_effect=RuntimeError('denied')) as apply: + with self.assertRaisesRegex(RuntimeError, 'denied'): + monitoring._apply(spec, component) + self.assertEqual(apply.call_count, 1) + self.assertTrue(all(o['kind'] == 'NetworkPolicy' for o in apply.call_args.args[0]['items'])) + + def test_missing_policy_stops_before_any_mutation(self): + with patch.object(monitoring.kube, 'apply') as apply: + for component in ['monitoring', 'alert-manager']: + with self.assertRaisesRegex(RuntimeError, 'Missing'): + monitoring._apply({'items': []}, component) + apply.assert_not_called() + + def test_rendered_specs_and_saved_state_keep_owned_policies(self): + for enabled in [False, True]: + for owned in [False, True]: + config.configure({'monitoring': {'alert-manager': {'enabled': enabled}}}) + owner = {'apiVersion': 'openserverless.org/v1', 'kind': 'Whisk', + 'metadata': {'name': 'controller', 'uid': 'synthetic'}} if owned else None + calls = [] + def rendered(where, *args, **kwargs): + return {'apiVersion': 'v1', 'kind': 'List', 'items': copy.deepcopy(documents(where))} + data = {'affinity': False, 'tolerations': False, 'applypodsecurity': False} + with patch.object(monitoring.ent_util, 'get_am_config_data', return_value=data), \ + patch.object(monitoring.ent_util, 'get_prometheus_config_data', return_value=data), \ + patch.object(monitoring.kus, 'renderTemplate'), \ + patch.object(monitoring.kus, 'patchTemplates', return_value=''), \ + patch.object(monitoring.kus, 'kustom_list', side_effect=rendered), \ + patch.object(monitoring.kube, 'apply', side_effect=lambda obj: calls.append(copy.deepcopy(obj)) or 'ok'), \ + patch.object(monitoring.kube, 'kubectl'), \ + patch.object(monitoring.util, 'wait_for_pod_ready'): + self.assertEqual(monitoring.create(owner), 'ok') + self.assertEqual(len(calls), 4 if enabled else 2) + if owned: + self.assertTrue(all(o['metadata'].get('ownerReferences') for c in calls for o in c['items'])) + else: + self.assertTrue(config.get('state.prometheus.spec')) + if enabled: + self.assertTrue(config.get('state.alertmanager.spec')) + + def test_resume_and_update_reconcile_before_success(self): + config.configure({'components': {'monitoring': True}}) + owner = {'metadata': {'uid': 'synthetic'}} + with patch.object(main.operator_util, 'config_from_spec'), \ + patch.object(main.kube, 'get', return_value=owner), \ + patch.object(monitoring, 'create', side_effect=RuntimeError('denied')), \ + patch.object(main.operator_util, 'whisk_post_resume') as post, \ + patch.object(main.patcher, 'patch') as patcher: + with self.assertRaisesRegex(RuntimeError, 'denied'): + main.whisk_resume({}, {}, 'controller') + with self.assertRaisesRegex(RuntimeError, 'denied'): + main.whisk_update({}, {}, 'openserverless', (), 'controller') + post.assert_not_called() + patcher.assert_not_called() + + def test_monitoring_disabled_and_missing_owner(self): + config.configure({'components': {'monitoring': False}}) + with patch.object(monitoring, 'create') as create: + monitoring.reconcile(None) + create.assert_not_called() + config.put('components.monitoring', True) + with self.assertRaisesRegex(RuntimeError, 'Whisk owner'): + monitoring.reconcile(None) + create.assert_not_called() + + def test_deletion_timeout_retains_policy(self): + spec = {'items': documents('monitoring') + [{'kind': 'Deployment'}]} + with patch.object(monitoring.kube, 'delete', return_value='deleted') as delete, \ + patch.object(monitoring.kube, 'kubectl', side_effect=['pod/synthetic', RuntimeError('timeout')]): + with self.assertRaisesRegex(RuntimeError, 'timeout'): + monitoring._delete(spec) + self.assertEqual(delete.call_count, 1) + self.assertTrue(all(o['kind'] != 'NetworkPolicy' for o in delete.call_args.args[0]['items'])) + + +if __name__ == '__main__': + unittest.main()