Validate runtime preloading against the configured catalog on 0.9.0

.github/workflows/check.yml

@@ -64,7 +64,7 @@ jobs:
         run: bash .github/cisetup.sh
       - name: Install
         run: task setup
-      - name: Security regression tests
+      - name: Regression tests
         run: poetry run python3 -m unittest discover -s tests -p 'test_*.py' -v
       - name: BuildAndLoad
         run: task build-and-load
@@ -78,20 +78,28 @@ jobs:
         continue-on-error: false
       - name: Reject new integration failures
         run: |
-          # The legacy runner writes failures without returning nonzero.
-          # These two tests also fail on unmodified 0.9.0 (run 38893750250).
+          # The legacy runner does not propagate failure. Only the exact original
+          # 0.9.0 test scripts (3ee0a79; reproduced in run 38893750250) may retain
+          # their known warning. An updated test must pass, so both exceptions
+          # automatically disappear once the two independent fixes are merged.
           unexpected_failures=0
           if [ -f _failed.txt ]; then
             while IFS= read -r failed_test; do
               case "$failed_test" in
-                tests/kind/ferretdb_test.ipy|tests/kind/runtimes_preloader_test.ipy)
-                  echo "::warning::Pre-existing 0.9.0 integration failure: $failed_test"
+                tests/kind/ferretdb_test.ipy)
+                  baseline_blob=90e70ffe1ae1b126e5bd651f32a6a6240b4def07
                   ;;
-                *)
-                  echo "::error::Integration regression: $failed_test"
-                  unexpected_failures=1
+                tests/kind/runtimes_preloader_test.ipy)
+                  baseline_blob=afbee5e8f49945f4737c507de36c731e1da6faee
                   ;;
+                *) baseline_blob= ;;
               esac
+              if [ -n "$baseline_blob" ] && [ "$(git hash-object -- "$failed_test")" = "$baseline_blob" ]; then
+                echo "::warning::Pre-existing 0.9.0 integration failure in unchanged baseline test: $failed_test"
+              else
+                echo "::error::Integration regression: $failed_test"
+                unexpected_failures=1
+              fi
             done < _failed.txt
           fi
           exit "$unexpected_failures"

tests/kind/runtimes_preloader_test.ipy

@@ -21,6 +21,7 @@ import openserverless.runtimes_util as rutil
 import openserverless.testutil as tu
 import openserverless.kube as kube
 import json
+from collections import Counter
 
 def cleanup():
     try:
@@ -46,11 +47,22 @@ for container in data['containers']:
     assert('name' in container), "Expected 'name' in container"
     assert('image' in container), "Expected 'image' in container"
 
-count_apache = sum(1 for container in data['containers'] if container['image'].startswith('apache/'))
-count_non_apache = len(data['containers']) - count_apache
-
-assert(count_apache > 0), "Expected at least one Apache runtime in the preloader data"
-assert(count_non_apache == 0), "Expected no non-Apache runtimes in the preloader data"
+# The configured catalog may include Apache, OpenWhisk or private images.
+# Check the full selected catalog instead of assuming a registry organization.
+expected=[]
+for family,versions in json.loads(runtimes_as_json)['runtimes'].items():
+    for runtime in versions:
+        is_default=runtime.get('default',False)
+        kind=runtime.get('kind','')
+        if not (is_default or kind.endswith(':sys')):
+            continue
+        image=runtime['image']
+        if not image.get('tag'):
+            continue
+        name=family if is_default else kind.replace(':','-').replace('.','-')
+        expected.append((name,f"{image['prefix']}/{image['name']}:{image['tag']}"))
+actual=[(container['name'],container['image']) for container in data['containers']]
+assert Counter(actual)==Counter(expected), "Preloader containers must match default and system runtimes from the configured catalog"
 
 
 cleanup()

tests/test_runtime_preloader_catalog.py

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+"""Run the real integration assertions with catalog fixtures and Kubernetes I/O mocked."""
+import copy
+import json
+from pathlib import Path
+import unittest
+from unittest.mock import Mock, patch
+
+from openserverless import config as cfg, kube, runtimes_util as runtimes, testutil, util
+
+
+class RuntimePreloaderCatalogTests(unittest.TestCase):
+    def catalog(self):
+        def runtime(kind,prefix,name,tag,default=False):
+            return {'kind':kind,'default':default,'image':{'prefix':prefix,'name':name,'tag':tag}}
+        return {'runtimes':{
+            'python':[runtime('python:3.12','apache','python','fixture',True),
+                      runtime('python:sys','ghcr.io/example','system-python','fixture'),
+                      runtime('python:3.11','apache','python','old')],
+            'ruby':[runtime('ruby:2.5','openwhisk','ruby','fixture',True)],
+            'unpublished':[runtime('unpublished:1','example','missing-tag','',True)]}}
+
+    def check_integration(self,mutate=None):
+        catalog=self.catalog();parse=runtimes.parse_runtimes
+        def selected(value):
+            result=copy.deepcopy(parse(value))
+            if mutate:mutate(result['containers'])
+            return result
+        script=Path(__file__).parent/'kind/runtimes_preloader_test.ipy'
+        with patch.object(cfg,'configure',return_value=True), \
+             patch.object(testutil,'load_sample_config',return_value={}), \
+             patch.object(util,'get_runtimes_json_from_config_map',return_value=json.dumps(catalog)), \
+             patch.object(kube,'kubectl',side_effect=AssertionError('unexpected Kubernetes access')), \
+             patch.object(runtimes,'parse_runtimes',side_effect=selected):
+            exec(compile(script.read_text(),str(script),'exec'),{'get_ipython':lambda:Mock()})
+
+    def test_mixed_registries_defaults_and_nondefault_system_runtime_are_valid(self):
+        self.check_integration()
+
+    def test_missing_default_runtime_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'must match'):
+            self.check_integration(lambda rows:rows.pop(0))
+
+    def test_missing_nondefault_system_runtime_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'must match'):
+            self.check_integration(lambda rows:rows.__setitem__(slice(None),[r for r in rows if r['name']!='python-sys']))
+
+    def test_incorrect_image_or_tag_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'must match'):
+            self.check_integration(lambda rows:rows[0].update(image='apache/python:wrong-tag'))
+
+    def test_incorrect_container_identity_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'must match'):
+            self.check_integration(lambda rows:rows[0].update(name='wrong-family'))
+
+    def test_duplicate_container_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'must match'):
+            self.check_integration(lambda rows:rows.append(dict(rows[0])))
+
+    def test_empty_result_is_rejected(self):
+        with self.assertRaisesRegex(AssertionError,'Expected one or more'):
+            self.check_integration(lambda rows:rows.clear())
+
+
+if __name__=='__main__':unittest.main()