diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 965ca64..3dca931 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -64,7 +64,7 @@ jobs: run: bash .github/cisetup.sh - name: Install run: task setup - - name: Security regression tests + - name: Regression tests run: poetry run python3 -m unittest discover -s tests -p 'test_*.py' -v - name: BuildAndLoad run: task build-and-load @@ -78,20 +78,28 @@ jobs: continue-on-error: false - name: Reject new integration failures run: | - # The legacy runner writes failures without returning nonzero. - # These two tests also fail on unmodified 0.9.0 (run 38893750250). + # The legacy runner does not propagate failure. Only the exact original + # 0.9.0 test scripts (3ee0a79; reproduced in run 38893750250) may retain + # their known warning. An updated test must pass, so both exceptions + # automatically disappear once the two independent fixes are merged. unexpected_failures=0 if [ -f _failed.txt ]; then while IFS= read -r failed_test; do case "$failed_test" in - tests/kind/ferretdb_test.ipy|tests/kind/runtimes_preloader_test.ipy) - echo "::warning::Pre-existing 0.9.0 integration failure: $failed_test" + tests/kind/ferretdb_test.ipy) + baseline_blob=90e70ffe1ae1b126e5bd651f32a6a6240b4def07 ;; - *) - echo "::error::Integration regression: $failed_test" - unexpected_failures=1 + tests/kind/runtimes_preloader_test.ipy) + baseline_blob=afbee5e8f49945f4737c507de36c731e1da6faee ;; + *) baseline_blob= ;; esac + if [ -n "$baseline_blob" ] && [ "$(git hash-object -- "$failed_test")" = "$baseline_blob" ]; then + echo "::warning::Pre-existing 0.9.0 integration failure in unchanged baseline test: $failed_test" + else + echo "::error::Integration regression: $failed_test" + unexpected_failures=1 + fi done < _failed.txt fi exit "$unexpected_failures" diff --git a/tests/kind/runtimes_preloader_test.ipy b/tests/kind/runtimes_preloader_test.ipy index afbee5e..df85566 100644 --- a/tests/kind/runtimes_preloader_test.ipy +++ b/tests/kind/runtimes_preloader_test.ipy @@ -21,6 +21,7 @@ import openserverless.runtimes_util as rutil import openserverless.testutil as tu import openserverless.kube as kube import json +from collections import Counter def cleanup(): try: @@ -46,11 +47,22 @@ for container in data['containers']: assert('name' in container), "Expected 'name' in container" assert('image' in container), "Expected 'image' in container" -count_apache = sum(1 for container in data['containers'] if container['image'].startswith('apache/')) -count_non_apache = len(data['containers']) - count_apache - -assert(count_apache > 0), "Expected at least one Apache runtime in the preloader data" -assert(count_non_apache == 0), "Expected no non-Apache runtimes in the preloader data" +# The configured catalog may include Apache, OpenWhisk or private images. +# Check the full selected catalog instead of assuming a registry organization. +expected=[] +for family,versions in json.loads(runtimes_as_json)['runtimes'].items(): + for runtime in versions: + is_default=runtime.get('default',False) + kind=runtime.get('kind','') + if not (is_default or kind.endswith(':sys')): + continue + image=runtime['image'] + if not image.get('tag'): + continue + name=family if is_default else kind.replace(':','-').replace('.','-') + expected.append((name,f"{image['prefix']}/{image['name']}:{image['tag']}")) +actual=[(container['name'],container['image']) for container in data['containers']] +assert Counter(actual)==Counter(expected), "Preloader containers must match default and system runtimes from the configured catalog" cleanup() diff --git a/tests/test_runtime_preloader_catalog.py b/tests/test_runtime_preloader_catalog.py new file mode 100644 index 0000000..5a04316 --- /dev/null +++ b/tests/test_runtime_preloader_catalog.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Run the real integration assertions with catalog fixtures and Kubernetes I/O mocked.""" +import copy +import json +from pathlib import Path +import unittest +from unittest.mock import Mock, patch + +from openserverless import config as cfg, kube, runtimes_util as runtimes, testutil, util + + +class RuntimePreloaderCatalogTests(unittest.TestCase): + def catalog(self): + def runtime(kind,prefix,name,tag,default=False): + return {'kind':kind,'default':default,'image':{'prefix':prefix,'name':name,'tag':tag}} + return {'runtimes':{ + 'python':[runtime('python:3.12','apache','python','fixture',True), + runtime('python:sys','ghcr.io/example','system-python','fixture'), + runtime('python:3.11','apache','python','old')], + 'ruby':[runtime('ruby:2.5','openwhisk','ruby','fixture',True)], + 'unpublished':[runtime('unpublished:1','example','missing-tag','',True)]}} + + def check_integration(self,mutate=None): + catalog=self.catalog();parse=runtimes.parse_runtimes + def selected(value): + result=copy.deepcopy(parse(value)) + if mutate:mutate(result['containers']) + return result + script=Path(__file__).parent/'kind/runtimes_preloader_test.ipy' + with patch.object(cfg,'configure',return_value=True), \ + patch.object(testutil,'load_sample_config',return_value={}), \ + patch.object(util,'get_runtimes_json_from_config_map',return_value=json.dumps(catalog)), \ + patch.object(kube,'kubectl',side_effect=AssertionError('unexpected Kubernetes access')), \ + patch.object(runtimes,'parse_runtimes',side_effect=selected): + exec(compile(script.read_text(),str(script),'exec'),{'get_ipython':lambda:Mock()}) + + def test_mixed_registries_defaults_and_nondefault_system_runtime_are_valid(self): + self.check_integration() + + def test_missing_default_runtime_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'must match'): + self.check_integration(lambda rows:rows.pop(0)) + + def test_missing_nondefault_system_runtime_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'must match'): + self.check_integration(lambda rows:rows.__setitem__(slice(None),[r for r in rows if r['name']!='python-sys'])) + + def test_incorrect_image_or_tag_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'must match'): + self.check_integration(lambda rows:rows[0].update(image='apache/python:wrong-tag')) + + def test_incorrect_container_identity_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'must match'): + self.check_integration(lambda rows:rows[0].update(name='wrong-family')) + + def test_duplicate_container_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'must match'): + self.check_integration(lambda rows:rows.append(dict(rows[0]))) + + def test_empty_result_is_rejected(self): + with self.assertRaisesRegex(AssertionError,'Expected one or more'): + self.check_integration(lambda rows:rows.clear()) + + +if __name__=='__main__':unittest.main()