Synchronise master with upstream

openstack_requirements/check.py

@@ -18,12 +18,9 @@
 import re
 import sys
 
-from packaging import markers
-
 from openstack_requirements.project import Project
 from openstack_requirements import requirement
 
-MIN_PY_VERSION = '3.5'
 PY3_GLOBAL_SPECIFIER_RE = re.compile(
     r'python_version(==|>=|>)[\'"]3\.\d+[\'"]'
 )
@@ -36,14 +33,10 @@ class RequirementsList:
     def __init__(self, name: str, project: Project) -> None:
         self.name = name
         self.reqs_by_file: dict[str, dict[str, set[str]]] = {}
+        self.optional_reqs_by_file: dict[str, dict[str, set[str]]] = {}
         self.project = project
         self.failed = False
 
-    @property
-    def reqs(self) -> dict[str, set[str]]:
-        """Flattens the list of per-file reqs."""
-        return {k: v for d in self.reqs_by_file.values() for k, v in d.items()}
-
     def extract_reqs(
         self, content: list[str], strict: bool
     ) -> dict[str, set[str]]:
@@ -112,17 +105,17 @@ def process(self, strict: bool = True) -> None:
             print(f"Processing {fname} (extras)")
             for name, content in extras.items():
                 print(f"  Processing {name!r} extra")
-                self.reqs_by_file[f'{fname} ({name!r} extra)'] = (
+                self.optional_reqs_by_file[f'{fname} ({name!r} extra)'] = (
                     self.extract_reqs(content, strict)
                 )
 
         for fname, groups in self.project['dependency_groups'].items():
             print(f"Processing {fname} (dependency-groups)")
             for name, content in groups.items():
                 print(f"  Processing {name!r} dependency group")
-                self.reqs_by_file[f'{fname} ({name!r} dependency group)'] = (
-                    self.extract_reqs(content, strict)
-                )
+                self.optional_reqs_by_file[
+                    f'{fname} ({name!r} dependency group)'
+                ] = self.extract_reqs(content, strict)
 
 
 def _get_exclusions(req):
@@ -234,29 +227,14 @@ def get_global_reqs(content):
     return global_reqs
 
 
-def _get_python3_reqs(reqs):
-    """Filters out the reqs that are less than our minimum version."""
-    results = []
-    for req in reqs:
-        if not req.markers:
-            results.append(req)
-        else:
-            req_markers = markers.Marker(req.markers)
-            if req_markers.evaluate(
-                {
-                    'python_version': MIN_PY_VERSION,
-                }
-            ):
-                results.append(req)
-    return results
-
-
 def _validate_one(
     name,
     reqs,
     denylist,
     global_reqs,
     backports,
+    *,
+    is_optional,
 ):
     """Returns True if there is a failure."""
 
@@ -267,6 +245,9 @@ def _validate_one(
         return False
 
     if name not in global_reqs:
+        if is_optional:
+            return False
+
         print(f"ERROR: Requirement '{reqs}' not in openstack/requirements")
         return True
 
@@ -319,18 +300,26 @@ def validate(
     failed = False
     # iterate through the changing entries and see if they match the global
     # equivalents we want enforced
-    for fname, freqs in head_reqs.reqs_by_file.items():
-        print(f"Validating {fname}")
-        for name, reqs in freqs.items():
-            failed = (
-                _validate_one(
-                    name,
-                    reqs,
-                    denylist,
-                    global_reqs,
-                    backports,
+    # note that extras and dependency groups are project-specific and may not
+    # be present in global-requirements, so we only note their absence rather
+    # than failing if so
+    for reqs_by_file, is_optional in (
+        (head_reqs.reqs_by_file, False),
+        (head_reqs.optional_reqs_by_file, True),
+    ):
+        for fname, freqs in reqs_by_file.items():
+            print(f"Validating {fname}")
+            for name, reqs in freqs.items():
+                failed = (
+                    _validate_one(
+                        name,
+                        reqs,
+                        denylist,
+                        global_reqs,
+                        backports,
+                        is_optional=is_optional,
+                    )
+                    or failed
                 )
-                or failed
-            )
 
     return failed

openstack_requirements/project.py

@@ -164,10 +164,14 @@ def verify_pyproject_toml(root: str) -> bool:
         return False
 
     if 'build-system' not in data:
-        print("pyproject.toml is missing 'build-system' table", file=sys.stderr)
+        print(
+            "pyproject.toml is missing 'build-system' table", file=sys.stderr
+        )
         return False
 
-    if (build_backend := data['build-system'].get('build-backend')) != 'pbr.build':
+    if (
+        build_backend := data['build-system'].get('build-backend')
+    ) != 'pbr.build':
         print(
             f"pyproject.toml has invalid 'build-system.build-backend'. "
             f"Expected 'pbr.build'; got {build_backend!r}",

openstack_requirements/tests/test_check.py

@@ -44,11 +44,15 @@ def test_extras__setup_cfg(self):
         req_list = check.RequirementsList('test-project', project_data)
         req_list.process(strict=False)
 
-        self.assertIn("setup.cfg ('test' extra)", req_list.reqs_by_file)
-        self.assertIn("setup.cfg ('dev' extra)", req_list.reqs_by_file)
+        self.assertIn(
+            "setup.cfg ('test' extra)", req_list.optional_reqs_by_file
+        )
+        self.assertIn(
+            "setup.cfg ('dev' extra)", req_list.optional_reqs_by_file
+        )
 
-        test_reqs = req_list.reqs_by_file["setup.cfg ('test' extra)"]
-        dev_reqs = req_list.reqs_by_file["setup.cfg ('dev' extra)"]
+        test_reqs = req_list.optional_reqs_by_file["setup.cfg ('test' extra)"]
+        dev_reqs = req_list.optional_reqs_by_file["setup.cfg ('dev' extra)"]
 
         self.assertEqual(2, len(test_reqs))
         self.assertIn('pytest', test_reqs)
@@ -88,30 +92,33 @@ def test_extras__pyproject_toml(self):
 
         self.assertIn('pyproject.toml (dependencies)', req_list.reqs_by_file)
         self.assertIn(
-            "pyproject.toml ('security' extra)", req_list.reqs_by_file
+            "pyproject.toml ('security' extra)",
+            req_list.optional_reqs_by_file,
         )
         self.assertIn(
-            "pyproject.toml ('test' dependency group)", req_list.reqs_by_file
+            "pyproject.toml ('test' dependency group)",
+            req_list.optional_reqs_by_file,
         )
         self.assertIn(
-            "pyproject.toml ('typing' dependency group)", req_list.reqs_by_file
+            "pyproject.toml ('typing' dependency group)",
+            req_list.optional_reqs_by_file,
         )
         self.assertIn(
             "pyproject.toml ('typing-test' dependency group)",
-            req_list.reqs_by_file,
+            req_list.optional_reqs_by_file,
         )
 
         main_reqs = req_list.reqs_by_file['pyproject.toml (dependencies)']
-        security_reqs = req_list.reqs_by_file[
+        security_reqs = req_list.optional_reqs_by_file[
             "pyproject.toml ('security' extra)"
         ]
-        test_reqs = req_list.reqs_by_file[
+        test_reqs = req_list.optional_reqs_by_file[
             "pyproject.toml ('test' dependency group)"
         ]
-        typing_reqs = req_list.reqs_by_file[
+        typing_reqs = req_list.optional_reqs_by_file[
             "pyproject.toml ('typing' dependency group)"
         ]
-        typing_test_reqs = req_list.reqs_by_file[
+        typing_test_reqs = req_list.optional_reqs_by_file[
             "pyproject.toml ('typing-test' dependency group)"
         ]
 
@@ -347,6 +354,7 @@ def test_unchanged(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -361,6 +369,7 @@ def test_denylisted(self):
                 denylist=requirement.parse('name'),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -376,6 +385,7 @@ def test_denylisted_mismatch(self):
                 denylist=requirement.parse('name'),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -390,6 +400,7 @@ def test_not_in_global_list(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -404,6 +415,7 @@ def test_new_item_matches_global_list(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -419,6 +431,7 @@ def test_new_item_lower_min(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -436,6 +449,7 @@ def test_new_item_extra_exclusion(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -451,6 +465,7 @@ def test_new_item_missing_exclusion(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -476,6 +491,7 @@ def test_new_item_matches_global_list_with_extra(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -500,6 +516,7 @@ def test_new_item_missing_extra_line(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
             )
         )
 
@@ -525,6 +542,73 @@ def test_new_item_mismatches_global_list_with_extra(self):
                 denylist=requirement.parse(''),
                 backports=self.backports,
                 global_reqs=global_reqs,
+                is_optional=False,
+            )
+        )
+
+    def test_optional_not_in_global_list(self):
+        # If an optional package is not in the global list, that is not an
+        # error.
+        reqs = [r for r, line in requirement.parse('name>=1.2,!=1.4')['name']]
+        global_reqs = check.get_global_reqs('')
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                denylist=requirement.parse(''),
+                backports=self.backports,
+                global_reqs=global_reqs,
+                is_optional=True,
+            )
+        )
+
+    def test_optional_matches_global_list(self):
+        # If an optional package is in the global list and matches,
+        # everything is OK.
+        reqs = [r for r, line in requirement.parse('name>=1.2,!=1.4')['name']]
+        global_reqs = check.get_global_reqs('name>=1.2,!=1.4')
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                denylist=requirement.parse(''),
+                backports=self.backports,
+                global_reqs=global_reqs,
+                is_optional=True,
+            )
+        )
+
+    def test_optional_mismatches_global_list(self):
+        # If an optional package is in the global list and does not match,
+        # that is an error.
+        reqs = [
+            r for r, line in requirement.parse('name>=1.2,!=1.4,!=1.5')['name']
+        ]
+        global_reqs = check.get_global_reqs('name>=1.2,!=1.4')
+        self.assertTrue(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                denylist=requirement.parse(''),
+                backports=self.backports,
+                global_reqs=global_reqs,
+                is_optional=True,
+            )
+        )
+
+    def test_optional_denylisted(self):
+        # If the optional package is denylisted, everything is OK even if absent
+        # from global-requirements.
+        reqs = [r for r, line in requirement.parse('name>=1.2,!=1.4')['name']]
+        global_reqs = check.get_global_reqs('')
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                denylist=requirement.parse('name'),
+                backports=self.backports,
+                global_reqs=global_reqs,
+                is_optional=True,
             )
         )