Synchronise master with upstream

doc/source/conf.py

@@ -1,7 +1,7 @@
-# -- General configuration -----------------------------------------------------
+# -- General configuration ----------------------------------------------------
 
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['openstackdocstheme']
 
 todo_include_todos = True
@@ -30,7 +30,7 @@
 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['requirements-doc.']
 
-# -- Options for HTML output ---------------------------------------------------
+# -- Options for HTML output --------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -42,10 +42,10 @@
 # If false, no index is generated.
 html_use_index = False
 
-# -- Options for LaTeX output --------------------------------------------------
+# -- Options for LaTeX output -------------------------------------------------
 
 # Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
+# (index file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
     (
         'index',
@@ -56,7 +56,7 @@
     ),
 ]
 
-# -- Options for openstackdocstheme extension ----------------------------------
+# -- Options for openstackdocstheme extension ---------------------------------
 
 openstackdocs_repo_name = 'openstack/requirements'
 openstackdocs_auto_name = False

global-requirements.txt

@@ -165,7 +165,6 @@ scipy  # BSD
 semantic-version  # BSD
 setproctitle  # BSD
 SQLAlchemy  # MIT
-sqlalchemy-filters  # Apache-2.0
 SQLAlchemy-Utils  # BSD License
 sshtunnel  # MIT
 statsd  # MIT

openstack_requirements/check.py

@@ -18,32 +18,26 @@
 import re
 import sys
 
-from packaging import markers
-
 from openstack_requirements.project import Project
 from openstack_requirements import requirement
 
-MIN_PY_VERSION = '3.5'
 PY3_GLOBAL_SPECIFIER_RE = re.compile(
     r'python_version(==|>=|>)[\'"]3\.\d+[\'"]'
 )
 PY3_LOCAL_SPECIFIER_RE = re.compile(
     r'python_version(==|>=|>|<=|<)[\'"]3\.\d+[\'"]'
 )
+WINDOWS_SPECIFIER_RE = re.compile(r'sys_platform!=[\'"]win32[\'"]')
 
 
 class RequirementsList:
     def __init__(self, name: str, project: Project) -> None:
         self.name = name
         self.reqs_by_file: dict[str, dict[str, set[str]]] = {}
+        self.optional_reqs_by_file: dict[str, dict[str, set[str]]] = {}
         self.project = project
         self.failed = False
 
-    @property
-    def reqs(self) -> dict[str, set[str]]:
-        """Flattens the list of per-file reqs."""
-        return {k: v for d in self.reqs_by_file.values() for k, v in d.items()}
-
     def extract_reqs(
         self, content: list[str], strict: bool
     ) -> dict[str, set[str]]:
@@ -112,17 +106,17 @@ def process(self, strict: bool = True) -> None:
             print(f"Processing {fname} (extras)")
             for name, content in extras.items():
                 print(f"  Processing {name!r} extra")
-                self.reqs_by_file[f'{fname} ({name!r} extra)'] = (
+                self.optional_reqs_by_file[f'{fname} ({name!r} extra)'] = (
                     self.extract_reqs(content, strict)
                 )
 
         for fname, groups in self.project['dependency_groups'].items():
             print(f"Processing {fname} (dependency-groups)")
             for name, content in groups.items():
                 print(f"  Processing {name!r} dependency group")
-                self.reqs_by_file[f'{fname} ({name!r} dependency group)'] = (
-                    self.extract_reqs(content, strict)
-                )
+                self.optional_reqs_by_file[
+                    f'{fname} ({name!r} dependency group)'
+                ] = self.extract_reqs(content, strict)
 
 
 def _get_exclusions(req):
@@ -136,7 +130,6 @@ def _get_exclusions(req):
 def _is_requirement_in_global_reqs(
     local_req,
     global_reqs,
-    backports,
 ):
     req_exclusions = _get_exclusions(local_req)
     for global_req in global_reqs:
@@ -145,49 +138,40 @@ def _is_requirement_in_global_reqs(
             local_req_val = getattr(local_req, aname)
             global_req_val = getattr(global_req, aname)
             if local_req_val != global_req_val:
-                # if a python 3 version is not specified in only one of
-                # global requirements or local requirements, allow it since
-                # python 3-only is okay
                 if matching and aname == 'markers':
+                    # if a Python version marker is specified globally but not
+                    # locally, allow it since this is unnecessary boilerplate
+                    # for projects to carry
                     if not local_req_val and PY3_GLOBAL_SPECIFIER_RE.match(
                         global_req_val
                     ):
                         continue
+
+                    # if a Python version marker is specified locally but not
+                    # globally, allow it since projects might only need the
+                    # package on specific Python versions
                     if (
                         not global_req_val
                         and local_req_val
                         and PY3_LOCAL_SPECIFIER_RE.match(local_req_val)
                     ):
                         continue
 
-                # likewise, if a package is one of the backport packages then
-                # we're okay with a potential marker (e.g. if a package
-                # requires a feature that is only available in a newer Python
-                # library, while other packages are happy without this feature
-                if (
-                    matching
-                    and aname == 'markers'
-                    and local_req.package in backports
-                ):
-                    if re.match(
-                        r'python_version(==|<=|<)[\'"]3\.\d+[\'"]',
-                        local_req_val,
+                    # OpenStack no longer supports Windows. If a package wants
+                    # to drop their sys_platform marker, let them.
+                    if not local_req_val and WINDOWS_SPECIFIER_RE.match(
+                        global_req_val
                     ):
-                        print(
-                            'Ignoring backport package with python_version '
-                            'marker'
-                        )
                         continue
 
                 print(
-                    f'WARNING: possible mismatch found for package "{local_req.package}"'
-                )  # noqa: E501
-                print(f'   Attribute "{aname}" does not match')
-                print(
-                    f'   "{local_req_val}" does not match "{global_req_val}"'
-                )  # noqa: E501
-                print(f'   {local_req}')
-                print(f'   {global_req}')
+                    f'WARNING: possible mismatch found for package '
+                    f'{local_req.package!r}\n'
+                    f'   Attribute "{aname}" does not match\n'
+                    f'   "{local_req_val}" does not match "{global_req_val}"\n'
+                    f'   {local_req}\n'
+                    f'   {global_req}'
+                )
                 matching = False
         if not matching:
             continue
@@ -202,8 +186,7 @@ def _is_requirement_in_global_reqs(
             difference = req_exclusions - global_exclusions
             print(
                 f"ERROR: Requirement for package {local_req.package} "
-                f"excludes a version not excluded in the "
-                f"global list.\n"
+                f"excludes a version not excluded in the global list.\n"
                 f"  Local settings : {list(req_exclusions)}\n"
                 f"  Global settings: {list(global_exclusions)}\n"
                 f"  Unexpected     : {list(difference)}"
@@ -234,29 +217,13 @@ def get_global_reqs(content):
     return global_reqs
 
 
-def _get_python3_reqs(reqs):
-    """Filters out the reqs that are less than our minimum version."""
-    results = []
-    for req in reqs:
-        if not req.markers:
-            results.append(req)
-        else:
-            req_markers = markers.Marker(req.markers)
-            if req_markers.evaluate(
-                {
-                    'python_version': MIN_PY_VERSION,
-                }
-            ):
-                results.append(req)
-    return results
-
-
 def _validate_one(
     name,
     reqs,
     denylist,
     global_reqs,
-    backports,
+    *,
+    is_optional,
 ):
     """Returns True if there is a failure."""
 
@@ -267,6 +234,9 @@ def _validate_one(
         return False
 
     if name not in global_reqs:
+        if is_optional:
+            return False
+
         print(f"ERROR: Requirement '{reqs}' not in openstack/requirements")
         return True
 
@@ -278,11 +248,7 @@ def _validate_one(
         else:
             counts[''] = counts.get('', 0) + 1
 
-        if not _is_requirement_in_global_reqs(
-            req,
-            global_reqs[name],
-            backports,
-        ):
+        if not _is_requirement_in_global_reqs(req, global_reqs[name]):
             return True
 
         # check for minimum being defined
@@ -314,23 +280,29 @@ def validate(
     head_reqs,
     denylist,
     global_reqs,
-    backports,
 ):
     failed = False
     # iterate through the changing entries and see if they match the global
     # equivalents we want enforced
-    for fname, freqs in head_reqs.reqs_by_file.items():
-        print(f"Validating {fname}")
-        for name, reqs in freqs.items():
-            failed = (
-                _validate_one(
-                    name,
-                    reqs,
-                    denylist,
-                    global_reqs,
-                    backports,
+    # note that extras and dependency groups are project-specific and may not
+    # be present in global-requirements, so we only note their absence rather
+    # than failing if so
+    for reqs_by_file, is_optional in (
+        (head_reqs.reqs_by_file, False),
+        (head_reqs.optional_reqs_by_file, True),
+    ):
+        for fname, freqs in reqs_by_file.items():
+            print(f"Validating {fname}")
+            for name, reqs in freqs.items():
+                failed = (
+                    _validate_one(
+                        name,
+                        reqs,
+                        denylist,
+                        global_reqs,
+                        is_optional=is_optional,
+                    )
+                    or failed
                 )
-                or failed
-            )
 
     return failed

openstack_requirements/cmds/check_exists.py

@@ -65,21 +65,24 @@ def main(args=None):
 
     for require_file, data in project_data.get('requirements', {}).items():
         print(
-            f'\nComparing {require_file} with global-requirements and upper-constraints'
+            f'\nComparing {require_file} with global-requirements and '
+            f'upper-constraints'
         )
         requirements = requirement.parse_lines(data)
         for name, spec_list in requirements.items():
             if not name or name in denylist:
                 continue
             if name not in global_requirements:
                 print(
-                    f'{name} from {require_file} not found in global-requirements'
+                    f'{name} from {require_file} not found in '
+                    f'global-requirements'
                 )
                 error_count += 1
                 continue
             if name not in upper_constraints:
                 print(
-                    f'{name} from {require_file} not found in upper-constraints'
+                    f'{name} from {require_file} not found in '
+                    f'upper-constraints'
                 )
                 error_count += 1
                 continue
@@ -95,8 +98,9 @@ def main(args=None):
                         # then something is wrong.
                         if Version(uc_spec.version) not in specs:
                             print(
-                                f'{name} must be <= {uc_spec.version} from upper-constraints and '
-                                'include the upper-constraints version'
+                                f'{name} must be <= {uc_spec.version} from '
+                                f'upper-constraints and include the '
+                                f'upper-constraints version'
                             )
                             error_count += 1
                             continue
@@ -109,8 +113,9 @@ def main(args=None):
                                 continue
                             if spec.version not in spec_gr:
                                 print(
-                                    f'Specifier {spec.version} from {name} is failing check '
-                                    f'from global-requirements specifiers {str(spec_gr)}'
+                                    f'Specifier {spec.version} from {name} is '
+                                    f'failing check from global-requirements '
+                                    f'specifiers {str(spec_gr)}'
                                 )
                                 error_count += 1
                                 continue

openstack_requirements/constraints.py

@@ -67,7 +67,8 @@ def check_format(parsed_constraints):
         for req, original_line in spec_list:
             if not req.specifiers.startswith('==='):
                 yield (
-                    f'Invalid constraint for {name} does not have 3 "=": {original_line}'
+                    f'Invalid constraint for {name} does not have 3 "=": '
+                    f'{original_line}'
                 )
 
 
@@ -106,7 +107,8 @@ def satisfied(reqs, name, version, failures):
                 return True
             tested.append(constraint.specifiers)
         failures.append(
-            f'Constraint {version} for {name} does not match requirement {tested}'
+            f'Constraint {version} for {name} does not match requirement '
+            f'{tested}'
         )
         return False
 

openstack_requirements/project.py

@@ -20,16 +20,10 @@
 import io
 import os
 import sys
+import tomllib
 from typing import Any
 from typing import TypedDict
 
-try:
-    # Python 3.11+
-    import tomllib
-except ImportError:
-    # Python 3.10 and lower
-    import tomli as tomllib  # type: ignore
-
 
 def _read_raw(root: str, filename: str) -> str | None:
     try:
@@ -164,10 +158,14 @@ def verify_pyproject_toml(root: str) -> bool:
         return False
 
     if 'build-system' not in data:
-        print("pyproject.toml is missing 'build-system' table", file=sys.stderr)
+        print(
+            "pyproject.toml is missing 'build-system' table", file=sys.stderr
+        )
         return False
 
-    if (build_backend := data['build-system'].get('build-backend')) != 'pbr.build':
+    if (
+        build_backend := data['build-system'].get('build-backend')
+    ) != 'pbr.build':
         print(
             f"pyproject.toml has invalid 'build-system.build-backend'. "
             f"Expected 'pbr.build'; got {build_backend!r}",