diff --git a/http2py/api_pkg_maker.py b/http2py/api_pkg_maker.py index a61f26c..4a42f74 100644 --- a/http2py/api_pkg_maker.py +++ b/http2py/api_pkg_maker.py @@ -15,13 +15,25 @@ function, not by ``import http2py``, and a virtualenv created by ``uv`` does not ship it. :func:`_check_build_backend` turns its absence into an actionable message instead of a subprocess exit code. + +Nothing taken from the spec is pasted into generated source as code: the spec +is written as a JSON data file the package loads at import, and every route +name must be a plain Python identifier (i2mint/http2py#21). A spec fetched from +a server therefore cannot inject code into the package built from it. """ +import json +import keyword import os +import re import shutil import subprocess import sys import tempfile +import warnings +from collections.abc import Callable +from importlib import import_module +from typing import Optional, Union from http2py.client import HttpClient from datetime import datetime, timezone @@ -33,22 +45,41 @@ INIT_FILE_TPL = """from .funcs import {funcs} """ -FUNCS_FILE_TPL = """from i2 import Sig -from http2py.client import HttpClient +SPEC_FILENAME = "openapi_spec.json" + +# Every name the generated module defines besides the routes starts with "_", +# except ``api``; route names may not start with "_" nor be "api" (_route_names). +FUNCS_FILE_TPL = """import json as _json +from importlib import import_module as _import_module +from pathlib import Path as _Path + +from i2 import Sig as _Sig +from http2py.client import HttpClient as _HttpClient + -api = HttpClient(openapi_spec={openapi_spec}, url={openapi_url}) +def _import_object(ref): + module_name, _, name = ref.rpartition(".") + return getattr(_import_module(module_name), name) -def apply_sig(meth): +_SPEC = _json.loads( + (_Path(__file__).parent / {spec_filename!r}).read_text(encoding="utf-8") +) +api = _HttpClient(openapi_spec=_SPEC, binary_loads={binary_loads}) + + +def _apply_sig(meth): def wrapper(func): - sig = Sig(meth) - 'self' + sig = _Sig(meth) - 'self' return sig(func) return wrapper {funcs}""" +_RESERVED_ROUTE_NAMES = {"api", "funcs"} # funcs: the module's own name + FUNC_TPL = """ -@apply_sig(api.{func_name}) +@_apply_sig(api.{func_name}) def {func_name}(*args, **kwargs): return api.{func_name}(*args, **kwargs) """ @@ -73,8 +104,137 @@ def {func_name}(*args, **kwargs): install_requires = i2 http2py + +[options.package_data] +* = {spec_filename} """ +_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*" # ASCII: Python normalizes other letters +_DOTTED_PATH = re.compile(rf"{_IDENTIFIER}(\.{_IDENTIFIER})+") + + +def _is_plain_identifier(name: str) -> bool: + return re.fullmatch(_IDENTIFIER, name) is not None and not keyword.iskeyword(name) + + +def object_ref(obj: Union[str, Callable]) -> str: + """The dotted import path (``"module.name"``) a generated package uses to + import ``obj``: ``obj`` itself when it is a string (checked to name a callable, + unless its module is not installed here: then a warning), else the path of an + importable callable, checked to import back to that very object. + + >>> object_ref(bytes) + 'builtins.bytes' + >>> object_ref("http2py.decorators.unsafe_pickle_loads") + 'http2py.decorators.unsafe_pickle_loads' + >>> object_ref(lambda b: b) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: Not an importable module.name path: '...' + """ + if isinstance(obj, str): + ref = obj + else: + ref = f"{getattr(obj, '__module__', '')}.{getattr(obj, '__qualname__', '')}" + if not _DOTTED_PATH.fullmatch(ref): + raise ValueError(f"Not an importable module.name path: {ref[:80]!r}") + module_name, _, name = ref.rpartition(".") + try: + imported = getattr(import_module(module_name), name) + except ModuleNotFoundError as error: + if isinstance(obj, str): # may be installed only where the package runs + warnings.warn(f"{ref!r} cannot be imported here ({error}); kept as given") + return ref + raise ValueError(f"{ref!r} does not import: {error}") from None + except Exception as error: + raise ValueError(f"{ref!r} does not import: {error!r}") from None + if not callable(imported): + raise ValueError(f"{ref!r} is not callable") + if not isinstance(obj, str) and imported is not obj: + raise ValueError(f"{ref!r} does not import back to {obj!r}") + return ref + + +def _route_names(api: HttpClient) -> list: + """The method names ``api`` registered, each a plain identifier, since each is + written into generated source.""" + names = [ + name + for name, method in vars(api).items() + if getattr(method, "method_spec", None) + ] + for name in names: + if not _is_plain_identifier(name): + raise ValueError( + f"Route name {name[:60]!r} is not an ASCII Python identifier: cannot " + "generate a function for it." + ) + if name.startswith("_") or name in _RESERVED_ROUTE_NAMES: + raise ValueError( + f"Route name {name!r} is reserved (it starts with '_' or is 'api')." + ) + if not names: + raise RuntimeError("The API is empty") + return names + + +def _cfg_value(text: str) -> str: + """``text`` as a setup.cfg value: one line, ``%`` escaped from interpolation.""" + return " ".join(str(text).split()).replace("%", "%%") + + +def api_pkg_files( + openapi_spec: dict = None, + openapi_url: str = None, + pkg_name: str = None, + pkg_version: str = None, + pkg_description: str = None, + pkg_author: str = None, + pkg_license: str = None, + *, + binary_loads: Optional[Union[str, Callable]] = None, +) -> dict: + """The source files of the package :func:`mk_api_pkg` builds, as + ``{relative path: text}`` (see :func:`mk_api_pkg` for the arguments).""" + if isinstance(openapi_spec, str): # e.g. given as JSON text on the command line + openapi_spec = json.loads(openapi_spec) + api = HttpClient(openapi_spec=openapi_spec, url=openapi_url) + func_names = _route_names(api) + loads_code = "None" if binary_loads is None else ( + f"_import_object({object_ref(binary_loads)!r})" + ) + pkg_name = pkg_name or _mk_pkg_name() + if not _is_plain_identifier(pkg_name): + raise ValueError(f"pkg_name must be an ASCII Python identifier: {pkg_name!r}") + server_url = api.openapi_spec["servers"][0]["url"] + setup_cfg = SETUP_CFG_TPL.format( + name=pkg_name, + version=_cfg_value(pkg_version or "1.0.0"), + description=_cfg_value( + pkg_description + or f"A client API to consume the webservices exposed at {server_url}" + ), + author=_cfg_value(pkg_author or "API Package Maker"), + license=_cfg_value(pkg_license or "Apache"), + spec_filename=SPEC_FILENAME, + ) + funcs_code = FUNCS_FILE_TPL.format( + spec_filename=SPEC_FILENAME, + binary_loads=loads_code, + funcs="".join(FUNC_TPL.format(func_name=name) for name in func_names), + ) + return { + "setup.py": SETUP_PY, + "setup.cfg": setup_cfg, + f"{pkg_name}/__init__.py": INIT_FILE_TPL.format(funcs=", ".join(func_names)), + f"{pkg_name}/funcs.py": funcs_code, + f"{pkg_name}/{SPEC_FILENAME}": json.dumps(api.openapi_spec, indent=1), + } + + +def _mk_pkg_name(): + return f"apipkg{int(datetime.now(timezone.utc).timestamp())}" + def mk_api_pkg( openapi_spec: dict = None, @@ -84,11 +244,17 @@ def mk_api_pkg( pkg_description: str = None, pkg_author: str = None, pkg_license: str = None, + *, + binary_loads: Optional[Union[str, Callable]] = None, + output_dir: str = None, ): """ Build a pip installable package containing the functions to consume a running webservice application executed with py2http. + The package holds the spec as it was when built (fetched from ``openapi_url`` + if no ``openapi_spec`` is given), as a JSON file it loads at import. + :param openapi_spec: The openapi specification for the webservice. :type openapi_spec: dict, optional :param openapi_url: The url to fetch the openapi specification for the webservice. @@ -107,60 +273,33 @@ def mk_api_pkg( :param pkg_license: The license for the package. If no license provided, default to 'Apache' :type pkg_license: str, optional + :param binary_loads: How the package decodes binary (octet-stream) responses: + an importable callable or its dotted path (``"builtins.bytes"`` for the raw + content, ``"http2py.decorators.unsafe_pickle_loads"`` for a fully trusted + server). By default they raise ``BinaryResponseNotDecoded``. + :param output_dir: Where the built ``.tar.gz`` is written (default ``OUTPUT_DIR``). """ - - def mk_str_element(value: str): - return value if value is None else f"'{value}'" - - def mk_pkg_name(): - dt = datetime.now(timezone.utc) - utc_time = dt.replace(tzinfo=timezone.utc) - utc_timestamp = int(utc_time.timestamp()) - return f"apipkg{utc_timestamp}" - - def create_file(filepath, content): - with open(filepath, "w") as file: - print(content, file=file) - _check_build_backend() + output_dir = OUTPUT_DIR if output_dir is None else output_dir + files = api_pkg_files( + openapi_spec, + openapi_url, + pkg_name, + pkg_version, + pkg_description, + pkg_author, + pkg_license, + binary_loads=binary_loads, + ) + pkg_name = next(path.split("/")[0] for path in files if "/" in path) tempdir = tempfile.mkdtemp() try: - api = HttpClient(openapi_spec=openapi_spec, url=openapi_url) - paths_spec = api.openapi_spec.get("paths") - if not paths_spec: - raise RuntimeError("The API is empty") - func_names = [path[1:] for path in paths_spec] - funcs_code = [FUNC_TPL.format(func_name=func_name) for func_name in func_names] - funcs_file_code = FUNCS_FILE_TPL.format( - openapi_spec=mk_str_element(openapi_spec), - openapi_url=mk_str_element(openapi_url), - funcs="".join(funcs_code), - ) - init_file_code = INIT_FILE_TPL.format(funcs=", ".join(func_names)) - pkg_name = pkg_name or mk_pkg_name() - module_dir = os.path.join(tempdir, pkg_name) - os.makedirs(module_dir) - create_file( - filepath=os.path.join(module_dir, f"__init__.py"), content=init_file_code - ) - create_file( - filepath=os.path.join(module_dir, f"funcs.py"), content=funcs_file_code - ) - server_url = api.openapi_spec["servers"][0]["url"] - pkg_version = pkg_version or "1.0.0" - setup_cfg_content = SETUP_CFG_TPL.format( - name=pkg_name, - version=pkg_version, - description=pkg_description - or f"A client API to consume the webservices exposed at {server_url}", - author=pkg_author or "API Package Maker", - license=pkg_license or "Apache", - ) - create_file( - filepath=os.path.join(tempdir, f"setup.cfg"), content=setup_cfg_content - ) - create_file(filepath=os.path.join(tempdir, f"setup.py"), content=SETUP_PY) + for relpath, content in files.items(): + filepath = os.path.join(tempdir, *relpath.split("/")) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as file: + file.write(content) build = subprocess.run( [sys.executable, "setup.py", "sdist"], cwd=tempdir, @@ -175,12 +314,12 @@ def create_file(filepath, content): f"{pkg_name!r}.\n--- stdout ---\n{build.stdout}" f"\n--- stderr ---\n{build.stderr}" ) - pkg_filename = f"{pkg_name}-{pkg_version}.tar.gz" - if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) - shutil.copy(src=os.path.join(tempdir, "dist", pkg_filename), dst=OUTPUT_DIR) + # setuptools normalizes the name and version in the file name it writes + (pkg_filename,) = os.listdir(os.path.join(tempdir, "dist")) + os.makedirs(output_dir, exist_ok=True) + shutil.copy(src=os.path.join(tempdir, "dist", pkg_filename), dst=output_dir) - return os.path.join(OUTPUT_DIR, pkg_filename) + return os.path.join(output_dir, pkg_filename) finally: shutil.rmtree(tempdir) diff --git a/http2py/cli_maker.py b/http2py/cli_maker.py index 15351b2..ba74574 100644 --- a/http2py/cli_maker.py +++ b/http2py/cli_maker.py @@ -24,6 +24,7 @@ from i2.io_trans import JSONAnnotAndDfltIoTrans from i2.signatures import set_signature_of_func, Sig, KO from http2py import HttpClient +from http2py.decorators import check_loads from http2py.authentication import mk_auth, DFLT_CONFIG_FILENAME #: How the command line is derived from a signature. @@ -87,6 +88,8 @@ def mk_cli( filename: str = "", parse_yaml: bool = False, config_filename: str = DFLT_CONFIG_FILENAME, + *, + binary_loads: Callable = None, ): """Creates a CLI parser that exposes all of the methods of an HTTP client defined by an OpenAPI spec. Accepts either an OpenAPI spec dict, a url, or path to a local file. @@ -97,7 +100,12 @@ def mk_cli( :param parse_yaml: A flag to indicate that the provided OpenAPI spec is in YAML format rather than JSON (default: False) :param config_filename: The path to a local config file that contains authentication details for the HTTP service in JSON format (default: ~/.http2cli/credentials.json + :param binary_loads: How commands decode binary (octet-stream) responses, passed to + ``HttpClient``: ``bytes`` for the raw content, a decoder of your own, or + ``http2py.decorators.unsafe_pickle_loads`` for a fully trusted server. By + default such a response raises ``BinaryResponseNotDecoded``. """ + check_loads(binary_loads) if not openapi_spec: if url: raw_spec = requests.get(url).text @@ -128,7 +136,13 @@ def mk_cli( expected_auth_kwargs = login_details.get("login_inputs", []) client_details = HttpClient(openapi_spec) cli_methods = [ - register_cli_method(openapi_spec, method, expected_auth_kwargs) + register_cli_method( + openapi_spec, + method, + expected_auth_kwargs, + config_filename, + binary_loads=binary_loads, + ) for methodname, method in client_details.__dict__.items() if getattr(method, "method_spec", None) ] @@ -157,6 +171,8 @@ def register_cli_method( expected_auth_kwargs: Iterable[str] = None, config_filename: str = DFLT_CONFIG_FILENAME, profile: str = "", + *, + binary_loads: Callable = None, ): """Creates a CLI-friendly function to instantiate an HttpClient with appropriate authentication arguments and call a particular method of the client instance @@ -168,6 +184,8 @@ def register_cli_method( :param config_filename: The path to a JSON file to read for authentication kwargs (defaults to ~/.http2py/credentials.json) :param profile: The named profile to find in the config file + :param binary_loads: How binary (octet-stream) responses are decoded (see + :func:`mk_cli`) """ methodname = client_method.__name__ method_sig = Sig(client_method) @@ -189,7 +207,9 @@ def cli_method(*args, **kwargs): auth_kwargs = mk_auth( auth_kwargs, expected_auth_kwargs, config_filename, profile ) - http_client = HttpClient(openapi_spec, **auth_kwargs) + http_client = HttpClient( + openapi_spec, binary_loads=binary_loads, **auth_kwargs + ) return getattr(http_client, methodname)(**kwargs) method_sig.wrap(cli_method) diff --git a/http2py/decorators.py b/http2py/decorators.py index 441d9d1..fa93062 100644 --- a/http2py/decorators.py +++ b/http2py/decorators.py @@ -48,6 +48,8 @@ def _handle_error(resp): raise InputError(resp.text) if resp.reason == "DuplicateRecordError": raise DuplicateRecordError(resp.text) + if resp.status_code == 401: # py2http answers AuthorizationError with 401 + raise AuthorizationError(resp.text) if resp.status_code == 403: raise ForbiddenError(resp.text) if resp.status_code == 404: diff --git a/http2py/tests/api_pkg_maker_test.py b/http2py/tests/api_pkg_maker_test.py index 0f5f113..b503a24 100644 --- a/http2py/tests/api_pkg_maker_test.py +++ b/http2py/tests/api_pkg_maker_test.py @@ -1,7 +1,7 @@ import os import re import pytest -from http2py.api_pkg_maker import OUTPUT_DIR, mk_api_pkg +from http2py.api_pkg_maker import mk_api_pkg from http2py.constants import JSON_CONTENT_TYPE @@ -131,6 +131,7 @@ ], ) def test_mk_api_pkg( + tmp_path, openapi_spec, openapi_url, pkg_name, @@ -147,9 +148,10 @@ def test_mk_api_pkg( pkg_description, pkg_author, pkg_license, + output_dir=str(tmp_path), ) output_pkg_dirpath = os.path.dirname(output_pkg_filepath) - assert output_pkg_dirpath == OUTPUT_DIR + assert output_pkg_dirpath == str(tmp_path) output_pkg_filename = os.path.basename(output_pkg_filepath) output_pkg_name = re.search(r"(.+?)-", output_pkg_filename).group(1) if pkg_name: diff --git a/tests/test_error_statuses.py b/tests/test_error_statuses.py new file mode 100644 index 0000000..f6b1542 --- /dev/null +++ b/tests/test_error_statuses.py @@ -0,0 +1,25 @@ +"""Server errors come back as the i2 error the server raised.""" + +from types import SimpleNamespace + +import pytest +from i2.errors import AuthorizationError, ForbiddenError, InputError, NotFoundError + +from http2py.decorators import _handle_error + + +@pytest.mark.parametrize( + "status, reason, error", + [ + (401, "Unauthorized", AuthorizationError), # py2http since i2mint/py2http#31 + (400, "AuthorizationError", AuthorizationError), # older py2http + (400, "InputError", InputError), + (403, "Forbidden", ForbiddenError), + (404, "Not Found", NotFoundError), + (500, "Internal Server Error", RuntimeError), + ], +) +def test_status_maps_to_error(status, reason, error): + resp = SimpleNamespace(status_code=status, reason=reason, text="msg") + with pytest.raises(error): + _handle_error(resp) diff --git a/tests/test_generated_clients.py b/tests/test_generated_clients.py new file mode 100644 index 0000000..a4c5afa --- /dev/null +++ b/tests/test_generated_clients.py @@ -0,0 +1,184 @@ +"""``mk_cli`` and ``api_pkg_maker`` pass ``binary_loads`` through, and a package +generated from a spec compiles and cannot be made to run code by that spec +(i2mint/http2py#21).""" + +from __future__ import annotations + +import copy +import importlib +import json +import sys +import tarfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +from spec_fixture import SPEC # noqa: E402 + +from http2py.api_pkg_maker import api_pkg_files, mk_api_pkg, object_ref # noqa: E402 + + +def _write(files: dict, root: Path) -> None: + for relpath, content in files.items(): + path = root / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +@pytest.fixture +def import_generated(tmp_path, monkeypatch): + """Write a generated package under tmp_path and import it.""" + + def _import(files): + _write(files, tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + pkg_name = next(path.split("/")[0] for path in files if "/" in path) + importlib.invalidate_caches() + module = importlib.import_module(f"{pkg_name}.funcs") + monkeypatch.delitem(sys.modules, pkg_name) + monkeypatch.delitem(sys.modules, f"{pkg_name}.funcs") + return module + + return _import + + +def test_generated_package_compiles_and_exposes_every_route(import_generated): + files = api_pkg_files(SPEC, pkg_name="genpkg_routes") + for relpath, content in files.items(): + if relpath.endswith(".py"): + compile(content, relpath, "exec") + funcs = import_generated(files) + assert {"get_thing", "make", "plain"} <= set(vars(funcs)) + assert funcs.api.binary_loads is None + + +def test_spec_is_data_not_source(): + spec = copy.deepcopy(SPEC) + spec["info"]["title"] = "'''\"\"\"); import os; os.system('x') #" + files = api_pkg_files(spec, pkg_name="genpkg_data") + assert "os.system" not in files["genpkg_data/funcs.py"] + assert json.loads(files["genpkg_data/openapi_spec.json"]) == spec + + +@pytest.mark.parametrize( + "bad_name", + [ + "x():\n import os\n os.system('x')\ndef y", + "class", + "a-b", + "\uff41\uff50\uff49", # full-width "api": an identifier Python normalizes to api + "api", + "_apply_sig", + "funcs", + ], +) +def test_route_name_that_is_not_an_identifier_is_refused(bad_name): + spec = copy.deepcopy(SPEC) + spec["paths"]["/plain"]["get"]["x-method_name"] = bad_name + with pytest.raises(ValueError, match="not an ASCII Python identifier|reserved"): + api_pkg_files(spec, pkg_name="genpkg_bad") + + +def test_setup_cfg_fields_stay_on_one_line(): + files = api_pkg_files( + SPEC, pkg_name="genpkg_cfg", pkg_description="one\n[options]\nscripts = x" + ) + assert "\n[options]\nscripts" not in files["setup.cfg"] + + +def test_binary_loads_reaches_the_generated_client(import_generated): + files = api_pkg_files(SPEC, pkg_name="genpkg_loads", binary_loads=bytes) + assert import_generated(files).api.binary_loads is bytes + + +@pytest.mark.parametrize( + "ref", + [ + lambda b: b, + "bytes", + "os.system('x') or bytes", + "a.b; import os", + "json.no_such_loads", + "json.decoder", # a module, not a callable + ], +) +def test_binary_loads_must_be_an_importable_path(ref): + with pytest.raises(ValueError): + object_ref(ref) + + +def test_mk_api_pkg_ships_the_spec(tmp_path): + pytest.importorskip("setuptools") + sdist = mk_api_pkg(SPEC, pkg_name="genpkg_sdist", output_dir=str(tmp_path)) + with tarfile.open(sdist) as tar: + names = tar.getnames() + assert "genpkg_sdist-1.0.0/genpkg_sdist/openapi_spec.json" in names + + +def test_dispatch_cli_passes_binary_loads_to_the_client(monkeypatch, tmp_path): + from http2py import cli_maker + + seen = {} + + class FakeClient: + def __init__(self, openapi_spec, **kwargs): + seen.update(kwargs) + + def plain(self, **kwargs): + return "ran" + + real_mk_cli = cli_maker.mk_cli + + def mk_cli_then_fake_client(*args, **kwargs): + parser = real_mk_cli(*args, **kwargs) # built from the real client + monkeypatch.setattr(cli_maker, "HttpClient", FakeClient) # run with the fake + return parser + + monkeypatch.setattr(cli_maker, "mk_cli", mk_cli_then_fake_client) + monkeypatch.setattr(sys, "argv", ["PROG", "plain", "--api-key", "k"]) + config = tmp_path / "credentials.json" + config.write_text("{}") + cli_maker.dispatch_cli( + openapi_spec=SPEC, config_filename=str(config), binary_loads=bytes + ) + assert seen["binary_loads"] is bytes + + +def test_mk_cli_refuses_a_non_callable_binary_loads(): + from http2py.cli_maker import mk_cli + + with pytest.raises(TypeError): + mk_cli(openapi_spec=SPEC, binary_loads="bytes") + + +def test_spec_without_routes_is_refused(): + spec = copy.deepcopy(SPEC) + spec["paths"] = {"/x": {}} + with pytest.raises(RuntimeError, match="empty"): + api_pkg_files(spec, pkg_name="genpkg_empty") + + +def test_spec_may_be_given_as_json_text(import_generated): + files = api_pkg_files(json.dumps(SPEC), pkg_name="genpkg_text") + assert "plain" in vars(import_generated(files)) + + +def test_awkward_names_and_values_still_build(tmp_path): + pytest.importorskip("setuptools") + spec = copy.deepcopy(SPEC) + spec["servers"] = [{"url": "https://h/a%20b"}] + sdist = mk_api_pkg( + spec, + pkg_name="MyPkg", + pkg_version="v1.0", + pkg_author="100% me", + output_dir=str(tmp_path), + ) + assert Path(sdist).is_file() + + +def test_binary_loads_ref_to_a_module_not_installed_here_is_kept(): + with pytest.warns(UserWarning, match="cannot be imported here"): + assert object_ref("no_such_module.loads") == "no_such_module.loads"