From 660d49a5d57487651a925ff327d5e5f9f6bcc20b Mon Sep 17 00:00:00 2001 From: Jonathan Payne Date: Thu, 22 Oct 2026 12:23:24 -0400 Subject: [PATCH 01/18] OpenConceptLab/ocl_online#451 | FHIR requests and errors use native FHIR JSON and XML FHIR views now accept and return FHIR JSON and XML through the shared core/fhir/base.py layer. _format overrides Accept, media aliases and quality values are honoured, and unsupported FHIR versions are refused. XML is converted locally with fhir.resources' R4B models (fhir.resources[xml] 8.3.0, up from 7.1.0), so XML conversion no longer calls the validator service and the FHIR_VALIDATOR_URL application setting is removed. The shared CodeSystem converter (core/code_systems/converter.py) moves to the R4B model and model_dump(). XML follows FHIR element order and preserves the values JSON carries. Unsupported elements and shapes are omitted, a single-valued element keeps its first item, and characters XML cannot represent become replacement characters. Errors from views, routing and middleware return OperationOutcome with terminology issue coding; server errors become 422 responses. FhirErrorMiddleware runs outermost, and core/wsgi.py installs FhirWSGIRequest, so a malformed Content-Type parameter or a non-text charset on a FHIR path is a FHIR JSON 400 rather than an error page; other paths are unchanged. metadata also answers with a trailing slash, and names the request's own FHIR base. Tests cover negotiation, JSON and XML input and output, malformed headers and queries, serializer outcomes and middleware failures. Work package WP1 of OpenConceptLab/ocl_online#451. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R8hw7w8753DDSCezrmrDA8 --- .env.example | 1 - .pylintrc | 1 + core/code_systems/converter.py | 4 +- core/code_systems/tests.py | 2 +- core/code_systems/tests_converter.py | 34 ++ core/code_systems/views.py | 35 +- core/common/fhir_helpers.py | 3 + core/concept_maps/tests.py | 5 +- core/concept_maps/views.py | 7 +- core/fhir/__init__.py | 0 core/fhir/base.py | 566 +++++++++++++++++++++ core/fhir/outcomes.py | 253 ++++++++++ core/fhir/tests.py | 9 +- core/fhir/tests_base.py | 726 +++++++++++++++++++++++++++ core/fhir/tests_middleware.py | 271 ++++++++++ core/fhir/tests_outcomes.py | 230 +++++++++ core/fhir/tests_protocol.py | 365 ++++++++++++++ core/fhir/urls.py | 1 + core/fhir/views.py | 8 +- core/middlewares/middlewares.py | 105 ++-- core/parameters/__init__.py | 0 core/parameters/tests.py | 20 +- core/settings.py | 6 +- core/value_sets/views.py | 7 +- core/wsgi.py | 38 ++ requirements.txt | 2 +- 26 files changed, 2587 insertions(+), 112 deletions(-) create mode 100644 core/code_systems/tests_converter.py create mode 100644 core/fhir/__init__.py create mode 100644 core/fhir/base.py create mode 100644 core/fhir/outcomes.py create mode 100644 core/fhir/tests_base.py create mode 100644 core/fhir/tests_middleware.py create mode 100644 core/fhir/tests_outcomes.py create mode 100644 core/fhir/tests_protocol.py create mode 100644 core/parameters/__init__.py diff --git a/.env.example b/.env.example index d0e096b39..f726c2c8a 100644 --- a/.env.example +++ b/.env.example @@ -82,7 +82,6 @@ FLOWER_PASSWORD=Root123 # ----------------------------------------------------------------------------- # FHIR Validator # ----------------------------------------------------------------------------- -FHIR_VALIDATOR_URL=http://fhir_validator:3500 # ----------------------------------------------------------------------------- # Error tracking (optional) diff --git a/.pylintrc b/.pylintrc index cc0e2c9ff..de482fed9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,6 +1,7 @@ [MESSAGES CONTROL] disable=missing-docstring,too-few-public-methods,no-member,too-many-ancestors,attribute-defined-outside-init,fixme,broad-except,import-outside-toplevel,too-many-instance-attributes,unsupported-membership-test,too-many-public-methods,too-many-lines,unused-private-member,consider-using-with,duplicate-code,unspecified-encoding,invalid-name,cyclic-import,missing-timeout,too-many-positional-arguments [MASTER] +extension-pkg-allow-list=lxml ignore=migrations,fixtures,scripts,commands,documents.py,password_validation.py,v1_dump [FORMAT] max-line-length=120 diff --git a/core/code_systems/converter.py b/core/code_systems/converter.py index c3ca2b2e0..d9cbe2872 100644 --- a/core/code_systems/converter.py +++ b/core/code_systems/converter.py @@ -1,4 +1,4 @@ -from fhir.resources.codesystem import CodeSystem +from fhir.resources.R4B.codesystem import CodeSystem # R4B models are the R4 wire models from core.settings import DEFAULT_LOCALE from core.sources.models import Source @@ -32,7 +32,7 @@ def to_fhir(source): 'language': source.default_locale, 'count': source.active_concepts, } - return CodeSystem(**code_system).dict() + return CodeSystem(**code_system).model_dump() @staticmethod def from_fhir(code_system): diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index ff5384f11..8026e3d5d 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -1,6 +1,6 @@ import json -from fhir.resources.codesystem import CodeSystem +from fhir.resources.R4B.codesystem import CodeSystem from mock.mock import patch, Mock from rest_framework.test import APIClient diff --git a/core/code_systems/tests_converter.py b/core/code_systems/tests_converter.py new file mode 100644 index 000000000..5fdbfd346 --- /dev/null +++ b/core/code_systems/tests_converter.py @@ -0,0 +1,34 @@ +import warnings + +from django.test import SimpleTestCase +from fhir.resources.R4B.codesystem import CodeSystem + +from core.code_systems.converter import CodeSystemConverter +from core.sources.tests.factories import OrganizationSourceFactory + + +class CodeSystemConverterR4Test(SimpleTestCase): + """The converter speaks R4 through fhir.resources 8's R4B models, without its deprecated API.""" + + def test_to_fhir_builds_an_r4_code_system(self): + source = OrganizationSourceFactory.build( + mnemonic='test', canonical_url='http://example.org/cs', name='Test', default_locale='en', released=True) + + with warnings.catch_warnings(): + warnings.simplefilter('error') + code_system = CodeSystemConverter.to_fhir(source) + + self.assertEqual(code_system['resourceType'], 'CodeSystem') + self.assertEqual(code_system['url'], 'http://example.org/cs') + self.assertEqual(code_system['status'], 'active') + CodeSystem.model_validate(code_system) + + def test_from_fhir_reads_an_r4_code_system(self): + code_system = CodeSystem(id='test', status='retired', content='complete', url='http://example.org/cs') + + self.assertTrue(CodeSystemConverter.can_convert_from_fhir(code_system)) + source = CodeSystemConverter.from_fhir(code_system.model_dump()) + + self.assertEqual(source['mnemonic'], 'test') + self.assertEqual(source['canonical_url'], 'http://example.org/cs') + self.assertTrue(source['retired']) diff --git a/core/code_systems/views.py b/core/code_systems/views.py index b216160d4..a3e7816fc 100644 --- a/core/code_systems/views.py +++ b/core/code_systems/views.py @@ -11,6 +11,8 @@ from core.common.serializers import IdentifierSerializer from core.concepts.permissions import CanViewParentDictionaryAsGuest from core.concepts.views import ConceptRetrieveUpdateDestroyView +from core.fhir.base import FhirViewMixin +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM from core.parameters.serializers import ParametersSerializer from core.sources.models import Source from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView @@ -28,7 +30,7 @@ def get_viewable_repo(repos, user): return None -class CodeSystemListView(SourceListView): +class CodeSystemListView(FhirViewMixin, SourceListView): serializer_class = CodeSystemDetailSerializer @staticmethod @@ -74,37 +76,14 @@ def __init__(self, code: str = None): { "severity": "error", "code": "not-found", - "details": {"text": message}, + "details": {"coding": [{"system": TX_ISSUE_TYPE_SYSTEM, "code": "not-found"}], "text": message}, } ], } ) -class FhirParameterRequiredError(ValidationError): - """A 400 OperationOutcome for an operation input that is required, naming it in `expression`.""" - def __init__(self, message: str, expression: str): - super().__init__( - detail={ - "resourceType": "OperationOutcome", - "id": "exception", - "text": { - "status": "additional", - "div": f'
{message}
', - }, - "issue": [ - { - "severity": "error", - "code": "invalid", - "details": {"text": message}, - "expression": [expression], - } - ], - } - ) - - -class CodeSystemListLookupView(ConceptRetrieveUpdateDestroyView): +class CodeSystemListLookupView(FhirViewMixin, ConceptRetrieveUpdateDestroyView): serializer_class = ParametersSerializer def is_container_version_specified(self): @@ -151,7 +130,7 @@ def get_serializer(self, instance=None): # pylint: disable=arguments-differ return None -class CodeSystemValidateCodeView(ConceptRetrieveUpdateDestroyView): +class CodeSystemValidateCodeView(FhirViewMixin, ConceptRetrieveUpdateDestroyView): serializer_class = ValidateCodeParametersSerializer def verify_scope(self): @@ -253,7 +232,7 @@ def get_object(self, queryset=None): } -class CodeSystemRetrieveUpdateView(SourceRetrieveUpdateDestroyView): +class CodeSystemRetrieveUpdateView(FhirViewMixin, SourceRetrieveUpdateDestroyView): serializer_class = CodeSystemDetailSerializer def get_filter_params(self, default_version_to_head=True): diff --git a/core/common/fhir_helpers.py b/core/common/fhir_helpers.py index 892092b34..8e41fbaa6 100644 --- a/core/common/fhir_helpers.py +++ b/core/common/fhir_helpers.py @@ -1,10 +1,13 @@ import json from core.common.exceptions import Http400 +from core.fhir.base import CONTROL_PARAMETERS def translate_fhir_query(fhir_query_fields, query_params, queryset): remaining_query_params = query_params.copy() + for name in CONTROL_PARAMETERS: # _format and the like choose the response format, they don't filter + remaining_query_params.pop(name, None) query_fields = fhir_query_fields.copy() url = remaining_query_params.get('url') if url and 'url' in query_fields: diff --git a/core/concept_maps/tests.py b/core/concept_maps/tests.py index 013a3858b..dc52ab6be 100644 --- a/core/concept_maps/tests.py +++ b/core/concept_maps/tests.py @@ -510,7 +510,10 @@ def test_put_concept_map_overwriting_version(self): # check if bad request self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'version': 'Version 1.0 already exists for CodeSystem test2.'}) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + self.assertEqual( + [(issue['code'], issue['expression'], issue['details']['text']) for issue in response.data['issue']], + [('invalid', ['version'], 'Version 1.0 already exists for CodeSystem test2.')]) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) diff --git a/core/concept_maps/views.py b/core/concept_maps/views.py index e1aa117fa..20b0677b9 100644 --- a/core/concept_maps/views.py +++ b/core/concept_maps/views.py @@ -14,6 +14,7 @@ from core.concept_maps.constants import RESOURCE_TYPE from core.concept_maps.serializers import ConceptMapDetailSerializer, ConceptMapParametersSerializer from core.concepts.permissions import CanAccessParentDictionary +from core.fhir.base import FhirViewMixin from core.mappings.constants import SAME_AS from core.mappings.views import MappingListView from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView @@ -21,7 +22,7 @@ logger = logging.getLogger('oclapi') -class ConceptMapListView(SourceListView): +class ConceptMapListView(FhirViewMixin, SourceListView): serializer_class = ConceptMapDetailSerializer @staticmethod @@ -51,7 +52,7 @@ def get_detail_serializer(self, obj): return self.serializer_class(obj) -class ConceptMapRetrieveUpdateView(SourceRetrieveUpdateDestroyView): +class ConceptMapRetrieveUpdateView(FhirViewMixin, SourceRetrieveUpdateDestroyView): serializer_class = ConceptMapDetailSerializer def get_filter_params(self, default_version_to_head=True): @@ -73,7 +74,7 @@ def get_detail_serializer(self, obj): return ConceptMapDetailSerializer(obj) -class ConceptMapTranslateView(MappingListView): +class ConceptMapTranslateView(FhirViewMixin, MappingListView): serializer_class = ConceptMapParametersSerializer def get_permissions(self): diff --git a/core/fhir/__init__.py b/core/fhir/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/base.py b/core/fhir/base.py new file mode 100644 index 000000000..fc17e72d1 --- /dev/null +++ b/core/fhir/base.py @@ -0,0 +1,566 @@ +""" +The FHIR view layer: what every FHIR view shares, whatever its operation. + +- Media types: FHIR JSON and XML in and out, with the aliases FHIR clients use, and `_format`. +- The `fhirVersion` MIME parameter: R4 (4.0) is served and any other version is refused. +- XML through fhir.resources' R4B models, which are wire-identical to R4 for the terminology resources, so + elements always come out in FHIR's order. +- One reader for operation parameters, from a GET query string or a POSTed Parameters resource. +- The FHIR base URL of a request, from its own host. +- Errors as OperationOutcome (core.fhir.outcomes), in views and in the middleware. +""" +import gzip +import json +import logging +import re +import types +import typing +import zlib +from functools import lru_cache + +from django.core.exceptions import BadRequest, SuspiciousOperation +from django.http import HttpResponse +from fhir.resources.R4B import get_fhir_model_class +from fhir_core.types import FhirBase +from lxml import etree +from pydantic import ValidationError +from rest_framework.exceptions import NotAcceptable, ParseError, UnsupportedMediaType +from rest_framework.negotiation import DefaultContentNegotiation +from rest_framework.parsers import BaseParser, FormParser, JSONParser, MultiPartParser +from rest_framework.renderers import BaseRenderer, JSONRenderer +from rest_framework.response import Response +from rest_framework.utils.encoders import JSONEncoder +from rest_framework.utils.mediatypes import _MediaType +from rest_framework.versioning import AcceptHeaderVersioning + +from core.fhir import outcomes + +logger = logging.getLogger('oclapi') + +FHIR_VERSION = '4.0' # the fhirVersion MIME parameter, and what $versions answers +FHIR_RELEASE = '4.0.1' # CapabilityStatement.fhirVersion +ACCEPTED_FHIR_VERSIONS = (FHIR_VERSION, FHIR_RELEASE) + +FHIR_JSON = 'application/fhir+json' +FHIR_XML = 'application/fhir+xml' +# Other names for the two formats: the short _format values, the DSTU2 types, and the generic XML types R4 says +# a server SHOULD accept. Plain application/json stays itself, so OCL's own clients keep getting it. +MEDIA_TYPE_ALIASES = { + 'json': FHIR_JSON, + 'application/json+fhir': FHIR_JSON, + 'xml': FHIR_XML, + 'application/xml+fhir': FHIR_XML, + 'application/xml': FHIR_XML, + 'text/xml': FHIR_XML, +} +FORMAT_PARAMETERS = ('_format', 'format') +# Protocol parameters: they choose the response format, and are never operation or search input +CONTROL_PARAMETERS = frozenset({'_format', 'format', '_pretty'}) +RESOURCE_TYPES = ('CodeSystem', 'ValueSet', 'ConceptMap') + +FHIR_NS = 'http://hl7.org/fhir' +XHTML_NS = 'http://www.w3.org/1999/xhtml' +EXTENSION_ELEMENTS = ('extension', 'modifierExtension') +EMPTY_VALUES = (None, '', {}, []) +# What XML 1.0 can't hold: C0 controls other than tab, LF and CR, lone surrogates, U+FFFE and U+FFFF +XML_FORBIDDEN_CHARACTERS = re.compile('[^\x09\x0a\x0d\x20-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]') +REPLACEMENT_CHARACTER = '\ufffd' +XML_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_.-]*$') +LOGGED_DROPPED_ELEMENTS = set() # what the XML has left out so far: each kind is logged once per process + + +def xml_parser(): + """A parser that never fetches or expands entities: a request body is untrusted.""" + return etree.XMLParser(resolve_entities=False, no_network=True, load_dtd=False, remove_comments=True) + + +def is_fhir_path(path): + """Whether a path is under a FHIR base: the global /fhir/, or a FHIR resource type at an owner base.""" + segments = path.strip('/').split('/') + return segments[0] == 'fhir' or (len(segments) >= 3 and segments[2] in RESOURCE_TYPES) + + +def fhir_base_url(request, kwargs=None): + """ + The FHIR base a request came in on, from its own scheme and host (the forwarded ones behind the load balancer), + so links work on every host OCL serves FHIR from: /fhir, or the owner base /orgs/{org} or /users/{user}. + """ + kwargs = kwargs or {} + if kwargs.get('org'): + path = f"/orgs/{kwargs['org']}" + elif kwargs.get('user'): + path = f"/users/{kwargs['user']}" + else: + path = '/fhir' + return request.build_absolute_uri(path) + + +def normalize_media_type(media_range): + """A media range from Accept, Content-Type or _format, with FHIR's other names replaced by the formal type.""" + main, separator, params = media_range.partition(';') + main = main.strip().lower().replace(' ', '+') # an unencoded + in _format arrives as a space + return MEDIA_TYPE_ALIASES.get(main, main) + separator + params + + +def parse_media_type(media_type): + """Treat undecodable MIME parameters as a bad request, including unknown RFC 2231 encodings.""" + try: + return _MediaType(media_type) + except (LookupError, ValueError) as exc: + raise BadRequest('The media type parameters could not be parsed.') from exc + + +def accepted_media_ranges(request): + """The media ranges a request accepts: its _format, else its Accept header.""" + ranges = [item.strip() for item in request.META.get('HTTP_ACCEPT', '').split(',') if item.strip()] + for media_range in ranges: + parse_media_type(media_range) # validate before _format and browser preference shortcuts + for name in FORMAT_PARAMETERS: + value = request.GET.get(name) + if value: + return [normalize_media_type(value)] + if any(item.startswith('text/html') for item in ranges) and not any('fhir' in item for item in ranges): + return ['application/json'] # a browser: JSON as before, not the application/xml it lists after text/html + return [normalize_media_type(item) for item in ranges] or ['*/*'] + + +def quality(media_range): + try: + return float(parse_media_type(media_range).params.get('q', 1)) + except ValueError: + return 0.0 + + +def media_type_with_params(media_type, media_range): + """The renderer's media type, carrying the parameters the client sent other than q (such as fhirVersion).""" + params = [param.strip() for param in media_range.split(';')[1:]] + params = [param for param in params if param and not param.lower().startswith('q=')] + return '; '.join([media_type, *params]) + + +class FhirContentNegotiation(DefaultContentNegotiation): + """ + Content negotiation for FHIR views: `_format` (or `format`) overrides Accept, FHIR's other media type names are + understood, and quality values are honoured. DRF's own `?format=` renderer filter is not used. + """ + + def select_parser(self, request, parsers): + content_type = parse_media_type(normalize_media_type(request.content_type or '')) + return next((parser for parser in parsers if parse_media_type(parser.media_type).match(content_type)), None) + + def select_renderer(self, request, renderers, format_suffix=None): + """Each renderer takes its quality from its most specific match, including q=0 exclusions.""" + # Reject a malformed Content-Type before choosing XML, so every parsing failure can use safe JSON. + parse_media_type(request.META.get('CONTENT_TYPE', request.META.get('HTTP_CONTENT_TYPE', ''))) + ranges = sorted(enumerate(accepted_media_ranges(request)), + key=lambda item: (-parse_media_type(item[1]).precedence, -quality(item[1]))) + candidates = [] + for renderer in renderers: + for position, media_range in ranges: + parsed_range = parse_media_type(media_range) + if parse_media_type(renderer.media_type).match(parsed_range): + preference = (quality(media_range), parsed_range.precedence, -position) + if preference[0] > 0: + candidates.append((preference, renderer, media_range)) + break # a less specific range cannot override this renderer's quality + if candidates: + _, renderer, media_range = max(candidates, key=lambda candidate: candidate[0]) + return renderer, media_type_with_params(renderer.media_type, media_range) + raise NotAcceptable(available_renderers=renderers) + + +class FhirVersioning(AcceptHeaderVersioning): + """ + The fhirVersion MIME parameter, on Accept (or _format) and on Content-Type. R4 is served: 4.0 (or 4.0.1, or no + parameter) is accepted, and any other version is refused, 406 on Accept and 415 on Content-Type. Django + lowercases parameter names, so the parameter is read without regard to case. + """ + version_param = 'fhirVersion' + default_version = FHIR_VERSION + allowed_versions = ACCEPTED_FHIR_VERSIONS + invalid_version_message = f'This server speaks FHIR R4: fhirVersion={FHIR_VERSION}.' + + def version_of(self, media_type): + return parse_media_type(media_type or '').params.get(self.version_param.lower()) + + def determine_version(self, request, *args, **kwargs): + content_version = self.version_of(request.content_type) + if content_version is not None and content_version not in self.allowed_versions: + raise UnsupportedMediaType(request.content_type, detail=self.invalid_version_message) + accept_version = self.version_of(request.accepted_media_type) + if accept_version is not None and accept_version not in self.allowed_versions: + raise NotAcceptable(self.invalid_version_message) + return self.default_version + + +def prune(value): + """FHIR has no empty values: drop None, '', {} and [] at every depth.""" + if isinstance(value, dict): + items = ((key, prune(item)) for key, item in value.items()) + return {key: item for key, item in items if item not in EMPTY_VALUES} + if isinstance(value, (list, tuple)): + return [item for item in (prune(item) for item in value) if item not in EMPTY_VALUES] + return value + + +def from_xml(content): + """FHIR XML -> FHIR JSON, through the R4B model of the root element. Raises ParseError.""" + try: + root = etree.fromstring(content, parser=xml_parser()) + name = etree.QName(root) + if name.namespace != FHIR_NS: + raise ValueError(f'the root element {root.tag} is not a FHIR resource') + resource = get_fhir_model_class(name.localname).model_validate_xml(content, xmlparser=xml_parser()) + except Exception as ex: + message = (str(ex).splitlines() or [type(ex).__name__])[0] + raise ParseError(f'The body is not a valid FHIR XML resource: {message}') from ex + return json.loads(resource.model_dump_json()) + + +def json_values(data): + """The data as the JSON response carries it: DRF's encoder writes datetimes, dates, decimals and UUIDs.""" + return json.loads(json.dumps(data, cls=JSONEncoder)) + + +def xml_safe(value): + """(value, how many characters were replaced): every character XML 1.0 forbids becomes U+FFFD.""" + if isinstance(value, str): + return XML_FORBIDDEN_CHARACTERS.subn(REPLACEMENT_CHARACTER, value) + if isinstance(value, dict): + items = [(xml_safe(key), xml_safe(item)) for key, item in value.items()] + return {key: item for (key, _), (item, _) in items}, sum(left + right for (_, left), (_, right) in items) + if isinstance(value, (list, tuple)): + items = [xml_safe(item) for item in value] + return [item for item, _ in items], sum(count for _, count in items) + return value, 0 + + +def element_type(annotation): + """(the R4B model of a field's values, or None for a primitive; whether the field repeats).""" + repeats = False + while True: + origin = typing.get_origin(annotation) + if origin in (typing.Union, types.UnionType): + annotation = next(arg for arg in typing.get_args(annotation) if arg is not type(None)) + elif origin is list: + repeats, annotation = True, typing.get_args(annotation)[0] + elif origin is typing.Annotated: + annotation = typing.get_args(annotation)[0] + else: + break + is_model = isinstance(annotation, type) and issubclass(annotation, FhirBase) + return (annotation.get_model_klass() if is_model else None), repeats + + +@lru_cache(maxsize=None) +def model_elements(model_class): + """An R4B model's elements by their JSON name: (field name, the model of its values or None, repeats).""" + return {field.alias or name: (name, *element_type(field.annotation)) + for name, field in model_class.model_fields.items() if name != 'fhir_comments'} + + +def resource_model(data): + """The R4B model of a FHIR JSON resource, or None when it isn't one.""" + if not isinstance(data, dict) or not isinstance(data.get('resourceType'), str): + return None + try: + return get_fhir_model_class(data['resourceType']) + except ValueError: + return None + + +def log_dropped(model_class, key, reason): + if (model_class.__name__, key, reason) not in LOGGED_DROPPED_ELEMENTS: + LOGGED_DROPPED_ELEMENTS.add((model_class.__name__, key, reason)) + logger.warning('FHIR XML: %s.%s %s', model_class.__name__, key, reason) + + +def xhtml_markup(text): + """A narrative div as XHTML markup: the text itself when it is an XHTML div, else the text inside one.""" + try: + if etree.fromstring(text, parser=xml_parser()).tag == f'{{{XHTML_NS}}}div': + return text + except etree.XMLSyntaxError: + pass + return etree.tostring(xhtml_div(text), encoding='unicode') + + +def conform_value(value_model, value): + """One value of an element, conformed, or None when it doesn't have the element's shape.""" + if value_model is None: + return None if isinstance(value, (dict, list)) else value + if not isinstance(value, dict): + return None + model_class = resource_model(value) if 'resourceType' in value else value_model # a resource names its type + return conform(model_class, value) if model_class else None + + +def conform(model_class, data): + """ + FHIR JSON reduced to what the R4 model defines, at every depth. A key R4 doesn't define (OCL's revisionDate) and + a value of the wrong shape are left out, each kind logged once: that is the content's problem, not the wire + format's. The JSON response itself is not changed. + """ + elements, result = model_elements(model_class), {} + for key, value in data.items(): + if key == 'resourceType': + result[key] = value + continue + if key not in elements: + log_dropped(model_class, key, 'is an element R4 does not define, so the XML leaves it out') + continue + _, value_model, repeats = elements[key] + if isinstance(value, list) and not repeats: # OCL's ConceptMap.identifier is a list; R4 allows one + if len(value) > 1: + log_dropped(model_class, key, 'repeats, but not in R4, so the XML keeps only the first') + value = value[:1] + items = value if isinstance(value, list) else [value] + kept = [item for item in (conform_value(value_model, item) for item in items) if item is not None] + if len(kept) < len(items): + log_dropped(model_class, key, 'has values of a shape R4 does not allow, so the XML leaves them out') + if key == 'div' and kept: + kept = [xhtml_markup(kept[0])] + if kept: + result[key] = kept if repeats else kept[0] + return result + + +def construct(model_class, data): + """ + A model instance from conformed FHIR JSON, nested models included, built without validation, so the model + still writes its elements in FHIR's order. For what the model refuses only on a pattern, such as an OCL id + with '_' or '@', or a code with a trailing space. + """ + elements, values = model_elements(model_class), {} + for key, value in data.items(): + if key == 'resourceType': + continue + name, value_model, repeats = elements[key] + items = [construct_value(value_model, item) for item in (value if repeats else [value])] + values[name] = items if repeats else items[0] + return model_class.model_construct(**values) + + +def construct_value(value_model, value): + if value_model is None: + return value if isinstance(value, bool) else str(value) # the writer takes text, and booleans as they are + return construct(resource_model(value) or value_model, value) + + +def to_xml(data): + """ + FHIR JSON -> FHIR XML, always through the R4B model, so in FHIR's element order. It starts from the values the + JSON response carries. Keys R4 doesn't define are left out; a resource the model refuses on a pattern is built + without validation; characters XML 1.0 forbids become U+FFFD. Data that isn't a FHIR resource is answered with + an OperationOutcome. + """ + if data is None: + return b'' + data = json_values(data) + model_class = resource_model(data) + if model_class is None: + data = outcomes.operation_outcome(outcomes.issue('not-supported', 'This response has no FHIR XML form.')) + model_class = resource_model(data) + data, replaced = xml_safe(data) + if replaced: + logger.warning('FHIR XML: %s character(s) XML cannot hold became U+FFFD in a %s', replaced, + data['resourceType']) + data = prune(conform(model_class, prune(data))) + try: + instance = model_class.model_validate(data) + except ValidationError: + instance = construct(model_class, data) + try: + return instance.model_dump_xml() + except Exception: + logger.exception('FHIR XML: the %s model could not write it; written element by element', model_class.__name__) + root = etree.Element(f"{{{FHIR_NS}}}{data['resourceType']}", nsmap={None: FHIR_NS}) + write_xml_children(root, data, resource=True) + return etree.tostring(root, xml_declaration=True, encoding='utf-8') + + +def write_xml_children(element, obj, resource=False, extension=False): + """The last resort writer: element by element, in the data's own order.""" + for key, value in obj.items(): + if key == 'resourceType' or not XML_NAME.match(key): + continue + if not resource and (key == 'id' or (extension and key == 'url')) and not isinstance(value, (dict, list)): + element.set(key, str(value)) # an element's id and an extension's url are XML attributes + elif key == 'div' and isinstance(value, str): + element.append(xhtml_div(value)) + else: + for item in value if isinstance(value, list) else [value]: + write_xml_element(element, key, item) + + +def write_xml_element(parent, name, value): + child = etree.SubElement(parent, f'{{{FHIR_NS}}}{name}') + if isinstance(value, dict) and value.get('resourceType'): + inner = etree.SubElement(child, f"{{{FHIR_NS}}}{value['resourceType']}") # a resource in a container + write_xml_children(inner, value, resource=True) + elif isinstance(value, dict): + write_xml_children(child, value, extension=name in EXTENSION_ELEMENTS) + else: + child.set('value', ('true' if value else 'false') if isinstance(value, bool) else str(value)) + + +def xhtml_div(text): + try: + div = etree.fromstring(text, parser=xml_parser()) + if div.tag == f'{{{XHTML_NS}}}div': + return div + except etree.XMLSyntaxError: + pass + div = etree.Element(f'{{{XHTML_NS}}}div', nsmap={None: XHTML_NS}) + div.text = text + return div + + +class FhirJsonParser(JSONParser): + media_type = FHIR_JSON + + +class FhirXmlParser(BaseParser): + media_type = FHIR_XML + + def parse(self, stream, media_type=None, parser_context=None): + return from_xml(stream.read()) + + +class FhirJsonRenderer(JSONRenderer): + media_type = FHIR_JSON + format = 'json' + + +class FhirXmlRenderer(BaseRenderer): + media_type = FHIR_XML + format = 'xml' + charset = 'utf-8' + + def render(self, data, accepted_media_type=None, renderer_context=None): + return to_xml(data) + + +def parameter_value(parameter): + """A Parameters.parameter's value: its value[x], resource, or parts (as FhirParameters).""" + for key, value in parameter.items(): + if key.startswith('value'): + return value + if 'resource' in parameter: + return parameter['resource'] + if 'part' in parameter: + return FhirParameters.from_resource({'parameter': parameter['part']}) + return None + + +class FhirParameters: + """ + An operation's input, read the same way from a GET query string and from a POSTed Parameters resource (FHIR JSON, + FHIR XML or a form). Names can repeat. Values keep their FHIR JSON form: a string, boolean or number, a dict for + a Coding, CodeableConcept or resource, and FhirParameters for parts. On a POST, the body's parameters come first, + then the query string's. Protocol parameters such as _format are left out. + """ + + def __init__(self, items=()): + self.items = list(items) + + @classmethod + def from_resource(cls, resource): + return cls((item.get('name'), parameter_value(item)) for item in resource.get('parameter') or []) + + @classmethod + def from_request(cls, request): + items = cls.body_items(request.data) if request.method in ('POST', 'PUT') else [] + return cls(items + cls.query_items(request.query_params)) + + @staticmethod + def query_items(query): + return [(name, value) for name, values in query.lists() if name not in CONTROL_PARAMETERS for value in values] + + @classmethod + def body_items(cls, data): + if not data: + return [] + if hasattr(data, 'lists'): # a form + return cls.query_items(data) + if isinstance(data, dict) and data.get('resourceType') == 'Parameters': + return cls.from_resource(data).items + raise outcomes.InvalidError('The request body must be a Parameters resource.') + + def get(self, name, default=None): + return next((value for key, value in self.items if key == name), default) + + def getlist(self, name): + return [value for key, value in self.items if key == name] + + def names(self): + return list(dict.fromkeys(key for key, _ in self.items)) + + def __contains__(self, name): + return any(key == name for key, _ in self.items) + + +def outcome_response(request, outcome, status_code, headers=None): + """A middleware outcome in the requested format, or safe JSON 400 for unparseable queries or MIME parameters.""" + renderer = FhirJsonRenderer() + try: + renderer, _ = FhirContentNegotiation().select_renderer( + request, [klass() for klass in FhirViewMixin.renderer_classes]) + except NotAcceptable: + pass + except (BadRequest, SuspiciousOperation) as exc: + status_code, outcome, _ = outcomes.outcome_for_exception(exc) + content_type = f'{renderer.media_type}; charset={renderer.charset}' if renderer.charset else renderer.media_type + response = HttpResponse( + renderer.render(outcome), status=outcomes.response_status(status_code), content_type=content_type) + for key, value in (headers or {}).items(): + response[key] = value + return response + + +def response_json(response): + """A response's JSON body, gunzipped if need be, or None.""" + if 'json' not in response.get('Content-Type', '') or getattr(response, 'streaming', False): + return None + content = response.content + try: + if response.get('Content-Encoding') == 'gzip': + content = gzip.decompress(content) + return json.loads(content) + except (OSError, EOFError, zlib.error, ValueError): + return None + + +def is_outcome_response(response): + """Whether a response is already FHIR: an error from a FHIR view, rather than Django's or another view's.""" + return 'fhir' in response.get('Content-Type', '') or outcomes.is_outcome(response_json(response)) + + +class FhirViewMixin: + """ + The first base of every FHIR view: FHIR JSON and XML in and out, `_format`, the fhirVersion parameter, and every + error as an OperationOutcome with a tx-issue-type, never a 5xx. + """ + parser_classes = (FhirJsonParser, JSONParser, FhirXmlParser, FormParser, MultiPartParser) + renderer_classes = (FhirJsonRenderer, JSONRenderer, FhirXmlRenderer) + content_negotiation_class = FhirContentNegotiation + versioning_class = FhirVersioning + + def get_exception_handler(self): + return outcomes.exception_handler + + def finalize_response(self, request, response, *args, **kwargs): + """Normalize returned outcomes at every status and depth, and wrap other error responses as outcomes.""" + if isinstance(response, Response): + if response.status_code >= 400: + if not outcomes.is_outcome(response.data): + response.data = outcomes.outcome_for_status(response.status_code, response.data) + response.status_code = outcomes.response_status(response.status_code) + response.data = outcomes.normalize_outcomes(response.data) + return super().finalize_response(request, response, *args, **kwargs) + + def get_fhir_parameters(self): + return FhirParameters.from_request(self.request) + + def get_fhir_base_url(self): + return fhir_base_url(self.request, self.kwargs) diff --git a/core/fhir/outcomes.py b/core/fhir/outcomes.py new file mode 100644 index 000000000..ba3c56d8b --- /dev/null +++ b/core/fhir/outcomes.py @@ -0,0 +1,253 @@ +""" +OperationOutcome for the FHIR API: typed errors, and the exception handler that answers every error a FHIR view +raises with an OperationOutcome. + +Every issue carries a tx-issue-type coding, as FHIRsmith and tx.fhir.org do, so clients and the parity suite can +compare errors by kind rather than by wording. No FHIR response is a 5xx: an error OCL can't attribute to the +request is answered with a 422 and the issue code `exception`, and logged. +""" +import logging +from http import HTTPStatus + +from django.core.exceptions import BadRequest, SuspiciousOperation, ValidationError as DjangoValidationError, \ + PermissionDenied as DjangoPermissionDenied +from django.http import Http404 +from rest_framework import exceptions, status +from rest_framework.response import Response +from rest_framework.views import set_rollback + +logger = logging.getLogger('oclapi') + +TX_ISSUE_TYPE_SYSTEM = 'http://hl7.org/fhir/tools/CodeSystem/tx-issue-type' +MESSAGE_ID_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/operationoutcome-message-id' + +# The tx-issue-type of an issue that names none, by its FHIR issue type. This is FHIRsmith's table +# (tx/library/operation-outcome.js), plus `throttled`; every other issue type gets `invalid-data`, as there. +DEFAULT_TX_ISSUE_TYPES = { + 'not-found': 'not-found', + 'code-invalid': 'invalid-code', + 'business-rule': 'business-rule', + 'too-costly': 'too-costly', + 'not-supported': 'not-supported', + 'throttled': 'too-costly', +} +FALLBACK_TX_ISSUE_TYPE = 'invalid-data' + +# The issue type of an error that has only an HTTP status +STATUS_ISSUE_CODES = { + 400: 'invalid', + 401: 'login', + 403: 'forbidden', + 404: 'not-found', + 405: 'not-supported', + 406: 'not-supported', + 410: 'deleted', + 415: 'not-supported', + 422: 'processing', + 429: 'throttled', +} +NON_FIELD_ERROR_KEYS = ('non_field_errors', '__all__') +UNEXPECTED_ERROR_STATUS = status.HTTP_422_UNPROCESSABLE_ENTITY +UNEXPECTED_ERROR_MESSAGE = 'An unexpected error occurred while processing the request.' + + +def tx_issue_type_for(issue_code): + """The tx-issue-type for an issue that doesn't name one.""" + return DEFAULT_TX_ISSUE_TYPES.get(issue_code, FALLBACK_TX_ISSUE_TYPE) + + +def issue( # pylint: disable=too-many-arguments + code, text, severity='error', expression=None, tx_issue_type=None, message_id=None, diagnostics=None): + """One OperationOutcome issue. `expression` is a FHIRPath (the parameter or element at fault) or a list of them.""" + result = { + 'severity': severity, + 'code': code, + 'details': { + 'coding': [{'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type or tx_issue_type_for(code)}], + 'text': text + } + } + if expression: + result['expression'] = [expression] if isinstance(expression, str) else list(expression) + if message_id: + result['extension'] = [{'url': MESSAGE_ID_EXTENSION, 'valueString': message_id}] + if diagnostics: + result['diagnostics'] = diagnostics + return result + + +def operation_outcome(*issues): + return {'resourceType': 'OperationOutcome', 'issue': list(issues)} + + +def is_outcome(data): + return isinstance(data, dict) and data.get('resourceType') == 'OperationOutcome' + + +def response_status(status_code): + """The status a FHIR error goes out with: a 5xx becomes a 422.""" + return UNEXPECTED_ERROR_STATUS if status_code >= 500 else status_code + + +def status_issue_code(status_code): + if status_code >= 500: + return 'exception' + return STATUS_ISSUE_CODES.get(status_code, 'invalid') + + +def status_text(status_code): + try: + return HTTPStatus(status_code).phrase + except ValueError: + return UNEXPECTED_ERROR_MESSAGE + + +def to_plain(value): + """DRF error details (ErrorDetail strings, nested) as plain JSON values.""" + if isinstance(value, dict): + return {str(key): to_plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [to_plain(item) for item in value] + return str(value) if isinstance(value, str) else value + + +def detail_issues(detail, code='invalid', field=None): + """Issues for a serializer-style error detail: one per message, naming the field it belongs to.""" + if isinstance(detail, dict): + issues = [] + for key, value in detail.items(): + issues += detail_issues(value, code, field or (None if key in NON_FIELD_ERROR_KEYS else str(key))) + return issues + if isinstance(detail, (list, tuple)): + return [item for value in detail for item in detail_issues(value, code, field)] + return [issue(code, str(detail), expression=field)] + + +def with_tx_issue_types(outcome): + """An OperationOutcome built elsewhere, with a tx-issue-type coding and a code on every issue.""" + outcome = to_plain(outcome) + for item in outcome.get('issue') or []: + item.setdefault('severity', 'error') + item.setdefault('code', 'processing') + details = item.get('details') + if not isinstance(details, dict): + details = item['details'] = {'text': str(details)} if details else {} + codings = details.setdefault('coding', []) + if not any(coding.get('system') == TX_ISSUE_TYPE_SYSTEM for coding in codings): + codings.append({'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type_for(item['code'])}) + return outcome + + +def normalize_outcomes(data): + """Normalize every returned OperationOutcome, including resources nested in Bundle entries.""" + if isinstance(data, dict): + data = {key: normalize_outcomes(value) for key, value in data.items()} + return with_tx_issue_types(data) if is_outcome(data) else data + if isinstance(data, (list, tuple)): + return [normalize_outcomes(value) for value in data] + return data + + +def outcome_for_status(status_code, data=None): + """An error response that isn't an OperationOutcome (an empty 404, a {'detail': ...} body) as one.""" + code = status_issue_code(status_code) + if isinstance(data, dict) and 'detail' in data: + return operation_outcome(issue(code, str(data['detail']))) + if data and isinstance(data, (dict, list)): + return operation_outcome(*detail_issues(data, code)) + return operation_outcome(issue(code, status_text(status_code))) + + +class FhirError(exceptions.APIException): + """An error a FHIR view raises on purpose. Each subclass names its issue type and HTTP status.""" + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'processing' + default_detail = 'The request could not be processed.' + + def __init__( # pylint: disable=too-many-arguments + self, message=None, expression=None, tx_issue_type=None, message_id=None, status_code=None): + super().__init__(message or self.default_detail) + self.expression = expression + self.tx_issue_type = tx_issue_type + self.message_id = message_id + if status_code: + self.status_code = status_code + + def outcome(self): + return operation_outcome(issue( + self.issue_code, str(self.detail), expression=self.expression, tx_issue_type=self.tx_issue_type, + message_id=self.message_id)) + + +class NotFoundError(FhirError): + status_code = status.HTTP_404_NOT_FOUND + issue_code = 'not-found' + default_detail = 'The resource was not found.' + + +class InvalidError(FhirError): + status_code = status.HTTP_400_BAD_REQUEST + issue_code = 'invalid' + default_detail = 'The request is not valid.' + + +class NotSupportedError(FhirError): + status_code = status.HTTP_400_BAD_REQUEST + issue_code = 'not-supported' + default_detail = 'The request uses something this server does not support.' + + +class MultipleMatchesError(FhirError): + status_code = status.HTTP_400_BAD_REQUEST + issue_code = 'multiple-matches' + default_detail = 'More than one resource matches the request.' + + +class TooCostlyError(FhirError): + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'too-costly' + default_detail = 'The request is too costly to process.' + + +class ProcessingError(FhirError): + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'processing' + + +def outcome_for_exception(exc): + """(status, OperationOutcome, headers) for an exception raised while answering a FHIR request.""" + headers = {} + if isinstance(exc, FhirError): + status_code, outcome = exc.status_code, exc.outcome() + elif isinstance(exc, (Http404, exceptions.NotFound)): + status_code = 404 + outcome = operation_outcome(issue('not-found', str(getattr(exc, 'detail', exc)) or status_text(404))) + elif isinstance(exc, DjangoPermissionDenied): + status_code, outcome = 403, operation_outcome(issue('forbidden', str(exc) or status_text(403))) + elif isinstance(exc, (BadRequest, SuspiciousOperation)): + status_code, outcome = 400, operation_outcome(issue('invalid', 'The request could not be parsed.')) + elif isinstance(exc, DjangoValidationError): + detail = exc.message_dict if hasattr(exc, 'error_dict') else exc.messages + status_code, outcome = 400, operation_outcome(*detail_issues(detail)) + elif isinstance(exc, exceptions.ValidationError): + status_code = exc.status_code + outcome = with_tx_issue_types(exc.detail) if is_outcome(exc.detail) else operation_outcome( + *detail_issues(exc.detail)) + elif isinstance(exc, exceptions.APIException): + if getattr(exc, 'auth_header', None): + headers['WWW-Authenticate'] = exc.auth_header + if getattr(exc, 'wait', None): + headers['Retry-After'] = str(int(exc.wait)) + status_code = exc.status_code + outcome = operation_outcome(*detail_issues(exc.detail, status_issue_code(exc.status_code))) + else: + logger.exception('Unexpected %s answering a FHIR request', type(exc).__name__) + status_code, outcome = UNEXPECTED_ERROR_STATUS, operation_outcome(issue('exception', UNEXPECTED_ERROR_MESSAGE)) + return status_code, outcome, headers + + +def exception_handler(exc, context): # pylint: disable=unused-argument + """The exception handler of FHIR views (FhirViewMixin): every error is an OperationOutcome, never a 5xx.""" + status_code, outcome, headers = outcome_for_exception(exc) + set_rollback() + return Response(outcome, status=response_status(status_code), headers=headers) diff --git a/core/fhir/tests.py b/core/fhir/tests.py index ac2ae745d..ef6f90fd6 100644 --- a/core/fhir/tests.py +++ b/core/fhir/tests.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - from rest_framework.test import APIClient from core.common.tests import OCLTestCase @@ -21,10 +19,9 @@ def test_public_can_vew_terminology_statement(self): self.assertEqual(response.status_code, 200) self.assertEqual(response.data['resourceType'], "TerminologyCapabilities") - def test_fhir_subdomain_replaces_api_base_url(self): - with patch('core.fhir.views.settings.FHIR_SUBDOMAIN', 'fhir'): - with patch('core.fhir.views.settings.API_BASE_URL', 'https://api.openconceptlab.org'): - response = self.client.get('/fhir/metadata?mode=terminology') + def test_implementation_url_is_the_fhir_base_of_the_request_host(self): + response = self.client.get( + '/fhir/metadata?mode=terminology', HTTP_HOST='fhir.openconceptlab.org', HTTP_X_FORWARDED_PROTO='https') self.assertEqual(response.status_code, 200) self.assertEqual( diff --git a/core/fhir/tests_base.py b/core/fhir/tests_base.py new file mode 100644 index 000000000..5d0b9e69c --- /dev/null +++ b/core/fhir/tests_base.py @@ -0,0 +1,726 @@ +import datetime +import json +import re +import uuid +from decimal import Decimal +from unittest.mock import patch + +from django.test import SimpleTestCase, override_settings +from django.http import HttpResponse, JsonResponse +from fhir.resources.R4B.bundle import Bundle +from fhir.resources.R4B.codesystem import CodeSystem, CodeSystemConcept +from fhir.resources.R4B.operationoutcome import OperationOutcome +from lxml import etree +from rest_framework.exceptions import NotAcceptable, ParseError, UnsupportedMediaType +from rest_framework.parsers import JSONParser +from rest_framework.renderers import JSONRenderer +from rest_framework.permissions import AllowAny +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.test import APIRequestFactory +from rest_framework.views import APIView + +from core.fhir import base, outcomes +from core.fhir.base import FhirContentNegotiation, FhirJsonParser, FhirJsonRenderer, FhirXmlParser, \ + FhirXmlRenderer, FhirParameters, FhirVersioning, FhirViewMixin + +FHIR = '{http://hl7.org/fhir}' +BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' + + +def child_names(element): + return [etree.QName(child).localname for child in element] + + +def assert_in_model_order(test, element, model_class): + """The element's children are elements of the R4B model, in the model's order (repeats kept together).""" + names, sequence = child_names(element), model_class.elements_sequence() + test.assertEqual([name for name in names if name not in sequence], [], 'elements the model does not define') + test.assertEqual(names, sorted(names, key=sequence.index)) + + +def released_code_system(**overrides): + """What CodeSystemDetailSerializer gives for a released source version, in its key order.""" + return { + 'resourceType': 'CodeSystem', 'url': 'http://example.org/cs', 'title': 'Codes', 'status': 'active', + 'id': 'codes', 'language': 'en', 'count': 1, 'content': 'complete', + 'property': [{'code': 'conceptclass', 'uri': 'http://testserver/orgs/OCL/sources/Classes/concepts', + 'description': 'Standard list of concept classes.', 'type': 'string'}], + 'meta': {'lastUpdated': '2026-10-14T10:00:00.123456Z'}, 'version': 'v1', + 'identifier': [{'system': 'http://testserver', 'value': '/orgs/X/CodeSystem/codes/', 'type': { + 'text': 'Accession ID', 'coding': [{'system': 'http://hl7.org/fhir/v2/0203', 'code': 'ACSN'}]}}], + 'name': 'codes', 'publisher': 'X', 'revisionDate': '2026-10-14T10:00:00Z', 'experimental': False, + 'caseSensitive': True, 'compositional': False, 'versionNeeded': False, + 'collectionReference': '/orgs/X/collections/c/', 'hierarchyMeaning': 'is-a', + 'concept': [{'code': 'c1', 'display': 'One', 'designation': [{'language': 'en', 'value': 'One'}], + 'property': [{'code': 'conceptclass', 'valueString': 'Diagnosis'}]}], + 'text': {'status': 'generated', 'div': '
Codes
'}, + 'filters': [{'code': 'x'}], + **overrides, + } + + +def renderers(): + return [klass() for klass in FhirViewMixin.renderer_classes] + + +def parsers(): + return [klass() for klass in FhirViewMixin.parser_classes] + + +class NegotiationTest(SimpleTestCase): + factory = APIRequestFactory() + + def select(self, path='/fhir/CodeSystem', **headers): + renderer, media_type = FhirContentNegotiation().select_renderer(Request(self.factory.get(path, **headers)), + renderers()) + return type(renderer), media_type + + def test_fhir_json_is_the_default(self): + self.assertEqual(self.select(), (FhirJsonRenderer, 'application/fhir+json')) + self.assertEqual(self.select(HTTP_ACCEPT='*/*'), (FhirJsonRenderer, 'application/fhir+json')) + + def test_accept_header(self): + self.assertEqual(self.select(HTTP_ACCEPT='application/fhir+xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='text/xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/json+fhir')[0], FhirJsonRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/json')[0], JSONRenderer) + + def test_quality_values_are_honoured(self): + accept = 'application/fhir+json;q=0.5, application/xml+fhir;q=0.9' + + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], FhirXmlRenderer) + + def test_explicit_exclusions_override_wildcards(self): + """A broader range cannot make an explicitly excluded renderer acceptable.""" + cases = [ + ('application/fhir+json;q=0, */*;q=1', JSONRenderer), + ('application/json+fhir;q=0, application/json;q=0, application/*;q=1', FhirXmlRenderer), + ] + for accept, expected in cases: + with self.subTest(accept=accept): + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], expected) + + def test_most_specific_range_sets_each_renderers_quality(self): + """Specific ranges determine quality before renderers compete with one another.""" + cases = [ + 'application/fhir+json;q=0.2, application/fhir+xml;q=0.5, application/json;q=0, */*;q=1', + 'application/fhir+xml;q=0.5, application/*;q=0, */*;q=1', + '*/*;q=1, application/fhir+xml;q=1', + ] + for accept in cases: + with self.subTest(accept=accept): + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], FhirXmlRenderer) + + def test_type_wildcard_exclusion_overrides_universal_wildcard(self): + """Reject the request when all available application renderers are excluded.""" + with self.assertRaises(NotAcceptable): + self.select(HTTP_ACCEPT='application/*;q=0, */*;q=1') + + def test_a_malformed_quality_value_ranks_last(self): + self.assertEqual(self.select(HTTP_ACCEPT='application/fhir+xml;q=high, application/fhir+json')[0], + FhirJsonRenderer) + + def test_a_browser_gets_json_rather_than_the_xml_it_also_lists(self): + self.assertEqual(self.select(HTTP_ACCEPT=BROWSER_ACCEPT), (JSONRenderer, 'application/json')) + + def test_format_parameter_overrides_accept(self): + cases = [ + ('_format=xml', FhirXmlRenderer), ('_format=json', FhirJsonRenderer), + ('_format=application/fhir%2Bxml', FhirXmlRenderer), + ('_format=application/fhir+xml', FhirXmlRenderer), # an unencoded + arrives as a space + ('_format=application/xml', FhirXmlRenderer), ('_format=text/xml', FhirXmlRenderer), + ('_format=application/json', JSONRenderer), ('format=xml', FhirXmlRenderer), + ] + for query, renderer in cases: + self.assertEqual( + self.select(f'/fhir/CodeSystem?{query}', HTTP_ACCEPT='application/fhir+json')[0], renderer, query) + + def test_the_fhir_version_parameter_is_kept_on_the_accepted_media_type(self): + self.assertEqual( + self.select(HTTP_ACCEPT='application/fhir+json; fhirVersion=4.0'), + (FhirJsonRenderer, 'application/fhir+json; fhirVersion=4.0')) + self.assertEqual( + self.select('/fhir/CodeSystem?_format=application/fhir%2Bxml;fhirVersion=4.0'), + (FhirXmlRenderer, 'application/fhir+xml; fhirVersion=4.0')) + + def test_unsupported_formats_are_not_acceptable(self): + for path, accept in (('/fhir/CodeSystem?_format=turtle', '*/*'), ('/fhir/CodeSystem', 'text/csv'), + ('/fhir/CodeSystem', 'application/fhir+json;q=0')): + with self.assertRaises(NotAcceptable, msg=(path, accept)): + self.select(path, HTTP_ACCEPT=accept) + + def test_parser_selection_understands_fhir_media_types(self): + cases = [ + ('application/fhir+json', FhirJsonParser), ('application/fhir+json; charset=utf-8', FhirJsonParser), + ('application/json+fhir', FhirJsonParser), ('application/json', JSONParser), + ('application/fhir+xml', FhirXmlParser), ('application/xml', FhirXmlParser), + ('text/xml; charset=utf-8', FhirXmlParser), ('application/fhir+json; fhirVersion=4.0', FhirJsonParser), + ] + for content_type, parser in cases: + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', '{}', content_type=content_type)) + self.assertIsInstance(FhirContentNegotiation().select_parser(request, parsers()), parser, content_type) + + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', 'x', content_type='text/plain')) + self.assertIsNone(FhirContentNegotiation().select_parser(request, parsers())) + + +class VersioningTest(SimpleTestCase): + factory = APIRequestFactory() + + def version(self, accepted_media_type, content_type=None): + if content_type: + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', '{}', content_type=content_type)) + else: + request = Request(self.factory.get('/fhir/CodeSystem')) + request.accepted_media_type = accepted_media_type + return FhirVersioning().determine_version(request) + + def test_r4_is_served(self): + self.assertEqual(self.version('application/fhir+json'), '4.0') + self.assertEqual(self.version('application/fhir+json; fhirVersion=4.0'), '4.0') + self.assertEqual(self.version('application/fhir+xml; fhirVersion=4.0.1'), '4.0') + self.assertEqual(self.version('application/fhir+json', 'application/fhir+json; fhirVersion=4.0'), '4.0') + + def test_another_version_is_refused(self): + with self.assertRaises(NotAcceptable): + self.version('application/fhir+json; fhirVersion=4.3') + with self.assertRaises(NotAcceptable): + self.version('application/fhir+json; fhirVersion=5.0') + with self.assertRaises(UnsupportedMediaType): + self.version('application/fhir+json', 'application/fhir+json; fhirVersion=5.0') + + +class XmlTest(SimpleTestCase): + def assert_in_model_order(self, element, model_class): + assert_in_model_order(self, element, model_class) + + def test_a_valid_resource_round_trips_through_its_model(self): + parameters = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'result', 'valueBoolean': True}, {'name': 'display', 'valueString': 'Malaria'}, + {'name': 'match', 'part': [ + {'name': 'concept', 'valueCoding': {'system': 'http://a', 'code': '1', 'display': 'One'}}]}, + {'name': 'issues', 'resource': {'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'code': 'invalid', 'details': {'text': 'Bad'}}]}}]} + + xml = base.to_xml(parameters) + + self.assertTrue(xml.startswith(b'

Codes

'}, + 'id': 'my_codes@v1', + } + + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(root.tag, f'{FHIR}CodeSystem') + self.assert_in_model_order(root, CodeSystem) + self.assertEqual(root.find(f'{FHIR}id').get('value'), 'my_codes@v1') + self.assertEqual(root.find(f'{FHIR}caseSensitive').get('value'), 'true') + self.assertEqual(root.find(f'{FHIR}count').get('value'), '2') + self.assertEqual(root.find(f'{FHIR}extension').get('url'), 'http://ext') + self.assertEqual(root.find(f'{FHIR}extension/{FHIR}valueString').get('value'), 'x') + self.assertEqual(root.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div/' + f'{{http://www.w3.org/1999/xhtml}}p').text, 'Codes') + concepts = root.findall(f'{FHIR}concept') + self.assertEqual(len(concepts), 2) + self.assert_in_model_order(concepts[0], CodeSystemConcept) + self.assertEqual(concepts[0].find(f'{FHIR}designation/{FHIR}language').get('value'), 'en') + + def test_contained_resources_and_narratives_that_are_not_xhtml(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'total': 1, 'unknownElement': 'x', + 'entry': [{'resource': {'resourceType': 'CodeSystem', 'id': 'a_b', 'status': 'active', + 'text': {'status': 'generated', 'div': 'plain Not a div

'}}}, + {'resource': {'resourceType': 'NoSuchResource', 'id': 'x'}}, + {'resource': 'not a resource'}]} + + root = etree.fromstring(base.to_xml(bundle)) + + self.assertEqual(child_names(root), ['type', 'total', 'entry', 'entry']) # the other two had nothing R4 + resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'a_b') + self.assertEqual(resource.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div').text, 'plain Not a div

') + + def test_data_that_is_not_a_resource_is_answered_with_an_outcome(self): + root = etree.fromstring(base.to_xml({'versions': ['4.0']})) + + self.assertEqual(root.tag, f'{FHIR}OperationOutcome') + self.assertEqual(base.to_xml(None), b'') + + def test_invalid_xml_bodies_are_parse_errors(self): + for body in (b'not xml', b'', b'', + b''): + with self.assertRaises(ParseError, msg=body): + base.from_xml(body) + + def test_entities_are_not_resolved(self): + body = (b']>' + b'' + b'') + try: + parsed = base.from_xml(body) + except ParseError: + return + self.assertNotIn('root:', json.dumps(parsed)) + + def test_the_xml_parser_and_renderer(self): + body = b'' \ + b'' + stream = type('Stream', (), {'read': lambda self: body})() + + parsed = FhirXmlParser().parse(stream) + + self.assertEqual(parsed, {'resourceType': 'Parameters', 'parameter': [{'name': 'code', 'valueCode': '1'}]}) + self.assertEqual(etree.fromstring(FhirXmlRenderer().render(parsed)).tag, f'{FHIR}Parameters') + + +class XmlElementOrderTest(SimpleTestCase): + """FHIR XML has a fixed element order: the R4B model's, whatever the serializer's key order.""" + + def assert_in_model_order(self, element, model_class): + assert_in_model_order(self, element, model_class) + + def test_a_released_code_system_comes_out_in_fhir_element_order(self): + xml = base.to_xml(released_code_system()) + + root = etree.fromstring(xml) + self.assert_in_model_order(root, CodeSystem) + self.assertEqual(child_names(root)[:4], ['id', 'meta', 'language', 'text']) + for name in ('revisionDate', 'collectionReference', 'filters'): + self.assertNotIn(name, child_names(root)) + concept = root.find(f'{FHIR}concept') + self.assert_in_model_order(concept, CodeSystemConcept) + self.assertEqual(base.from_xml(xml)['concept'][0]['display'], 'One') # valid R4B XML + + def test_a_search_bundle_holding_one_comes_out_in_fhir_element_order(self): + code_system = released_code_system(id='my_codes') # an owner-base id the model refuses + bundle = {'resourceType': 'Bundle', 'meta': {'lastUpdated': '2026-10-14T10:00:00Z'}, 'type': 'searchset', + 'total': 1, 'entry': [{'resource': code_system}], + 'link': [{'relation': 'self', 'url': 'http://testserver/fhir/CodeSystem/?page=1'}]} + + root = etree.fromstring(base.to_xml(bundle)) + + self.assertEqual(child_names(root), ['meta', 'type', 'total', 'link', 'entry']) + resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assert_in_model_order(resource, CodeSystem) + self.assertEqual(child_names(resource)[:4], ['id', 'meta', 'language', 'text']) + self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'my_codes') + + def test_bundle_children_keep_the_model_order_with_an_id(self): + root = etree.fromstring(base.to_xml({ + 'resourceType': 'Bundle', 'entry': [{'resource': {'resourceType': 'CodeSystem', 'id': 'codes@1', + 'status': 'active', 'content': 'complete'}}], + 'link': [{'relation': 'self', 'url': 'http://x'}], 'total': 1, 'type': 'searchset', 'id': 'b1', + 'meta': {'lastUpdated': '2026-10-14T10:00:00Z'}})) + + self.assertEqual(child_names(root), ['id', 'meta', 'type', 'total', 'link', 'entry']) + + def test_keys_r4_does_not_define_are_dropped_at_every_depth(self): + code_system = {'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', + 'contact': [{'name': 'Jo', 'my key': 'x', 'telecom': [{'value': 'a@b.c', 'oops': 1}]}], + 'concept': [{'code': 'a', 'concept_class': 'Misc'}]} + + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(child_names(root.find(f'{FHIR}contact')), ['name', 'telecom']) + self.assertEqual(child_names(root.find(f'{FHIR}contact/{FHIR}telecom')), ['value']) + self.assertEqual(child_names(root.find(f'{FHIR}concept')), ['code']) + + def test_each_dropped_key_is_logged_once(self): + base.LOGGED_DROPPED_ELEMENTS.clear() + with self.assertLogs('oclapi', level='WARNING') as logs: + base.to_xml(released_code_system()) + base.to_xml(released_code_system()) + + dropped = [line for line in logs.output if 'does not define' in line] + self.assertEqual(len(dropped), 3, dropped) + self.assertTrue(any('CodeSystem.revisionDate' in line for line in dropped), dropped) + + def test_values_of_the_wrong_shape_are_dropped(self): + outcome = {'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'details': 'Failed to represent "/x" as CodeSystem', 'expression': 'code'}], + 'text': ['not', 'a', 'narrative'], 'id': {'not': 'an id'}} + + root = etree.fromstring(base.to_xml(outcome)) + + self.assertEqual(child_names(root), ['issue']) + self.assertEqual(child_names(root.find(f'{FHIR}issue')), ['severity', 'expression']) + + def test_a_resource_the_model_cannot_write_falls_back_to_the_element_writer(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'entry': [{'resource': { + 'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', 'experimental': False, + 'text': {'status': 'generated', 'div': '

A

'}, + 'extension': [{'url': 'http://ext', 'valueString': 'x'}], 'concept': [{'id': 'c1', 'code': 'x'}]}}]} + with patch.object(Bundle, 'model_dump_xml', side_effect=ValueError('boom')), \ + self.assertLogs('oclapi', level='ERROR'): + root = etree.fromstring(base.to_xml(bundle)) + + code_system = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assertEqual(code_system.find(f'{FHIR}id').get('value'), 'a') + self.assertEqual(code_system.find(f'{FHIR}experimental').get('value'), 'false') + self.assertEqual(code_system.find(f'{FHIR}extension').get('url'), 'http://ext') + self.assertEqual(code_system.find(f'{FHIR}concept').get('id'), 'c1') + self.assertEqual(code_system.find(f'{FHIR}concept/{FHIR}code').get('value'), 'x') + self.assertEqual(code_system.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div/' + f'{{http://www.w3.org/1999/xhtml}}p').text, 'A') + + +# FHIR's instant: a full date and time with seconds and a zone +FHIR_INSTANT = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$') +UPDATED = datetime.datetime(2026, 10, 14, 12, 34, 56, 123456, tzinfo=datetime.timezone.utc) + + +def serialized_value_set(id_): + """What ValueSetDetailSerializer gives: meta.lastUpdated is the model's raw datetime.""" + return {'resourceType': 'ValueSet', 'id': id_, 'version': 'v1', 'url': f'http://example.org/vs/{id_}', + 'status': 'active', 'meta': {'lastUpdated': UPDATED}, 'date': UPDATED} + + +def json_of(data): + """What the JSON response carries for the same data.""" + return json.loads(FhirJsonRenderer().render(data)) + + +class XmlValuesTest(SimpleTestCase): + """XML carries the values the JSON response carries, on both the model and the unvalidated path.""" + + def last_updated(self, element): + return element.find(f'{FHIR}meta/{FHIR}lastUpdated').get('value') + + def test_a_refused_value_set_writes_its_datetimes_as_the_json_does(self): + data = serialized_value_set('my_vs') # an owner-base id the model refuses + + root = etree.fromstring(base.to_xml(data)) + + expected = json_of(data)['meta']['lastUpdated'] + self.assertEqual(self.last_updated(root), expected) + self.assertRegex(self.last_updated(root), FHIR_INSTANT) + self.assertEqual(root.find(f'{FHIR}date').get('value'), json_of(data)['date']) + + def test_every_entry_of_a_bundle_holding_a_refused_one(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'total': 2, + 'entry': [{'resource': serialized_value_set('good')}, {'resource': serialized_value_set('bad_id')}]} + + root = etree.fromstring(base.to_xml(bundle)) + + expected = json_of(bundle)['entry'][0]['resource']['meta']['lastUpdated'] + for resource in root.iterfind(f'{FHIR}entry/{FHIR}resource/{FHIR}ValueSet'): + self.assertEqual(self.last_updated(resource), expected) + self.assertRegex(self.last_updated(resource), FHIR_INSTANT) + + def test_a_valid_resource_writes_the_same_instant(self): + data = serialized_value_set('good') + + written = self.last_updated(etree.fromstring(base.to_xml(data))) + + self.assertRegex(written, FHIR_INSTANT) + self.assertEqual(datetime.datetime.fromisoformat(written), + datetime.datetime.fromisoformat(json_of(data)['meta']['lastUpdated'])) + + def test_other_python_values_are_written_as_json_writes_them(self): + code = uuid.UUID('12345678-1234-5678-1234-567812345678') + parameters = {'resourceType': 'Parameters', 'id': 'p_1', 'parameter': [ + {'name': 'amount', 'valueDecimal': Decimal('1.50')}, {'name': 'when', 'valueDate': UPDATED.date()}, + {'name': 'code', 'valueString': code}]} + + values = [element.get('value') for element in etree.fromstring(base.to_xml(parameters)).iter() + if element.get('value') and etree.QName(element).localname.startswith('value')] + + self.assertEqual(values, ['1.5', '2026-10-14', str(code)]) + + +class XmlSingleElementTest(SimpleTestCase): + """A list for an element R4 allows once keeps its first item.""" + + def concept_map(self, identifiers): + return {'resourceType': 'ConceptMap', 'id': 'cm', 'status': 'active', 'identifier': identifiers} + + def test_a_concept_map_keeps_its_identifier(self): + first = {'system': 'http://testserver', 'value': '/orgs/X/ConceptMap/cm/'} + for identifiers in ([first], [first, {'value': 'another'}]): + root = etree.fromstring(base.to_xml(self.concept_map(identifiers))) + + self.assertEqual(len(root.findall(f'{FHIR}identifier')), 1, identifiers) + self.assertEqual(root.find(f'{FHIR}identifier/{FHIR}value').get('value'), '/orgs/X/ConceptMap/cm/') + + def test_the_items_left_out_are_logged_once(self): + base.LOGGED_DROPPED_ELEMENTS.clear() + identifiers = [{'value': 'a'}, {'value': 'b'}, {'value': 'c'}] + with self.assertLogs('oclapi', level='WARNING') as logs: + base.to_xml(self.concept_map(identifiers)) + base.to_xml(self.concept_map(identifiers)) + + self.assertEqual(len([line for line in logs.output if 'ConceptMap.identifier' in line]), 1, logs.output) + + +class XmlCharactersTest(SimpleTestCase): + """Characters XML 1.0 forbids never stop a response: each becomes U+FFFD, the replacement character.""" + + def test_a_control_character_in_a_value_becomes_the_replacement_character(self): + parameters = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'display', 'valueString': 'Mal\x0baria\x00'}, + {'name': 'name', 'valueString': 'Tab\tand\nnew line'}]} + + with self.assertLogs('oclapi', level='WARNING') as logs: + xml = base.to_xml(parameters) + + values = [element.get('value') for element in etree.fromstring(xml).iter(f'{FHIR}valueString')] + self.assertEqual(values, ['Mal\ufffdaria\ufffd', 'Tab\tand\nnew line']) + self.assertIn('2', logs.output[0]) + + def test_lone_surrogates_and_non_characters_in_text_and_narrative(self): + code_system = {'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', + 'title': 'A\ud800B\uffffC', 'text': {'status': 'generated', 'div': 'x\x1fy'}} + + with self.assertLogs('oclapi', level='WARNING'): + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(root.find(f'{FHIR}title').get('value'), 'A\ufffdB\ufffdC') + self.assertEqual(root.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div').text, 'x\ufffdy') + + def test_the_element_writer_is_safe_too(self): + with patch.object(CodeSystem, 'model_dump_xml', side_effect=ValueError('boom')), \ + self.assertLogs('oclapi', level='WARNING'): + root = etree.fromstring(base.to_xml({'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', + 'content': 'complete', 'title': 'bad\x0b'})) + + self.assertEqual(root.find(f'{FHIR}title').get('value'), 'bad\ufffd') + + +class FhirParametersTest(SimpleTestCase): + factory = APIRequestFactory() + + def read(self, django_request): + request = Request(django_request, parsers=parsers(), negotiator=FhirContentNegotiation()) + return FhirParameters.from_request(request) + + def test_query_parameters(self): + params = self.read(self.factory.get( + '/fhir/CodeSystem/$lookup?system=http://a&code=1&property=x&property=y&_format=xml&format=json&_pretty=1')) + + self.assertEqual(params.get('system'), 'http://a') + self.assertEqual(params.getlist('property'), ['x', 'y']) + self.assertEqual(params.names(), ['system', 'code', 'property']) + self.assertNotIn('_format', params) + self.assertIsNone(params.get('version')) + self.assertEqual(params.get('version', 'v1'), 'v1') + + def test_a_posted_parameters_resource(self): + body = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'url', 'valueUri': 'http://a'}, {'name': 'activeOnly', 'valueBoolean': False}, + {'name': 'count', 'valueInteger': 10}, + {'name': 'coding', 'valueCoding': {'system': 'http://a', 'code': '1'}}, + {'name': 'valueSet', 'resource': {'resourceType': 'ValueSet', 'status': 'active'}}, + {'name': 'dependency', 'part': [{'name': 'element', 'valueUri': 'http://e'}]}, + {'name': 'empty'}]} + + params = self.read(self.factory.post( + '/fhir/ValueSet/$expand?_format=xml&offset=5', json.dumps(body), content_type='application/fhir+json')) + + self.assertEqual(params.get('url'), 'http://a') + self.assertIs(params.get('activeOnly'), False) + self.assertEqual(params.get('count'), 10) + self.assertEqual(params.get('coding'), {'system': 'http://a', 'code': '1'}) + self.assertEqual(params.get('valueSet')['resourceType'], 'ValueSet') + self.assertEqual(params.get('dependency').get('element'), 'http://e') + self.assertIsNone(params.get('empty')) + self.assertEqual(params.get('offset'), '5') + self.assertNotIn('_format', params) + + def test_a_posted_xml_parameters_resource(self): + body = '' \ + '' + + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup', body, content_type='application/fhir+xml')) + + self.assertEqual(params.get('code'), '1') + + def test_a_form_post(self): + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup', {'code': '1', 'system': 'http://a'})) + + self.assertEqual(params.get('code'), '1') + self.assertEqual(params.get('system'), 'http://a') + + def test_a_post_without_a_body_reads_the_query(self): + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup?code=1')) + + self.assertEqual(params.get('code'), '1') + + def test_a_posted_resource_that_is_not_parameters_is_invalid(self): + with self.assertRaises(outcomes.InvalidError): + self.read(self.factory.post( + '/fhir/CodeSystem/$lookup', json.dumps({'resourceType': 'CodeSystem'}), + content_type='application/fhir+json')) + + +class BaseUrlTest(SimpleTestCase): + factory = APIRequestFactory() + + def test_the_base_comes_from_the_request_host(self): + request = self.factory.get( + '/orgs/CIEL/CodeSystem/', HTTP_HOST='fhir.openconceptlab.org', HTTP_X_FORWARDED_PROTO='https') + + self.assertEqual(base.fhir_base_url(request), 'https://fhir.openconceptlab.org/fhir') + self.assertEqual(base.fhir_base_url(request, {'org': 'CIEL'}), 'https://fhir.openconceptlab.org/orgs/CIEL') + self.assertEqual(base.fhir_base_url(request, {'user': 'jo'}), 'https://fhir.openconceptlab.org/users/jo') + + def test_fhir_paths(self): + for path in ('/fhir', '/fhir/', '/fhir/metadata', '/fhir/CodeSystem/$lookup/', '/orgs/CIEL/CodeSystem', + '/orgs/CIEL/ValueSet/x/$expand/', '/users/jo/ConceptMap/$translate'): + self.assertTrue(base.is_fhir_path(path), path) + for path in ('/', '/orgs/CIEL/', '/orgs/CIEL/sources/', '/fhirx/', '/version/'): + self.assertFalse(base.is_fhir_path(path), path) + + +class OutcomeResponseTest(SimpleTestCase): + factory = APIRequestFactory() + outcome = outcomes.operation_outcome(outcomes.issue('not-found', 'Nothing here')) + + def test_json_by_default(self): + response = base.outcome_response(self.factory.get('/fhir/Patient'), self.outcome, 404) + + self.assertEqual(response.status_code, 404) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + self.assertEqual(json.loads(response.content), self.outcome) + + def test_xml_when_asked_and_never_a_5xx(self): + response = base.outcome_response( + self.factory.get('/fhir/Patient?_format=xml'), self.outcome, 500, {'Retry-After': '1'}) + + self.assertEqual(response.status_code, 422) + self.assertEqual(response['Content-Type'], 'application/fhir+xml; charset=utf-8') + self.assertEqual(response['Retry-After'], '1') + self.assertEqual(etree.fromstring(response.content).tag, f'{FHIR}OperationOutcome') + + def test_an_unsupported_format_still_gets_json(self): + response = base.outcome_response(self.factory.get('/fhir/Patient?_format=turtle'), self.outcome, 404) + + self.assertEqual(response['Content-Type'], 'application/fhir+json') + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_unparseable_query_uses_a_400_json_outcome(self): + """Error negotiation must not repeat a failed query parse or retain the original error status.""" + query = '&'.join(['_format=xml'] + ['x='] * 1001) + response = base.outcome_response( + self.factory.get(f'/fhir/missing?{query}', HTTP_ACCEPT='application/fhir+xml'), + self.outcome, 404, {'Retry-After': '1'}) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + self.assertEqual(response['Retry-After'], '1') + outcome = OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome.issue[0].code, 'invalid') + self.assertEqual(outcome.issue[0].details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(outcome.issue[0].details.coding[0].code, 'invalid-data') + + def test_is_outcome_response(self): + self.assertTrue(base.is_outcome_response(base.outcome_response( + self.factory.get('/fhir/x'), self.outcome, 404))) + self.assertTrue(base.is_outcome_response(JsonResponse(self.outcome, status=404))) + self.assertFalse(base.is_outcome_response(JsonResponse({'detail': 'Not found.'}, status=404))) + self.assertFalse(base.is_outcome_response(HttpResponse(b'

Not Found

', status=404))) + self.assertFalse(base.is_outcome_response(HttpResponse(b'{oops', content_type='application/json'))) + + +class EchoView(FhirViewMixin, APIView): + """A FHIR view that answers with its parameters, or with the status it is asked for.""" + permission_classes = (AllowAny,) + authentication_classes = () + + def get(self, request): + status = int(request.query_params.get('status', 200)) + if status != 200: + return Response({'detail': 'As asked.'} if status == 409 else None, status=status) + return self.post(request) + + def post(self, _request): + params = self.get_fhir_parameters() + return Response({'resourceType': 'Parameters', 'parameter': [ + {'name': name, 'valueString': str(params.get(name))} for name in params.names()] + [ + {'name': 'base', 'valueUri': self.get_fhir_base_url()}]}) + + +class ReturnedOutcomeView(FhirViewMixin, APIView): + """Return the legacy serializer error shape with a configurable HTTP status.""" + permission_classes = (AllowAny,) + authentication_classes = () + status_code = 200 + + def get(self, _request): + """Leave normalization to the production view mixin.""" + return Response({'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'details': 'Failed to serialize the resource.'}]}, status=self.status_code) + + +class FhirViewMixinTest(SimpleTestCase): + factory = APIRequestFactory() + + def test_returned_outcomes_are_normalized_at_every_status(self): + """Successful and error statuses both retain the diagnostic in valid JSON and XML outcomes.""" + for status, expected in ((200, 200), (201, 201), (400, 400), (503, 422)): + for media_type in ('application/fhir+json', 'application/fhir+xml'): + with self.subTest(status=status, media_type=media_type): + response = ReturnedOutcomeView.as_view(status_code=status)( + self.factory.get('/fhir/x', HTTP_ACCEPT=media_type)) + response.render() + + self.assertEqual(response.status_code, expected) + self.assertEqual(response['Content-Type'].split(';')[0], media_type) + outcome = (OperationOutcome.model_validate_xml(response.content) if media_type.endswith('xml') + else OperationOutcome.model_validate_json(response.content)) + issue = outcome.issue[0] + self.assertEqual(issue.code, 'processing') + self.assertEqual(issue.details.text, 'Failed to serialize the resource.') + self.assertEqual(issue.details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(issue.details.coding[0].code, 'invalid-data') + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_query_parsing_failure_in_a_view_is_a_400_json_outcome(self): + """A query rejected during initial negotiation is a client error, even when XML was requested.""" + query = '&'.join(['x='] * 1001) + response = EchoView.as_view()(self.factory.get(f'/fhir/x?{query}', HTTP_ACCEPT='application/fhir+xml')) + response.render() + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + outcome = OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome.issue[0].code, 'invalid') + self.assertEqual(outcome.issue[0].details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(outcome.issue[0].details.coding[0].code, 'invalid-data') + + def test_a_view_reads_its_parameters_the_same_way_from_get_and_post(self): + get = EchoView.as_view()(self.factory.get('/fhir/x?code=1')) + post = EchoView.as_view()(self.factory.post( + '/fhir/x', json.dumps({'resourceType': 'Parameters', 'parameter': [{'name': 'code', 'valueCode': '1'}]}), + content_type='application/fhir+json')) + + self.assertEqual(get.data, post.data) + self.assertEqual(get.data['parameter'][0], {'name': 'code', 'valueString': '1'}) + self.assertEqual(get.data['parameter'][1], {'name': 'base', 'valueUri': 'http://testserver/fhir'}) + + def test_errors_a_view_returns_go_out_as_outcomes_and_never_as_a_5xx(self): + for status, expected, code in ((404, 404, 'not-found'), (409, 409, 'invalid'), (503, 422, 'exception')): + response = EchoView.as_view()(self.factory.get(f'/fhir/x?status={status}')) + + self.assertEqual(response.status_code, expected, status) + self.assertEqual(response.data['resourceType'], 'OperationOutcome', status) + self.assertEqual(response.data['issue'][0]['code'], code, status) diff --git a/core/fhir/tests_middleware.py b/core/fhir/tests_middleware.py new file mode 100644 index 000000000..557b9b13c --- /dev/null +++ b/core/fhir/tests_middleware.py @@ -0,0 +1,271 @@ +"""The middleware's answers on FHIR paths are OperationOutcomes (OpenConceptLab/ocl_online#451).""" +import gzip +import json +from unittest.mock import patch + +import requests +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse, JsonResponse, HttpResponseNotFound +from django.test import RequestFactory, SimpleTestCase, TestCase, override_settings +from fhir.resources.R4B.operationoutcome import OperationOutcome +from lxml import etree +from rest_framework.test import APIClient + +from core.common.exceptions import DeactivatedAccountLoginRefused +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.middlewares.middlewares import FhirErrorMiddleware, FhirMiddleware, RequireAuthenticationMiddleware +from core.wsgi import application + +REQUIRE_AUTHENTICATION = 'core.middlewares.middlewares.RequireAuthenticationMiddleware' + + +def outcome_issue(response): + """The first issue of an OperationOutcome response, as severity|code|tx-issue-type.""" + outcome = json.loads(response.content) + issue = outcome['issue'][0] + tx_issue_type = next(c['code'] for c in issue['details']['coding'] if c['system'] == TX_ISSUE_TYPE_SYSTEM) + return f"{issue['severity']}|{issue['code']}|{tx_issue_type}" + + +class FhirMiddlewareTest(SimpleTestCase): + factory = RequestFactory() + + def answer(self, path, response=None, host='api.example.org', **headers): + middleware = FhirMiddleware(lambda request: response or HttpResponse('ok')) + return middleware(self.factory.get(path, HTTP_HOST=host, **headers)) + + @override_settings(FHIR_SUBDOMAIN='fhir') + def test_fhir_paths_off_the_fhir_host_are_a_404_outcome(self): + for path in ('/fhir/CodeSystem/', '/orgs/CIEL/CodeSystem/$lookup', '/users/jo/ValueSet/x/$expand/'): + response = self.answer(path) + + self.assertEqual(response.status_code, 404, path) + self.assertEqual(response['Content-Type'], 'application/fhir+json', path) + self.assertEqual(outcome_issue(response), 'error|not-found|not-found', path) + + self.assertEqual(self.answer('/orgs/CIEL/sources/').content, b'ok') + + @override_settings(FHIR_SUBDOMAIN='fhir') + def test_other_paths_on_the_fhir_host_are_a_404_outcome(self): + response = self.answer('/orgs/CIEL/sources/?q=x', host='fhir.example.org', HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 404) + self.assertEqual(etree.fromstring(response.content).tag, '{http://hl7.org/fhir}OperationOutcome') + + for path in ('/fhir/CodeSystem/?url=http://x', '/orgs/CIEL/CodeSystem?url=http://x', '/version/'): + self.assertEqual(self.answer(path, host='fhir.example.org').content, b'ok', path) + + + +class FhirErrorMiddlewareTest(SimpleTestCase): + """The outermost middleware: an error anywhere in the stack on a FHIR path goes out as an OperationOutcome.""" + factory = RequestFactory() + + def answer(self, path, response, **headers): + return FhirErrorMiddleware(lambda request: response)(self.factory.get(path, **headers)) + + def test_is_the_outermost_middleware(self): + self.assertEqual(settings.MIDDLEWARE[0], 'core.middlewares.middlewares.FhirErrorMiddleware') + + def test_an_error_no_fhir_view_answered_becomes_an_outcome(self): + response = self.answer('/fhir/Patient/', HttpResponseNotFound(b'

Not Found

')) + + self.assertEqual(response.status_code, 404) + self.assertEqual(outcome_issue(response), 'error|not-found|not-found') + self.assertEqual( + json.loads(response.content)['issue'][0]['details']['text'], 'Unknown FHIR path: GET /fhir/Patient/') + + server_error = JsonResponse({'error': 'Server Error (500)'}, status=500) + server_error['Access-Control-Allow-Origin'] = '*' + response = self.answer('/fhir/CodeSystem/', server_error, HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 422) + self.assertEqual(response['Access-Control-Allow-Origin'], '*') # the stack's headers are kept + issue = etree.fromstring(response.content).find('{http://hl7.org/fhir}issue/{http://hl7.org/fhir}code') + self.assertEqual(issue.get('value'), 'exception') + + def test_a_json_error_keeps_its_message(self): + response = self.answer('/orgs/CIEL/CodeSystem/', JsonResponse({'detail': 'Slow down.'}, status=429)) + + self.assertEqual(response.status_code, 429) + self.assertEqual(json.loads(response.content)['issue'][0]['details']['text'], 'Slow down.') + + def test_answers_left_alone(self): + outcome = JsonResponse({'resourceType': 'OperationOutcome', 'issue': []}, status=404) + self.assertIs(self.answer('/fhir/CodeSystem/x', outcome), outcome) + + compressed = HttpResponse(gzip.compress(json.dumps({'resourceType': 'OperationOutcome', 'issue': []}).encode()), + status=404, content_type='application/json') + compressed['Content-Encoding'] = 'gzip' + self.assertIs(self.answer('/fhir/CodeSystem/x', compressed), compressed) + + not_found = HttpResponseNotFound(b'

Not Found

') + self.assertIs(self.answer('/orgs/CIEL/sources/x/', not_found), not_found) + ok = HttpResponse('ok') + self.assertIs(self.answer('/fhir/CodeSystem/', ok), ok) + + +@override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000, FHIR_SUBDOMAIN=None, REQUIRE_AUTHENTICATION=False, + DEBUG=False, DEBUG_PROPAGATE_EXCEPTIONS=False) +class QueryParsingFailureTest(TestCase): + """Malformed query strings remain FHIR errors through the complete middleware stack.""" + + def test_too_many_query_fields_return_a_400_json_outcome(self): + """Both routed and unknown paths use safe JSON when the query cannot be parsed.""" + query = '&'.join(['_format=xml'] + ['x='] * 1001) + client = APIClient(raise_request_exception=False) + for path in ('/fhir/missing', '/fhir/metadata'): + with self.subTest(path=path): + response = client.get(f'{path}?{query}', HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome_issue(response), 'error|invalid|invalid-data') + + +@override_settings(FHIR_SUBDOMAIN=None, REQUIRE_AUTHENTICATION=False, + DEBUG=False, DEBUG_PROPAGATE_EXCEPTIONS=False) +class MediaTypeParsingFailureTest(TestCase): + """Malformed MIME parameters remain client errors through the complete middleware stack.""" + + def assert_bad_request_outcome(self, response): + """Malformed media types always use safe FHIR JSON, including when XML was requested.""" + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome_issue(response), 'error|invalid|invalid-data') + + def test_unknown_accept_encoding_on_unknown_path_is_a_400_json_outcome(self): + """An unrouted path must not fail again while negotiating its error response.""" + response = APIClient(raise_request_exception=False).get( + '/fhir/missing', HTTP_ACCEPT="application/fhir+xml; foo*=bad''%41") + + self.assert_bad_request_outcome(response) + + def test_unknown_accept_encoding_on_routed_path_is_a_400_json_outcome(self): + """Initial view negotiation rejects the same malformed Accept as middleware negotiation.""" + client = APIClient(raise_request_exception=False) + for path in ('/fhir/metadata', '/fhir/metadata?_format=xml'): + for accept in ("application/fhir+xml; foo*=bad''%41", "text/html; foo*=bad''%41, application/xml"): + with self.subTest(path=path, accept=accept): + response = client.get(path, HTTP_ACCEPT=accept) + + self.assert_bad_request_outcome(response) + + def test_unknown_content_type_encoding_on_post_is_a_400_json_outcome(self): + """Use the production request class so Content-Type parsing reaches the middleware stack.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + for path in ('/fhir/missing', '/fhir/CodeSystem/$lookup'): + with self.subTest(path=path): + response = client.post( + path, b'{"resourceType":"Parameters"}', + content_type="application/fhir+json; foo*=bad''%41", HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_too_many_query_fields_with_charset_are_a_400_json_outcome(self): + """Query parsing in the production request's encoding setter remains a client error.""" + query = '&'.join(['x='] * 1001) + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get( + f'/fhir/metadata?{query}', content_type='application/fhir+json; charset=utf-8', + HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + + def test_unknown_charset_name_is_ignored_as_django_does(self): + """An unknown charset name was never an error: Django ignores it and decodes with the default.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get('/fhir/metadata', content_type='application/fhir+json; charset=not-a-charset') + + self.assertEqual(response.status_code, 200) + + + def test_non_text_charset_with_query_is_a_400_json_outcome(self): + """A failed charset is cleared and cannot break error rendering on an unknown path.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get( + '/fhir/missing?x=1', content_type='application/fhir+json; charset=base64', + HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + self.assertIsNone(response.wsgi_request.encoding) + + +@override_settings(REQUIRE_AUTHENTICATION=True, APPROVED_ANONYMOUS_CLIENTS=[], APPROVED_ANONYMOUS_API_KEYS=[], + APPROVED_ANONYMOUS_IPS=[], MIDDLEWARE=[ + *settings.MIDDLEWARE[:settings.MIDDLEWARE.index( + 'django.contrib.auth.middleware.AuthenticationMiddleware') + 1], + REQUIRE_AUTHENTICATION, + *[name for name in settings.MIDDLEWARE[settings.MIDDLEWARE.index( + 'django.contrib.auth.middleware.AuthenticationMiddleware') + 1:] + if name != REQUIRE_AUTHENTICATION]]) +class OuterMiddlewareFailureTest(TestCase): + """An outer middleware failing on a FHIR path (the sign-in provider down) is not a 5xx.""" + + @patch('core.middlewares.middlewares.OCLAuthentication.authenticate', + side_effect=requests.exceptions.ConnectionError('sign-in provider unreachable')) + def test_an_outer_middleware_error_on_a_fhir_path_is_a_4xx_outcome(self, _authenticate): + client = APIClient(raise_request_exception=False) + with self.assertLogs('django.request', level='ERROR'): + response = client.get('/orgs/CIEL/CodeSystem/$lookup?system=http://x&code=1', HTTP_AUTHORIZATION='Bearer x') + + self.assertEqual(response.status_code, 422) + self.assertEqual(outcome_issue(response), 'error|exception|invalid-data') + + with self.assertLogs('django.request', level='ERROR'): + response = client.get('/orgs/CIEL/sources/', HTTP_AUTHORIZATION='Bearer x') + + self.assertEqual(response.status_code, 500) # other paths keep Django's answer + self.assertEqual(json.loads(response.content), {'error': 'Server Error (500)'}) + + +@override_settings(REQUIRE_AUTHENTICATION=True, APPROVED_ANONYMOUS_CLIENTS=[], APPROVED_ANONYMOUS_API_KEYS=[], + APPROVED_ANONYMOUS_IPS=[]) +class RequireAuthenticationOnFhirPathsTest(SimpleTestCase): + factory = RequestFactory() + + def answer(self, path, **headers): + request = self.factory.get(path, **headers) + request.user = AnonymousUser() + return RequireAuthenticationMiddleware(lambda request: HttpResponse('ok'))(request) + + def test_anonymous_access_to_an_owner_base_is_a_403_outcome(self): + response = self.answer('/orgs/CIEL/CodeSystem/$lookup?system=x&code=1') + + self.assertEqual(response.status_code, 403) + self.assertEqual(outcome_issue(response), 'error|login|invalid-data') + issue = json.loads(response.content)['issue'][0] + self.assertEqual(issue['details']['text'], RequireAuthenticationMiddleware.forbidden_response['detail']) + self.assertIn(RequireAuthenticationMiddleware.forbidden_response['upgrade_url'], issue['diagnostics']) + + def test_other_paths_keep_their_json_403(self): + response = self.answer('/orgs/CIEL/sources/') + + self.assertEqual(response.status_code, 403) + self.assertEqual(json.loads(response.content), RequireAuthenticationMiddleware.forbidden_response) + + def test_the_global_fhir_base_stays_open(self): + self.assertEqual(self.answer('/fhir/CodeSystem/$lookup?system=x&code=1').content, b'ok') + + def test_a_refused_sign_in_on_a_fhir_path_is_an_outcome(self): + middleware = RequireAuthenticationMiddleware(lambda request: HttpResponse('ok')) + middleware.is_request_allowed = lambda request: (_ for _ in ()).throw(DeactivatedAccountLoginRefused()) + + request = self.factory.get('/orgs/CIEL/CodeSystem/', HTTP_AUTHORIZATION='Bearer x') + response = middleware(request) + + self.assertEqual(response.status_code, 401) + self.assertEqual(response['WWW-Authenticate'], 'Bearer realm="api"') + self.assertEqual(outcome_issue(response), 'error|login|invalid-data') + + response = middleware(self.factory.get('/orgs/CIEL/sources/', HTTP_AUTHORIZATION='Bearer x')) + self.assertEqual(response.status_code, 401) + self.assertIn('detail', json.loads(response.content)) diff --git a/core/fhir/tests_outcomes.py b/core/fhir/tests_outcomes.py new file mode 100644 index 000000000..1779901cf --- /dev/null +++ b/core/fhir/tests_outcomes.py @@ -0,0 +1,230 @@ +from django.core.exceptions import ValidationError as DjangoValidationError, PermissionDenied as DjangoPermissionDenied +from django.http import Http404 +from django.test import SimpleTestCase +from fhir.resources.R4B.operationoutcome import OperationOutcome +from rest_framework import exceptions + +from core.common.exceptions import Http400 +from core.fhir import outcomes +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM + + +def issue_summary(outcome): + """severity|code|tx-issue-type|expression for each issue: what the parity suite compares.""" + summary = [] + for issue in outcome['issue']: + coding = [c['code'] for c in issue['details']['coding'] if c['system'] == TX_ISSUE_TYPE_SYSTEM] + expression = ','.join(issue.get('expression', [])) + summary.append(f"{issue['severity']}|{issue['code']}|{','.join(coding)}|{expression}") + return summary + + +class TypedErrorTest(SimpleTestCase): + def test_each_typed_error_has_its_code_tx_issue_type_and_status(self): + cases = [ + (outcomes.NotFoundError, 404, 'not-found', 'not-found'), + (outcomes.InvalidError, 400, 'invalid', 'invalid-data'), + (outcomes.NotSupportedError, 400, 'not-supported', 'not-supported'), + (outcomes.MultipleMatchesError, 400, 'multiple-matches', 'invalid-data'), + (outcomes.TooCostlyError, 422, 'too-costly', 'too-costly'), + (outcomes.ProcessingError, 422, 'processing', 'invalid-data'), + ] + for klass, status, code, tx_issue_type in cases: + error = klass('Something is wrong', expression='system') + self.assertEqual(error.status_code, status, klass) + outcome = error.outcome() + self.assertEqual(issue_summary(outcome), [f'error|{code}|{tx_issue_type}|system'], klass) + self.assertEqual(outcome['issue'][0]['details']['text'], 'Something is wrong') + OperationOutcome.model_validate(outcome) + + def test_a_typed_error_can_name_its_tx_issue_type_status_and_message_id(self): + error = outcomes.NotFoundError( + "Unknown code 'x'", expression=['code', 'coding.code'], tx_issue_type='invalid-code', + message_id='Unknown_Code_in_Version', status_code=422) + + outcome = error.outcome() + + self.assertEqual(error.status_code, 422) + self.assertEqual(issue_summary(outcome), ['error|not-found|invalid-code|code,coding.code']) + self.assertEqual(outcome['issue'][0]['extension'], [ + {'url': outcomes.MESSAGE_ID_EXTENSION, 'valueString': 'Unknown_Code_in_Version'}]) + OperationOutcome.model_validate(outcome) + + def test_a_typed_error_without_a_message_or_expression(self): + outcome = outcomes.NotSupportedError().outcome() + + self.assertEqual(issue_summary(outcome), ['error|not-supported|not-supported|']) + self.assertNotIn('expression', outcome['issue'][0]) + self.assertTrue(outcome['issue'][0]['details']['text']) + + def test_issue_codes_without_a_tx_issue_type_of_their_own_get_invalid_data(self): + self.assertEqual(outcomes.tx_issue_type_for('forbidden'), 'invalid-data') + self.assertEqual(outcomes.tx_issue_type_for('code-invalid'), 'invalid-code') + self.assertEqual(outcomes.tx_issue_type_for('throttled'), 'too-costly') + + +class OutcomeNormalizationTest(SimpleTestCase): + """Returned outcomes are normalized throughout the response without changing neighboring resources.""" + + def test_nested_outcomes_preserve_existing_codings_and_other_resources(self): + """An outcome inside another outcome still gains its missing code and structured details.""" + coding = {'system': TX_ISSUE_TYPE_SYSTEM, 'code': 'invalid-code'} + nested = {'resourceType': 'OperationOutcome', 'id': 'nested', 'issue': [{'details': 'Nested failure'}]} + outer = {'resourceType': 'OperationOutcome', 'contained': [nested], 'issue': [ + {'severity': 'error', 'code': 'code-invalid', 'details': {'coding': [coding], 'text': 'Unknown code'}}]} + code_system = {'resourceType': 'CodeSystem', 'status': 'active', 'content': 'not-present'} + bundle = {'resourceType': 'Bundle', 'type': 'collection', 'entry': [ + {'resource': outer}, {'resource': code_system}]} + + normalized = outcomes.normalize_outcomes(bundle) + + result = normalized['entry'][0]['resource'] + OperationOutcome.model_validate(result) + self.assertEqual(result['issue'], outer['issue']) + self.assertEqual(issue_summary(result['contained'][0]), ['error|processing|invalid-data|']) + self.assertEqual(result['contained'][0]['issue'][0]['details']['text'], 'Nested failure') + self.assertEqual(normalized['entry'][1]['resource'], code_system) + self.assertEqual(nested['issue'], [{'details': 'Nested failure'}]) + + +class ExceptionHandlerTest(SimpleTestCase): + def handle(self, exc): + response = outcomes.exception_handler(exc, {}) + OperationOutcome.model_validate(response.data) + return response + + def test_typed_error(self): + response = self.handle(outcomes.NotFoundError('No such system', expression='system')) + + self.assertEqual(response.status_code, 404) + self.assertEqual(issue_summary(response.data), ['error|not-found|not-found|system']) + + def test_not_found_exceptions(self): + for exc in (Http404('nope'), exceptions.NotFound()): + response = self.handle(exc) + + self.assertEqual(response.status_code, 404, exc) + self.assertEqual(issue_summary(response.data), ['error|not-found|not-found|'], exc) + + def test_drf_exceptions_keep_their_status(self): + cases = [ + (exceptions.NotAuthenticated(), 401, 'login'), + (exceptions.AuthenticationFailed(), 401, 'login'), + (exceptions.PermissionDenied(), 403, 'forbidden'), + (DjangoPermissionDenied(), 403, 'forbidden'), + (exceptions.MethodNotAllowed('DELETE'), 405, 'not-supported'), + (exceptions.NotAcceptable(), 406, 'not-supported'), + (exceptions.UnsupportedMediaType('text/plain'), 415, 'not-supported'), + (exceptions.ParseError(), 400, 'invalid'), + (Http400('The following query params are not supported: x'), 400, 'invalid'), + (exceptions.Throttled(wait=3), 429, 'throttled'), + ] + for exc, status, code in cases: + response = self.handle(exc) + + self.assertEqual(response.status_code, status, exc) + self.assertEqual(response.data['issue'][0]['code'], code, exc) + self.assertEqual( + response.data['issue'][0]['details']['coding'][0]['code'], outcomes.tx_issue_type_for(code), exc) + + def test_throttled_and_unauthenticated_keep_their_headers(self): + self.assertEqual(self.handle(exceptions.Throttled(wait=3))['Retry-After'], '3') + + exc = exceptions.NotAuthenticated() + exc.auth_header = 'Token' + self.assertEqual(self.handle(exc)['WWW-Authenticate'], 'Token') + + def test_validation_error_dict_names_each_field(self): + response = self.handle(exceptions.ValidationError( + {'version': ['Version 1.0 already exists.'], 'non_field_errors': ['Bad.'], 'url': 'Required.'})) + + self.assertEqual(response.status_code, 400) + self.assertEqual(issue_summary(response.data), [ + 'error|invalid|invalid-data|version', 'error|invalid|invalid-data|', 'error|invalid|invalid-data|url']) + self.assertEqual( + [issue['details']['text'] for issue in response.data['issue']], + ['Version 1.0 already exists.', 'Bad.', 'Required.']) + + def test_validation_error_list_and_nested_dict(self): + response = self.handle(exceptions.ValidationError(['First.', 'Second.'])) + self.assertEqual(issue_summary(response.data), ['error|invalid|invalid-data|', 'error|invalid|invalid-data|']) + + response = self.handle(exceptions.ValidationError({'parameter': [{'name': ['Required.']}]})) + self.assertEqual(issue_summary(response.data), ['error|invalid|invalid-data|parameter']) + self.assertEqual(response.data['issue'][0]['details']['text'], 'Required.') + + def test_validation_error_carrying_an_operation_outcome_is_kept_and_gets_a_tx_issue_type(self): + detail = {'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'code': 'not-found', 'details': {'text': "Code 'x' not found"}}]} + + response = self.handle(exceptions.ValidationError(detail=detail)) + + self.assertEqual(response.status_code, 400) + self.assertEqual(issue_summary(response.data), ['error|not-found|not-found|']) + self.assertEqual(response.data['issue'][0]['details']['text'], "Code 'x' not found") + + def test_an_outcome_without_codes_gets_them(self): + detail = {'resourceType': 'OperationOutcome', 'issue': [{'severity': 'error', 'details': 'Failed'}, {}]} + + response = self.handle(exceptions.ValidationError(detail=detail)) + + self.assertEqual(issue_summary(response.data), ['error|processing|invalid-data|'] * 2) + self.assertEqual(response.data['issue'][0]['details']['text'], 'Failed') + + def test_django_validation_error(self): + response = self.handle(DjangoValidationError({'url': ['Enter a valid URL.']})) + self.assertEqual(response.status_code, 400) + self.assertEqual(issue_summary(response.data), ['error|invalid|invalid-data|url']) + + response = self.handle(DjangoValidationError('Bad parameters')) + self.assertEqual(issue_summary(response.data), ['error|invalid|invalid-data|']) + self.assertEqual(response.data['issue'][0]['details']['text'], 'Bad parameters') + + def test_server_side_api_exceptions_never_return_a_5xx(self): + class Unavailable(exceptions.APIException): + status_code = 503 + + for exc in (exceptions.APIException(), Unavailable()): + response = self.handle(exc) + + self.assertEqual(response.status_code, 422, exc) + self.assertEqual(issue_summary(response.data), ['error|exception|invalid-data|'], exc) + + def test_unexpected_exception_is_logged_and_answered_with_a_4xx_outcome(self): + with self.assertLogs('oclapi', level='ERROR') as logs: + response = self.handle(TypeError('get_serializer() got multiple values for argument')) + + self.assertEqual(response.status_code, 422) + self.assertEqual(issue_summary(response.data), ['error|exception|invalid-data|']) + self.assertNotIn('get_serializer', response.data['issue'][0]['details']['text']) + self.assertIn('TypeError', logs.output[0]) + + +class OutcomeForStatusTest(SimpleTestCase): + def test_an_error_status_without_a_body(self): + self.assertEqual(issue_summary(outcomes.outcome_for_status(404)), ['error|not-found|not-found|']) + self.assertEqual(issue_summary(outcomes.outcome_for_status(405)), ['error|not-supported|not-supported|']) + self.assertEqual(issue_summary(outcomes.outcome_for_status(409)), ['error|invalid|invalid-data|']) + self.assertEqual(issue_summary(outcomes.outcome_for_status(500)), ['error|exception|invalid-data|']) + self.assertEqual(outcomes.outcome_for_status(499)['issue'][0]['details']['text'], + outcomes.UNEXPECTED_ERROR_MESSAGE) + + def test_a_body_with_a_detail_keeps_its_message(self): + outcome = outcomes.outcome_for_status(403, {'detail': 'You do not have permission.'}) + + self.assertEqual(issue_summary(outcome), ['error|forbidden|invalid-data|']) + self.assertEqual(outcome['issue'][0]['details']['text'], 'You do not have permission.') + + def test_a_field_error_body_names_its_fields(self): + outcome = outcomes.outcome_for_status(400, {'version': ['Version 1.0 already exists.']}) + + self.assertEqual(issue_summary(outcome), ['error|invalid|invalid-data|version']) + + def test_is_outcome(self): + self.assertTrue(outcomes.is_outcome({'resourceType': 'OperationOutcome', 'issue': []})) + self.assertFalse(outcomes.is_outcome({'detail': 'x'})) + self.assertFalse(outcomes.is_outcome(None)) + + def test_response_status_maps_5xx_to_422(self): + self.assertEqual(outcomes.response_status(503), 422) + self.assertEqual(outcomes.response_status(404), 404) diff --git a/core/fhir/tests_protocol.py b/core/fhir/tests_protocol.py new file mode 100644 index 000000000..ced0bf63e --- /dev/null +++ b/core/fhir/tests_protocol.py @@ -0,0 +1,365 @@ +""" +The FHIR protocol through the real views (OpenConceptLab/ocl_online#451): FHIR JSON and XML in and out, _format, +the fhirVersion parameter, trailing slashes on operations, and errors as OperationOutcome, never a 5xx. +""" +import json +from datetime import datetime +from unittest.mock import patch, Mock + +from fhir.resources.R4B.bundle import Bundle +from fhir.resources.R4B.codesystem import CodeSystem +from fhir.resources.R4B.operationoutcome import OperationOutcome +from django.utils import timezone +from lxml import etree +from rest_framework.test import APIClient + +from core.collections.tests.factories import OrganizationCollectionFactory, ExpansionFactory +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory +from core.fhir import base +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.tests_base import assert_in_model_order, child_names +from core.mappings.tests.factories import MappingFactory +from core.orgs.tests.factories import OrganizationFactory +from core.sources.models import Source +from core.sources.tests.factories import OrganizationSourceFactory +from core.users.tests.factories import UserProfileFactory + +FHIR = '{http://hl7.org/fhir}' +SYSTEM = 'http://example.org/fhir/CodeSystem/proto' +VALUE_SET = 'http://example.org/fhir/ValueSet/proto' +RELEASED = 'http://example.org/fhir/CodeSystem/released' + + +def parameters(**values): + """A Parameters resource from name=(valueType, value) pairs.""" + return {'resourceType': 'Parameters', + 'parameter': [{'name': name, f'value{kind}': value} for name, (kind, value) in values.items()]} + + +def parameters_xml(**values): + """The same Parameters resource as FHIR XML.""" + items = ''.join(f'' + for name, (kind, value) in values.items()) + return f'{items}' + + +class FhirProtocolTest(OCLTestCase): + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def setUp(self): + super().setUp() + self.org = OrganizationFactory(mnemonic='ProtoOrg') + self.source = OrganizationSourceFactory( + organization=self.org, mnemonic='proto', canonical_url=SYSTEM, full_name='Protocol codes') + self.concept = ConceptFactory(parent=self.source, mnemonic='c1', names=1, names__name='Malaria') + self.other = ConceptFactory(parent=self.source, mnemonic='c2') + MappingFactory(parent=self.source, from_concept=self.concept, to_concept=self.other) + self.source_v1 = OrganizationSourceFactory.build( + version='v1', mnemonic=self.source.mnemonic, organization=self.org, canonical_url=SYSTEM) + Source.persist_new_version(self.source_v1, self.source.created_by) + self.collection = OrganizationCollectionFactory( + organization=self.org, mnemonic='protovs', canonical_url=VALUE_SET, version='HEAD') + self.collection_v1 = OrganizationCollectionFactory( + organization=self.org, mnemonic='protovs', canonical_url=VALUE_SET, version='v1', is_latest_version=True) + expansion = ExpansionFactory(mnemonic='e1', collection_version=self.collection_v1, parameters={}) + self.collection_v1.expansion_uri = expansion.uri + self.collection_v1.save() + self.user = UserProfileFactory(username='protouser') + self.client = APIClient() + + def assert_fhir_answer(self, response, msg=None): + """Never a 5xx; an error is an OperationOutcome whose every issue has a tx-issue-type coding.""" + self.assertLess(response.status_code, 500, msg) + if response.status_code >= 400: + outcome = self.body(response) + self.assertEqual(outcome['resourceType'], 'OperationOutcome', msg) + for issue in outcome['issue']: + self.assertIn(TX_ISSUE_TYPE_SYSTEM, [coding['system'] for coding in issue['details']['coding']], msg) + + @staticmethod + def body(response): + if 'xml' in response['Content-Type']: + return base.from_xml(response.content) + return json.loads(response.content) + + def post(self, path, body, content_type='application/fhir+json', **headers): + data = body if isinstance(body, str) else json.dumps(body) + return self.client.post(path, data, content_type=content_type, **headers) + + # FHIR JSON in + + def test_fhir_json_posts_are_read(self): + validate = parameters(url=('Uri', SYSTEM), code=('Code', 'c1'), version=('String', 'v1')) + for content_type in ('application/fhir+json', 'application/fhir+json; charset=utf-8', + 'application/json+fhir', 'application/fhir+json; fhirVersion=4.0'): + response = self.post('/fhir/CodeSystem/$validate-code', validate, content_type) + + self.assertEqual(response.status_code, 200, content_type) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + self.assertEqual(response.data['parameter'][0], {'name': 'result', 'valueBoolean': True}) + + response = self.post('/fhir/ValueSet/$validate-code', parameters( + url=('Uri', VALUE_SET), system=('Uri', SYSTEM), code=('Code', 'c1'))) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['resourceType'], 'Parameters') + + def test_operations_answer_posts_without_a_5xx(self): + cases = [ + ('/fhir/CodeSystem/$lookup', parameters(system=('Uri', SYSTEM), code=('Code', 'c1'))), + ('/fhir/ConceptMap/$translate', + parameters(url=('Uri', SYSTEM), system=('Uri', SYSTEM), code=('Code', 'c1'))), + ('/fhir/CodeSystem/$validate-code', {'resourceType': 'Parameters', 'parameter': [{'name': 'code'}]}), + ('/fhir/CodeSystem/$validate-code', '{not json'), + ] + for path, body in cases: + self.assert_fhir_answer(self.post(path, body), path) + + # XML in and out + + def test_format_parameter_on_search(self): + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}&_format=json') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['resourceType'], 'Bundle') + self.assertEqual(len(response.data['entry']), 1) + + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}&_format=xml') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/fhir+xml; charset=utf-8') + root = etree.fromstring(response.content) + self.assertEqual(root.tag, f'{FHIR}Bundle') + self.assertEqual(root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem/{FHIR}url').get('value'), SYSTEM) + + def test_xml_out_on_operations_and_metadata(self): + lookup = f'/fhir/CodeSystem/$lookup?system={SYSTEM}&code=c1' + for path, headers in ((lookup + '&_format=xml', {}), (lookup, {'HTTP_ACCEPT': 'application/fhir+xml'}), + (lookup, {'HTTP_ACCEPT': 'application/xml'})): + response = self.client.get(path, **headers) + + self.assertEqual(response.status_code, 200, (path, headers)) + self.assertEqual(response['Content-Type'], 'application/fhir+xml; charset=utf-8') + parameters_ = base.from_xml(response.content) + self.assertEqual(parameters_['resourceType'], 'Parameters') + self.assertIn({'name': 'display', 'valueString': 'Malaria'}, parameters_['parameter']) + + response = self.client.get(f'/fhir/CodeSystem/$validate-code?url={SYSTEM}&code=c1&_format=xml') + self.assertEqual(response.status_code, 200) + self.assertEqual(base.from_xml(response.content)['parameter'][0], {'name': 'result', 'valueBoolean': True}) + + for search in ('ValueSet', 'ConceptMap'): + response = self.client.get(f'/fhir/{search}/?url={VALUE_SET}&_format=xml') + self.assertEqual(response.status_code, 200, search) + self.assertEqual(etree.fromstring(response.content).tag, f'{FHIR}Bundle', search) + + response = self.client.get(f'/orgs/{self.org.mnemonic}/ValueSet/protovs/$expand?_format=xml') + self.assertEqual(response.status_code, 200) + self.assertEqual(etree.fromstring(response.content).tag, f'{FHIR}ValueSet') + + response = self.client.get('/fhir/metadata?_format=xml') + self.assertEqual(response.status_code, 200) + self.assertEqual(etree.fromstring(response.content).tag, f'{FHIR}CapabilityStatement') + + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def released_source(self): + """A released source version with an owner-base id FHIR doesn't allow and a display with a control character.""" + source = OrganizationSourceFactory(organization=self.org, mnemonic='rel_codes', canonical_url=RELEASED) + ConceptFactory(parent=source, mnemonic='r1', names=1, names__name='Mal\x0baria') + Source.persist_new_version(OrganizationSourceFactory.build( + version='v1', mnemonic='rel_codes', organization=self.org, canonical_url=RELEASED, released=True), + source.created_by) + # a release through the API sets the revision date (ConceptContainerModel.clean) + Source.objects.filter(mnemonic='rel_codes', version='v1').update(revision_date=timezone.now()) + + def test_xml_of_released_code_systems_is_in_fhir_element_order_and_json_is_unchanged(self): + self.released_source() + + response = self.client.get(f'/fhir/CodeSystem/?url={RELEASED}') + self.assertIn('revisionDate', response.data['entry'][0]['resource']) + + response = self.client.get(f'/fhir/CodeSystem/?url={RELEASED}&_format=xml') + self.assertEqual(response.status_code, 200) + root = etree.fromstring(response.content) + assert_in_model_order(self, root, Bundle) + self.assertLess(child_names(root).index('link'), child_names(root).index('entry')) + resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + assert_in_model_order(self, resource, CodeSystem) + self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'rel_codes') + + response = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/rel_codes/?_format=xml') + self.assertEqual(response.status_code, 200) + root = etree.fromstring(response.content) + assert_in_model_order(self, root, CodeSystem) + self.assertEqual(child_names(root)[:2], ['id', 'meta']) + + def test_xml_values_match_json_for_a_value_set_and_a_concept_map_at_an_owner_base(self): + OrganizationCollectionFactory( + organization=self.org, mnemonic='my_vs', canonical_url='http://example.org/vs/my_vs', version='v1', + is_latest_version=True) + for path in (f'/orgs/{self.org.mnemonic}/ValueSet/my_vs/', f'/orgs/{self.org.mnemonic}/ConceptMap/proto/'): + as_json = json.loads(self.client.get(path).content) + response = self.client.get(path + '?_format=xml') + + self.assertEqual(response.status_code, 200, path) + root = etree.fromstring(response.content) + last_updated = root.find(f'{FHIR}meta/{FHIR}lastUpdated').get('value') + self.assertEqual(datetime.fromisoformat(last_updated), + datetime.fromisoformat(as_json['meta']['lastUpdated']), path) + self.assertEqual(root.find(f'{FHIR}identifier/{FHIR}value').get('value'), + as_json['identifier'][0]['value'], path) + + def test_a_control_character_never_stops_an_xml_response(self): + self.released_source() + lookup = f'/fhir/CodeSystem/$lookup?system={RELEASED}&code=r1' + + self.assertIn({'name': 'display', 'valueString': 'Mal\x0baria'}, self.client.get(lookup).data['parameter']) + + response = self.client.get(lookup + '&_format=xml') + self.assertEqual(response.status_code, 200) + self.assertIn({'name': 'display', 'valueString': 'Mal\ufffdaria'}, base.from_xml(response.content)['parameter']) + + def test_xml_in_and_out(self): + body = parameters_xml(url=('Uri', SYSTEM), code=('Code', 'c1'), version=('String', 'v1')) + + response = self.post( + '/fhir/CodeSystem/$validate-code', body, 'application/fhir+xml', HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 200) + self.assertEqual(base.from_xml(response.content)['parameter'][0], {'name': 'result', 'valueBoolean': True}) + + def test_an_invalid_xml_body_is_a_400_outcome_in_xml(self): + response = self.post('/fhir/CodeSystem/$validate-code', ' 2 else '' - is_fhir_domain = domain.startswith(settings.FHIR_SUBDOMAIN + '.') - resource_type = uri[5] if len(uri) > 5 else None - global_space = uri[3] if len(uri) > 3 else None - is_fhir_resource = (global_space == 'fhir' or resource_type == 'CodeSystem' or - resource_type == 'ValueSet' or resource_type == 'ConceptMap') - - if is_fhir_domain: - if global_space != 'version' and not is_fhir_resource: - return HttpResponseNotFound() - elif is_fhir_resource: - return HttpResponseNotFound() - - if settings.FHIR_VALIDATOR_URL and ('/fhir/' in absolute_uri or '/CodeSystem/' in absolute_uri or - '/ValueSet/' in absolute_uri or '/ConceptMap' in absolute_uri): - accept_content_type = request.headers.get('Accept', '') - content_type = request.headers.get('Content-Type', '') - - if content_type.startswith('application/xml') or content_type.startswith('application/fhir+xml'): - request.META['CONTENT_TYPE'] = "application/json" - if request.method in ['POST', 'PUT']: - json_request = requests.post(settings.FHIR_VALIDATOR_URL + - '/convert?version=4.0&type=xml&toType=json', data=request.body) - request._body = json_request - - if accept_content_type.startswith( - 'application/xml') or accept_content_type.startswith('application/fhir+xml'): - request.META['HTTP_ACCEPT'] = "application/json" - response = self.get_response(request) +class FhirMiddleware(BaseMiddleware): + """ + Keeps FHIR on the FHIR host when FHIR_SUBDOMAIN is set: FHIR resource paths 404 elsewhere, and other paths 404 on + the FHIR host, each with an OperationOutcome. + """ - if accept_content_type.startswith( - 'application/xml') or accept_content_type.startswith('application/fhir+xml'): - xml_response = requests.post(settings.FHIR_VALIDATOR_URL + '/convert?version=4.0&type=json&toType=xml', - data=response.content) - response = HttpResponse(xml_response, content_type=accept_content_type) - else: - response = self.get_response(request) + def __call__(self, request): + fhir_path = is_fhir_path(request.path) + if settings.FHIR_SUBDOMAIN: + on_fhir_host = request.get_host().startswith(settings.FHIR_SUBDOMAIN + '.') + if on_fhir_host and not fhir_path and request.path.strip('/').split('/')[0] != 'version': + return self.not_found(request, 'This path is not part of the FHIR API.') + if not on_fhir_host and fhir_path: + return self.not_found(request, 'The FHIR API is served on its own host.') - return response + return self.get_response(request) + + @staticmethod + def not_found(request, message): + return outcome_response(request, operation_outcome(issue('not-found', message)), 404) class ThrottleHeadersMiddleware(MiddlewareMixin): diff --git a/core/parameters/__init__.py b/core/parameters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/parameters/tests.py b/core/parameters/tests.py index 4c3c8f8f5..d83d4ccc9 100644 --- a/core/parameters/tests.py +++ b/core/parameters/tests.py @@ -1,12 +1,12 @@ from core.common.tests import OCLTestCase -from core.concepts.tests.factories import LocalizedTextFactory, ConceptFactory +from core.concepts.tests.factories import ConceptNameFactory, ConceptFactory from core.parameters.serializers import ParametersSerializer from core.sources.tests.factories import OrganizationSourceFactory class ParametersTest(OCLTestCase): class CustomParametersSerializer(ParametersSerializer): - allowed_parameters = { + allowed_input_parameters = { 'url': 'valueUri', 'code': 'valueCode', 'system': 'valueUri' @@ -15,19 +15,15 @@ class CustomParametersSerializer(ParametersSerializer): def test_parse_query_params(self): query_params = {'url': 'https://openconceptlab.org', 'code': 'testCode', 'otherParam': 'otherValue'} serializer = self.CustomParametersSerializer.parse_query_params(query_params) - serializer.is_valid() - params = serializer.validated_data['parameter'] + self.assertTrue(serializer.is_valid()) self.assertIsInstance(serializer, self.CustomParametersSerializer) - self.assertEqual(len(params), 2) - self.assertEqual(params[0]['name'], 'url') - self.assertEqual(params[0]['valueUri'], 'https://openconceptlab.org') - self.assertEqual(params[1]['name'], 'code') - self.assertEqual(params[1]['valueCode'], 'testCode') + self.assertEqual( + serializer.validated_data, {'parameters': {'url': 'https://openconceptlab.org', 'code': 'testCode'}}) def test_from_concept(self): source = OrganizationSourceFactory(default_locale='fr', supported_locales=['fr', 'ti']) - ch_locale = LocalizedTextFactory(locale_preferred=True, locale='ch') - en_locale = LocalizedTextFactory(locale_preferred=True, locale='en') + ch_locale = ConceptNameFactory.build(locale_preferred=True, locale='ch') + en_locale = ConceptNameFactory.build(locale_preferred=True, locale='en') concept = ConceptFactory(names=[ch_locale, en_locale], parent=source) serializer = ParametersSerializer.from_concept(concept) params = serializer.data['parameter'] @@ -39,4 +35,4 @@ def test_from_concept(self): self.assertEqual(params[1]['name'], 'version') self.assertEqual(params[1]['valueString'], source.version) self.assertEqual(params[2]['name'], 'display') - self.assertEqual(params[2]['valueString'], concept.name) \ No newline at end of file + self.assertEqual(params[2]['valueString'], concept.display_name) diff --git a/core/settings.py b/core/settings.py index 63c89575e..f260c445a 100644 --- a/core/settings.py +++ b/core/settings.py @@ -194,6 +194,7 @@ def get_set_from_env(name): } MIDDLEWARE = [ + 'core.middlewares.middlewares.FhirErrorMiddleware', # outermost: a FHIR path's error is an OperationOutcome 'django.middleware.gzip.GZipMiddleware', 'cid.middleware.CidMiddleware', 'core.middlewares.middlewares.CustomLoggerMiddleware', @@ -421,8 +422,6 @@ def get_set_from_env(name): API_SUPERUSER_PASSWORD = os.environ.get('API_SUPERUSER_PASSWORD', 'Root123') # password for ocladmin superuser API_SUPERUSER_TOKEN = os.environ.get('API_SUPERUSER_TOKEN', '891b4b17feab99f3ff7e5b5d04ccc5da7aa96da6') -FHIR_VALIDATOR_URL = os.environ.get('FHIR_VALIDATOR_URL', None) - # Redis REDIS_CONNECTION_OPTIONS = { 'socket_timeout': 5.0, @@ -598,7 +597,8 @@ def get_set_from_env(name): if ENV and ENV != 'development': # Serving swagger static files (inserted after SecurityMiddleware) - MIDDLEWARE.insert(1, 'whitenoise.middleware.WhiteNoiseMiddleware') + MIDDLEWARE.insert(MIDDLEWARE.index('django.middleware.gzip.GZipMiddleware') + 1, + 'whitenoise.middleware.WhiteNoiseMiddleware') EMAIL_SUBJECT_PREFIX = os.environ.get('EMAIL_SUBJECT_PREFIX', None) if not EMAIL_SUBJECT_PREFIX: diff --git a/core/value_sets/views.py b/core/value_sets/views.py index 1faaea35c..810d63b8c 100644 --- a/core/value_sets/views.py +++ b/core/value_sets/views.py @@ -12,6 +12,7 @@ from core.common.constants import HEAD, LATEST from core.common.fhir_helpers import translate_fhir_query from core.concepts.models import Concept +from core.fhir.base import FhirViewMixin from core.sources.models import Source from core.value_sets.serializers import ValueSetDetailSerializer, \ ValueSetExpansionParametersSerializer, ValueSetExpansionSerializer @@ -19,7 +20,7 @@ logger = logging.getLogger('oclapi') -class ValueSetListView(CollectionListView): +class ValueSetListView(FhirViewMixin, CollectionListView): serializer_class = ValueSetDetailSerializer @staticmethod @@ -130,7 +131,7 @@ def get_queryset(self): return queryset -class ValueSetRetrieveUpdateView(CollectionRetrieveUpdateDestroyView): +class ValueSetRetrieveUpdateView(FhirViewMixin, CollectionRetrieveUpdateDestroyView): serializer_class = ValueSetDetailSerializer def get_filter_params(self, default_version_to_head=True): @@ -147,7 +148,7 @@ def get_detail_serializer(self, obj): return ValueSetDetailSerializer(obj) -class ValueSetExpandView(CollectionVersionExpansionsView): +class ValueSetExpandView(FhirViewMixin, CollectionVersionExpansionsView): sync = True def get_serializer_class(self): diff --git a/core/wsgi.py b/core/wsgi.py index 883d9f9c1..6c1add058 100644 --- a/core/wsgi.py +++ b/core/wsgi.py @@ -7,10 +7,48 @@ https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ """ +import codecs import os +from django.core.exceptions import BadRequest, SuspiciousOperation +from django.core.handlers.wsgi import WSGIRequest from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + +class FhirWSGIRequest(WSGIRequest): + """Let FHIR middleware answer malformed Content-Type parameters rejected during request construction.""" + + def _set_content_type_params(self, meta): + """Preserve the invalid raw header and mark FHIR requests for a safe JSON 400 response.""" + from core.fhir.base import is_fhir_path + + try: + super()._set_content_type_params(meta) + except (LookupError, ValueError, BadRequest, SuspiciousOperation): + if not is_fhir_path(self.path): + raise + self._mark_bad_content_type() + return + charset = self.content_params.get('charset') + # Django accepts any codec as the charset, but decoding the query with a non-text one (base64) fails later. + if charset and is_fhir_path(self.path) and not self._is_text_encoding(charset): + self._mark_bad_content_type() + + @staticmethod + def _is_text_encoding(charset): + try: + return codecs.lookup(charset)._is_text_encoding # pylint: disable=protected-access + except LookupError: + return True # Django ignores an unknown charset name, so the query decodes with the default + + def _mark_bad_content_type(self): + self.fhir_content_type_error = True + self.content_type = '' + self.content_params = {} + # Bypass the encoding setter, which can parse the rejected query again. + self._encoding = None + application = get_wsgi_application() +application.request_class = FhirWSGIRequest diff --git a/requirements.txt b/requirements.txt index 7bfa0b1fc..b9b67dd03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ django-celery-beat==2.9.0 jsondiff==2.0.0 django-silk==5.5.0 azure-storage-blob==12.19.0 -fhir.resources==7.1.0 +fhir.resources[xml]==8.3.0 ijson==3.2.3 responses==0.25.0 pydantic==2.10.6 From 986fced4ff131e5706d641334e76f9812c37160d Mon Sep 17 00:00:00 2001 From: Jonathan Payne Date: Thu, 22 Oct 2026 12:23:24 -0400 Subject: [PATCH 02/18] OpenConceptLab/ocl_online#451 | Canonical lookups preserve registry entries and support FHIR version rules Canonical resolution can select a repository type and require a released default version. The new core/fhir/resolve.py applies the FHIR base's namespace and access rules, preserves the resource type of OCL paths, and uses HEAD only when named. Repositories without a release are absent. Unavailable repositories receive the same not-found answer as missing ones. $resolveReference accepts namespace for expressions that name none when the URL registry is enabled; an expression's own namespace wins. Existing shared-resolver callers keep their default resolution rules. Registry reads no longer save or reindex entries. Repository writes refresh cached links, and ConceptContainerModel.delete in core/common/models.py preserves entries and resolves their links again after deletion. Link updates save only their own fields, preserving concurrent edits. Tests cover both registry settings, resource types, namespaces, named versions and query counts, plus link maintenance, concurrent edits and retries of interrupted repository or version deletes. Work package WP2 of OpenConceptLab/ocl_online#451. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R8hw7w8753DDSCezrmrDA8 --- core/collections/models.py | 7 +- core/collections/views.py | 20 +- core/common/mixins.py | 11 +- core/common/models.py | 103 ++++- core/fhir/resolve.py | 92 ++++ core/fhir/tests_resolve.py | 301 +++++++++++++ core/repos/models.py | 6 +- core/sources/models.py | 7 +- core/url_registry/models.py | 35 +- core/url_registry/tests_resolution.py | 606 ++++++++++++++++++++++++++ 10 files changed, 1156 insertions(+), 32 deletions(-) create mode 100644 core/fhir/resolve.py create mode 100644 core/fhir/tests_resolve.py create mode 100644 core/url_registry/tests_resolution.py diff --git a/core/collections/models.py b/core/collections/models.py index ffb013656..74b880de5 100644 --- a/core/collections/models.py +++ b/core/collections/models.py @@ -167,8 +167,11 @@ def save( super().save(*args, force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields) - if self.id and 'canonical_url' in dirty_fields and self.active_url_registry_entries.exists(): - resolve_url_registry_entries.apply_async((self.id, self.resource_type), queue='default', permanent=False) + if self.id and 'canonical_url' in dirty_fields: + if self.active_url_registry_entries.exists(): + resolve_url_registry_entries.apply_async( + (self.id, self.resource_type), queue='default', permanent=False) + self.link_url_registry_entries() def post_create_actions(self): if self.should_auto_expand: diff --git a/core/collections/views.py b/core/collections/views.py index e45e6b8dc..395b64217 100644 --- a/core/collections/views.py +++ b/core/collections/views.py @@ -71,6 +71,7 @@ from core.tasks.mixins import TaskMixin, IndexingTaskMixin from core.tasks.models import Task from core.tasks.serializers import TaskBriefSerializer +from core.toggles.models import Toggle logger = logging.getLogger('oclapi') @@ -1298,15 +1299,26 @@ class ReferenceExpressionResolveView(APIView): def get_throttles(self): return ThrottleUtil.get_throttles_by_user_plan(self.request.user) + def get_namespace(self): + """ + ?namespace= is the namespace of every expression that doesn't name its own. Clients such as the Mapper send + their project's owner to choose which URL registries apply, so it is used only while the resolver uses the + registries. Without them a namespace limits a canonical URL to that owner's repositories, and such a client + would stop resolving every canonical URL its owner doesn't hold. + """ + namespace = self.request.query_params.get('namespace') + return namespace if namespace and Toggle.get('URL_REGISTRY_IN_RESOLVE_REFERENCE_TOGGLE') else None + def get_results(self): data = self.request.data if not isinstance(data, list): data = [data] from core.common.models import ConceptContainerModel + namespace = self.get_namespace() results = [] for expression in data: - instance, registry_entry = ConceptContainerModel.resolve_expression_to_version(expression) + instance, registry_entry = ConceptContainerModel.resolve_expression_to_version(expression, namespace) if instance: result = { **ReferenceExpressionResolveSerializer(instance).data, @@ -1333,6 +1345,12 @@ def get_results(self): responses={ 200: ReferenceExpressionResolveSerializer(many=True) }, + manual_parameters=[ + openapi.Parameter( + 'namespace', openapi.IN_QUERY, type=openapi.TYPE_STRING, + description='Namespace (/orgs/{org}/ or /users/{user}/) for the expressions that name none. ' + 'Used only while canonical URLs resolve through the URL registries') + ], request_body=openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema( diff --git a/core/common/mixins.py b/core/common/mixins.py index 57ea324e5..206739f7c 100644 --- a/core/common/mixins.py +++ b/core/common/mixins.py @@ -562,8 +562,17 @@ def update(self, request, *args, **kwargs): # pylint: disable=unused-argument class SourceContainerMixin: - def find_repo_by_canonical_url(self, canonical_url): + def find_repo_by_canonical_url(self, canonical_url, repo_type=None): + """ + This owner's lowest-id repository row with the canonical URL, a Source before a Collection; `url|version` + names a version. With repo_type (Source or Collection), only that type's active, non-retired rows, and the + HEAD unless a version is named. + """ url, version = canonical_url_to_url_and_version(canonical_url) + if repo_type: + repos = {'Source': self.source_set, 'Collection': self.collection_set}[repo_type.OBJECT_TYPE] + queryset = repos.filter(canonical_url=url, is_active=True, retired=False) + return queryset.filter(version=decode_string(version) if version else HEAD).first() queryset = self.source_set.filter(canonical_url=url) if not queryset.exists(): queryset = self.collection_set.filter(canonical_url=url) diff --git a/core/common/models.py b/core/common/models.py index 743f2f586..0452d3e11 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -6,6 +6,7 @@ from celery_once import AlreadyQueued from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import ArrayField from django.core.cache import cache from django.core.exceptions import ValidationError @@ -712,8 +713,12 @@ def get_latest_version(self): def get_last_version(self): return self.active_versions.order_by('-created_at').first() - def get_latest_released_version(self): - return self.released_versions.filter(retired=False).order_by('-created_at').first() + def get_latest_released_version(self, exclude_head=False): + """The newest released version that isn't retired; exclude_head skips a HEAD flagged released""" + versions = self.released_versions.filter(retired=False) + if exclude_head: + versions = versions.exclude(version=HEAD) + return versions.order_by('-created_at').first() def get_prev_released_version(self): return self.released_versions.exclude(id=self.id).order_by('-created_at').first() @@ -922,13 +927,16 @@ def versions(self): def delete(self, using=None, keep_parents=False, force=False, sync=False): # pylint: disable=arguments-differ export_paths = self.get_export_paths_to_delete() deleted_urls = [self.url] + deleted_ids = [self.id] + canonical_urls = {self.canonical_url} if self.is_head: other_versions = list(self.versions.exclude(id=self.id)) for other_version in other_versions: other_version.clear_cache() export_paths += other_version.get_export_paths_to_delete() deleted_urls.append(other_version.url) - self.versions.exclude(id=self.id).delete() + deleted_ids.append(other_version.id) + canonical_urls.add(other_version.canonical_url) elif self.is_latest_version: prev_version = self.prev_version if not force and not prev_version: @@ -937,6 +945,10 @@ def delete(self, using=None, keep_parents=False, force=False, sync=False): # py prev_version.is_latest_version = True prev_version.save() + registry_entry_ids = self.unlink_url_registry_entries(deleted_ids) + if self.is_head: + self.versions.exclude(id=self.id).delete() + self.delete_pins() self.delete_following() self.delete_version_changelogs(deleted_urls) @@ -945,6 +957,32 @@ def delete(self, using=None, keep_parents=False, force=False, sync=False): # py super().delete(using=using, keep_parents=keep_parents) self.delete_export_paths(export_paths, sync) self.post_delete_actions() + self.relink_url_registry_entries(registry_entry_ids, canonical_urls - {None, ''}) + + def unlink_url_registry_entries(self, repo_ids): + """ + Unlinks (saved, so reindexed) the registry entries linked to these rows of this repository before the rows + are deleted: url_registry_entries is a GenericRelation, which would delete them with the rows. Returns + their ids for relink_url_registry_entries. + """ + from core.url_registry.models import URLRegistry + entries = list(URLRegistry.objects.filter( + repo_type=ContentType.objects.get_for_model(self), repo_id__in=repo_ids)) + for entry in entries: + entry.unlink() + return [entry.id for entry in entries] + + @staticmethod + def relink_url_registry_entries(entry_ids, canonical_urls): + """ + After a delete: every active entry that was linked to a deleted row, and every active entry naming one of the + deleted rows' canonical URLs, linked or not, is re-resolved (lookup_entry). A retry of a delete that failed + part-way so finds, by URL, the entries the failed attempt unlinked. The link is a cache that resolution never + reads, so a failure here only leaves it stale until the next delete, retry or write that relinks. + """ + from core.url_registry.models import URLRegistry + for entry in URLRegistry.objects.filter(Q(id__in=entry_ids) | Q(url__in=canonical_urls), is_active=True): + entry.lookup_entry() @staticmethod def delete_version_changelogs(urls): @@ -1399,6 +1437,20 @@ def is_exporting(self): def active_url_registry_entries(self): return self.url_registry_entries.filter(is_active=True) + def link_url_registry_entries(self): + """ + For a repository row (HEAD or version) created with, or given, a canonical URL: the active registry entries + that name it in this repository's namespace are pointed at what they resolve to now (lookup_entry: the + namespace's lowest-id row with the URL, as the registry has always resolved), so an entry made before the row + that answers it gets it. The entries already linked to this row are re-resolved by the + resolve_url_registry_entries task. + """ + if not self.canonical_url: + return + from core.url_registry.models import URLRegistry + for entry in URLRegistry.objects.filter(is_active=True, url=self.canonical_url, namespace=self.parent_url): + entry.lookup_entry() + @cached_property def version_export_path(self): last_update = self.last_child_update.strftime('%Y-%m-%d_%H%M%S') @@ -1472,22 +1524,23 @@ def can_view_all_content(self, user): ) or self.public_can_view or self.user_id == user.id or self.organization.members.filter(id=user.id).exists() @classmethod - def resolve_expression_to_version(cls, expression): + def resolve_expression_to_version(cls, expression, namespace=None): + """`namespace` is the namespace of an expression that doesn't name its own (a dict without the key, or a url)""" url = expression - namespace = None version = None instance = None resolved_registry_entry = None if isinstance(expression, dict) and get(expression, 'url'): url = expression['url'] - namespace = expression.get('namespace', None) + namespace = expression.get('namespace', namespace) version = expression.get('version', None) if url: instance, resolved_registry_entry = cls.resolve_reference_expression(url, namespace, version) return instance, resolved_registry_entry @classmethod - def resolve_reference_expression(cls, url, namespace=None, version=None): + def resolve_reference_expression( # pylint: disable=too-many-arguments + cls, url, namespace=None, version=None, repo_type=None, allow_head=True): """ resolves to repository version according to this process: @@ -1511,6 +1564,16 @@ def resolve_reference_expression(cls, url, namespace=None, version=None): is defined somewhere else in OCL) 2. Else if relative URL provided: - Return the repository directly using the relative URL, or return 404 if not found + + Resolving reads only: it never writes a registry entry. + + Two flags make the answer stricter, for FHIR (core.fhir.resolve); their defaults keep the behaviour above: + - repo_type (Source or Collection): resolve only to a repository of that class. A canonical URL matches the + active, non-retired HEAD of a repository (its current canonical URL) in every step, so a registry entry whose + namespace holds no such repository resolves to nothing, and nothing else is tried. A named version must be + active. Unresolved is an unsaved repo_type. + - allow_head=False: with no version named, the latest released version other than HEAD (even a HEAD flagged + released) or nothing. A version named HEAD is still served. """ resolution_url, version, is_canonical = cls.__get_resolution_url(url, version) @@ -1527,36 +1590,44 @@ def resolve_reference_expression(cls, url, namespace=None, version=None): owner = SourceContainerMixin.get_object_from_namespace(namespace) if owner: registry_entry = owner.url_registry_entries.filter(is_active=True, url=resolution_url).first() - instance = registry_entry.lookup_entry() if registry_entry else owner.find_repo_by_canonical_url( # pylint: disable=line-too-long - resolution_url) + instance = registry_entry.find_repo(repo_type) if registry_entry else \ + owner.find_repo_by_canonical_url(resolution_url, repo_type) if is_global_namespace or (not registry_entry and not instance): registry_entry = URLRegistry.get_active_global_entries().filter(url=resolution_url).first() if registry_entry: - instance = registry_entry.lookup_entry() + instance = registry_entry.find_repo(repo_type) - return cls.resolve_repo(instance, version, is_canonical, resolution_url), registry_entry + return cls.resolve_repo( + instance, version, is_canonical, resolution_url, repo_type, allow_head), registry_entry criteria &= models.Q(canonical_url=resolution_url) + if repo_type: + criteria &= models.Q(version=HEAD) if namespace: criteria &= models.Q(models.Q(user__uri=namespace) | models.Q(organization__uri=namespace)) else: criteria &= models.Q(uri=(resolution_url if resolution_url.endswith('/') else resolution_url + '/')) from core.repos.models import Repository - return cls.resolve_repo(Repository.get(criteria), version, is_canonical, resolution_url), registry_entry + return cls.resolve_repo( + Repository.get(criteria, repo_type), version, is_canonical, resolution_url, repo_type, allow_head + ), registry_entry @classmethod - def resolve_repo(cls, instance, version, is_canonical, resolution_url): + def resolve_repo( # pylint: disable=too-many-arguments + cls, instance, version, is_canonical, resolution_url, repo_type=None, allow_head=True): if instance: if version: - instance = instance.versions.filter(version=decode_string(version)).first() + versions = instance.active_versions if repo_type else instance.versions + instance = versions.filter(version=decode_string(version)).first() elif instance.is_head: - instance = instance.get_latest_released_version() or instance + instance = (instance.get_latest_released_version() or instance) if allow_head else \ + instance.get_latest_released_version(exclude_head=True) if not instance: from core.sources.models import Source - instance = Source() + instance = (repo_type or Source)() instance.is_fqdn = is_canonical instance.resolution_url = resolution_url diff --git a/core/fhir/resolve.py b/core/fhir/resolve.py new file mode 100644 index 000000000..c8d408b84 --- /dev/null +++ b/core/fhir/resolve.py @@ -0,0 +1,92 @@ +""" +FHIR's canonical resolution: the resolver $resolveReference uses (ConceptContainerModel.resolve_reference_expression, +with or without the URL registry), with FHIR's rules on top. + +- Resource type: CodeSystem and ConceptMap resolve to a Source, ValueSet to a Collection. +- Namespace: the base URL's. /fhir/ is global; /orgs/{org}/ and /users/{user}/ are that owner's. It comes from the + route, never from the request. +- Version: the one named (the operation's version parameter, else url|version), HEAD included. With none named, the + latest released version, never HEAD. A repository with nothing released isn't visible, whatever version is named. +- Access, after resolving: at the global base public repositories only, at an owner base what the caller may view. + A repository the caller can't see is not found, and no other repository is tried. + +Every failure is the same UnresolvedError, so it never tells a caller that a repository they can't see exists. +""" +from core.collections.models import Collection +from core.common.constants import HEAD +from core.common.mixins import SourceContainerMixin +from core.fhir.outcomes import NotFoundError +from core.sources.models import Source + +REPO_TYPES = {'CodeSystem': Source, 'ConceptMap': Source, 'ValueSet': Collection} +REPO_PATHS = {'CodeSystem': 'sources', 'ConceptMap': 'sources', 'ValueSet': 'collections'} + + +class UnresolvedError(NotFoundError): + """ + No repository version of the resource type the caller may see for a canonical URL (and version). `url` is the + url without a |version, `version` the one asked for: the version parameter, else the url's. + """ + + def __init__(self, resource_type, url, version=None, expression=None): + self.resource_type, self.url, self.version = resource_type, url, version + named = f" version '{version}'" if version else '' + super().__init__( + f"A definition for {resource_type} '{url}'{named} could not be found", expression=expression, + tx_issue_type='not-found') + + +def namespace_of(kwargs): + """The namespace of the FHIR base a route's kwargs name: the owner's URI, or None for the global base.""" + kwargs = kwargs or {} + if kwargs.get('org'): + return f"/orgs/{kwargs['org']}/" + if kwargs.get('user'): + return f"/users/{kwargs['user']}/" + return None + + +def resolve(resource_type, url, version=None, kwargs=None, user=None, expression=None): # pylint: disable=too-many-arguments + """ + The repository version a FHIR request names: `url` (a canonical URL, or an OCL path such as + /orgs/{org}/CodeSystem/{id}/) and `version`, at the base `kwargs` names, for `user`. A view passes its own + self.kwargs and self.request.user. Raises UnresolvedError, whose `expression` is the parameter the url came in. + """ + url = url or '' + lookup_url, _, piped_version = url.partition('|') + error = UnresolvedError(resource_type, lookup_url, version or piped_version or None, expression) + namespace = namespace_of(kwargs) + if not lookup_url or '|' in piped_version or ( + namespace and not SourceContainerMixin.get_object_from_namespace(namespace)): + raise error + + repo_type = REPO_TYPES[resource_type] + instance, _ = repo_type.resolve_reference_expression( + to_ocl_path(url), namespace, version, repo_type=repo_type, allow_head=False) + + if not instance.id or not is_visible(instance, namespace, user) or not has_release(instance): + raise error + return instance + + +def to_ocl_path(url): + """ + An OCL FHIR path (/orgs/{org}/CodeSystem/{id}/...) as the OCL path of the repository it names, by its own + resource type, so a reference to the other type finds nothing; anything else as it is. + """ + parts = url.split('/') + if len(parts) > 3 and not parts[0] and parts[1] in ('orgs', 'users') and parts[3] in REPO_PATHS: + parts[3] = REPO_PATHS[parts[3]] + return '/'.join(parts) + + +def is_visible(instance, namespace, user): + """At the global base, public repositories to anyone; at an owner base, what the user may view.""" + return instance.public_can_view if namespace is None else instance.has_view_access(user) + + +def has_release(instance): + """Whether the repository of a version has a released version that isn't retired; HEAD's released flag isn't one""" + if instance.released and not instance.retired and not instance.is_head: + return True + return instance.released_versions.filter(retired=False).exclude(version=HEAD).exists() diff --git a/core/fhir/tests_resolve.py b/core/fhir/tests_resolve.py new file mode 100644 index 000000000..637823f44 --- /dev/null +++ b/core/fhir/tests_resolve.py @@ -0,0 +1,301 @@ +""" +core.fhir.resolve: FHIR's canonical resolution through the shared resolver, with the URL registry toggle off and on. +Shared canonicals (ICD-10, ATC), a Source and a Collection sharing one, a repository with nothing released, the +fixture org's private namespace, unknown systems and versions, and HEAD. OpenConceptLab/ocl_online#451. +""" +from django.contrib.auth.models import AnonymousUser + +from core.collections.models import Collection +from core.common.constants import ACCESS_TYPE_NONE, HEAD +from core.common.tests import OCLTestCase +from core.fhir import resolve as fhir_resolve +from core.fhir.outcomes import NotFoundError, TX_ISSUE_TYPE_SYSTEM +from core.orgs.tests.factories import OrganizationFactory +from core.sources.tests.factories import UserSourceFactory +from core.url_registry.factories import GlobalURLRegistryFactory +from core.url_registry.models import URLRegistry +from core.url_registry.tests_resolution import ( + set_registry_toggle, source, collection, ResolverCorpusMixin, ICD10, PBD, DRAFT) +from core.users.models import UserProfile +from core.users.tests.factories import UserProfileFactory + +ATC = 'http://www.whocc.no/atc' +CIEL = 'https://CIELterminology.org' +FIXTURE = 'https://openconceptlab.org/fhir-fixtures/CodeSystem/fixture-codes' +UNKNOWN = 'http://example.org/ocl-fhir-conformance/unknown-system' +ANONYMOUS = AnonymousUser() + + +class NamespaceTest(OCLTestCase): + def test_namespace_is_the_base_url_owner(self): + self.assertIsNone(fhir_resolve.namespace_of({})) + self.assertIsNone(fhir_resolve.namespace_of(None)) + self.assertEqual(fhir_resolve.namespace_of({'org': 'CIEL', 'source': 'x'}), '/orgs/CIEL/') + self.assertEqual(fhir_resolve.namespace_of({'user': 'jdoe'}), '/users/jdoe/') + + def test_unresolved_error(self): + error = fhir_resolve.UnresolvedError('CodeSystem', UNKNOWN, expression='system') + self.assertIsInstance(error, NotFoundError) + self.assertEqual(error.status_code, 404) + self.assertEqual( + error.outcome()['issue'], + [{ + 'severity': 'error', 'code': 'not-found', + 'details': { + 'coding': [{'system': TX_ISSUE_TYPE_SYSTEM, 'code': 'not-found'}], + 'text': f"A definition for CodeSystem '{UNKNOWN}' could not be found" + }, + 'expression': ['system'] + }]) + self.assertEqual((error.resource_type, error.url, error.version), ('CodeSystem', UNKNOWN, None)) + + error = fhir_resolve.UnresolvedError('ValueSet', UNKNOWN, '1.0') + self.assertEqual(str(error.detail), f"A definition for ValueSet '{UNKNOWN}' version '1.0' could not be found") + self.assertNotIn('expression', error.outcome()['issue'][0]) + + +class FhirResolveCases(ResolverCorpusMixin): # pylint: disable=too-many-public-methods + """ + Every case runs with the toggle off and on (the subclasses); `registry` says which. The resolver's corpus (ICD-10 + shared by WHO and sehatapp, padam's Source and Collection, an unreleased draft), plus ATC shared the same way, + CIEL, and the private fixture org. + """ + registry = None + + def setUp(self): + set_registry_toggle(self.registry) + super().setUp() + self.admin = UserProfile.objects.get(username='ocladmin') + self.ciel = OrganizationFactory(mnemonic='CIEL') + self.hl7 = OrganizationFactory(mnemonic='HL7') + self.fixtures = OrganizationFactory(mnemonic='FHIR-Fixtures', public_access=ACCESS_TYPE_NONE) + self.member = UserProfileFactory(username='fixture-member') + self.outsider = UserProfileFactory(username='outsider') + self.fixtures.members.add(self.member) + + # WHO first, so WHO has the lower ids, as in production (the corpus does the same for ICD-10) + self.who_atc = source(self.who, 'WHOATC', canonical_url=ATC) + self.who_atc_2023 = source(self.who, 'WHOATC', '2023AA', released=True, canonical_url=ATC) + self.who_atc_2024 = source(self.who, 'WHOATC', 'ATC_2024_01_19', canonical_url=ATC) # newest, unreleased + self.sehat_atc = source(self.sehatapp, 'ATC', canonical_url=ATC) + self.sehat_atc_v2 = source(self.sehatapp, 'ATC', 'v2', released=True, canonical_url=ATC) + self.ciel_source = source(self.ciel, 'CIEL', canonical_url=CIEL) + self.ciel_v1 = source(self.ciel, 'CIEL', 'v2026-09-24', released=True, canonical_url=CIEL) + self.fixture = source( + self.fixtures, 'fixture-codes', canonical_url=FIXTURE, public_access=ACCESS_TYPE_NONE) + self.fixture_v1 = source( + self.fixtures, 'fixture-codes', '1.0.0', released=True, canonical_url=FIXTURE, + public_access=ACCESS_TYPE_NONE) + self.fixture_at = source( + self.fixtures, 'codes@fixture', canonical_url=f'{FIXTURE}-at', public_access=ACCESS_TYPE_NONE) + self.fixture_at_v1 = source( + self.fixtures, 'codes@fixture', '1.0.0', released=True, canonical_url=f'{FIXTURE}-at', + public_access=ACCESS_TYPE_NONE) + if self.registry: + # the curated global registry (design D24): shared canonicals to the owners chosen, none for private repos + for url, owner in ((ICD10, self.who), (ATC, self.who), (CIEL, self.ciel), (PBD, self.padam), + (DRAFT, self.drafts)): + GlobalURLRegistryFactory(url=url, namespace=owner.uri) + + resolve = staticmethod(fhir_resolve.resolve) + + def assertUnresolved(self, resource_type, url, version=None, kwargs=None, user=ANONYMOUS): # pylint: disable=invalid-name,too-many-arguments + with self.assertRaises(fhir_resolve.UnresolvedError) as context: + self.resolve(resource_type, url, version, kwargs, user) + self.assertEqual( + (context.exception.resource_type, context.exception.url), (resource_type, url.partition('|')[0])) + return context.exception + + def test_shared_canonicals_at_the_global_base(self): + version = self.resolve('CodeSystem', ICD10) + self.assertEqual(version.id, self.who_icd_2019.id) + self.assertEqual(version.canonical_url, ICD10) + + # the latest released version, not the newest one + self.assertEqual(self.resolve('CodeSystem', ATC).id, self.who_atc_2023.id) + self.assertEqual(self.resolve('CodeSystem', CIEL).id, self.ciel_v1.id) + + def test_shared_canonicals_at_an_owner_base(self): + sehatapp = {'org': 'sehatapp'} + self.assertEqual(self.resolve('CodeSystem', ICD10, kwargs=sehatapp).id, self.sehat_icd_v1.id) + self.assertEqual(self.resolve('CodeSystem', ATC, kwargs=sehatapp).id, self.sehat_atc_v2.id) + self.assertEqual(self.resolve('CodeSystem', ATC, kwargs={'org': 'WHO'}).id, self.who_atc_2023.id) + + def test_resource_type(self): + self.assertEqual(self.resolve('CodeSystem', PBD).id, self.pbd_source_v1.id) + self.assertEqual(self.resolve('ConceptMap', PBD).id, self.pbd_source_v1.id) + version = self.resolve('ValueSet', PBD) + self.assertEqual(version.id, self.pbd_collection_v1.id) + self.assertIsInstance(version, Collection) + + self.assertUnresolved('ValueSet', ICD10) + self.assertUnresolved('ValueSet', ICD10, kwargs={'org': 'sehatapp'}) + + def test_unknown_system(self): + error = self.assertUnresolved('CodeSystem', UNKNOWN) + self.assertEqual(error.status_code, 404) + self.assertIsNone(error.version) + self.assertUnresolved('CodeSystem', UNKNOWN, kwargs={'org': 'WHO'}) + self.assertUnresolved('ValueSet', UNKNOWN) + self.assertUnresolved('CodeSystem', '') + + def test_unknown_version(self): + error = self.assertUnresolved('CodeSystem', ICD10, 'nope') + self.assertEqual(error.version, 'nope') + + # the version of a repository that lost the resolution: the resolver never tries another one + self.assertUnresolved('CodeSystem', ICD10, 'v1.0') + self.assertEqual(self.resolve('CodeSystem', ICD10, 'v1.0', {'org': 'sehatapp'}).id, self.sehat_icd_v1.id) + + def test_named_version(self): + self.assertEqual(self.resolve('CodeSystem', ICD10, 'ICD102019').id, self.who_icd_2019.id) + # an unreleased version is served when named + self.assertEqual(self.resolve('CodeSystem', ATC, 'ATC_2024_01_19').id, self.who_atc_2024.id) + self.assertEqual(self.resolve('CodeSystem', f'{ATC}|2023AA').id, self.who_atc_2023.id) + # the version parameter wins over url|version, as in the resolver + self.assertEqual(self.resolve('CodeSystem', f'{ATC}|nope', '2023AA').id, self.who_atc_2023.id) + error = self.assertUnresolved('CodeSystem', f'{ATC}|nope') + self.assertEqual((error.url, error.version), (ATC, 'nope')) + # a second pipe is no version OCL holds: not found, never a server error + self.assertEqual(self.assertUnresolved('CodeSystem', f'{ATC}|2023AA|x').version, '2023AA|x') + + def test_head_only_when_named(self): + self.assertEqual(self.resolve('CodeSystem', ICD10, HEAD).id, self.who_icd.id) + self.assertEqual(self.resolve('CodeSystem', f'{ICD10}|HEAD').id, self.who_icd.id) + self.assertNotEqual(self.resolve('CodeSystem', ICD10).version, HEAD) + + def test_a_repository_with_nothing_released_is_not_visible(self): + self.assertUnresolved('CodeSystem', DRAFT) + self.assertUnresolved('CodeSystem', DRAFT, HEAD) + self.assertUnresolved('CodeSystem', DRAFT, HEAD, {'org': 'drafts'}, self.admin) + + source(self.drafts, 'draft', 'v1', released=True, canonical_url=DRAFT) + self.assertEqual(self.resolve('CodeSystem', DRAFT, HEAD).id, self.draft.id) + + def test_a_head_flagged_released_is_no_release(self): + """HEAD's released flag neither makes HEAD the default nor makes a repository visible""" + flagged = 'http://example.org/flagged-head' + head = source(self.hl7, 'flagged', released=True, canonical_url=flagged) + if self.registry: + GlobalURLRegistryFactory(url=flagged, namespace=self.hl7.uri) + for version, kwargs, user in ((None, None, ANONYMOUS), (HEAD, None, ANONYMOUS), + (None, {'org': 'HL7'}, self.admin), (HEAD, {'org': 'HL7'}, self.admin)): + self.assertUnresolved('CodeSystem', flagged, version, kwargs, user) + + released = source(self.hl7, 'flagged', 'v1', released=True, canonical_url=flagged) + self.assertEqual(self.resolve('CodeSystem', flagged).id, released.id) + self.assertEqual(self.resolve('CodeSystem', flagged, HEAD).id, head.id) + + def test_a_relative_url_keeps_its_own_resource_type(self): + """An OCL path names a Source or a Collection by itself; a reference of the other type is not found""" + acme = OrganizationFactory(mnemonic='acme') + source(acme, 'codes') + codes = source(acme, 'codes', 'v1', released=True) + collection(acme, 'codes') + value_set = collection(acme, 'codes', 'v1', released=True) + for url in ('/orgs/acme/CodeSystem/codes/', '/orgs/acme/ConceptMap/codes/', '/orgs/acme/sources/codes/'): + self.assertEqual(self.resolve('CodeSystem', url).id, codes.id) + self.assertEqual(self.resolve('ConceptMap', url).id, codes.id) + self.assertUnresolved('ValueSet', url) + for url in ('/orgs/acme/ValueSet/codes/', '/orgs/acme/collections/codes/'): + self.assertEqual(self.resolve('ValueSet', url).id, value_set.id) + self.assertUnresolved('CodeSystem', url) + self.assertUnresolved('ConceptMap', url) + self.assertEqual(self.resolve('CodeSystem', '/orgs/acme/CodeSystem/codes/|v1').id, codes.id) + self.assertUnresolved('ValueSet', '/orgs/acme/CodeSystem/codes/|v1') + + def test_fixture_namespace(self): + fixtures = {'org': 'FHIR-Fixtures'} + for user in (self.member, self.admin): + self.assertEqual(self.resolve('CodeSystem', FIXTURE, kwargs=fixtures, user=user).id, self.fixture_v1.id) + self.assertEqual( + self.resolve('CodeSystem', f'{FIXTURE}-at', '1.0.0', fixtures, user).id, self.fixture_at_v1.id) + self.assertEqual( + self.resolve('CodeSystem', '/orgs/FHIR-Fixtures/CodeSystem/codes@fixture/', kwargs=fixtures, + user=user).id, + self.fixture_at_v1.id) + for user in (ANONYMOUS, self.outsider, None): + self.assertUnresolved('CodeSystem', FIXTURE, kwargs=fixtures, user=user) + + # the global base shows public repositories only, to anyone + for user in (ANONYMOUS, self.member, self.admin): + self.assertUnresolved('CodeSystem', FIXTURE, user=user) + + def test_a_user_base(self): + user = UserProfileFactory(username='jdoe') + head = UserSourceFactory(user=user, mnemonic='mine', canonical_url='http://example.org/mine') + released = UserSourceFactory( + user=user, mnemonic='mine', version='1', released=True, canonical_url='http://example.org/mine') + self.assertEqual(self.resolve('CodeSystem', 'http://example.org/mine', kwargs={'user': 'jdoe'}).id, released.id) + self.assertEqual( + self.resolve('CodeSystem', 'http://example.org/mine', HEAD, kwargs={'user': 'jdoe'}).id, head.id) + + def test_an_unknown_owner_base(self): + self.assertUnresolved('CodeSystem', ICD10, kwargs={'org': 'nobody'}) + self.assertUnresolved('CodeSystem', ICD10, kwargs={'user': 'nobody'}) + + def test_a_deleted_repository(self): + self.ciel_source.is_active = False + self.ciel_source.save() + self.assertUnresolved('CodeSystem', CIEL) + + def test_another_owners_canonical_at_an_owner_base(self): + """With the registry, an owner base falls through to the global entry; without it, an owner base is strict.""" + if self.registry: + self.assertEqual(self.resolve('CodeSystem', CIEL, kwargs={'org': 'sehatapp'}).id, self.ciel_v1.id) + else: + self.assertUnresolved('CodeSystem', CIEL, kwargs={'org': 'sehatapp'}) + + def test_a_private_repository_that_resolves_hides_a_public_one(self): + """ + Access is checked after resolving, and nothing else is tried: when the private copy of a shared canonical + resolves (the lower id without the registry, the global entry's namespace with it), the public copy is not + found at the global base, while each owner base still finds its own. + """ + shared = 'http://example.org/shared-codes' + private = source(self.fixtures, 'shared', canonical_url=shared, public_access=ACCESS_TYPE_NONE) + private_v1 = source(self.fixtures, 'shared', 'v1', released=True, canonical_url=shared, + public_access=ACCESS_TYPE_NONE) + source(self.hl7, 'shared', canonical_url=shared) + public_v1 = source(self.hl7, 'shared', 'v1', released=True, canonical_url=shared) + self.assertLess(private.id, public_v1.id) + if self.registry: + GlobalURLRegistryFactory(url=shared, namespace=self.fixtures.uri) + + for user in (ANONYMOUS, self.member, self.admin): + self.assertUnresolved('CodeSystem', shared, user=user) + self.assertEqual( + self.resolve('CodeSystem', shared, kwargs={'org': 'FHIR-Fixtures'}, user=self.member).id, private_v1.id) + self.assertUnresolved('CodeSystem', shared, kwargs={'org': 'FHIR-Fixtures'}, user=self.outsider) + self.assertEqual(self.resolve('CodeSystem', shared, kwargs={'org': 'HL7'}).id, public_v1.id) + + def test_query_counts(self): + """ + A hot path: these counts may only fall. Without the registry: the toggle, the repository and its latest + released (or named) version; an owner base adds the owner, and a private repository its owner and the + caller's membership. The registry adds the entry and the owner of its namespace; at an owner base, the + owner's own entry instead of the global one, and the owner a second time. + """ + extra = 2 if self.registry else 0 + for count, args in ( + (3 + extra, ('CodeSystem', ICD10)), + (3 + extra, ('CodeSystem', ICD10, 'ICD102019')), + (4 + extra, ('ValueSet', PBD, None, {'org': 'padam'})), + (6 + extra, ('CodeSystem', FIXTURE, None, {'org': 'FHIR-Fixtures'}, self.member)), + ): + with self.subTest(args=args), self.assertNumQueries(count): + self.resolve(*args) + + +class FhirResolveWithoutRegistryTest(FhirResolveCases, OCLTestCase): + registry = False + + +class FhirResolveWithRegistryTest(FhirResolveCases, OCLTestCase): + registry = True + + def test_an_unregistered_canonical_is_unresolved_at_the_global_base(self): + """The registry's rule: at the global base only a global entry resolves, whatever repositories hold.""" + URLRegistry.objects.filter(url=ICD10).delete() + self.assertUnresolved('CodeSystem', ICD10) + self.assertEqual(self.resolve('CodeSystem', ICD10, kwargs={'org': 'WHO'}).id, self.who_icd_2019.id) diff --git a/core/repos/models.py b/core/repos/models.py index 540415ec3..9e2a99b61 100644 --- a/core/repos/models.py +++ b/core/repos/models.py @@ -17,7 +17,11 @@ def count(self): class Repository: @classmethod - def get(cls, criteria): + def get(cls, criteria, repo_type=None): + """The lowest-id repository matching criteria: a Source before a Collection, or only repo_type's.""" + if repo_type: + return repo_type.objects.filter(criteria).first() + repo = Source.objects.filter(criteria).first() if not repo: diff --git a/core/sources/models.py b/core/sources/models.py index 079a473ba..513263669 100644 --- a/core/sources/models.py +++ b/core/sources/models.py @@ -699,8 +699,11 @@ def save( else: self.__update_sequences(dirty_fields) - if self.id and 'canonical_url' in dirty_fields and self.active_url_registry_entries.exists(): - resolve_url_registry_entries.apply_async((self.id, self.resource_type), queue='default', permanent=False) + if self.id and 'canonical_url' in dirty_fields: + if self.active_url_registry_entries.exists(): + resolve_url_registry_entries.apply_async( + (self.id, self.resource_type), queue='default', permanent=False) + self.link_url_registry_entries() def __update_sequences(self, dirty_fields=[]): # pylint: disable=dangerous-default-value def should_update(is_seq, field): diff --git a/core/url_registry/models.py b/core/url_registry/models.py index c8f793f04..eb40f57d5 100644 --- a/core/url_registry/models.py +++ b/core/url_registry/models.py @@ -107,25 +107,42 @@ def lookup(cls, url, registry_owner=None): repo = None if entry: - repo = entry.lookup_entry() + repo = entry.find_repo() return repo, entry - def lookup_entry(self): + def find_repo(self, repo_type=None): + """ + The repository the entry points at now: the one with its url in its namespace (none without a namespace). + repo_type limits it to a Source or a Collection (find_repo_by_canonical_url). Reads only. + """ namespace_owner = self.namespace_owner - if namespace_owner: - repo = namespace_owner.find_repo_by_canonical_url(self.url) - self._set_repo(repo) - return repo + return namespace_owner.find_repo_by_canonical_url(self.url, repo_type) if namespace_owner else None - return None + def lookup_entry(self): + """ + find_repo, kept on the entry as `repo` (cleared when nothing is found): for an entry's create or update and + for a repository's creation or canonical URL change, never for a read + """ + repo = self.find_repo() + self._set_repo(repo) + return repo @classmethod def get_entry(cls, url, owner=None): entries = owner.url_registry_entries.filter(is_active=True) if owner else cls.get_active_global_entries() return entries.filter(url=url).first() + def unlink(self): + """Clears the cached repository, as before that repository is deleted""" + self._set_repo(None) + def _set_repo(self, repo): - if repo: + """ + Points the entry at repo, or at nothing; saves (and so reindexes) only when that changes, and only the link's + own columns, so an instance read before a concurrent edit (a new namespace, say) can't undo it + """ + link = (ContentType.objects.get_for_model(repo).id, repo.id) if repo else (None, None) + if (self.repo_type_id, self.repo_id) != link: self.repo = repo - self.save() + self.save(update_fields=['repo_type', 'repo_id', 'updated_at']) diff --git a/core/url_registry/tests_resolution.py b/core/url_registry/tests_resolution.py new file mode 100644 index 000000000..f50f9c566 --- /dev/null +++ b/core/url_registry/tests_resolution.py @@ -0,0 +1,606 @@ +""" +The shared canonical resolver (ConceptContainerModel.resolve_reference_expression) with the URL registry toggle off +and on: today's answers for its existing callers, the flags FHIR resolves with (repo_type, allow_head), reads that +never write a registry entry, the registry links kept on the write side instead, and $resolveReference's ?namespace=. +OpenConceptLab/ocl_online#451. +""" +from django.test import override_settings +from mock import patch +from pydash import get + +from core.collections.models import Collection +from core.collections.tests.factories import OrganizationCollectionFactory +from core.common.constants import HEAD +from core.common.models import ConceptContainerModel +from core.common.tasks import resolve_url_registry_entries +from core.common.tests import OCLTestCase, OCLAPITestCase +from core.orgs.tests.factories import OrganizationFactory +from core.sources.models import Source +from core.sources.tests.factories import OrganizationSourceFactory +from core.toggles.models import Toggle +from core.url_registry.documents import URLRegistryDocument +from core.url_registry.factories import GlobalURLRegistryFactory, OrganizationURLRegistryFactory +from core.url_registry.models import URLRegistry +from core.users.models import UserProfile + +REGISTRY_TOGGLE = 'URL_REGISTRY_IN_RESOLVE_REFERENCE_TOGGLE' +ICD10 = 'http://hl7.org/fhir/sid/icd-10' +PBD = 'http://pbd.com.np' +DRAFT = 'https://example.org/draft' +FLAGGED = 'https://example.org/flagged-head' + + +def set_registry_toggle(on): + """The toggle in every environment's column, so a test doesn't depend on which one settings.ENV reads.""" + Toggle.objects.filter(name=REGISTRY_TOGGLE).update( + dev=on, qa=on, demo=on, staging=on, who_staging=on, production=on) + assert Toggle.get(REGISTRY_TOGGLE) is on + + +def source(org, mnemonic, version=HEAD, **kwargs): + return OrganizationSourceFactory(organization=org, mnemonic=mnemonic, version=version, **kwargs) + + +def collection(org, mnemonic, version=HEAD, **kwargs): + return OrganizationCollectionFactory(organization=org, mnemonic=mnemonic, version=version, **kwargs) + + +class ResolverCorpusMixin: + """ + ICD-10 shared by WHO (first, so the lower id) and sehatapp, with WHO's released version row lacking the + canonical as in production; a Source and a Collection sharing a canonical in one org (padam); a repository with + no released version. + """ + def setUp(self): + super().setUp() + self.who = OrganizationFactory(mnemonic='WHO') + self.sehatapp = OrganizationFactory(mnemonic='sehatapp') + self.padam = OrganizationFactory(mnemonic='padam') + self.drafts = OrganizationFactory(mnemonic='drafts') + + self.who_icd = source(self.who, 'ICD-10-WHO', canonical_url=ICD10) + self.who_icd_2019 = source(self.who, 'ICD-10-WHO', 'ICD102019', released=True, canonical_url=None) + self.sehat_icd = source(self.sehatapp, 'ICD-10', canonical_url=ICD10) + self.sehat_icd_v1 = source(self.sehatapp, 'ICD-10', 'v1.0', released=True, canonical_url=ICD10) + + self.pbd_source = source(self.padam, 'pbd', canonical_url=PBD) + self.pbd_source_v1 = source(self.padam, 'pbd', 'v1', released=True, canonical_url=PBD) + self.pbd_collection = collection(self.padam, 'pbd-vs', canonical_url=PBD) + self.pbd_collection_v1 = collection(self.padam, 'pbd-vs', 'v1', released=True, canonical_url=PBD) + + self.draft = source(self.drafts, 'draft', canonical_url=DRAFT) + + +class StrictDefaultCases: + """With the toggle off and on (the toggle classes mix this in).""" + def test_a_head_flagged_released_is_never_the_strict_default(self): + """allow_head=False skips HEAD even when HEAD is flagged released; the default keeps today's answer""" + head = source(self.drafts, 'flagged', released=True, canonical_url=FLAGGED) + GlobalURLRegistryFactory(url=FLAGGED, namespace=self.drafts.uri) # read only with the toggle on + + version, _ = Source.resolve_reference_expression(FLAGGED) + self.assertEqual(version.id, head.id) + for repo_type in (None, Source): + version, _ = Source.resolve_reference_expression(FLAGGED, repo_type=repo_type, allow_head=False) + self.assertIsNone(version.id) + self.assertTrue(version.is_fqdn) + version, _ = Source.resolve_reference_expression( + FLAGGED, version=HEAD, repo_type=repo_type, allow_head=False) + self.assertEqual(version.id, head.id) + + released = source(self.drafts, 'flagged', 'v1', released=True, canonical_url=FLAGGED) + version, _ = Source.resolve_reference_expression(FLAGGED, repo_type=Source, allow_head=False) + self.assertEqual(version.id, released.id) + + +class ResolverToggleOffTest(StrictDefaultCases, ResolverCorpusMixin, OCLTestCase): + """No registry: the lowest-id active repository with the canonical, Source before Collection.""" + def setUp(self): + set_registry_toggle(False) + super().setUp() + + def test_defaults_keep_todays_answers(self): + version, entry = Source.resolve_reference_expression(ICD10) + self.assertEqual(version.id, self.who_icd_2019.id) + self.assertEqual(version.canonical_url, ICD10) # filled in from the request, as before + self.assertTrue(version.is_fqdn) + self.assertIsNone(entry) + + # a namespace limits the match to that owner's repositories + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/') + self.assertEqual(version.id, self.sehat_icd_v1.id) + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/padam/') + self.assertIsNone(version.id) + + # type-blind: a Source wins over a Collection with the same canonical, whatever class asks + version, _ = Collection.resolve_reference_expression(PBD) + self.assertEqual(version.id, self.pbd_source_v1.id) + self.assertIsInstance(version, Source) + + # nothing released: HEAD + version, _ = Source.resolve_reference_expression(DRAFT) + self.assertEqual(version.id, self.draft.id) + + # a version row's own canonical matches, and is returned as it is + self.sehat_icd.canonical_url = 'http://example.org/new-icd' + self.sehat_icd.save() + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/') + self.assertEqual(version.id, self.sehat_icd_v1.id) + + # unknown: an unsaved Source + version, _ = Collection.resolve_reference_expression('http://example.org/unknown') + self.assertIsNone(version.id) + self.assertIsInstance(version, Source) + + def test_repo_type_resolves_only_to_that_type(self): + version, _ = Source.resolve_reference_expression(PBD, repo_type=Collection) + self.assertEqual(version.id, self.pbd_collection_v1.id) + self.assertIsInstance(version, Collection) + + version, _ = Source.resolve_reference_expression(PBD, repo_type=Source) + self.assertEqual(version.id, self.pbd_source_v1.id) + + version, _ = Source.resolve_reference_expression(ICD10, repo_type=Collection) + self.assertIsNone(version.id) + self.assertIsInstance(version, Collection) + + version, _ = Source.resolve_reference_expression(self.pbd_collection.uri, repo_type=Source) + self.assertIsNone(version.id) + version, _ = Source.resolve_reference_expression(self.pbd_collection.uri, repo_type=Collection) + self.assertEqual(version.id, self.pbd_collection_v1.id) + + def test_repo_type_matches_the_repository_canonical_on_its_head(self): + self.sehat_icd.canonical_url = 'http://example.org/new-icd' + self.sehat_icd.save() + + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/', repo_type=Source) + self.assertIsNone(version.id) + + version, _ = Source.resolve_reference_expression( + 'http://example.org/new-icd', '/orgs/sehatapp/', repo_type=Source) + self.assertEqual(version.id, self.sehat_icd_v1.id) + + def test_repo_type_serves_only_active_versions(self): + self.sehat_icd_v1.is_active = False + self.sehat_icd_v1.save() + + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/', 'v1.0') + self.assertEqual(version.id, self.sehat_icd_v1.id) + + version, _ = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/', 'v1.0', repo_type=Source) + self.assertIsNone(version.id) + + +class ResolverToggleOnTest(StrictDefaultCases, ResolverCorpusMixin, OCLTestCase): + """The registry's order: the owner's entry, the owner's repositories, the global entry; nothing else.""" + def setUp(self): + set_registry_toggle(True) + super().setUp() + self.icd_entry = GlobalURLRegistryFactory(url=ICD10, namespace=self.who.uri) + self.pbd_entry = GlobalURLRegistryFactory(url=PBD, namespace=self.padam.uri) + GlobalURLRegistryFactory(url=DRAFT, namespace=self.drafts.uri) + + def test_defaults_keep_todays_answers(self): + version, entry = Source.resolve_reference_expression(ICD10) + self.assertEqual(version.id, self.who_icd_2019.id) + self.assertEqual(entry, self.icd_entry) + + version, entry = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/') + self.assertEqual(version.id, self.sehat_icd_v1.id) + self.assertIsNone(entry) + + version, entry = Collection.resolve_reference_expression(PBD) + self.assertEqual(version.id, self.pbd_source_v1.id) + self.assertEqual(entry, self.pbd_entry) + + version, _ = Source.resolve_reference_expression(DRAFT) + self.assertEqual(version.id, self.draft.id) + + version, entry = Source.resolve_reference_expression('http://example.org/unregistered') + self.assertIsNone(version.id) + self.assertIsNone(entry) + + def test_repo_type_finds_that_type_in_the_entry_namespace(self): + version, entry = Source.resolve_reference_expression(PBD, repo_type=Collection) + self.assertEqual(version.id, self.pbd_collection_v1.id) + self.assertEqual(entry, self.pbd_entry) + + version, entry = Source.resolve_reference_expression(PBD, repo_type=Source) + self.assertEqual(version.id, self.pbd_source_v1.id) + self.assertEqual(entry, self.pbd_entry) + + def test_an_entry_without_a_repository_of_the_type_is_unresolved_without_falling_through(self): + collection(self.sehatapp, 'icd-vs', canonical_url=ICD10) + collection(self.sehatapp, 'icd-vs', 'v1', released=True, canonical_url=ICD10) + + version, entry = Source.resolve_reference_expression(ICD10, repo_type=Collection) + self.assertIsNone(version.id) + self.assertIsInstance(version, Collection) + self.assertEqual(entry, self.icd_entry) + + def test_an_owner_entry_resolves_by_type_in_its_namespace(self): + owner_entry = OrganizationURLRegistryFactory(organization=self.sehatapp, url=PBD, namespace=self.padam.uri) + + version, entry = Source.resolve_reference_expression(PBD, '/orgs/sehatapp/', repo_type=Collection) + self.assertEqual(version.id, self.pbd_collection_v1.id) + self.assertEqual(entry, owner_entry) + + version, entry = Source.resolve_reference_expression(ICD10, '/orgs/sehatapp/', repo_type=Source) + self.assertEqual(version.id, self.sehat_icd_v1.id) + self.assertIsNone(entry) + + def test_repo_type_skips_a_deleted_repository(self): + self.who_icd.is_active = False + self.who_icd.save() + + version, _ = Source.resolve_reference_expression(ICD10) + self.assertEqual(version.id, self.who_icd_2019.id) + + version, entry = Source.resolve_reference_expression(ICD10, repo_type=Source) + self.assertIsNone(version.id) + self.assertEqual(entry, self.icd_entry) + + def test_resolving_never_writes_a_registry_entry(self): + OrganizationURLRegistryFactory(organization=self.sehatapp, url=PBD, namespace=self.padam.uri) + + with patch.object(URLRegistry, 'save') as save_mock: + Source.resolve_reference_expression(ICD10) + Source.resolve_reference_expression(PBD, '/orgs/sehatapp/') + Source.resolve_reference_expression(PBD, repo_type=Collection) + + save_mock.assert_not_called() + self.icd_entry.refresh_from_db() + self.assertIsNone(self.icd_entry.repo_id) + + +class URLRegistryReadsTest(ResolverCorpusMixin, OCLTestCase): + def setUp(self): + super().setUp() + self.entry = GlobalURLRegistryFactory(url=PBD, namespace=self.padam.uri) + + def test_lookup_reads_without_writing(self): + with patch.object(URLRegistry, 'save') as save_mock: + repo, entry = URLRegistry.lookup(PBD) + + save_mock.assert_not_called() + self.assertEqual(repo, self.pbd_source) + self.assertEqual(entry, self.entry) + + def test_find_repo(self): + self.assertEqual(self.entry.find_repo(), self.pbd_source) + self.assertEqual(self.entry.find_repo(Source), self.pbd_source) + self.assertEqual(self.entry.find_repo(Collection), self.pbd_collection) + self.assertIsNone(URLRegistry(url=PBD).find_repo()) + + def test_lookup_entry_keeps_the_link_and_saves_only_on_a_change(self): + """The write path: lookup_entry links the repository found, clears a link that no longer resolves, and saves + (so reindexes) only when the link changes""" + with patch.object(URLRegistry, 'save', wraps=self.entry.save) as save_mock: + self.assertEqual(self.entry.lookup_entry(), self.pbd_source) + self.assertEqual(self.entry.lookup_entry(), self.pbd_source) + self.assertEqual(save_mock.call_count, 1) + self.entry.refresh_from_db() + self.assertEqual(self.entry.repo, self.pbd_source) + + self.entry.namespace = self.drafts.uri + self.entry.save() # as the entry's update does, before its lookup_entry + self.assertIsNone(self.entry.lookup_entry()) + self.entry.refresh_from_db() + self.assertEqual( + (self.entry.namespace, self.entry.repo_id, self.entry.repo_type_id), (self.drafts.uri, None, None)) + + with patch.object(URLRegistry, 'save') as save_mock: + self.assertIsNone(URLRegistry(url='http://example.org/nowhere', namespace=self.padam.uri).lookup_entry()) + save_mock.assert_not_called() + + +def run_inline(args, **_): + """resolve_url_registry_entries.apply_async, run in the test's own transaction (tests have no broker)""" + return resolve_url_registry_entries(*args) + + +@override_settings(ES_SYNC=True, ELASTICSEARCH_DSL_AUTOSYNC=True, ELASTICSEARCH_DSL_AUTO_REFRESH=True) +class URLRegistryLinkTest(OCLAPITestCase): + """ + An entry's cached repository is kept on the write side, since reads no longer write: a repository created after + its entry links it, and a canonical URL change clears the entries of the old URL and fills those of the new one, + in the registry API and in the registry's search index. Indexing is switched on here because a test that pauses + it leaves it paused (resume_indexing does nothing in TEST_MODE). + """ + OLD, NEW = 'https://example.org/acme/codes', 'https://example.org/acme/codes-v2' + + def setUp(self): + super().setUp() + self.auth = {'HTTP_AUTHORIZATION': 'Token ' + UserProfile.objects.get(username='ocladmin').get_token()} + self.org = OrganizationFactory(mnemonic='acme-registry') + + def create_entry(self, base, url): + response = self.client.post( + f'{base}url-registry/', {'name': url, 'url': url, 'namespace': self.org.uri}, format='json', **self.auth) + self.assertEqual(response.status_code, 201, response.data) + return f"{base}url-registry/{response.data['id']}/", response.data['id'] + + def linked_repo(self, entry): + """The entry's repository in the registry API and in its search document (the repo and repoOwner facets)""" + path, entry_id = entry + response = self.client.get(path, **self.auth) + self.assertEqual(response.status_code, 200) + URLRegistryDocument._index.refresh() # pylint: disable=protected-access + hits = URLRegistryDocument.search().filter('ids', values=[str(entry_id)]).execute() + self.assertEqual(len(hits), 1) + return get(response.data, 'repo.url'), (get(hits[0], 'repo'), get(hits[0], 'repo_owner')) + + def test_a_repository_created_after_its_entries_links_them_and_a_canonical_change_moves_the_links(self): + old_entries = [self.create_entry('/', self.OLD), self.create_entry(self.org.uri, self.OLD)] + new_entry = self.create_entry(self.org.uri, self.NEW) + for entry in old_entries + [new_entry]: + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + + response = self.client.post(f'{self.org.uri}sources/', { + 'id': 'codes', 'name': 'codes', 'short_code': 'codes', 'canonical_url': self.OLD, + 'default_locale': 'en', 'supported_locales': 'en'}, format='json', **self.auth) + self.assertEqual(response.status_code, 201, response.data) + codes = f'{self.org.uri}sources/codes/' + for entry in old_entries: + self.assertEqual(self.linked_repo(entry), (codes, ('codes', 'acme-registry'))) + self.assertEqual(self.linked_repo(new_entry), (None, (None, None))) + + with patch.object(resolve_url_registry_entries, 'apply_async', side_effect=run_inline) as queued: + response = self.client.put(codes, {'canonical_url': self.NEW}, format='json', **self.auth) + self.assertEqual(response.status_code, 200, response.data) + queued.assert_called_once() # the entries the repository was linked to, through the existing task + for entry in old_entries: + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + self.assertEqual(self.linked_repo(new_entry), (codes, ('codes', 'acme-registry'))) + + def test_a_version_given_an_entrys_canonical_links_it_and_loses_it_again(self): + """A version row that gains the canonical an unlinked entry names is what the registry's lookup finds""" + head_entry = self.create_entry(self.org.uri, self.OLD) + head = source(self.org, 'codes', canonical_url=self.OLD) + source(self.org, 'codes', 'v1', released=True, canonical_url=self.OLD) + head_link = (head.uri, ('codes', 'acme-registry')) + self.assertEqual(self.linked_repo(head_entry), head_link) # a new version doesn't take HEAD's entry + + new_entries = [self.create_entry('/', self.NEW), self.create_entry(self.org.uri, self.NEW)] + version_url = f'{head.uri}v1/' + response = self.client.put(version_url, {'canonical_url': self.NEW}, format='json', **self.auth) + self.assertEqual(response.status_code, 200, response.data) + for entry in new_entries: + self.assertEqual(self.linked_repo(entry), (version_url, ('codes', 'acme-registry'))) + self.assertEqual(self.linked_repo(head_entry), head_link) + + with patch.object(resolve_url_registry_entries, 'apply_async', side_effect=run_inline) as queued: + response = self.client.put( + version_url, {'canonical_url': 'https://example.org/acme/elsewhere'}, format='json', **self.auth) + self.assertEqual(response.status_code, 200, response.data) + queued.assert_called_once() + for entry in new_entries: + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + self.assertEqual(self.linked_repo(head_entry), head_link) + + def entry_state(self, entry): + """(exists, active) for an entry, read from the database""" + row = URLRegistry.objects.filter(id=entry[1]).values_list('is_active', flat=True).first() + return row is not None, bool(row) + + def create_source(self, canonical_url): + response = self.client.post(f'{self.org.uri}sources/', { + 'id': 'codes', 'name': 'codes', 'short_code': 'codes', 'canonical_url': canonical_url, + 'default_locale': 'en', 'supported_locales': 'en'}, format='json', **self.auth) + self.assertEqual(response.status_code, 201, response.data) + return f'{self.org.uri}sources/codes/' + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_deleting_a_repository_unlinks_its_entries_instead_of_deleting_them(self, _): + """ + A hard delete of a repository (HEAD and versions) keeps every registry entry linked to it, global and + owner-scoped alike, unlinked; recreating the repository links them again + """ + entries = [self.create_entry('/', self.OLD), self.create_entry(self.org.uri, self.OLD)] + codes = self.create_source(self.OLD) + source(self.org, 'codes', 'v1', released=True, canonical_url=self.OLD) + for entry in entries: + self.assertEqual(self.linked_repo(entry), (codes, ('codes', 'acme-registry'))) + + response = self.client.delete(codes, **self.auth) + self.assertEqual(response.status_code, 204, response.data) + self.assertFalse(Source.objects.filter(mnemonic='codes', organization=self.org).exists()) + for entry in entries: + self.assertEqual(self.entry_state(entry), (True, True)) + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + + self.assertEqual(self.create_source(self.OLD), codes) + for entry in entries: + self.assertEqual(self.linked_repo(entry), (codes, ('codes', 'acme-registry'))) + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_deleting_a_version_relinks_its_entry_to_what_it_resolves_to_now(self, _): + """An entry linked to a version row survives that version's delete and moves to the released sibling""" + source(self.org, 'codes', canonical_url=self.OLD) + source(self.org, 'codes', 'v1', released=True, canonical_url=self.NEW, is_latest_version=False) + source(self.org, 'codes', 'v2', released=True, canonical_url=self.NEW, is_latest_version=True) + entry = self.create_entry('/', self.NEW) + v1_url, v2_url = f'{self.org.uri}sources/codes/v1/', f'{self.org.uri}sources/codes/v2/' + self.assertEqual(self.linked_repo(entry), (v1_url, ('codes', 'acme-registry'))) + + response = self.client.delete(v1_url, **self.auth) + self.assertEqual(response.status_code, 204, getattr(response, 'data', None)) + self.assertEqual(self.entry_state(entry), (True, True)) + self.assertEqual(self.linked_repo(entry), (v2_url, ('codes', 'acme-registry'))) + + def fail_once(self): + """A delete that fails part-way: after the entries are unlinked and, for a HEAD, its versions deleted""" + return patch.object(ConceptContainerModel, 'delete_version_checksum_maps', side_effect=RuntimeError('fail')) + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_a_failed_repository_delete_keeps_its_entries_and_a_retry_finishes_it(self, _): + entries = [self.create_entry('/', self.OLD), self.create_entry(self.org.uri, self.OLD)] + codes = self.create_source(self.OLD) + source(self.org, 'codes', 'v1', released=True, canonical_url=self.OLD) + + with self.fail_once(): + response = self.client.delete(codes, **self.auth) + self.assertEqual(response.status_code, 400, response.data) # delete_source reports the failure + for entry in entries: + self.assertEqual(self.entry_state(entry), (True, True)) + + response = self.client.delete(codes, **self.auth) + self.assertEqual(response.status_code, 204, response.data) + for entry in entries: + self.assertEqual(self.entry_state(entry), (True, True)) + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + + self.assertEqual(self.create_source(self.OLD), codes) + for entry in entries: + self.assertEqual(self.linked_repo(entry), (codes, ('codes', 'acme-registry'))) + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_a_retried_repository_delete_relinks_entries_its_failed_attempt_unlinked(self, _): + """The retry finds the entries by the deleted rows' canonical URL and links them to what is left""" + entry = self.create_entry('/', self.OLD) + codes = self.create_source(self.OLD) + other = source(self.org, 'codes-2', canonical_url=self.OLD) + self.assertEqual(self.linked_repo(entry), (codes, ('codes', 'acme-registry'))) + + with self.fail_once(): + self.assertEqual(self.client.delete(codes, **self.auth).status_code, 400) + self.assertEqual(self.entry_state(entry), (True, True)) + + self.assertEqual(self.client.delete(codes, **self.auth).status_code, 204) + self.assertEqual(URLRegistry.objects.get(id=entry[1]).repo_id, other.id) + self.assertEqual(self.linked_repo(entry), (other.uri, ('codes-2', 'acme-registry'))) + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_a_failed_version_delete_keeps_its_entry_and_a_retry_moves_it(self, _): + source(self.org, 'codes', canonical_url=self.OLD) + source(self.org, 'codes', 'v1', released=True, canonical_url=self.NEW, is_latest_version=False) + v2 = source(self.org, 'codes', 'v2', released=True, canonical_url=self.NEW, is_latest_version=True) + entry = self.create_entry('/', self.NEW) + v1_url, v2_url = f'{self.org.uri}sources/codes/v1/', f'{self.org.uri}sources/codes/v2/' + self.assertEqual(self.linked_repo(entry), (v1_url, ('codes', 'acme-registry'))) + + with self.fail_once(), self.assertRaises(RuntimeError): + self.client.delete(v1_url, **self.auth) + self.assertEqual(self.entry_state(entry), (True, True)) + + response = self.client.delete(v1_url, **self.auth) + self.assertEqual(response.status_code, 204, getattr(response, 'data', None)) + self.assertEqual(self.entry_state(entry), (True, True)) + self.assertEqual(URLRegistry.objects.get(id=entry[1]).repo_id, v2.id) + self.assertEqual(self.linked_repo(entry), (v2_url, ('codes', 'acme-registry'))) + + def test_a_stale_entry_saves_only_its_link(self): + """Saving the cached link writes only the link columns, so a stale instance can't undo a concurrent edit""" + other = OrganizationFactory(mnemonic='acme-other') + codes = source(self.org, 'codes', canonical_url=self.OLD) + entry = self.create_entry('/', self.OLD) + stale = URLRegistry.objects.get(id=entry[1]) + self.assertEqual(stale.repo, codes) + + URLRegistry.objects.filter(id=stale.id).update(namespace=other.uri, name='redirected') + stale.unlink() + row = URLRegistry.objects.get(id=stale.id) + self.assertEqual( + (row.namespace, row.name, row.repo_type_id, row.repo_id), (other.uri, 'redirected', None, None)) + self.assertEqual(self.linked_repo(entry), (None, (None, None))) + + stale.lookup_entry() # the stale namespace still finds codes; only the link is written + row = URLRegistry.objects.get(id=stale.id) + self.assertEqual((row.namespace, row.name, row.repo_id), (other.uri, 'redirected', codes.id)) + self.assertEqual(self.linked_repo(entry), (codes.uri, ('codes', 'acme-registry'))) + + @patch('core.common.models.delete_s3_objects') # export files: no broker in tests + def test_a_namespace_redirected_during_a_delete_is_kept_and_resolved(self, _): + """The entry is redirected to another namespace between the delete loading it and unlinking it""" + other = OrganizationFactory(mnemonic='acme-other') + source(self.org, 'codes', canonical_url=self.OLD) + other_codes = source(other, 'codes', canonical_url=self.OLD) + entry = self.create_entry('/', self.OLD) + unlink = URLRegistry.unlink + + def unlink_after_a_concurrent_redirect(instance): + URLRegistry.objects.filter(id=instance.id).update(namespace=other.uri) + unlink(instance) + + with patch.object(URLRegistry, 'unlink', autospec=True, side_effect=unlink_after_a_concurrent_redirect): + self.assertEqual(self.client.delete(f'{self.org.uri}sources/codes/', **self.auth).status_code, 204) + row = URLRegistry.objects.get(id=entry[1]) + self.assertEqual((row.namespace, row.repo_id), (other.uri, other_codes.id)) + self.assertEqual(self.linked_repo(entry), (other_codes.uri, ('codes', 'acme-other'))) + + def test_a_collection_created_after_its_entry_links_it(self): + entry = GlobalURLRegistryFactory(url=self.OLD, namespace=self.org.uri) + source(self.org, 'codes', canonical_url=self.OLD.replace('codes', 'other')) + value_set = collection(self.org, 'codes-vs', canonical_url=self.OLD) + entry.refresh_from_db() + self.assertEqual(entry.repo, value_set) + + # neither a new version row nor a later repository with the same canonical takes the entry over + collection(self.org, 'codes-vs', 'v1', released=True, canonical_url=self.OLD) + collection(self.org, 'codes-vs-2', canonical_url=self.OLD) + entry.refresh_from_db() + self.assertEqual(entry.repo, value_set) + + def test_entries_elsewhere_are_left_alone(self): + other = OrganizationFactory(mnemonic='acme-other') + entry = GlobalURLRegistryFactory(url=self.OLD, namespace=other.uri) + inactive = GlobalURLRegistryFactory(url=self.NEW, namespace=self.org.uri, is_active=False) + source(self.org, 'codes', canonical_url=self.OLD) + source(self.org, 'codes-2', canonical_url=self.NEW) + for unlinked in (entry, inactive): + unlinked.refresh_from_db() + self.assertIsNone(unlinked.repo_id) + + +class ResolveReferenceNamespaceTest(OCLAPITestCase): + """$resolveReference's ?namespace=: the namespace of every expression that doesn't name its own.""" + def setUp(self): + super().setUp() + self.token = UserProfile.objects.get(username='ocladmin').get_token() + self.org_a = OrganizationFactory(mnemonic='org-a') + self.org_b = OrganizationFactory(mnemonic='org-b') + self.source_a = source(self.org_a, 'codes', canonical_url=ICD10) + self.source_a_v1 = source(self.org_a, 'codes', 'v1', released=True, canonical_url=ICD10) + self.source_b = source(self.org_b, 'codes', canonical_url=ICD10) + self.source_b_v1 = source(self.org_b, 'codes', 'v1', released=True, canonical_url=ICD10) + GlobalURLRegistryFactory(url=ICD10, namespace=self.org_a.uri) + + def resolve(self, expressions, query=''): + response = self.client.post( + f'/$resolveReference/{query}', expressions, HTTP_AUTHORIZATION='Token ' + self.token, format='json') + self.assertEqual(response.status_code, 200) + return [item['result']['version_url'] if item['resolved'] else None for item in response.data] + + def test_with_the_registry(self): + set_registry_toggle(True) + a_v1, b_v1 = self.source_a_v1.uri, self.source_b_v1.uri + + self.assertEqual(self.resolve([{'url': ICD10}]), [a_v1]) + self.assertEqual(self.resolve([{'url': ICD10}, ICD10], '?namespace=/orgs/org-b/'), [b_v1, b_v1]) + self.assertEqual( + self.resolve([{'url': ICD10, 'namespace': '/orgs/org-a/'}, {'url': ICD10, 'namespace': None}], + '?namespace=/orgs/org-b/'), + [a_v1, a_v1]) + self.assertEqual(self.resolve([{'url': ICD10}], '?namespace='), [a_v1]) + + def test_one_expression_query_count(self): + """ + A hot path: these counts may only fall. Without the registry: the token (3), the toggle, the repository, its + latest released version and the serializer (2); ?namespace= adds a toggle read. The registry adds the global + entry, its namespace owner and a Source check before the read; there ?namespace= also swaps the global entry + and its owner for the namespace owner and its own entry. + """ + for registry, query, count in ((False, '', 8), (False, '?namespace=/orgs/org-b/', 9), + (True, '', 11), (True, '?namespace=/orgs/org-b/', 12)): + with self.subTest(registry=registry, query=query): + set_registry_toggle(registry) + with self.assertNumQueries(count): + self.resolve([{'url': ICD10}], query) + + def test_without_the_registry_the_parameter_is_not_used(self): + set_registry_toggle(False) + a_v1, b_v1 = self.source_a_v1.uri, self.source_b_v1.uri + + self.assertEqual(self.resolve([{'url': ICD10}], '?namespace=/orgs/org-b/'), [a_v1]) + self.assertEqual(self.resolve([{'url': ICD10}], '?namespace=/orgs/nobody/'), [a_v1]) + self.assertEqual(self.resolve([{'url': ICD10, 'namespace': '/orgs/org-b/'}], '?namespace=/orgs/org-a/'), [b_v1]) From e7eaa21d08bea1bddbe92ba8d3c73926888502b6 Mon Sep 17 00:00:00 2001 From: Jonathan Payne Date: Thu, 22 Oct 2026 12:23:24 -0400 Subject: [PATCH 03/18] OpenConceptLab/ocl_online#451 | FHIR search and read return the releases a caller may view CodeSystem, ValueSet and ConceptMap search and read share core/fhir/search.py. Search lists released versions with real totals, paging links and SUBSETTED summaries. Global reads use an opaque id per release; owner reads use the mnemonic and a named version or the newest visible release. meta.versionId distinguishes releases at both bases. Search supports SVCM parameters, token systems, date comparisons, comma alternatives and repeated criteria. String searches ignore case and accents except with :exact; identifiers and canonical URLs match the values resources show. CodeSystem reads include all concepts up to 1,000, then report not-present. A release is visible only with a viewable active HEAD. The global base shows public content; owner bases apply the caller's rights. Source and reference filters can match any owner's source the caller may view. ConceptMap creation by PUT or PATCH checks ownership, and deletion of a missing map returns 404. Search evaluates its criteria once, batches canonical lookups, accepts at most 20 values per parameter and 50 overall, and bounds database work to five seconds. Tests cover search semantics, paging, read content, access, write ownership, query counts and too-costly outcomes. Work package WP3 of OpenConceptLab/ocl_online#451. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R8hw7w8753DDSCezrmrDA8 --- core/bundles/serializers.py | 22 +- core/code_systems/serializers.py | 48 +- core/code_systems/tests.py | 47 +- core/code_systems/views.py | 30 +- core/common/fhir_helpers.py | 54 - core/common/models.py | 3 + core/common/tests.py | 53 - core/concept_maps/serializers.py | 26 +- core/concept_maps/tests.py | 27 +- core/concept_maps/views.py | 41 +- core/fhir/resolve.py | 155 ++- core/fhir/search.py | 662 ++++++++++ core/fhir/tests_protocol.py | 9 +- core/fhir/tests_resolve.py | 23 + core/fhir/tests_search.py | 1127 +++++++++++++++++ core/integration_tests/tests_code_systems.py | 1 + .../tests_fhir_owner_access.py | 66 + core/value_sets/serializers.py | 7 +- core/value_sets/tests.py | 6 +- core/value_sets/views.py | 32 +- 20 files changed, 2165 insertions(+), 274 deletions(-) create mode 100644 core/fhir/search.py create mode 100644 core/fhir/tests_search.py diff --git a/core/bundles/serializers.py b/core/bundles/serializers.py index 8fbd307cd..be9a7ee24 100644 --- a/core/bundles/serializers.py +++ b/core/bundles/serializers.py @@ -21,7 +21,7 @@ class BundleLinkSerializer(CommonSerializer): class BundleMetaSerializer(CommonSerializer): - lastUpdated = DateTimeField(read_only=True, default=datetime.now()) + lastUpdated = DateTimeField(read_only=True, default=datetime.now) class BundleExtensionSerializer(CommonSerializer): @@ -33,7 +33,9 @@ class BundleExtensionSerializer(CommonSerializer): class BundleEntrySerializer(CommonSerializer): + fullUrl = CharField(required=False) resource = JSONField() + search = JSONField(required=False) class FHIRBundleSerializer(CommonSerializer): @@ -53,17 +55,29 @@ def get_resource_type(cls, _obj): return 'Bundle' @classmethod - def convert_to_entry(cls, resources): + def convert_to_entry(cls, resources, base_url): + """ + Searchset entries: each resource a match with its fullUrl (base_url is the base and resource type), and a + resource that couldn't be represented (an OperationOutcome) an outcome. + """ entry = [] for resource in resources: - entry.append({'resource': resource}) + if resource.get('resourceType') == 'OperationOutcome': + entry.append({'resource': resource, 'search': {'mode': 'outcome'}}) + else: + entry.append( + {'fullUrl': f"{base_url}/{resource['id']}", 'resource': resource, 'search': {'mode': 'match'}}) return entry def get_link(self, _): + """self, previous and next, for the page the paginator holds; only self without one (_count=0)""" paginator = self.context.get('paginator') + if paginator is None: + return BundleLinkSerializer( + [{'relation': 'self', 'url': self.context['request'].build_absolute_uri()}], many=True).data links = [{'relation': 'self', 'url': paginator.get_current_page_url()}] if paginator.has_previous(): - links.append({'relation': 'prev', 'url': paginator.get_previous_page_url()}) + links.append({'relation': 'previous', 'url': paginator.get_previous_page_url()}) if paginator.has_next(): links.append({'relation': 'next', 'url': paginator.get_next_page_url()}) return BundleLinkSerializer(links, many=True).data diff --git a/core/code_systems/serializers.py b/core/code_systems/serializers.py index 635d15261..e81819d2c 100644 --- a/core/code_systems/serializers.py +++ b/core/code_systems/serializers.py @@ -13,6 +13,8 @@ from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer from core.concepts.models import Concept, ConceptName from core.concepts.serializers import ConceptDetailSerializer +from core.fhir import search as fhir_search +from core.fhir.search import CanonicalUrlField, FhirRepositorySerializerMixin from core.parameters.serializers import ParametersSerializer from core.sources.models import Source from core.sources.serializers import SourceCreateOrUpdateSerializer @@ -89,10 +91,10 @@ def to_internal_value(self, data): return {'descriptions': [{'description': data, 'locale': settings.DEFAULT_LOCALE, 'locale_preferred': True}]} def to_representation(self, value): - descriptions = value.descriptions_for_default_locale - if descriptions and len(descriptions) > 0: - return value.descriptions_for_default_locale[0] - return None + """The first description in the default locale, from the descriptions a read prefetches.""" + descriptions = [description.name for description in value.descriptions.all() + if description.locale == settings.DEFAULT_LOCALE and not description.retired] + return descriptions[0] if descriptions else None class CodeSystemConceptSerializer(ReadSerializerMixin, serializers.Serializer): @@ -149,13 +151,9 @@ def to_internal_value(self, data): return {'concepts': concepts.validated_data} def to_representation(self, value): - # limit to 1000 concepts by default - # TODO: support graphQL to go around the limit - if self.context.get('has_many', False): - limit = 25 - else: - limit = 1000 - return CodeSystemConceptSerializer(value.concepts.order_by('id')[:limit], many=True).data + """Every concept, retired ones included, up to the limit; none above it (content not-present).""" + concepts = fhir_search.code_system_concepts(value)[1] + return CodeSystemConceptSerializer(concepts, many=True).data if concepts else [] class TextField(ReadSerializerMixin, serializers.Serializer): @@ -178,10 +176,11 @@ def to_representation(self, instance): return super().to_representation(obj) -class CodeSystemDetailSerializer(serializers.ModelSerializer): +class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.ModelSerializer): + content_field = 'concept' resourceType = SerializerMethodField(method_name='get_resource_type') id = CharField(source='mnemonic') - url = CharField(source='canonical_url', required=False) + url = CanonicalUrlField(source='canonical_url', required=False) title = CharField(source='full_name', required=False) status = StatusField(source='*') language = CharField(source='default_locale', required=False) @@ -210,21 +209,20 @@ class Meta: 'copyright', 'revisionDate', 'experimental', 'caseSensitive', 'compositional', 'versionNeeded', 'collectionReference', 'hierarchyMeaning', 'concept', 'text', 'filters') - def __new__(cls, *args, **kwargs): - if kwargs.get('many', False): - context = kwargs.get('context', {}) - context.update({'has_many': True}) - kwargs.update({'context': context}) - - return super().__new__(cls, *args, **kwargs) - @staticmethod def get_resource_type(_): return RESOURCE_TYPE + def get_total_count(self, obj): + """A read's concept count; a search's summary has none (it only checks the limit).""" + return None if self.context.get('has_many') else fhir_search.code_system_concepts(obj)[0] + @staticmethod - def get_total_count(obj): - return obj.concepts.count() + def get_content(obj): + """complete (or the content mode the CodeSystem was saved with) up to the concept limit, else not-present""" + if fhir_search.concepts_over_limit(obj): + return 'not-present' + return obj.content_type if obj.content_type in ('example', 'fragment', 'supplement') else 'complete' @staticmethod def get_property(obj): @@ -264,11 +262,11 @@ def get_meta(obj): def to_representation(self, instance): try: - if not instance.content_type: - instance.content_type = 'complete' rep = super().to_representation(instance) + rep['content'] = self.get_content(instance) delete_empty_fields(rep) IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep) + self.with_fhir_view(instance, rep) except (Exception, ): msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}' logger.exception(msg) diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index 8026e3d5d..d1fa73a8c 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -32,14 +32,14 @@ def setUp(self): self.concept_1 = ConceptFactory(parent=self.org_source, names=1, names__name="concept_1_name") self.concept_2 = ConceptFactory(parent=self.org_source) self.org_source_v2 = OrganizationSourceFactory.build( - version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) + version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent, released=True) Source.persist_new_version(self.org_source_v2, self.org_source.created_by) self.user = UserProfileFactory() self.user_token = self.user.get_token() self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url='/some/url') self.user_source_v1 = UserSourceFactory.build( - version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent) + version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent, released=True) Source.persist_new_version(self.user_source_v1, self.user_source.created_by) self.client = APIClient() @@ -56,28 +56,36 @@ def test_public_can_view(self): '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v2') + self.assertNotIn('concept', resource) # a search entry is a summary; the read has the concepts + + resource = self.client.get(response.data['entry'][0]['fullUrl']).data self.assertEqual(len(resource['concept']), 2) self.assertEqual(resource['concept'][0]['code'], self.concept_1.mnemonic) self.assertEqual(resource['concept'][1]['code'], self.concept_2.mnemonic) def test_private_can_view(self): + # the global base shows public repositories only, whoever asks; the owner base shows the owner its own response = self.client.get('/fhir/CodeSystem/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 2) - self.assertEqual(response.data['total'], 2) + self.assertEqual(response.data['total'], 1) + resource = response.data['entry'][0]['resource'] + self.assertEqual( + resource['identifier'][0]['value'], + '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' + ) + self.assertEqual(resource['version'], 'v2') + + response = self.client.get( + f'/users/{self.user.mnemonic}/CodeSystem/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v1') - resource_2 = response.data['entry'][1]['resource'] - self.assertEqual( - resource_2['identifier'][0]['value'], - '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' - ) - self.assertEqual(resource_2['version'], 'v2') + self.assertEqual(self.client.get(f'/users/{self.user.mnemonic}/CodeSystem/?url=/some/url').data['total'], 0) def test_public_can_list(self): response = self.client.get('/fhir/CodeSystem/') @@ -132,19 +140,24 @@ def test_find_by_non_supported_param(self): def test_private_can_list(self): response = self.client.get('/fhir/CodeSystem/', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 2) + self.assertEqual(len(response.data['entry']), 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], - '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' + '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' ) - self.assertEqual(resource['version'], 'v1') - resource_2 = response.data['entry'][1]['resource'] + self.assertEqual(resource['version'], 'v2') + + response = self.client.get( + f'/users/{self.user.mnemonic}/CodeSystem/', HTTP_AUTHORIZATION='Token ' + self.user_token) + + self.assertEqual(len(response.data['entry']), 1) + resource = response.data['entry'][0]['resource'] self.assertEqual( - resource_2['identifier'][0]['value'], - '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' + resource['identifier'][0]['value'], + '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' ) - self.assertEqual(resource_2['version'], 'v2') + self.assertEqual(resource['version'], 'v1') def test_get_code_system(self): response = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/{self.org_source.mnemonic}/') diff --git a/core/code_systems/views.py b/core/code_systems/views.py index a3e7816fc..99518162b 100644 --- a/core/code_systems/views.py +++ b/core/code_systems/views.py @@ -3,16 +3,15 @@ from django.db.models import F from rest_framework.exceptions import ValidationError, NotAuthenticated, PermissionDenied -from core.bundles.serializers import FHIRBundleSerializer from core.code_systems.serializers import CodeSystemDetailSerializer, \ ValidateCodeParametersSerializer from core.common.constants import HEAD -from core.common.fhir_helpers import translate_fhir_query from core.common.serializers import IdentifierSerializer from core.concepts.permissions import CanViewParentDictionaryAsGuest from core.concepts.views import ConceptRetrieveUpdateDestroyView from core.fhir.base import FhirViewMixin from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.parameters.serializers import ParametersSerializer from core.sources.models import Source from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView @@ -30,30 +29,10 @@ def get_viewable_repo(repos, user): return None -class CodeSystemListView(FhirViewMixin, SourceListView): +class CodeSystemListView(FhirViewMixin, FhirSearchMixin, SourceListView): + resource_type = 'CodeSystem' serializer_class = CodeSystemDetailSerializer - @staticmethod - def bundle_response(data, paginator): - bundle = FHIRBundleSerializer( - {'meta': {}, 'total': len(data), 'type': 'searchset', 'entry': FHIRBundleSerializer.convert_to_entry(data)}, - context={'paginator': paginator} - ) - return bundle.data - - def get_filter_params(self, default_version_to_head=True): - return super().get_filter_params(False) - - def apply_query_filters(self, queryset): - query_fields = list(self.serializer_class.Meta.fields) - query_fields.extend(['code']) - queryset = translate_fhir_query(query_fields, self.request.query_params, queryset) - return queryset - - def apply_filters(self, queryset): - queryset = queryset.exclude(version=HEAD).filter(is_latest_version=True) - return self.apply_query_filters(queryset) - def get_serializer_class(self): return self.serializer_class @@ -232,7 +211,8 @@ def get_object(self, queryset=None): } -class CodeSystemRetrieveUpdateView(FhirViewMixin, SourceRetrieveUpdateDestroyView): +class CodeSystemRetrieveUpdateView(FhirViewMixin, FhirReadMixin, SourceRetrieveUpdateDestroyView): + resource_type = 'CodeSystem' serializer_class = CodeSystemDetailSerializer def get_filter_params(self, default_version_to_head=True): diff --git a/core/common/fhir_helpers.py b/core/common/fhir_helpers.py index 8e41fbaa6..70d405db4 100644 --- a/core/common/fhir_helpers.py +++ b/core/common/fhir_helpers.py @@ -1,57 +1,3 @@ -import json - -from core.common.exceptions import Http400 -from core.fhir.base import CONTROL_PARAMETERS - - -def translate_fhir_query(fhir_query_fields, query_params, queryset): - remaining_query_params = query_params.copy() - for name in CONTROL_PARAMETERS: # _format and the like choose the response format, they don't filter - remaining_query_params.pop(name, None) - query_fields = fhir_query_fields.copy() - url = remaining_query_params.get('url') - if url and 'url' in query_fields: - queryset = queryset.filter(canonical_url=url) - remaining_query_params.pop('url') - query_fields.remove('url') - language = remaining_query_params.get('language') - if language and 'language' in query_fields: - queryset = queryset.filter(default_locale=language) - remaining_query_params.pop('language') - query_fields.remove('language') - status = remaining_query_params.get('status') - if status and 'status' in query_fields: - query_fields.remove('status') - remaining_query_params.pop('status') - if status == 'retired': - queryset = queryset.filter(retired=True) - elif status == 'active': - queryset = queryset.filter(released=True) - elif status == 'draft': - queryset = queryset.filter(released=False) - title = remaining_query_params.get('title') - if title and 'title' in query_fields: - remaining_query_params.pop('title') - query_fields.remove('title') - queryset = queryset.filter(full_name=title) - code = remaining_query_params.get('code') - if code and 'code' in query_fields: - remaining_query_params.pop('code') - query_fields.remove('code') - queryset = queryset.filter(concepts__mnemonic=code) - for query_field in query_fields: - query_value = remaining_query_params.get(query_field) - if query_value: - remaining_query_params.pop(query_field, None) - kwargs = {query_field: query_value} - queryset = queryset.filter(**kwargs) - - if remaining_query_params: - raise Http400('The following query params are not supported: ' + json.dumps(remaining_query_params)) - - return queryset - - def delete_empty_fields(obj): if isinstance(obj, dict): for field in list(obj.keys()): diff --git a/core/common/models.py b/core/common/models.py index 0452d3e11..abb1b1eab 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -1618,12 +1618,15 @@ def resolve_reference_expression( # pylint: disable=too-many-arguments def resolve_repo( # pylint: disable=too-many-arguments cls, instance, version, is_canonical, resolution_url, repo_type=None, allow_head=True): if instance: + head = instance if repo_type and instance.is_head else None if version: versions = instance.active_versions if repo_type else instance.versions instance = versions.filter(version=decode_string(version)).first() elif instance.is_head: instance = (instance.get_latest_released_version() or instance) if allow_head else \ instance.get_latest_released_version(exclude_head=True) + if instance and head: # the repository's access (its HEAD's), which FHIR checks with the version's + instance.repository_public_access = head.public_access if not instance: from core.sources.models import Source diff --git a/core/common/tests.py b/core/common/tests.py index af3a169d7..f4ec11a45 100644 --- a/core/common/tests.py +++ b/core/common/tests.py @@ -75,10 +75,8 @@ from .backends import OCLOIDCAuthenticationBackend from .exceptions import SSOIdentityMismatchRefused from .checksums import Checksum -from .fhir_helpers import translate_fhir_query from .serializers import IdentifierSerializer from .validators import URIValidator -from ..code_systems.serializers import CodeSystemDetailSerializer from ..concepts.serializers import ConceptDetailSerializer from ..concepts.tests.factories import ConceptFactory, ConceptNameFactory from ..mappings.serializers import MappingDetailSerializer @@ -325,57 +323,6 @@ def setUp(self): self.maxDiff = None -class FhirHelpersTest(OCLTestCase): - def test_language_to_default_locale(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'language': 'eng'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('"sources"."default_locale" = eng' in str(query_set.query)) - - def test_status_retired(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'retired'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."retired"' in str(query_set.query)) - - def test_status_active(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'active'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."released"' in str(query_set.query)) - - def test_status_draft(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'draft'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE NOT "sources"."released"' in str(query_set.query)) - - def test_title_to_full_name(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'title': 'some title'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."full_name" = some title' in str(query_set.query)) - - def test_other_fields(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'version': 'v1', 'id': '2'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('"sources"."version" = v1' in str(query_set.query)) - self.assertTrue('"sources"."id" = 2' in str(query_set.query)) - - class IdentifierSerializerTest(OCLTestCase): def test_deserialize(self): data = {'system': '/org/OCL/test', diff --git a/core/concept_maps/serializers.py b/core/concept_maps/serializers.py index 28f126fa1..53884fbc7 100644 --- a/core/concept_maps/serializers.py +++ b/core/concept_maps/serializers.py @@ -9,6 +9,7 @@ from core.concept_maps.constants import RESOURCE_TYPE from core.common.constants import HEAD from core.common.serializers import StatusField, IdentifierSerializer +from core.fhir.search import CanonicalUrlField, FhirRepositorySerializerMixin from core.mappings.constants import SAME_AS from core.mappings.models import Mapping from core.mappings.serializers import MappingDetailSerializer @@ -44,8 +45,8 @@ def to_internal_value(self, data): def to_representation(self, value): # limit to 1000 mappings by default # TODO: support graphQL to go around the limit - limit = self.get_limit() - mappings = value.get_mappings_queryset().filter(retired=False).order_by('id')[:limit] + mappings = value.get_mappings_queryset().filter(retired=False).select_related( + 'from_source', 'to_source').order_by('id')[:1000] groups = {} for mapping in mappings: key = (mapping.from_source_url or '') + (mapping.to_source_url or '') @@ -91,18 +92,12 @@ def to_representation(self, value): }) return [*groups.values()] - def get_limit(self): - if self.context.get('has_many', False): - limit = 25 - else: - limit = 1000 - return limit - -class ConceptMapDetailSerializer(serializers.ModelSerializer): +class ConceptMapDetailSerializer(FhirRepositorySerializerMixin, serializers.ModelSerializer): + content_field = 'group' resourceType = SerializerMethodField(method_name='get_resource_type') id = CharField(source='mnemonic') - url = CharField(source='canonical_url', required=False) + url = CanonicalUrlField(source='canonical_url', required=False) title = CharField(source='full_name', required=False) status = StatusField(source='*') language = CharField(source='default_locale', required=False) @@ -117,14 +112,6 @@ class Meta: 'version', 'identifier', 'contact', 'jurisdiction', 'name', 'description', 'publisher', 'purpose', 'copyright', 'date', 'experimental', 'group') - def __new__(cls, *args, **kwargs): - if kwargs.get('many', False): - context = kwargs.get('context', {}) - context.update({'has_many': True}) - kwargs.update({'context': context}) - - return super().__new__(cls, *args, **kwargs) - @staticmethod def get_resource_type(_): return RESOURCE_TYPE @@ -138,6 +125,7 @@ def to_representation(self, instance): rep = super().to_representation(instance) delete_empty_fields(rep) IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep) + self.with_fhir_view(instance, rep) except (Exception, ): msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}' logger.exception(msg) diff --git a/core/concept_maps/tests.py b/core/concept_maps/tests.py index dc52ab6be..83ded029f 100644 --- a/core/concept_maps/tests.py +++ b/core/concept_maps/tests.py @@ -35,7 +35,7 @@ def setUp(self): self.user_token = self.user.get_token() self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url='/some/url') self.user_source_v1 = UserSourceFactory.build( - version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent) + version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent, released=True) Source.persist_new_version(self.user_source_v1, self.user_source.created_by) self.org_source_B = OrganizationSourceFactory(organization=self.org, canonical_url='/some/url/B', @@ -44,7 +44,7 @@ def setUp(self): names=[ConceptNameFactory.build(name="concept_B_1_name")]) self.concept_B_2 = ConceptFactory(parent=self.org_source_B, mnemonic='concept_B_2') self.org_source_B_v1 = OrganizationSourceFactory.build( - version='v1', mnemonic=self.org_source_B.mnemonic, organization=self.org_source_B.parent) + version='v1', mnemonic=self.org_source_B.mnemonic, organization=self.org_source_B.parent, released=True) Source.persist_new_version(self.org_source_B_v1, self.org_source_B.created_by) self.mapping_1 = MappingFactory(parent=self.org_source, to_concept=self.concept_1, @@ -61,7 +61,7 @@ def setUp(self): self.mapping_3.save() self.org_source_v2 = OrganizationSourceFactory.build( - version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) + version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent, released=True) Source.persist_new_version(self.org_source_v2, self.org_source.created_by) for mapping in self.org_source_v2.get_mappings_queryset().all(): @@ -82,6 +82,9 @@ def test_public_can_view(self): '/orgs/' + self.org.mnemonic + '/ConceptMap/' + self.org_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v2') + self.assertNotIn('group', resource) # a search entry is a summary; the read has the groups + + resource = self.client.get(response.data['entry'][0]['fullUrl']).data self.assertEqual(len(resource['group']), 1) self.assertEqual(resource['group'], [{'source': self.org_source_B_v1.canonical_url, 'target': self.org_source.canonical_url, @@ -94,19 +97,22 @@ def test_public_can_view(self): 'target': [{'code': 'concept_1', 'equivalence': 'equivalent'}]}]}]) def test_private_can_view(self): + # the global base shows public repositories only, whoever asks; the owner base shows the owner its own response = self.client.get('/fhir/ConceptMap/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 2) - self.assertEqual(response.data['total'], 2) + self.assertEqual(len(response.data['entry']), 1) + self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], '/orgs/' + self.org.mnemonic + '/ConceptMap/' + self.org_source.mnemonic + '/' ) - self.assertEqual(resource['version'], 'v2') - resource2 = response.data['entry'][1]['resource'] + + response = self.client.get( + f'/users/{self.user.mnemonic}/ConceptMap/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + resource2 = response.data['entry'][0]['resource'] self.assertEqual( resource2['identifier'][0]['value'], '/users/' + self.user.mnemonic + '/ConceptMap/' + self.user_source.mnemonic + '/' @@ -132,7 +138,7 @@ def test_public_can_list(self): self.assertEqual(resource2['version'], 'v1') def test_find_by_title(self): - response = self.client.get('/fhir/ConceptMap/?title=source name') + response = self.client.get('/fhir/ConceptMap/?title:exact=source name') # title alone is 'starts with' self.assertEqual(len(response.data['entry']), 1) resource = response.data['entry'][0]['resource'] @@ -145,7 +151,10 @@ def test_find_by_title(self): def test_private_can_list(self): response = self.client.get('/fhir/ConceptMap/', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 3) + self.assertEqual(len(response.data['entry']), 2) # public only at the global base + response = self.client.get( + f'/users/{self.user.mnemonic}/ConceptMap/', HTTP_AUTHORIZATION='Token ' + self.user_token) + self.assertEqual(len(response.data['entry']), 1) def test_get_concept_map(self): response = self.client.get(f'/orgs/{self.org.mnemonic}/ConceptMap/{self.org_source.mnemonic}/') diff --git a/core/concept_maps/views.py b/core/concept_maps/views.py index 20b0677b9..9ce92b260 100644 --- a/core/concept_maps/views.py +++ b/core/concept_maps/views.py @@ -5,16 +5,14 @@ from django.http import Http404 from pydash import get -from core.bundles.serializers import FHIRBundleSerializer -from core.code_systems.views import FhirParameterRequiredError from core.common.constants import HEAD -from core.common.fhir_helpers import translate_fhir_query -from core.common.permissions import CanViewConceptDictionary +from core.common.permissions import CanViewConceptDictionary, HasOwnership from core.common.serializers import IdentifierSerializer from core.concept_maps.constants import RESOURCE_TYPE from core.concept_maps.serializers import ConceptMapDetailSerializer, ConceptMapParametersSerializer from core.concepts.permissions import CanAccessParentDictionary from core.fhir.base import FhirViewMixin +from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.mappings.constants import SAME_AS from core.mappings.views import MappingListView from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView @@ -22,29 +20,10 @@ logger = logging.getLogger('oclapi') -class ConceptMapListView(FhirViewMixin, SourceListView): +class ConceptMapListView(FhirViewMixin, FhirSearchMixin, SourceListView): + resource_type = 'ConceptMap' serializer_class = ConceptMapDetailSerializer - @staticmethod - def bundle_response(data, paginator): - bundle = FHIRBundleSerializer( - {'meta': {}, 'total': len(data), 'type': 'searchset', 'entry': FHIRBundleSerializer.convert_to_entry(data)}, - context={'paginator': paginator} - ) - return bundle.data - - def get_filter_params(self, default_version_to_head=True): - return super().get_filter_params(False) - - def apply_query_filters(self, queryset): - query_fields = list(self.serializer_class.Meta.fields) - queryset = translate_fhir_query(query_fields, self.request.query_params, queryset) - return queryset - - def apply_filters(self, queryset): - queryset = queryset.exclude(version=HEAD).filter(is_latest_version=True) - return self.apply_query_filters(queryset) - def get_serializer_class(self): return self.serializer_class @@ -52,7 +31,8 @@ def get_detail_serializer(self, obj): return self.serializer_class(obj) -class ConceptMapRetrieveUpdateView(FhirViewMixin, SourceRetrieveUpdateDestroyView): +class ConceptMapRetrieveUpdateView(FhirViewMixin, FhirReadMixin, SourceRetrieveUpdateDestroyView): + resource_type = 'ConceptMap' serializer_class = ConceptMapDetailSerializer def get_filter_params(self, default_version_to_head=True): @@ -65,9 +45,18 @@ def get_queryset(self): return queryset.exclude(version=HEAD) def get_object(self, queryset=None): + """ + A PUT or PATCH to a ConceptMap that doesn't exist gets None, which creates it, so it needs what a POST needs: + staff, the owner, or a member of the owning org (HasOwnership on the owner). Anyone else is refused as they are + for an existing one. Any other request for one is a 404 (a read's from FhirReadMixin). + """ try: return super().get_object(queryset) except Http404: + if self.request.method not in ('PUT', 'PATCH'): + raise + if not HasOwnership().has_object_permission(self.request, self, self.parent_resource): + self.permission_denied(self.request) return None def get_detail_serializer(self, obj): diff --git a/core/fhir/resolve.py b/core/fhir/resolve.py index c8d408b84..075446c20 100644 --- a/core/fhir/resolve.py +++ b/core/fhir/resolve.py @@ -8,18 +8,24 @@ - Version: the one named (the operation's version parameter, else url|version), HEAD included. With none named, the latest released version, never HEAD. A repository with nothing released isn't visible, whatever version is named. - Access, after resolving: at the global base public repositories only, at an owner base what the caller may view. - A repository the caller can't see is not found, and no other repository is tried. + The version and its repository (HEAD) must both be viewable. A repository the caller can't see is not found, and + no other repository is tried. Every failure is the same UnresolvedError, so it never tells a caller that a repository they can't see exists. """ +from django.db.models import Exists, OuterRef, Q, Subquery +from django.db.models.functions import Coalesce + from core.collections.models import Collection -from core.common.constants import HEAD +from core.common.constants import HEAD, ACCESS_TYPE_VIEW, RETIRED_ACCESS_TYPE_EDIT from core.common.mixins import SourceContainerMixin +from core.common.utils import normalize_public_access from core.fhir.outcomes import NotFoundError from core.sources.models import Source REPO_TYPES = {'CodeSystem': Source, 'ConceptMap': Source, 'ValueSet': Collection} REPO_PATHS = {'CodeSystem': 'sources', 'ConceptMap': 'sources', 'ValueSet': 'collections'} +PUBLIC_ACCESS = (ACCESS_TYPE_VIEW, RETIRED_ACCESS_TYPE_EDIT) # what public_can_view reads as public class UnresolvedError(NotFoundError): @@ -64,7 +70,7 @@ def resolve(resource_type, url, version=None, kwargs=None, user=None, expression instance, _ = repo_type.resolve_reference_expression( to_ocl_path(url), namespace, version, repo_type=repo_type, allow_head=False) - if not instance.id or not is_visible(instance, namespace, user) or not has_release(instance): + if not instance.id or not is_visible(instance, namespace, user) or not has_release(instance, namespace, user): raise error return instance @@ -81,12 +87,143 @@ def to_ocl_path(url): def is_visible(instance, namespace, user): - """At the global base, public repositories to anyone; at an owner base, what the user may view.""" - return instance.public_can_view if namespace is None else instance.has_view_access(user) + """ + Whether a base shows a repository version: the version and its repository (its active HEAD, which governs access + to the repository's concepts and mappings) must both be viewable there. At the global base both are public, to + anyone; at an owner base the same, or any of the owner's versions to those who may view its private repositories + (staff, the org's members, the user). + """ + access = repository_access(instance) + if access is None: + return False + return (instance.public_can_view and is_public(access)) or may_view_private(instance, namespace, user) -def has_release(instance): - """Whether the repository of a version has a released version that isn't retired; HEAD's released flag isn't one""" - if instance.released and not instance.retired and not instance.is_head: +def has_release(instance, namespace=None, user=None): + """ + Whether the repository of a version has a released version that isn't retired, and that the base shows the + caller: a public one, or any to those who may view the owner's private repositories. HEAD's released flag isn't + one. (The repository's own access, HEAD's, is is_visible's to check.) + """ + if instance.released and not instance.retired and not instance.is_head and ( + instance.public_can_view or may_view_private(instance, namespace, user)): return True - return instance.released_versions.filter(retired=False).exclude(version=HEAD).exists() + live = instance.released_versions.filter(retired=False).exclude(version=HEAD) + if not may_view_private(instance, namespace, user): + live = live.filter(public_access__in=PUBLIC_ACCESS) + return live.exists() + + +def is_public(public_access): + """A public_access value as public_can_view reads it.""" + return normalize_public_access(public_access).lower() == ACCESS_TYPE_VIEW.lower() + + +def repository_access(instance): + """ + The public_access of a version's repository, its active HEAD's, or None without one. A read (with_repository_access) + and the resolver (resolve_repo) set it on the version they find, so it isn't looked up again. + """ + if instance.is_head: + return instance.public_access if instance.is_active else None + if hasattr(instance, 'repository_public_access'): + return instance.repository_public_access + head = instance.head + return head.public_access if head else None + + +def may_view_private(instance, namespace, user): + """At an owner base, whether `user` may view the owner's private repositories; asked once per version.""" + if namespace is None or not (user and user.is_authenticated): + return False + if not hasattr(instance, 'fhir_may_view_private'): + instance.fhir_may_view_private = instance.has_parent_edit_access(user) + return instance.fhir_may_view_private + + +def repository_rows(model, **filters): + """ + For a subquery: the rows of the outer row's repository (same mnemonic and owner) matching `filters`, as two + querysets, one for an org-owned outer row and one for a user-owned one (an owner column is NULL on the other). + """ + rows = model.objects.filter(mnemonic=OuterRef('mnemonic'), **filters) + return (rows.filter(organization_id=OuterRef('organization_id'), user_id__isnull=True), + rows.filter(user_id=OuterRef('user_id'), organization_id__isnull=True)) + + +def with_head(model, *conditions, **filters): + """ + For a subquery: whether the outer row's repository has an active HEAD matching `conditions` and `filters`. One + EXISTS, the owner matched inside it (an owner column is NULL on the other kind of row, and NULL equals nothing), + which the planner runs as one semi-join; two EXISTS joined by OR it plans as an index scan costing far more. + """ + return Exists(model.objects.filter( + *conditions, mnemonic=OuterRef('mnemonic'), version=HEAD, is_active=True, **filters).filter( + Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id')))) + + +def with_repository_access(queryset): + """Each row annotated with repository_public_access, its repository's (active HEAD's) public_access.""" + org_heads, user_heads = repository_rows(queryset.model, version=HEAD, is_active=True) + return queryset.annotate(repository_public_access=Coalesce( + Subquery(org_heads.values('public_access')[:1]), Subquery(user_heads.values('public_access')[:1]))) + + +def owner_access(kwargs, user): + """ + (owner filter, may view private) of the base `kwargs` names, for `user`: ({}, False) at the global base; at an + owner base, the filter on the owner's rows (None when the owner doesn't exist) and whether `user` may view its + private repositories (staff, the org's members, the user). + """ + namespace = namespace_of(kwargs) + if namespace is None: + return {}, False + owner = SourceContainerMixin.get_object_from_namespace(namespace) + if not owner: + return None, False + is_org = bool(kwargs.get('org')) + return {'organization_id' if is_org else 'user_id': owner.id}, bool(user and user.is_authenticated) and ( + user.is_staff or (owner.is_member(user) if is_org else owner.id == user.id)) + + +def viewable_rows(model, namespace, user): + """ + The queryset form of is_visible(row, namespace, user) over every row of `model`, HEAD and versions alike: anywhere + a public row of a public repository (active HEAD); at an owner base also, whoever owns them, the rows `user` may + view by their own rights (all to staff, their orgs' and their own), with an active HEAD. For what a version refers + to (a mapping's source, the code systems an expansion drew on), so only repositories the caller is shown match. + + A row and its HEAD share their owner, so "both public, or the caller's own" is "the row public or the caller's own, + and its HEAD public or the caller's own": one EXISTS, which the planner runs as a semi-join. + """ + public = Q(public_access__in=PUBLIC_ACCESS) + if namespace is None or not (user and user.is_authenticated): + return model.objects.filter(public, with_head(model, public)) + if user.is_staff: + return model.objects.filter(with_head(model)) + shown = public | Q(organization_id__in=user.organizations.values('id')) | Q(user_id=user.id) + return model.objects.filter(shown, with_head(model, shown)) + + +def visible_versions(resource_type, kwargs=None, user=None, access=None): + """ + The queryset form of is_visible and has_release, for search: the active released versions, never HEAD, that the + base `kwargs` names shows `user`. A row is in it exactly when is_visible(row, namespace, user) and + has_release(row, namespace, user): at the global base public versions of public repositories (HEAD), whoever asks; + at an owner base the owner's, private ones only to those who may view them (staff, the org's members, the user). + An owner base whose owner doesn't exist shows nothing. `access`, owner_access's, saves looking it up again. + + A retired release counts only beside a live one the caller is shown; retired releases are so few that they're + checked on their own, which keeps the planner on a plain scan of the table. + """ + model = REPO_TYPES[resource_type] + owner, private = access or owner_access(kwargs, user) + if owner is None: + return model.objects.none() + access = {} if private else {'public_access__in': PUBLIC_ACCESS} + releases = model.objects.filter(is_active=True, released=True, **owner, **access).exclude(version=HEAD).filter( + with_head(model, **access)) + org_live, user_live = repository_rows(model, is_active=True, released=True, retired=False, **access) + retired_beside_live = releases.filter(retired=True).filter( + Exists(org_live.exclude(version=HEAD)) | Exists(user_live.exclude(version=HEAD))) + return releases.filter(Q(retired=False) | Q(id__in=retired_beside_live.values('id'))) diff --git a/core/fhir/search.py b/core/fhir/search.py new file mode 100644 index 000000000..cf697229c --- /dev/null +++ b/core/fhir/search.py @@ -0,0 +1,662 @@ +""" +FHIR search and read of CodeSystem, ValueSet and ConceptMap: SVCM's Query transactions (ITI-95, ITI-96, ITI-100). + +- What a base shows is core.fhir.resolve's rule: released versions only, never HEAD; at the global base /fhir/ the + public ones of public repositories (HEAD), whoever asks; at an owner base the owner's, private ones to those who may + view them. Search uses its queryset form (visible_versions), a read checks is_visible and has_release on the version + it finds. +- Ids: at the global base every released version has an opaque, stable id, its database key; an owner base keeps the + repository's mnemonic (so its versions share one), and a read there serves the latest release the requester is + shown, or the `version` named. +- Search parameters: SVCM's, with :exact and :contains on the string ones (no modifier is starts-with; both it and + :contains ignore case and accents, :exact neither). + A comma separates alternatives, a repeated parameter must hold every time. Anything else is a 400 not-supported. +- Paging: _count (at most MAX_COUNT, which is also the default) and page; Bundle.total is the full count, and every + entry has a fullUrl and a search mode. The search runs once, for the ids of every match; the page's rows are then + read by id. +- A search's queries are bounded together by SEARCH_TIMEOUT_MS: one the database cancels makes the search too costly + (422). A parameter takes at most MAX_ALTERNATIVES values and a search MAX_SEARCH_ALTERNATIVES (else a 400 + too-costly), and the canonical URLs each parameter names are looked up in one query. +- A search entry is the resource's summary, marked SUBSETTED: no concept, compose or group, which only a read returns. + A CodeSystem is `complete` (with every concept, retired ones flagged) up to CONCEPT_LIMIT concepts, else + `not-present`. +- A version released before its repository had a canonical URL has the repository's (HEAD's), in its `url` and for + the url parameter. +""" +import json +import math +import re +import time +from datetime import datetime, timedelta +from functools import reduce +from operator import or_ + +from dateutil.relativedelta import relativedelta +from django.conf import settings +from django.db import connection, transaction, OperationalError +from django.db.models import BooleanField, Exists, Func, OuterRef, Q, Subquery, TextField, Value +from django.db.models.functions import Coalesce +from django.db.models.lookups import Exact, IContains, IStartsWith +from django.utils import timezone +from psycopg2 import errorcodes +from rest_framework.fields import CharField +from rest_framework.response import Response + +from core.bundles.serializers import FHIRBundleSerializer +from core.collections.models import Expansion +from core.common.constants import HEAD +from core.common.db_functions import ImmutableUnaccent +from core.common.mixins import CustomPaginator +from core.common.utils import to_int +from core.concepts.models import Concept +from core.fhir import outcomes +from core.fhir.base import CONTROL_PARAMETERS, fhir_base_url +from core.fhir.resolve import ( + PUBLIC_ACCESS, REPO_TYPES, namespace_of, owner_access, visible_versions, viewable_rows, is_visible, has_release, + repository_rows, with_repository_access) +from core.mappings.models import Mapping +from core.sources.models import Source + +MAX_COUNT = 100 # the most entries a search page holds, and how many it holds when _count isn't given +SEARCH_TIMEOUT_MS = 5000 # the database time a search may take; past it the search is too costly +CONCEPT_LIMIT = 1000 # a CodeSystem is `complete` up to this many concepts, else `not-present` (design D9) +SUBSETTED = {'system': 'http://terminology.hl7.org/CodeSystem/v3-ObservationValue', 'code': 'SUBSETTED'} +PAGING_PARAMETERS = frozenset({'_count', 'page'}) +OPAQUE_ID = re.compile(r'[1-9][0-9]{0,17}') # a database key, as str() writes it +NOTHING = Q(pk__in=[]) +EVERYTHING = Q(pk__isnull=False) # unlike Q(), it holds as an alternative too (Q() | x is x) + +# string parameters: the field each searches, and the lookup of each modifier (:exact is the field's own equality) +STRING_FIELDS = {'name': 'name', 'title': 'full_name', 'description': 'description', 'publisher': 'publisher'} +STRING_LOOKUPS = {'': IStartsWith, 'exact': None, 'contains': IContains} +# the other parameters, by resource type: the Search method each one's values go to +COMMON_PARAMETERS = { + '_id': 'by_id', '_lastUpdated': 'by_last_updated', 'url': 'by_url', 'version': 'by_version', + 'status': 'by_status', 'identifier': 'by_identifier'} +PARAMETERS = { + 'CodeSystem': {**COMMON_PARAMETERS, 'system': 'by_url', 'code': 'by_code', 'language': 'by_language'}, + 'ValueSet': {**COMMON_PARAMETERS, 'reference': 'by_reference'}, + 'ConceptMap': { + **COMMON_PARAMETERS, 'source-system': 'by_source', 'source-uri': 'by_source', 'target-system': 'by_target', + 'target-uri': 'by_target'}, +} +# A stored OCL accession identifier, as IdentifierSerializer.find_ocl_identifier finds one in the serialized list: a +# value that isn't empty, and a type coding that is ACSN of v2-0203. IdentifierTypeCodingSerializer shows a missing code +# as ACSN and a missing system as v2-0203 (a null one as nothing). +STORED_ACCESSION = ( + 'lax $[*] ? (@.value.type() == "string" && @.value != "" && exists(@.type.coding[*] ? (' + '(@.code == "ACSN" || !exists(@.code)) && (@.system == "http://hl7.org/fhir/v2/0203" || !exists(@.system)))))') +# token parameters, by the system their codes belong to: None for none defined (|v is then v), CODE_SYSTEM for the +# CodeSystem itself (code=|). identifier is a token too, with its own rules (by_identifier). +CODE_SYSTEM = 'the code system searched' +TOKEN_SYSTEMS = { + '_id': None, 'version': None, 'status': 'http://hl7.org/fhir/publication-status', 'language': 'urn:ietf:bcp:47', + 'code': CODE_SYSTEM} +# what system| (any code of the system) means for a parameter with a system of its own +ANY_CODE = {'status': EVERYTHING, 'language': Q(default_locale__isnull=False) & ~Q(default_locale='')} +STATUSES = {'active': Q(released=True, retired=False), 'retired': Q(retired=True)} # search shows no draft +# the parameters whose values are canonical URLs, looked up before the search (Search.look_up), by what they're among +LOOKUPS = { + 'url': 'repositories', 'system': 'repositories', 'code': 'repositories', 'reference': 'referenced', + 'source-system': 'sources', 'source-uri': 'sources', 'target-system': 'sources', 'target-uri': 'sources'} +MAX_ALTERNATIVES = 20 # the values a search parameter takes: its alternatives and repeats together +MAX_SEARCH_ALTERNATIVES = 50 # the values a search takes, every parameter's together + +# a FHIR date or dateTime search value: prefix, year, month, day, hour, minute, second, fraction, zone +DATE = re.compile( + r'(eq|ne|gt|lt|ge|le|sa|eb)?(\d{4})(?:-(\d{2})(?:-(\d{2})' + r'(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}:\d{2})?)?)?)?') + + +def split(value, separator): + """A search value split on the separators that aren't escaped; escapes are kept for unescape().""" + parts, start, index = [], 0, 0 + while index < len(value): + if value[index] == '\\': + index += 2 + continue + if value[index] == separator: + parts.append(value[start:index]) + start = index + 1 + index += 1 + return parts + [value[start:]] + + +def token(value): + """(system, code) of a token search value, unescaped: system None without a '|', '' for an explicit none.""" + system, *code = split(value, '|') + return (None, unescape(system)) if not code else (unescape(system), unescape('|'.join(code))) + + +def unescape(value): + """FHIR's search escapes: a backslash makes the next character (, | $ or \\) literal.""" + return re.sub(r'\\(.)', r'\1', value) + + +def date_range(value): + """ + (prefix, start, end): the instants [start, end) a date search value covers at its precision, UTC by default. end is + None when it would fall past year 9999, which datetime can't hold: the range runs on unbounded. + """ + match = DATE.fullmatch(value) + if not match: + raise outcomes.InvalidError(f"'{value}' is not a supported FHIR date search value.", expression='_lastUpdated') + prefix, year, month, day, hour, minute, second, fraction, zone = match.groups() + try: + start = datetime( + int(year), int(month or 1), int(day or 1), int(hour or 0), int(minute or 0), int(second or 0), + int((fraction or '0')[:6].ljust(6, '0')), tzinfo=datetime.strptime(zone or 'Z', '%z').tzinfo) + except ValueError as ex: + raise outcomes.InvalidError(f"'{value}' is not a valid date.", expression='_lastUpdated') from ex + unit = 'seconds' if second else 'minutes' if minute else 'days' if day else 'months' if month else 'years' + try: + end = start + ( + timedelta(microseconds=10 ** max(0, 6 - len(fraction))) if fraction else relativedelta(**{unit: 1})) + except (ValueError, OverflowError): + end = None + return prefix or 'eq', start, end + + +def canonical_q(url, repositories): + """ + Rows whose canonical URL is `url` as CanonicalUrlField shows it: their own, or for a row with none (a version + released before its repository had one), their repository's (HEAD's). `repositories` are the (mnemonic, + organization_id, user_id) of the repositories whose HEAD has it, looked up first (repositories_with), so the + planner sees plain conditions rather than a correlated subquery it can't estimate. + """ + if not repositories: + return Q(canonical_url=url) + return Q(canonical_url=url) | ((Q(canonical_url__isnull=True) | Q(canonical_url='')) & reduce(or_, ( + Q(mnemonic=mnemonic, organization_id=organization_id, user_id=user_id) + for mnemonic, organization_id, user_id in repositories))) + + +def repositories_with(rows, urls): + """ + For each of `urls`, the (mnemonic, organization_id, user_id) of the repositories among `rows` whose HEAD has that + canonical URL: one query for them all. + """ + found = {url: [] for url in urls} + for url, *repository in rows.filter(version=HEAD, canonical_url__in=urls).values_list( + 'canonical_url', 'mnemonic', 'organization_id', 'user_id'): + found[url].append(tuple(repository)) + return found + + +def sources_with(rows, urls): + """For each of `urls`, the ids of the Source rows among `rows` with that canonical URL: one query for them all.""" + found = {url: [] for url in urls} + for url, source_id in rows.filter(canonical_url__in=urls).values_list('canonical_url', 'id'): + found[url].append(source_id) + return found + + +def with_repository_canonical(queryset): + """Each row annotated with its repository's (HEAD's) canonical URL, for a version that has none of its own.""" + org_heads, user_heads = repository_rows(queryset.model, version=HEAD) + return queryset.annotate(repository_canonical_url=Coalesce( + Subquery(org_heads.values('canonical_url')[:1]), Subquery(user_heads.values('canonical_url')[:1]))) + + +def code_system_concepts(source): + """ + (count, concepts) for a CodeSystem read, cached on the version: its concepts, retired ones included, when it has + at most CONCEPT_LIMIT, else (its number of concepts, None). The number is counted on the version's concept links + alone (an index-only count, cheap even for LOINC); within the limit it's the concepts listed. + """ + if not hasattr(source, 'fhir_concepts'): + count, concepts = Concept.sources.through.objects.filter(source_id=source.id).count(), None + if count <= CONCEPT_LIMIT: + concepts = list(source.concepts.filter(is_active=True).select_related('parent').prefetch_related( + 'names', 'descriptions').order_by('id')) + count = len(concepts) + source.fhir_concepts = count, concepts + return source.fhir_concepts + + +def concepts_over_limit(source): + """Whether a CodeSystem has more than CONCEPT_LIMIT concepts: a search annotates it, a read counts them.""" + over = getattr(source, 'concepts_over_limit', None) + return code_system_concepts(source)[1] is None if over is None else over + + +def with_concepts_over_limit(queryset): + """Each Source row annotated with concepts_over_limit, reading at most CONCEPT_LIMIT + 1 of its concept links.""" + links = Concept.sources.through.objects.filter(source_id=OuterRef('id')) + return queryset.annotate(concepts_over_limit=Exists(links[CONCEPT_LIMIT:CONCEPT_LIMIT + 1])) + + +def defines_concept(**filters): + """ + A CodeSystem version with a concept (matching `filters`), as its read lists them: linked to the version and not + deleted. A retired concept counts; the read shows it flagged inactive. + """ + return Exists(Concept.sources.through.objects.filter(source_id=OuterRef('id'), concept__is_active=True, **filters)) + + +class JsonPathExists(Func): # pylint: disable=abstract-method + """jsonb_path_exists(field, path, vars, silent): whether the path finds anything, an error being no match.""" + function = 'jsonb_path_exists' + output_field = BooleanField() + + def __init__(self, field, path, variables): + super().__init__(field, Value(path), Value(json.dumps(variables)), Value(True)) + + +class JsonbTypeof(Func): # pylint: disable=abstract-method + function = 'jsonb_typeof' + output_field = TextField() + + +def held_identifier(system, value): + """ + Rows holding an identifier shown as system|value, both checked on the same identifier: `system` None is any + system, '' none; `value` None is any value. IdentifierSerializer shows an identifier stored without a system key + under API_BASE_URL, and one with a null or empty system without one. Only a list of identifiers is shown; lax mode + reads a missing key as nothing (strict mode would make exists() unknown). + """ + conditions, variables = [], {} + if value is not None: + conditions.append('@.value == $value') + variables['value'] = value + if system == '': + conditions.append('exists(@.system) && (@.system == null || @.system == "")') + elif system is not None: + no_key = ' || !exists(@.system)' if system == settings.API_BASE_URL else '' + conditions.append(f'(@.system == $system{no_key})') + variables['system'] = system + return Exact(JsonbTypeof('identifier'), 'array') & JsonPathExists( + 'identifier', f'lax $[*] ? ({" && ".join(conditions)})', variables) + + +def mapped(side, value, sources): + """ + A ConceptMap version holding a live mapping whose group source (side 'from') or target ('to') shows `value`, as + ConceptMapGroupField builds groups: retired mappings aren't read; a group's system is the from or to source's + canonical URL, or for one without, its OCL URL as a ConceptMap path (/orgs/{org}/ConceptMap/{id}/), or the + mapping's own URL when it names no OCL source. A search over many mappings is bounded by SEARCH_TIMEOUT_MS. + OCL keeps no source or target value set on a ConceptMap, so source-uri and target-uri are read as the systems. + + `sources` are the ids of the sources with that canonical, listed first (sources_with), so the planner can read the + mappings by their source index. Only those the base shows the caller (viewable_rows) are listed: a canonical only + other sources have answers as one no source has, so how long the search takes doesn't depend on mappings the + caller isn't shown. + """ + field = f'mapping__{side}_source' + shown = Q(**{f'{field}_id__in': sources}) | (Q(**{f'{field}_url': value}) & Q(**{f'{field}__isnull': True})) + path = re.fullmatch(r'/(orgs|users)/([^/]+)/ConceptMap/([^/]+)/', value) + if path: + stored = reduce(or_, (Q(**{f'{field}_url__startswith': f'/{path[1]}/{path[2]}/{kind}/{path[3]}/'}) + for kind in ('sources', 'collections', 'ConceptMap'))) + without_canonical = Q(**{f'{field}__isnull': True}) | Q(**{f'{field}__canonical_url__isnull': True}) | Q( + **{f'{field}__canonical_url': ''}) + shown |= stored & without_canonical + return Exists(Mapping.sources.through.objects.filter( + source_id=OuterRef('id'), mapping__retired=False).filter(shown)) + + +class Search: + """The criteria of one search's parameters, on the versions of a resource type at a base, for a caller.""" + + def __init__(self, resource_type, kwargs, user=None): + self.resource_type = resource_type + self.model = REPO_TYPES[resource_type] + self.parameters = PARAMETERS[resource_type] + self.namespace, self.user = namespace_of(kwargs), user + self.access = owner_access(kwargs, user) + self.found = {} # (lookup, canonical URL): what look_up found for it + + def viewable_sources(self): + """ + The Source rows the base shows the caller, whoever owns them: what source-system, target-system and reference + match. At an owner base that's another owner's private source too, if the caller may view it themselves. + """ + return viewable_rows(Source, self.namespace, self.user) + + def criteria(self, query): + """ + One Q for a query string: every parameter holds (a repeated one each time), any of a value's comma-separated + alternatives does. An empty value isn't a criterion; the format and paging parameters aren't criteria. + """ + criteria = Q() + for name, modifier, repeats in self.parameters_of(query): + self.look_up(name, [item for alternatives in repeats for item in alternatives]) + for alternatives in repeats: + if alternatives: + criteria &= reduce(or_, [self.criterion(name, modifier, item) for item in alternatives]) + return criteria + + def parameters_of(self, query): + """ + (name, modifier, each repeat's alternatives) for every search parameter of a query string, each supported and + within MAX_ALTERNATIVES values (alternatives and repeats together) and the search within + MAX_SEARCH_ALTERNATIVES, else a 400 too-costly naming the parameter. Checked before any query runs. + """ + parameters, given, total = [], {}, 0 + for key, values in query.lists(): + if key in CONTROL_PARAMETERS or key in PAGING_PARAMETERS: + continue + name, _, modifier = key.partition(':') + self.check(key, name, modifier) + repeats = [[item for item in split(value, ',') if item] for value in values] + count = sum(map(len, repeats)) + given[name], total = given.get(name, 0) + count, total + count + if given[name] > MAX_ALTERNATIVES or total > MAX_SEARCH_ALTERNATIVES: + raise outcomes.TooCostlyError( + f'A search takes at most {MAX_ALTERNATIVES} values of a parameter, its alternatives and repeats ' + f'together, and {MAX_SEARCH_ALTERNATIVES} in all.', expression=key, status_code=400) + parameters.append((name, modifier, repeats)) + return parameters + + def check(self, key, name, modifier): + modifiers = STRING_LOOKUPS if name in STRING_FIELDS else ('',) if name in self.parameters else () + if modifier not in modifiers: + raise outcomes.NotSupportedError(f"The search parameter '{key}' is not supported.", expression=key) + + def look_up(self, name, alternatives): + """ + The canonical URLs a parameter's alternatives (still escaped) name, looked up together in one query, for its + criteria: the repositories with them (url, system, code's system), or among the sources the base shows the + caller, the repositories (reference) or the sources (source-system, source-uri, target-system, target-uri). + """ + lookup = LOOKUPS.get(name) + urls = {token(item)[0] if name == 'code' else unescape(item) for item in alternatives} - {None, ''} + if not lookup or not urls: + return + if lookup == 'repositories': + found = repositories_with(self.model.objects, urls) + elif lookup == 'referenced': + found = repositories_with(self.viewable_sources(), urls) + else: + found = sources_with(self.viewable_sources(), urls) + self.found.update({(lookup, url): value for url, value in found.items()}) + + def criterion(self, name, modifier, value): + """The Q for one alternative (still escaped) of a parameter's value.""" + if name in STRING_FIELDS: + return self.by_string(STRING_FIELDS[name], modifier, unescape(value)) + if name == 'identifier': + return self.by_identifier(value) + if name in TOKEN_SYSTEMS: + return self.by_token(name, *token(value)) + return getattr(self, self.parameters[name])(unescape(value)) + + def by_token(self, name, system, code): + """ + A token parameter: code (any system), system|code, system| (any code of that system) or |code (explicitly no + system), against the system its codes belong to (TOKEN_SYSTEMS). A parameter with no system takes |code as + code; any other system is no match. + """ + handler, implicit = getattr(self, self.parameters[name]), TOKEN_SYSTEMS[name] + if system is None or (system == '' and implicit is None): + return handler(code) + if not system or implicit is None: + return NOTHING + if implicit == CODE_SYSTEM: # system| is any code: a version of that code system with a concept + return self.by_url(system) & (handler(code) if code else defines_concept()) + if system != implicit: + return NOTHING + return handler(code) if code else ANY_CODE[name] + + @staticmethod + def by_string(field, modifier, value): + """ + FHIR string search: :exact is the whole value, case and accents included; no modifier (starts with) and + :contains ignore case and accents, unaccenting both sides in the database (LIKE's wildcards are escaped). + """ + lookup = STRING_LOOKUPS[modifier] + if lookup is None: + return Q(**{field: value}) + return lookup(ImmutableUnaccent(field), ImmutableUnaccent(Value(value))) + + def by_id(self, value): + """The opaque id at the global base, the mnemonic at an owner base.""" + if self.namespace is None: + return Q(id=int(value)) if OPAQUE_ID.fullmatch(value) else NOTHING + return Q(mnemonic=value) + + @staticmethod + def by_last_updated(value): + prefix, start, end = date_range(value) + before_end = Q(updated_at__lt=end) if end else EVERYTHING + after_end = Q(updated_at__gte=end) if end else NOTHING + return { + 'eq': Q(updated_at__gte=start) & before_end, + 'ne': Q(updated_at__lt=start) | after_end, + 'gt': after_end, 'sa': after_end, + 'ge': Q(updated_at__gte=start), + 'lt': Q(updated_at__lt=start), 'eb': Q(updated_at__lt=start), + 'le': before_end, + }[prefix] + + def by_url(self, value): + return canonical_q(value, self.found[('repositories', value)]) + + @staticmethod + def by_version(value): + return Q(version=value) + + @staticmethod + def by_status(value): + return STATUSES.get(value, NOTHING) + + def by_identifier(self, value): + """ + A token, matched on one identifier as it's shown: value (any system), system|value, |value (no system) or + system| (any value). Besides the identifiers a repository holds, every resource without a stored accession + identifier shows a generated one: system API_BASE_URL, value /{orgs|users}/{owner}/{type}/{id}/. + """ + system, value = token(value) + value = value or None + if system == '' and value is None: + return NOTHING + criterion = held_identifier(system, value) + if system not in (None, settings.API_BASE_URL): + return criterion + generated = Q(identifier__isnull=True) | ~Q( + Exact(JsonbTypeof('identifier'), 'array') & JsonPathExists('identifier', STORED_ACCESSION, {})) + if value is None: + return criterion | generated + accession = re.fullmatch(r'/(orgs|users)/([^/]+)/([^/]+)/([^/]+)/?', value) + if accession and accession.group(3) == self.resource_type: + owner = 'organization__mnemonic' if accession.group(1) == 'orgs' else 'user__username' + criterion |= Q(Q(**{owner: accession.group(2), 'mnemonic': accession.group(4)}) & generated) + return criterion + + @staticmethod + def by_code(value): + """A CodeSystem that defines the code.""" + return defines_concept(concept__mnemonic=value) + + @staticmethod + def by_language(value): + return Q(default_locale=value) + + def by_reference(self, value): + """ + A ValueSet whose stored default expansion draws on a version of the code system (canonical URL) named, among + those the base shows the caller. An uncorrelated IN, which the planner hashes once, rather than a correlated + subquery. + """ + sources = self.viewable_sources().filter(canonical_q(value, self.found[('referenced', value)])).values('id') + return reduce(or_, ( + Q(expansion_uri__in=versions.through.objects.filter(source_id__in=sources).values('expansion__uri')) + for versions in (Expansion.explicit_source_versions, Expansion.evaluated_source_versions))) + + def by_source(self, value): + """A group's source is the system named.""" + return mapped('from', value, self.found[('sources', value)]) + + def by_target(self, value): + """A group's target is the system named.""" + return mapped('to', value, self.found[('sources', value)]) + + +def page_size(query): + value = query.get('_count') + if value is None: + return MAX_COUNT + if not re.fullmatch(r'[0-9]{1,9}', value.strip()): + raise outcomes.InvalidError(f"_count must be a whole number, got '{value}'.", expression='_count') + return min(int(value), MAX_COUNT) + + +def within_search_timeout(build): + """ + build() within SEARCH_TIMEOUT_MS in all: each query it runs may take only the time the search has left, so a + search of several queries is bounded as a whole. A query the database cancels, or one with no time left, makes the + search too costly (422). Inside a transaction (tests) the timeout would outlive the search, so it isn't set there. + """ + if connection.vendor != 'postgresql' or connection.in_atomic_block: + return build() + deadline = time.monotonic() + SEARCH_TIMEOUT_MS / 1000 + + def within_time_left(execute, sql, params, many, context): + left = math.ceil((deadline - time.monotonic()) * 1000) + if left <= 0: + raise too_costly() + with context['connection'].connection.cursor() as cursor: # the driver's own cursor, past this wrapper + cursor.execute('SET LOCAL statement_timeout = %s', [left]) + return execute(sql, params, many, context) + + try: + with transaction.atomic(), connection.execute_wrapper(within_time_left): + return build() + except OperationalError as ex: + if getattr(ex.__cause__, 'pgcode', None) != errorcodes.QUERY_CANCELED: + raise + raise too_costly() from ex + + +def too_costly(): + return outcomes.TooCostlyError( + 'The search would take too long to answer. Narrow it, for example with url, _id or version.') + + +def search_bundle(view): + """The searchset Bundle a FHIR search view answers: the base's matching versions, one page of them.""" + request, resource_type = view.request, view.resource_type + count = page_size(request.query_params) + search = Search(resource_type, view.kwargs, request.user) + ids = list(visible_versions(resource_type, access=search.access).filter( + search.criteria(request.query_params)).order_by('-id').values_list('id', flat=True)) + paginator, entries = None, [] + if count: + page = to_int(request.GET.get('page', '1'), 1) + rows = with_repository_canonical(REPO_TYPES[resource_type].objects.filter( + id__in=ids[(page - 1) * count:page * count])).order_by('-id') + if resource_type == 'CodeSystem': + rows = with_concepts_over_limit(rows) + paginator = CustomPaginator( + request=request, total_count=len(ids), queryset=list(rows), page_size=count, is_sliced=True) + resources = view.get_serializer(paginator.current_page_results, many=True).data + entries = FHIRBundleSerializer.convert_to_entry( + resources, f'{fhir_base_url(request, view.kwargs)}/{resource_type}') + bundle = {'meta': {'lastUpdated': timezone.now()}, 'type': 'searchset', 'total': len(ids)} + if entries: + bundle['entry'] = entries + return FHIRBundleSerializer(bundle, context={'paginator': paginator, 'request': request}).data + + +def read(view): + """ + The repository version a FHIR read view names: at the global base by its opaque id, at an owner base by its + mnemonic, the version ?version= names (an OCL extension) or else the latest release the requester is shown. It + must be one search lists: released, never HEAD, visible to the requester there. Raises NotFoundError. + """ + resource_type, resource_id = view.resource_type, view.kwargs[view.lookup_field] + namespace, user = namespace_of(view.kwargs), view.request.user + versions = REPO_TYPES[resource_type].objects.filter(is_active=True, released=True).exclude(version=HEAD) + latest = False + if namespace is None: + versions = versions.filter(id=int(resource_id)) if OPAQUE_ID.fullmatch(resource_id) else versions.none() + else: + versions = view.filter_queryset_by_owner(versions.filter(mnemonic=resource_id)) + version = view.request.query_params.get('version') + latest = not version + versions = versions.filter(version=version) if version else versions.filter( + retired=False).order_by('-created_at') + + def shown(instance): + return instance and is_visible(instance, namespace, user) and has_release(instance, namespace, user) + + instance = with_repository_access(with_repository_canonical(versions)).first() + if latest and instance and not shown(instance): # the latest is private: the latest public one, if shown + instance = with_repository_access(with_repository_canonical( + versions.filter(public_access__in=PUBLIC_ACCESS))).first() + if not shown(instance): + raise outcomes.NotFoundError(f'{resource_type}/{resource_id} not found') + return instance + + +class FhirRepositoryViewMixin: + """The search and read views of CodeSystem, ValueSet and ConceptMap (after FhirViewMixin).""" + resource_type = None + + def get_serializer_context(self): + return {**super().get_serializer_context(), 'opaque_ids': namespace_of(self.kwargs) is None} + + +class FhirSearchMixin(FhirRepositoryViewMixin): + """GET on a resource type is its search.""" + + def get(self, request, *args, **kwargs): # pylint: disable=unused-argument + """The searchset Bundle, in at most SEARCH_TIMEOUT_MS of database time.""" + return Response(within_search_timeout(lambda: search_bundle(self))) + + head = get + + +class FhirReadMixin(FhirRepositoryViewMixin): + """ + GET on an id is its read, at the global base without the owner check the view's writes keep (verify_scope), and + without the source and collection views' recounts. + """ + + def verify_scope(self): + if self.request.method not in ('GET', 'HEAD'): + super().verify_scope() + + def get_object(self, queryset=None): + if self.request.method not in ('GET', 'HEAD'): + return super().get_object(queryset) + return read(self) + + +class CanonicalUrlField(CharField): + """A resource's canonical URL: the version's own, else its repository's (repository_canonical_url, on a read).""" + + def get_attribute(self, instance): + return super().get_attribute(instance) or getattr(instance, 'repository_canonical_url', None) + + +class FhirRepositorySerializerMixin: + """ + For the CodeSystem, ValueSet and ConceptMap serializers: the opaque id at the global base (context opaque_ids), and + in a search (many=True, context has_many) the summary, marked SUBSETTED, without `content_field`. + """ + content_field = None + + @classmethod + def many_init(cls, *args, **kwargs): + kwargs['context'] = {**kwargs.get('context', {}), 'has_many': True} + return super().many_init(*args, **kwargs) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.context.get('has_many'): + self.fields.pop(self.content_field, None) + + def with_fhir_view(self, instance, rep): + """ + The id (opaque at the global base), and meta.versionId: the version's opaque id at both bases, so the + releases of one repository, which share their mnemonic id at an owner base, stay distinct in a Bundle (bdl-7). + A search entry is also tagged SUBSETTED. meta keeps what the serializer put there (lastUpdated). + """ + if instance.id: + if self.context.get('opaque_ids'): + rep['id'] = str(instance.id) + rep['meta'] = {'versionId': str(instance.id), **(rep.get('meta') or {})} + if self.context.get('has_many'): + rep['meta'] = {**(rep.get('meta') or {}), 'tag': [SUBSETTED]} + return rep diff --git a/core/fhir/tests_protocol.py b/core/fhir/tests_protocol.py index ced0bf63e..5bc7cf4ad 100644 --- a/core/fhir/tests_protocol.py +++ b/core/fhir/tests_protocol.py @@ -56,7 +56,7 @@ def setUp(self): self.other = ConceptFactory(parent=self.source, mnemonic='c2') MappingFactory(parent=self.source, from_concept=self.concept, to_concept=self.other) self.source_v1 = OrganizationSourceFactory.build( - version='v1', mnemonic=self.source.mnemonic, organization=self.org, canonical_url=SYSTEM) + version='v1', mnemonic=self.source.mnemonic, organization=self.org, canonical_url=SYSTEM, released=True) Source.persist_new_version(self.source_v1, self.source.created_by) self.collection = OrganizationCollectionFactory( organization=self.org, mnemonic='protovs', canonical_url=VALUE_SET, version='HEAD') @@ -186,7 +186,8 @@ def test_xml_of_released_code_systems_is_in_fhir_element_order_and_json_is_uncha self.assertLess(child_names(root).index('link'), child_names(root).index('entry')) resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') assert_in_model_order(self, resource, CodeSystem) - self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'rel_codes') + released = Source.objects.get(mnemonic='rel_codes', version='v1') + self.assertEqual(resource.find(f'{FHIR}id').get('value'), str(released.id)) # opaque at the global base response = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/rel_codes/?_format=xml') self.assertEqual(response.status_code, 200) @@ -195,9 +196,11 @@ def test_xml_of_released_code_systems_is_in_fhir_element_order_and_json_is_uncha self.assertEqual(child_names(root)[:2], ['id', 'meta']) def test_xml_values_match_json_for_a_value_set_and_a_concept_map_at_an_owner_base(self): + OrganizationCollectionFactory( + organization=self.org, mnemonic='my_vs', canonical_url='http://example.org/vs/my_vs', version='HEAD') OrganizationCollectionFactory( organization=self.org, mnemonic='my_vs', canonical_url='http://example.org/vs/my_vs', version='v1', - is_latest_version=True) + is_latest_version=True, released=True) for path in (f'/orgs/{self.org.mnemonic}/ValueSet/my_vs/', f'/orgs/{self.org.mnemonic}/ConceptMap/proto/'): as_json = json.loads(self.client.get(path).content) response = self.client.get(path + '?_format=xml') diff --git a/core/fhir/tests_resolve.py b/core/fhir/tests_resolve.py index 637823f44..d697ccfda 100644 --- a/core/fhir/tests_resolve.py +++ b/core/fhir/tests_resolve.py @@ -269,6 +269,29 @@ def test_a_private_repository_that_resolves_hides_a_public_one(self): self.assertUnresolved('CodeSystem', shared, kwargs={'org': 'FHIR-Fixtures'}, user=self.outsider) self.assertEqual(self.resolve('CodeSystem', shared, kwargs={'org': 'HL7'}).id, public_v1.id) + def test_a_public_release_of_a_private_repository(self): + """ + A repository's HEAD must be viewable for its releases to be: a release made public before its repository was + made private resolves only at its owner's base, for those who may view the owner's private repositories. + """ + url = 'http://moh.example/fhir/CodeSystem/hiv' + moh = OrganizationFactory(mnemonic='MOH') + moh_member = UserProfileFactory(username='moh-member') + moh.members.add(moh_member) + source(moh, 'hiv', canonical_url=url, public_access=ACCESS_TYPE_NONE) + release = source(moh, 'hiv', 'v1', released=True, canonical_url=url) + if self.registry: + GlobalURLRegistryFactory(url=url, namespace=moh.uri) + + for user in (ANONYMOUS, self.outsider, moh_member, self.admin): + for reference in (url, f'{url}|v1', '/orgs/MOH/CodeSystem/hiv/'): + self.assertUnresolved('CodeSystem', reference, user=user) + for user in (ANONYMOUS, self.outsider): + self.assertUnresolved('CodeSystem', url, kwargs={'org': 'MOH'}, user=user) + for user in (moh_member, self.admin): + for reference in (url, f'{url}|v1', '/orgs/MOH/CodeSystem/hiv/'): + self.assertEqual(self.resolve('CodeSystem', reference, kwargs={'org': 'MOH'}, user=user).id, release.id) + def test_query_counts(self): """ A hot path: these counts may only fall. Without the registry: the toggle, the repository and its latest diff --git a/core/fhir/tests_search.py b/core/fhir/tests_search.py new file mode 100644 index 000000000..3d10ad2e2 --- /dev/null +++ b/core/fhir/tests_search.py @@ -0,0 +1,1127 @@ +""" +core.fhir.search: FHIR search and read of CodeSystem, ValueSet and ConceptMap (SVCM ITI-95, ITI-96 and ITI-100) at the +global base /fhir/ and at owner bases. Released versions only, never HEAD; public ones at the global base whoever asks; +an opaque id per released version at the global base, the mnemonic at an owner base; SVCM's search parameters with +:exact and :contains; _count, paging, the real total and a fullUrl on every entry; a CodeSystem read is complete up to +the concept limit, else not-present. OpenConceptLab/ocl_online#451. +""" +import time +from datetime import datetime, timezone as dt_timezone +from urllib.parse import parse_qs, urlsplit + +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.db import connection, OperationalError +from django.test import TransactionTestCase +from django.test.utils import CaptureQueriesContext +from mock import patch +from rest_framework.test import APIClient + +from core.collections.models import Collection +from core.collections.tests.factories import OrganizationCollectionFactory, ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE, HEAD +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory, ConceptDescriptionFactory +from core.fhir import resolve as fhir_resolve, search as fhir_search +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM, TooCostlyError +from core.mappings.tests.factories import MappingFactory +from core.orgs.tests.factories import OrganizationFactory +from core.sources.models import Source +from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory +from core.users.models import UserProfile +from core.users.tests.factories import UserProfileFactory + +ICD10 = 'http://hl7.org/fhir/sid/icd-10' +CIEL = 'https://CIELterminology.org' +FIXTURE = 'https://openconceptlab.org/fhir-fixtures/CodeSystem/fixture-codes' +ICD10_VS = 'http://sahra.tools/fhir/ValueSet/icd10-all' +CIEL_VS = 'https://example.org/ValueSet/ciel-set' +HIV = 'http://moh.example/fhir/CodeSystem/hiv' +HIV_VS = 'http://moh.example/fhir/ValueSet/hiv' +SUBSETTED = {'system': 'http://terminology.hl7.org/CodeSystem/v3-ObservationValue', 'code': 'SUBSETTED'} +HL7_OID = {'system': 'urn:ietf:rfc:3986', 'value': 'urn:oid:2.16.840.1.113883.6.90'} +ACSN = {'text': 'Accession ID', 'coding': [{'system': 'http://hl7.org/fhir/v2/0203', 'code': 'ACSN'}]} + + +def source(owner, mnemonic, version=HEAD, **kwargs): + if isinstance(owner, UserProfile): + return UserSourceFactory(user=owner, mnemonic=mnemonic, name=mnemonic, version=version, **kwargs) + return OrganizationSourceFactory(organization=owner, mnemonic=mnemonic, name=mnemonic, version=version, **kwargs) + + +def collection(owner, mnemonic, version=HEAD, **kwargs): + return OrganizationCollectionFactory( + organization=owner, mnemonic=mnemonic, name=mnemonic, version=version, **kwargs) + + +def at(year): + return datetime(year, 6, 15, 12, tzinfo=dt_timezone.utc) + + +class SearchCorpusMixin: + """ + ICD-10 shared by WHO (whose released version has no canonical of its own, as in production) and sehatapp (with a + newer unreleased version); CIEL with a retired release and a live one, with concepts and mappings; repositories + nothing visible comes from (nothing released, only a retired release, a HEAD flagged released); the private fixture + org with two releases; a user's repository; MOH's code system and value set, made private after a public release; + three value sets with stored expansions. + """ + def setUp(self): + super().setUp() + self.client = APIClient() + self.admin = UserProfile.objects.get(username='ocladmin') + self.who = OrganizationFactory(mnemonic='WHO') + self.sehatapp = OrganizationFactory(mnemonic='sehatapp') + self.ciel = OrganizationFactory(mnemonic='CIEL') + self.misc = OrganizationFactory(mnemonic='misc') + self.fixtures = OrganizationFactory(mnemonic='FHIR-Fixtures', public_access=ACCESS_TYPE_NONE) + self.member = UserProfileFactory(username='fixture-member') + self.outsider = UserProfileFactory(username='outsider') + self.fixtures.members.add(self.member) + self.jdoe = UserProfileFactory(username='jdoe') + + self.who_icd = source(self.who, 'ICD-10-WHO', canonical_url=ICD10, full_name='ICD-10 WHO Edition') + self.who_icd_2019 = source( + self.who, 'ICD-10-WHO', 'ICD102019', released=True, canonical_url=None, full_name='ICD-10 WHO Edition', + publisher='World Health Organization', identifier=[HL7_OID]) + self.sehat_icd = source(self.sehatapp, 'ICD-10', canonical_url=ICD10, full_name='ICD-10 for Sehat') + self.sehat_icd_v1 = source( + self.sehatapp, 'ICD-10', 'v1.0', released=True, canonical_url=ICD10, full_name='ICD-10 for Sehat') + self.sehat_icd_v2 = source(self.sehatapp, 'ICD-10', 'v2.0', canonical_url=ICD10) # newest, unreleased + + self.ciel_source = source( + self.ciel, 'CIEL', canonical_url=CIEL, full_name='Columbia International eHealth Laboratory', + description='Concepts for clinical care') + self.ciel_v1 = source( + self.ciel, 'CIEL', 'v1', released=True, retired=True, canonical_url=CIEL, + full_name='Columbia International eHealth Laboratory', description='Concepts for clinical care') + self.ciel_v2 = source( + self.ciel, 'CIEL', 'v2', released=True, canonical_url=CIEL, + full_name='Columbia International eHealth Laboratory', description='Concepts for clinical care') + + # nothing visible comes from these + self.draft = source(self.misc, 'draft', canonical_url='https://example.org/draft') + self.draft_v1 = source(self.misc, 'draft', 'v1', canonical_url='https://example.org/draft') + self.gone = source(self.misc, 'gone', canonical_url='https://example.org/gone') + self.gone_v1 = source( + self.misc, 'gone', 'v1', released=True, retired=True, canonical_url='https://example.org/gone') + self.flagged = source(self.misc, 'flagged', released=True, canonical_url='https://example.org/flagged') + + self.fixture = source(self.fixtures, 'fixture-codes', canonical_url=FIXTURE, public_access=ACCESS_TYPE_NONE) + self.fixture_v1 = source( + self.fixtures, 'fixture-codes', '1.0.0', released=True, canonical_url=FIXTURE, + public_access=ACCESS_TYPE_NONE) + self.fixture_v2 = source( + self.fixtures, 'fixture-codes', '2.0.0', released=True, canonical_url=FIXTURE, + public_access=ACCESS_TYPE_NONE) + + self.jdoe_codes = source(self.jdoe, 'my-codes', canonical_url='https://example.org/my-codes') + self.jdoe_codes_v1 = source( + self.jdoe, 'my-codes', 'v1', released=True, canonical_url='https://example.org/my-codes') + + # repositories made private after a public release: the release keeps its own public access, but its + # repository (HEAD) is private, so it's shown only at MOH's base, to MOH's members and staff + self.moh = OrganizationFactory(mnemonic='MOH') + self.moh_member = UserProfileFactory(username='moh-member') + self.moh.members.add(self.moh_member) + self.hiv = source(self.moh, 'hiv-codes', canonical_url=HIV, public_access=ACCESS_TYPE_NONE) + self.hiv_v1 = source(self.moh, 'hiv-codes', 'v1', released=True, canonical_url=None) # the repository's + + # the value sets: sehatapp's draws on its ICD-10 v1.0, the CIEL one on CIEL's HEAD; the released version of + # the ICD-10 one has no canonical of its own + self.icd_vs = collection(self.sehatapp, 'icd10-all', canonical_url=ICD10_VS, full_name='All ICD-10 codes') + self.icd_vs_v1 = collection( + self.sehatapp, 'icd10-all', 'v1.0', released=True, canonical_url=None, full_name='All ICD-10 codes') + self.with_expansion(self.icd_vs_v1, explicit=[self.sehat_icd_v1]) + self.ciel_vs = collection(self.ciel, 'ciel-set', canonical_url=CIEL_VS) + self.ciel_vs_v1 = collection(self.ciel, 'ciel-set', 'v1', released=True, canonical_url=CIEL_VS) + self.with_expansion(self.ciel_vs_v1, evaluated=[self.ciel_source]) + self.ciel_vs_v2 = collection(self.ciel, 'ciel-set', 'v2', canonical_url=CIEL_VS) # unreleased + self.hiv_vs = collection(self.moh, 'hiv-set', canonical_url=HIV_VS, public_access=ACCESS_TYPE_NONE) + self.hiv_vs_v1 = collection(self.moh, 'hiv-set', 'v1', released=True, canonical_url=HIV_VS) + self.with_expansion(self.hiv_vs_v1, explicit=[self.ciel_v2]) + + @staticmethod + def with_expansion(collection_version, explicit=(), evaluated=()): + expansion = ExpansionFactory(collection_version=collection_version) + expansion.explicit_source_versions.set(explicit) + expansion.evaluated_source_versions.set(evaluated) + Collection.objects.filter(id=collection_version.id).update(expansion_uri=expansion.uri) + + def get(self, path, user=None, **params): + client = APIClient() + if user: + client.force_authenticate(user) + return client.get(path, params) + + def read(self, path, user=None, **params): + response = self.get(path, user, **params) + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def search(self, resource_type, base='/fhir', user=None, **params): + response = self.get(f'{base}/{resource_type}/', user, **params) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual((response.data['resourceType'], response.data['type']), ('Bundle', 'searchset')) + return response.data + + @staticmethod + def keys(bundle): + return sorted(f"{entry['resource'].get('url')}|{entry['resource']['version']}" + for entry in bundle.get('entry', [])) + + @staticmethod + def ids(bundle): + return sorted(entry['resource']['id'] for entry in bundle.get('entry', [])) + + def assertOutcome(self, response, status, code, tx_issue_type): # pylint: disable=invalid-name + self.assertEqual(response.status_code, status, response.data) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + issue = response.data['issue'][0] + self.assertEqual(issue['code'], code) + self.assertIn({'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type}, issue['details']['coding']) + return issue + + def assertNotFound(self, path, user=None, **params): # pylint: disable=invalid-name + response = self.get(path, user, **params) + self.assertOutcome(response, 404, 'not-found', 'not-found') + + +class CodeSystemSearchTest(SearchCorpusMixin, OCLTestCase): # pylint: disable=too-many-public-methods + PUBLIC_RELEASES = sorted([ + f'{CIEL}|v1', f'{CIEL}|v2', f'{ICD10}|ICD102019', f'{ICD10}|v1.0', 'https://example.org/my-codes|v1']) + + def test_global_search_lists_every_released_public_version_with_an_opaque_id(self): + bundle = self.search('CodeSystem') + + # never HEAD, an unreleased version, a repository with nothing (or only a retired release) released, or a + # private one; a retired release of a repository with a live one is listed + self.assertEqual(self.keys(bundle), self.PUBLIC_RELEASES) + self.assertEqual(bundle['total'], 5) + versions = [self.ciel_v1, self.ciel_v2, self.who_icd_2019, self.sehat_icd_v1, self.jdoe_codes_v1] + self.assertEqual(self.ids(bundle), sorted(str(version.id) for version in versions)) + for entry in bundle['entry']: + self.assertEqual(entry['fullUrl'], f"http://testserver/fhir/CodeSystem/{entry['resource']['id']}") + self.assertEqual(entry['search'], {'mode': 'match'}) + self.assertEqual(bundle['link'][0]['relation'], 'self') + + def test_global_base_is_public_only_whoever_asks(self): + for user in (self.member, self.admin, self.jdoe): + self.assertEqual(self.keys(self.search('CodeSystem', user=user)), self.PUBLIC_RELEASES, user.username) + + def test_a_version_with_its_own_canonical_is_found_by_that_one_only(self): + """The repository's (HEAD's) canonical stands in only for a version with none, as CanonicalUrlField shows it.""" + old = 'https://old.example.org/ciel' + Source.objects.filter(id=self.ciel_v1.id).update(canonical_url=old) + ConceptFactory(parent=self.ciel_source, mnemonic='1427', sources=[self.ciel_v1, self.ciel_v2]) + + self.assertEqual(self.keys(self.search('CodeSystem', url=CIEL)), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', url=old)), [f'{old}|v1']) + self.assertEqual(self.keys(self.search('CodeSystem', system=old)), [f'{old}|v1']) + self.assertEqual(self.keys(self.search('CodeSystem', code=f'{CIEL}|1427')), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', code=f'{old}|1427')), [f'{old}|v1']) + Source.objects.filter(id=self.ciel_v1.id).update(canonical_url='') # an empty one is none + self.assertEqual(self.keys(self.search('CodeSystem', url=CIEL)), [f'{CIEL}|v1', f'{CIEL}|v2']) + + def test_url_matches_the_version_canonical_or_its_repository_canonical(self): + bundle = self.search('CodeSystem', url=ICD10) + + self.assertEqual(self.keys(bundle), [f'{ICD10}|ICD102019', f'{ICD10}|v1.0']) + who = next(entry['resource'] for entry in bundle['entry'] if entry['resource']['version'] == 'ICD102019') + self.assertEqual(who['url'], ICD10) # the repository's canonical, as the version has none + self.assertEqual(self.keys(self.search('CodeSystem', system=ICD10)), self.keys(bundle)) + + bundle = self.search('CodeSystem', url=ICD10, version='ICD102019') + self.assertEqual((self.keys(bundle), bundle['total']), ([f'{ICD10}|ICD102019'], 1)) + self.assertEqual(self.search('CodeSystem', url='http://example.org/unknown')['total'], 0) + self.assertNotIn('entry', self.search('CodeSystem', url='http://example.org/unknown')) + + def test_id_is_the_opaque_id_at_the_global_base_and_the_mnemonic_at_an_owner_base(self): + bundle = self.search('CodeSystem', _id=str(self.sehat_icd_v1.id)) + self.assertEqual(self.keys(bundle), [f'{ICD10}|v1.0']) + for missing in ('ICD-10', str(self.sehat_icd.id), str(self.sehat_icd_v2.id), 'ocl-no-such-id'): + self.assertEqual(self.search('CodeSystem', _id=missing)['total'], 0, missing) + + bundle = self.search('CodeSystem', base='/orgs/CIEL', _id='CIEL') + self.assertEqual(self.keys(bundle), [f'{CIEL}|v1', f'{CIEL}|v2']) + self.assertEqual(self.ids(bundle), ['CIEL', 'CIEL']) + self.assertEqual(self.search('CodeSystem', base='/orgs/CIEL', _id=str(self.ciel_v2.id))['total'], 0) + + def test_string_parameters_and_their_modifiers(self): + # no modifier: starts with, ignoring case + self.assertEqual(self.keys(self.search('CodeSystem', title='icd-10')), [f'{ICD10}|ICD102019', f'{ICD10}|v1.0']) + self.assertEqual(self.keys(self.search('CodeSystem', name='ciel')), [f'{CIEL}|v1', f'{CIEL}|v2']) + self.assertEqual(self.search('CodeSystem', title='WHO')['total'], 0) + # :exact is the whole value, case included + self.assertEqual( + self.keys(self.search('CodeSystem', **{'title:exact': 'ICD-10 WHO Edition'})), [f'{ICD10}|ICD102019']) + self.assertEqual(self.search('CodeSystem', **{'title:exact': 'icd-10 who edition'})['total'], 0) + self.assertEqual(self.search('CodeSystem', **{'title:exact': 'ICD-10'})['total'], 0) + # :contains anywhere, ignoring case + self.assertEqual(self.keys(self.search('CodeSystem', **{'title:contains': 'who'})), [f'{ICD10}|ICD102019']) + self.assertEqual( + self.keys(self.search('CodeSystem', **{'description:contains': 'CLINICAL'})), [f'{CIEL}|v1', f'{CIEL}|v2']) + self.assertEqual( + self.keys(self.search('CodeSystem', publisher='world health')), [f'{ICD10}|ICD102019']) + self.assertEqual(self.keys(self.search('CodeSystem', **{'name:exact': 'CIEL'})), [f'{CIEL}|v1', f'{CIEL}|v2']) + + def test_strings_ignore_accents_except_exact(self): + Source.objects.filter(id=self.ciel_v2.id).update(full_name='Édition française') + + self.assertEqual(self.keys(self.search('CodeSystem', title='edition')), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', title='ÉDITION')), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', **{'title:contains': 'francaise'})), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', **{'title:contains': 'FRANÇAISE'})), [f'{CIEL}|v2']) + self.assertEqual(self.keys(self.search('CodeSystem', **{'title:exact': 'Édition française'})), [f'{CIEL}|v2']) + self.assertEqual(self.search('CodeSystem', **{'title:exact': 'Edition francaise'})['total'], 0) + for wildcard in ('%', '_', 'É_ition'): # LIKE's wildcards are plain characters + self.assertEqual(self.search('CodeSystem', **{'title:contains': wildcard})['total'], 0, wildcard) + + def test_a_comma_is_or_and_a_repeated_parameter_is_and(self): + self.assertEqual( + self.keys(self.search('CodeSystem', version='v1,v1.0')), + sorted([f'{CIEL}|v1', f'{ICD10}|v1.0', 'https://example.org/my-codes|v1'])) + response = self.client.get('/fhir/CodeSystem/?name=ci&name=ICD') + self.assertEqual(response.data['total'], 0) + response = self.client.get('/fhir/CodeSystem/?name=ci&name=CIEL') + self.assertEqual(self.keys(response.data), [f'{CIEL}|v1', f'{CIEL}|v2']) + # an escaped comma is part of the value + Source.objects.filter(id=self.ciel_v2.id).update(full_name='Columbia, International') + self.assertEqual(self.keys(self.search('CodeSystem', title=r'Columbia\, Int')), [f'{CIEL}|v2']) + + def test_status(self): + self.assertEqual( + self.keys(self.search('CodeSystem', status='active')), + sorted([f'{CIEL}|v2', f'{ICD10}|ICD102019', f'{ICD10}|v1.0', 'https://example.org/my-codes|v1'])) + self.assertEqual(self.keys(self.search('CodeSystem', status='retired')), [f'{CIEL}|v1']) + self.assertEqual(self.search('CodeSystem', status='draft')['total'], 0) + self.assertEqual(self.search('CodeSystem', status='unknown')['total'], 0) + self.assertEqual(self.search('CodeSystem', status='retired,active')['total'], 5) + + def test_identifier(self): + for token in ('urn:ietf:rfc:3986|urn:oid:2.16.840.1.113883.6.90', 'urn:oid:2.16.840.1.113883.6.90', + 'urn:ietf:rfc:3986|'): + self.assertEqual(self.keys(self.search('CodeSystem', identifier=token)), [f'{ICD10}|ICD102019'], token) + for token in ('urn:ietf:rfc:3986|urn:oid:1.2.3', 'http://example.org|urn:oid:2.16.840.1.113883.6.90'): + self.assertEqual(self.search('CodeSystem', identifier=token)['total'], 0, token) + + # the OCL accession identifier every resource carries + bundle = self.search('CodeSystem', identifier='/orgs/sehatapp/CodeSystem/ICD-10/') + self.assertEqual(self.keys(bundle), [f'{ICD10}|v1.0']) + accession = bundle['entry'][0]['resource']['identifier'][0] + self.assertEqual( + self.keys(self.search('CodeSystem', identifier=f"{accession['system']}|{accession['value']}")), + [f'{ICD10}|v1.0']) + self.assertEqual(self.search('CodeSystem', identifier='/orgs/sehatapp/ValueSet/ICD-10/')['total'], 0) + self.assertEqual( + self.search('CodeSystem', identifier='/orgs/FHIR-Fixtures/CodeSystem/fixture-codes/')['total'], 0) + + def test_a_stored_accession_without_a_value_leaves_the_generated_one(self): + """IdentifierSerializer.find_ocl_identifier needs an accession value; without one the generated id shows.""" + api = settings.API_BASE_URL + for stored in ([{'system': 'urn:external', 'type': ACSN}], + [{'system': 'urn:external', 'value': '', 'type': ACSN}], + [{'system': 'urn:external', 'value': None, 'type': ACSN}]): + Source.objects.filter(id=self.who_icd_2019.id).update(identifier=stored) + resource = self.search('CodeSystem', url=ICD10, version='ICD102019')['entry'][0]['resource'] + self.assertIn({'system': api, 'value': '/orgs/WHO/CodeSystem/ICD-10-WHO/'}, + [{'system': item.get('system'), 'value': item.get('value')} + for item in resource['identifier']], stored) + for token in (f'{api}|', '/orgs/WHO/CodeSystem/ICD-10-WHO/', f'{api}|/orgs/WHO/CodeSystem/ICD-10-WHO/'): + self.assertIn(f'{ICD10}|ICD102019', self.keys(self.search('CodeSystem', identifier=token)), token) + + # with a value, the stored one is shown instead; a coding's missing code and system show ACSN's defaults + path = '/orgs/WHO/CodeSystem/ICD-10-WHO/' + for coding in (ACSN['coding'][0], {}, {'code': 'ACSN'}): + Source.objects.filter(id=self.who_icd_2019.id).update( + identifier=[{'system': 'urn:external', 'value': path, 'type': {'coding': [coding]}}]) + self.assertNotIn(f'{ICD10}|ICD102019', self.keys(self.search('CodeSystem', identifier=f'{api}|')), coding) + self.assertEqual( + self.keys(self.search('CodeSystem', identifier=f'urn:external|{path}')), [f'{ICD10}|ICD102019']) + # another type, or a coding whose code or system is null, is no accession + for coding in ({'code': 'MR'}, {'code': None}, {'code': 'ACSN', 'system': None}): + Source.objects.filter(id=self.who_icd_2019.id).update( + identifier=[{'system': 'urn:external', 'value': path, 'type': {'coding': [coding]}}]) + self.assertIn(f'{ICD10}|ICD102019', self.keys(self.search('CodeSystem', identifier=f'{api}|')), coding) + + def test_identifier_system_is_matched_on_the_same_identifier(self): + """ + |value is an identifier without a system; system| is any value of that system. Both are checked on one + identifier, as it's shown: a stored identifier without a system key shows API_BASE_URL's, a null or empty one + none; the generated accession identifier (API_BASE_URL) shows when no accession identifier is stored. + """ + oid = 'urn:oid:2.16.840.1.113883.6.90' + Source.objects.filter(id=self.ciel_v2.id).update(identifier=[{'system': None, 'value': oid}]) + Source.objects.filter(id=self.sehat_icd_v1.id).update(identifier=[{'system': '', 'value': oid}]) + Source.objects.filter(id=self.ciel_v1.id).update(identifier=[{'value': oid}, {'system': 'urn:x', 'value': 'y'}]) + Source.objects.filter(id=self.jdoe_codes_v1.id).update(identifier=[ + {'system': 'https://fhir.openconceptlab.org', 'value': '/users/jdoe/CodeSystem/my-codes/', 'type': ACSN}]) + api = settings.API_BASE_URL + + self.assertEqual( + self.keys(self.search('CodeSystem', identifier=f'|{oid}')), sorted([f'{CIEL}|v2', f'{ICD10}|v1.0'])) + self.assertEqual(self.keys(self.search('CodeSystem', identifier=oid)), sorted( + [f'{CIEL}|v1', f'{CIEL}|v2', f'{ICD10}|ICD102019', f'{ICD10}|v1.0'])) + self.assertEqual(self.keys(self.search('CodeSystem', identifier=f'{api}|{oid}')), [f'{CIEL}|v1']) + self.assertEqual(self.search('CodeSystem', identifier=f'urn:x|{oid}')['total'], 0) # not the same identifier + self.assertEqual(self.search('CodeSystem', identifier='|/orgs/sehatapp/CodeSystem/ICD-10/')['total'], 0) + self.assertEqual(self.search('CodeSystem', identifier='|')['total'], 0) + + # system|: the generated accession identifiers too, for a repository with no stored identifier at all + Source.objects.filter(id=self.who_icd_2019.id).update(identifier={}) + bundle = self.search('CodeSystem', identifier=f'{api}|') + self.assertEqual( + self.keys(bundle), sorted([f'{CIEL}|v1', f'{CIEL}|v2', f'{ICD10}|ICD102019', f'{ICD10}|v1.0'])) + for entry in bundle['entry']: + self.assertIn(api, [identifier.get('system') for identifier in entry['resource']['identifier']]) + # a stored accession identifier is shown instead of the generated one + self.assertEqual( + self.keys(self.search('CodeSystem', identifier='https://fhir.openconceptlab.org|')), + ['https://example.org/my-codes|v1']) + self.assertEqual(self.search('CodeSystem', identifier=f'{api}|/users/jdoe/CodeSystem/my-codes/')['total'], 0) + self.assertEqual( + self.keys(self.search('CodeSystem', identifier='/users/jdoe/CodeSystem/my-codes/')), + ['https://example.org/my-codes|v1']) + self.assertEqual(self.keys(self.search('CodeSystem', identifier='urn:x|')), [f'{CIEL}|v1']) + + # identifiers that aren't a list aren't shown, so they aren't found either + Source.objects.filter(id=self.ciel_v1.id).update(identifier={'system': 'urn:x', 'value': 'y'}) + self.assertEqual(self.search('CodeSystem', identifier='urn:x|')['total'], 0) + + def test_language_system_only_needs_a_language(self): + Source.objects.filter(id=self.ciel_v2.id).update(default_locale='') + self.assertEqual(self.keys(self.search('CodeSystem', language='urn:ietf:bcp:47|')), sorted( + [f'{CIEL}|v1', f'{ICD10}|ICD102019', f'{ICD10}|v1.0', 'https://example.org/my-codes|v1'])) + + def test_code_needs_a_concept_the_read_lists(self): + """ + code searches concept.code, so code=| is a release of that code system with a concept, and code= a + release defining it. A concept counts as the read lists it: linked and not deleted; a retired one counts. + """ + ConceptFactory(parent=self.who_icd, mnemonic='A00', retired=True, sources=[self.who_icd_2019]) + ConceptFactory(parent=self.ciel_source, mnemonic='gone', is_active=False, sources=[self.ciel_v1]) + + self.assertEqual(self.keys(self.search('CodeSystem', code=f'{ICD10}|')), [f'{ICD10}|ICD102019']) + self.assertEqual(self.keys(self.search('CodeSystem', code=f'{ICD10}|A00')), [f'{ICD10}|ICD102019']) + self.assertEqual(self.search('CodeSystem', code=f'{CIEL}|')['total'], 0) + self.assertEqual(self.search('CodeSystem', code='gone')['total'], 0) + + def test_token_parameters_take_their_system(self): + """ + R4 tokens: code, system|code, or |code (no system). status's system is publication-status, language's BCP 47, + code's the code system itself; version and _id have none, so |v is v, and any system finds nothing. + """ + status, bcp47 = 'http://hl7.org/fhir/publication-status', 'urn:ietf:bcp:47' + Source.objects.filter(id=self.ciel_v2.id).update(default_locale='fr') + Source.objects.filter(id=self.ciel_v1.id).update(version='v1|a,b') + ConceptFactory(parent=self.ciel_source, mnemonic='1427', sources=[self.ciel_v2]) + ConceptFactory(parent=self.sehat_icd, mnemonic='1427', sources=[self.sehat_icd_v1]) + active = sorted([f'{CIEL}|v2', f'{ICD10}|ICD102019', f'{ICD10}|v1.0', 'https://example.org/my-codes|v1']) + ciel_v2, icd_v1, ciel_v1 = [f'{CIEL}|v2'], [f'{ICD10}|v1.0'], [f'{CIEL}|v1|a,b'] + pk = str(self.sehat_icd_v1.id) + cases = [ + ('status', 'active', active), ('status', f'{status}|active', active), + ('status', f'{status}|retired', ciel_v1), + ('status', f'{status}|active,{status}|retired', self.keys(self.search('CodeSystem'))), + ('status', '|active', []), ('status', 'http://example.org/other|active', []), + ('status', f'{status}|', self.keys(self.search('CodeSystem'))), ('status', 'http://example.org/other|', []), + ('language', f'{bcp47}|', self.keys(self.search('CodeSystem'))), ('language', 'urn:other|', []), + ('code', f'{CIEL}|', ciel_v2), ('code', f'{ICD10}|', icd_v1), # releases with concepts only + ('code', 'http://example.org/other|', []), ('version', 'urn:x|', []), ('_id', 'urn:x|', []), + ('language', 'fr', ciel_v2), ('language', f'{bcp47}|fr', ciel_v2), ('language', '|fr', []), + ('language', 'urn:other|fr', []), + ('code', '1427', sorted(ciel_v2 + icd_v1)), ('code', f'{CIEL}|1427', ciel_v2), + ('code', f'{ICD10}|1427', icd_v1), + ('code', f'{CIEL}|1427,{ICD10}|1427', sorted(ciel_v2 + icd_v1)), ('code', '|1427', []), + ('code', 'http://example.org/other|1427', []), ('code', f'{CIEL}|A00', []), + ('version', 'v1.0', icd_v1), ('version', '|v1.0', icd_v1), ('version', 'urn:x|v1.0', []), + ('version', r'v1\|a\,b', ciel_v1), ('version', r'|v1\|a\,b', ciel_v1), ('version', 'v1|a', []), + ('_id', pk, icd_v1), ('_id', f'|{pk}', icd_v1), ('_id', f'urn:x|{pk}', []), + ] + for name, value, keys in cases: + self.assertEqual(self.keys(self.search('CodeSystem', **{name: value})), keys, f'{name}={value}') + owner = [('_id', 'CIEL', 2), ('_id', '|CIEL', 2), ('_id', 'urn:x|CIEL', 0)] + for name, value, total in owner: + self.assertEqual(self.search('CodeSystem', base='/orgs/CIEL', **{name: value})['total'], total, value) + + def test_last_updated_and_its_prefixes(self): + Source.objects.filter(id=self.ciel_v2.id).update(updated_at=at(2020)) + Source.objects.filter(id=self.sehat_icd_v1.id).update(updated_at=at(2022)) + pinned = {'url': CIEL, 'version': 'v2'} + for value, found in (('2020', True), ('eq2020-06', True), ('2020-06-15', True), ('2020-06-16', False), + ('gt2019', True), ('gt2020', False), ('sa2019-12-31', True), ('ge2020-06-15', True), + ('ge2020-06-16', False), ('lt2021', True), ('lt2020-06-15', False), ('eb2020-06-16', True), + ('le2020-06-15', True), ('le2020-06-14', False), ('ne2020', False), ('ne2021', True), + ('ge2020-06-15T12:00:00Z', True), ('gt2020-06-15T12:00:00Z', False), + ('lt2020-06-15T13:00:00+01:00', False), ('le2020-06-15T13:00+01:00', True)): + self.assertEqual(self.search('CodeSystem', _lastUpdated=value, **pinned)['total'], int(found), value) + + response = self.client.get('/fhir/CodeSystem/?_lastUpdated=ge2020&_lastUpdated=lt2021') + self.assertEqual(self.keys(response.data), [f'{CIEL}|v2']) + self.assertEqual( + self.keys(self.search('CodeSystem', _lastUpdated='2020,2022')), sorted([f'{CIEL}|v2', f'{ICD10}|v1.0'])) + + def test_last_updated_at_the_end_of_time(self): + """A value in year 9999 has no end to build (year 10000): its range runs on unbounded.""" + for value, total in (('lt9999', 5), ('eb9999', 5), ('le9999', 5), ('le9999-12', 5), ('le9999-12-31', 5), + ('le9999-12-31T23:59:59Z', 5), ('le9999-12-31T23:59:59.999999Z', 5), ('ne9999-12', 5), + ('9999', 0), ('9999-12', 0), ('9999-12-31', 0), ('gt9999', 0), ('sa9999-12', 0), + ('gt9999-12-31', 0), ('ge9999-12-31', 0), ('gt9999-12-31T23:59:59Z', 0)): + self.assertEqual(self.search('CodeSystem', _lastUpdated=value)['total'], total, value) + self.assertEqual(self.search('CodeSystem', _lastUpdated='le9999,lt2000')['total'], 5) # an alternative too + + def test_unsupported_parameters_and_values_are_400_outcomes(self): + for query in ('unsupported_parameter=1', 'q=CIEL', 'limit=10', 'name:missing=true', 'url:below=http://x', + '_sort=name', 'reference=http://x', 'source-system=http://x'): + issue = self.assertOutcome(self.client.get(f'/fhir/CodeSystem/?{query}'), 400, 'not-supported', + 'not-supported') + self.assertIn(query.split('=', maxsplit=1)[0].split(':')[0], issue['details']['text'], query) + for query in ('_count=-1', '_count=ten', '_lastUpdated=yesterday', '_lastUpdated=ap2020', + '_lastUpdated=2020-13', '_lastUpdated=2020-06-15T12:00%2B25:00', 'page=0'): + self.assertOutcome(self.client.get(f'/fhir/CodeSystem/?{query}'), 400, 'invalid', 'invalid-data') + # the format parameters choose the response, they aren't search input + self.assertEqual(self.search('CodeSystem', _format='json', _pretty='true')['total'], 5) + self.assertEqual(self.search('CodeSystem', name='')['total'], 5) # an empty value is no criterion + + def test_count_paging_total_and_links(self): + bundle = self.search('CodeSystem', _count='2') + self.assertEqual((len(bundle['entry']), bundle['total']), (2, 5)) + links = {link['relation']: link['url'] for link in bundle['link']} + self.assertEqual(set(links), {'self', 'next'}) + query = parse_qs(urlsplit(links['next']).query) + self.assertEqual((query['page'], query['_count']), (['2'], ['2'])) + self.assertTrue(links['next'].startswith('http://testserver/fhir/CodeSystem/?')) + + page_2 = self.get(links['next']).data + self.assertEqual({link['relation'] for link in page_2['link']}, {'self', 'next', 'previous'}) + page_3 = self.search('CodeSystem', _count='2', page='3') + self.assertEqual({link['relation'] for link in page_3['link']}, {'self', 'previous'}) + seen = [entry['resource']['id'] for page in (bundle, page_2, page_3) for entry in page['entry']] + self.assertEqual(sorted(seen), self.ids(self.search('CodeSystem'))) # every version once, in a stable order + + bundle = self.search('CodeSystem', _count='0') + self.assertEqual(bundle['total'], 5) + self.assertNotIn('entry', bundle) + + with patch('core.fhir.search.MAX_COUNT', 3): + bundle = self.search('CodeSystem', _count='1000') + self.assertEqual((len(bundle['entry']), bundle['total']), (3, 5)) + self.assertEqual(len(self.search('CodeSystem')['entry']), 3) # the default is the maximum + + def test_entries_are_summaries_marked_subsetted(self): + ConceptFactory(parent=self.ciel_source, sources=[self.ciel_v2]) + for entry in self.search('CodeSystem')['entry']: + resource = entry['resource'] + self.assertNotIn('concept', resource) + self.assertNotIn('count', resource) + self.assertEqual(resource['content'], 'complete') + self.assertIn(SUBSETTED, resource['meta']['tag']) + self.assertIn('lastUpdated', resource['meta']) + + with patch('core.fhir.search.CONCEPT_LIMIT', 0): + contents = {entry['resource']['version']: entry['resource']['content'] + for entry in self.search('CodeSystem', url=CIEL)['entry']} + self.assertEqual(contents, {'v1': 'complete', 'v2': 'not-present'}) + + def test_owner_base_search(self): + bundle = self.search('CodeSystem', base='/orgs/CIEL') + self.assertEqual(self.keys(bundle), [f'{CIEL}|v1', f'{CIEL}|v2']) + for entry in bundle['entry']: + self.assertEqual(entry['resource']['id'], 'CIEL') + self.assertEqual(entry['fullUrl'], 'http://testserver/orgs/CIEL/CodeSystem/CIEL') + + self.assertEqual(self.keys(self.search('CodeSystem', base='/users/jdoe')), ['https://example.org/my-codes|v1']) + self.assertEqual(self.search('CodeSystem', base='/orgs/nobody')['total'], 0) + self.assertEqual(self.search('CodeSystem', base='/orgs/misc')['total'], 0) + + # a private owner's repositories: to its members and staff only + fixture_releases = [f'{FIXTURE}|1.0.0', f'{FIXTURE}|2.0.0'] + for user, keys in ((None, []), (self.outsider, []), (self.member, fixture_releases), + (self.admin, fixture_releases)): + self.assertEqual( + self.keys(self.search('CodeSystem', base='/orgs/FHIR-Fixtures', user=user)), keys, user) + bundle = self.search('CodeSystem', base='/orgs/FHIR-Fixtures', user=self.member, url=FIXTURE, version='1.0.0') + self.assertEqual(self.ids(bundle), ['fixture-codes']) + + def test_no_query_per_entry(self): + def count_queries(expected, **params): + with self.assertNumQueries(expected): + return self.client.get('/fhir/CodeSystem/', params).data + + # the count and the page, its summaries included (and the url's repositories) + one = count_queries(3, url=CIEL, version='v2') + every = count_queries(2) + self.assertEqual((one['total'], every['total']), (1, 5)) + with self.assertNumQueries(3): # + the owner + self.client.get('/orgs/CIEL/CodeSystem/') + + +class ValueSetSearchTest(SearchCorpusMixin, OCLTestCase): + def test_global_search_and_the_repository_canonical(self): + bundle = self.search('ValueSet') + + self.assertEqual(self.keys(bundle), [f'{ICD10_VS}|v1.0', f'{CIEL_VS}|v1']) + self.assertEqual(self.ids(bundle), sorted([str(self.icd_vs_v1.id), str(self.ciel_vs_v1.id)])) + self.assertEqual(self.keys(self.search('ValueSet', url=ICD10_VS)), [f'{ICD10_VS}|v1.0']) + for entry in bundle['entry']: + self.assertEqual(entry['fullUrl'], f"http://testserver/fhir/ValueSet/{entry['resource']['id']}") + self.assertNotIn('compose', entry['resource']) + self.assertIn(SUBSETTED, entry['resource']['meta']['tag']) + + def test_reference_is_a_code_system_the_stored_expansion_draws_on(self): + self.assertEqual(self.keys(self.search('ValueSet', reference=ICD10)), [f'{ICD10_VS}|v1.0']) + self.assertEqual(self.keys(self.search('ValueSet', reference=CIEL)), [f'{CIEL_VS}|v1']) + self.assertEqual(self.search('ValueSet', reference='http://loinc.org')['total'], 0) + + def test_owner_base(self): + bundle = self.search('ValueSet', base='/orgs/CIEL') + self.assertEqual((self.keys(bundle), self.ids(bundle)), ([f'{CIEL_VS}|v1'], ['ciel-set'])) + + def test_unsupported(self): + self.assertOutcome(self.client.get('/fhir/ValueSet/?system=http://x'), 400, 'not-supported', 'not-supported') + + def test_status_takes_its_system(self): + for resource_type in ('ValueSet', 'ConceptMap'): + bare = self.search(resource_type, status='active')['total'] + self.assertGreater(bare, 0) + qualified = self.search(resource_type, status='http://hl7.org/fhir/publication-status|active') + self.assertEqual(qualified['total'], bare, resource_type) + self.assertEqual(self.search(resource_type, status='urn:x|active')['total'], 0, resource_type) + + def test_no_query_per_entry(self): + with self.assertNumQueries(2): + self.client.get('/fhir/ValueSet/') + + +class ConceptMapSearchTest(SearchCorpusMixin, OCLTestCase): + def setUp(self): + super().setUp() + icd_code = ConceptFactory(parent=self.sehat_icd, mnemonic='A00') + ciel_code = ConceptFactory(parent=self.ciel_source, mnemonic='1427') + MappingFactory( + parent=self.ciel_source, from_concept=ciel_code, to_concept=icd_code, from_source=self.ciel_source, + to_source=self.sehat_icd, from_concept_code='1427', to_concept_code='A00', sources=[self.ciel_v2]) + + def test_source_and_target(self): + everything = self.search('ConceptMap') + self.assertEqual(self.keys(everything), CodeSystemSearchTest.PUBLIC_RELEASES) + for entry in everything['entry']: + self.assertNotIn('group', entry['resource']) + self.assertEqual(entry['fullUrl'], f"http://testserver/fhir/ConceptMap/{entry['resource']['id']}") + + for name in ('source-system', 'source-uri'): + self.assertEqual(self.keys(self.search('ConceptMap', **{name: CIEL})), [f'{CIEL}|v2'], name) + self.assertEqual(self.search('ConceptMap', **{name: ICD10})['total'], 0, name) + for name in ('target-system', 'target-uri'): + self.assertEqual(self.keys(self.search('ConceptMap', **{name: ICD10})), [f'{CIEL}|v2'], name) + self.assertEqual(self.search('ConceptMap', **{name: CIEL})['total'], 0, name) + + def test_source_system_is_the_mappings_source_as_the_read_shows_it(self): + """ + group.source is each live mapping's from_source canonical, or for a source without one its OCL path as a + ConceptMap; group.target the same for to_source. A container's own canonical isn't its source system. + """ + uncanonical = source(self.misc, 'local-codes') + local = ConceptFactory(parent=uncanonical, mnemonic='L1') + icd_code = ConceptFactory(parent=self.sehat_icd, mnemonic='B01') + mine = ConceptFactory(parent=self.jdoe_codes, mnemonic='m1') + for from_source, from_concept in ((self.sehat_icd, icd_code), (uncanonical, local)): + MappingFactory( + parent=self.jdoe_codes, from_concept=from_concept, to_concept=mine, from_source=from_source, + to_source=self.jdoe_codes, from_source_url=from_source.uri, to_source_url=self.jdoe_codes.uri, + sources=[self.jdoe_codes_v1]) + my_codes = 'https://example.org/my-codes|v1' + groups = self.read(f'/fhir/ConceptMap/{self.jdoe_codes_v1.id}')['group'] + self.assertEqual( + sorted(group['source'] for group in groups), sorted([ICD10, '/orgs/misc/ConceptMap/local-codes/'])) + + for name in ('source-system', 'source-uri'): + self.assertEqual(self.keys(self.search('ConceptMap', **{name: ICD10})), [my_codes], name) + self.assertEqual( + self.keys(self.search('ConceptMap', **{name: '/orgs/misc/ConceptMap/local-codes/'})), [my_codes], name) + self.assertEqual(self.search('ConceptMap', **{name: 'https://example.org/my-codes'})['total'], 0, name) + self.assertEqual(self.keys(self.search('ConceptMap', **{name: CIEL})), [f'{CIEL}|v2'], name) + self.assertEqual( + self.keys(self.search('ConceptMap', **{'target-system': 'https://example.org/my-codes'})), [my_codes]) + # a target with a canonical is shown by it, not by its OCL path + self.assertEqual(self.search('ConceptMap', **{'target-system': '/users/jdoe/ConceptMap/my-codes/'})['total'], 0) + + def test_retired_mappings_are_no_part_of_a_concept_map(self): + """A read leaves retired mappings out (ConceptMapGroupField), so source and target searches don't count them.""" + loinc = source(self.misc, 'LOINC', canonical_url='http://loinc.org') + loinc_code = ConceptFactory(parent=loinc, mnemonic='1234-5') + MappingFactory( + parent=self.ciel_source, from_concept=ConceptFactory(parent=self.ciel_source), to_concept=loinc_code, + from_source=self.ciel_source, to_source=loinc, to_concept_code='1234-5', retired=True, + sources=[self.ciel_v2]) + jdoe_code = ConceptFactory(parent=self.jdoe_codes, mnemonic='x1') + retired = MappingFactory( + parent=self.jdoe_codes, from_concept=jdoe_code, to_concept=loinc_code, from_source=self.jdoe_codes, + to_source=loinc, retired=True, sources=[self.jdoe_codes_v1]) + + for name in ('target-system', 'target-uri'): + self.assertEqual(self.search('ConceptMap', **{name: 'http://loinc.org'})['total'], 0, name) + self.assertEqual(self.keys(self.search('ConceptMap', **{name: ICD10})), [f'{CIEL}|v2'], name) + resource = self.read(f'/fhir/ConceptMap/{self.ciel_v2.id}') + self.assertEqual([group['target'] for group in resource['group']], [ICD10]) + for name in ('source-system', 'source-uri'): + self.assertEqual(self.search('ConceptMap', **{name: 'https://example.org/my-codes'})['total'], 0, name) + + type(retired).objects.filter(id=retired.id).update(retired=False) + self.assertEqual( + self.keys(self.search('ConceptMap', **{'source-system': 'https://example.org/my-codes'})), + ['https://example.org/my-codes|v1']) + self.assertEqual(self.keys(self.search('ConceptMap', **{'target-system': 'http://loinc.org'})), + ['https://example.org/my-codes|v1']) + + def test_no_query_per_entry(self): + with self.assertNumQueries(3): # the sources with the target's canonical, the matches' ids, the page + self.assertEqual(self.client.get('/fhir/ConceptMap/', {'target-system': ICD10}).data['total'], 1) + + +class ReferencedSourceAccessTest(SearchCorpusMixin, OCLTestCase): + """ + source-system, target-system (and their -uri forms) and reference look a canonical up only among the sources the + base shows the caller: public ones at /fhir/; at an owner base also, whoever owns them, those the caller may view. + A mapping whose source is another one is matched only by the URL it stores. So the canonical of a source the caller + isn't shown answers exactly as one no source has: the same total, the same queries. + """ + PRIVATE = 'http://moh.example/fhir/CodeSystem/private' + UNKNOWN = 'http://moh.example/fhir/CodeSystem/unknown' + + def setUp(self): + super().setUp() + self.private = source(self.moh, 'private-codes', canonical_url=self.PRIVATE, public_access=ACCESS_TYPE_NONE) + self.private_v1 = source(self.moh, 'private-codes', 'v1', released=True, canonical_url=self.PRIVATE, + public_access=ACCESS_TYPE_NONE) + self.private_code = ConceptFactory(parent=self.private, mnemonic='P1') + for _ in range(3): + MappingFactory(parent=self.private, from_source=self.private, to_source=self.private, + from_concept=self.private_code, to_concept=self.private_code, sources=[self.private_v1]) + drawing = 'https://example.org/ValueSet/drawing' # a public value set whose stored expansion drew on it + collection(self.sehatapp, 'drawing', canonical_url=drawing) + self.with_expansion(collection(self.sehatapp, 'drawing', 'v1', released=True, canonical_url=drawing), + explicit=[self.private_v1]) + + def with_partner(self): + """ + Partner's public ConceptMap maps to and from MOH's private source, and its public value set drew on it; a + member of both orgs, and a member of Partner only. + """ + partner = OrganizationFactory(mnemonic='Partner') + self.both = UserProfileFactory(username='both-member') + self.partner_only = UserProfileFactory(username='partner-member') + partner.members.add(self.both, self.partner_only) + self.moh.members.add(self.both) + maps = source(partner, 'partner-map', canonical_url='https://example.org/partner-map') + maps_v1 = source(partner, 'partner-map', 'v1', released=True, canonical_url=maps.canonical_url) + own = ConceptFactory(parent=maps, mnemonic='X1') + MappingFactory(parent=maps, from_source=maps, to_source=self.private, from_concept=own, + to_concept=self.private_code, sources=[maps_v1]) + MappingFactory(parent=maps, from_source=self.private, to_source=maps, from_concept=self.private_code, + to_concept=own, sources=[maps_v1]) + collection(partner, 'partner-set', canonical_url='https://example.org/partner-set') + self.with_expansion( + collection(partner, 'partner-set', 'v1', released=True, canonical_url='https://example.org/partner-set'), + explicit=[self.private_v1]) + + def answer(self, path, user, name, value, base='/fhir'): # pylint: disable=too-many-arguments + """The total and the SQL of a search, with the value searched for written as URL.""" + with CaptureQueriesContext(connection) as context: + total = self.search(path, base=base, user=user, **{name: value})['total'] + return total, [query['sql'].replace(value, 'URL') for query in context.captured_queries] + + def test_a_source_the_caller_is_not_shown_answers_as_an_unknown_one(self): + for user in (None, self.outsider, self.admin): + for resource_type, name in (('ConceptMap', 'source-system'), ('ConceptMap', 'target-system'), + ('ConceptMap', 'source-uri'), ('ConceptMap', 'target-uri'), + ('ValueSet', 'reference')): + private = self.answer(resource_type, user, name, self.PRIVATE) + self.assertEqual(private, self.answer(resource_type, user, name, self.UNKNOWN), (user, name)) + self.assertEqual(private[0], 0, (user, name)) + + def test_at_its_owner_base_to_those_who_may_view_it(self): + for name in ('source-system', 'target-system'): + for user, keys in ((None, []), (self.outsider, []), (self.moh_member, [f'{self.PRIVATE}|v1']), + (self.admin, [f'{self.PRIVATE}|v1'])): + self.assertEqual(self.keys(self.search( + 'ConceptMap', base='/orgs/MOH', user=user, **{name: self.PRIVATE})), keys, (name, user)) + + def test_across_owners_by_the_callers_own_rights(self): + """ + At an owner base, another owner's private source is matched for a caller who may view it by their own rights + (staff, a member of its org). A member of the base's org only, or an anonymous caller, gets what an unknown + canonical gets. The global base stays public only, whoever asks. + """ + self.with_partner() + for resource_type, name, key in ( + ('ConceptMap', 'target-system', 'https://example.org/partner-map|v1'), + ('ConceptMap', 'source-system', 'https://example.org/partner-map|v1'), + ('ConceptMap', 'target-uri', 'https://example.org/partner-map|v1'), + ('ConceptMap', 'source-uri', 'https://example.org/partner-map|v1'), + ('ValueSet', 'reference', 'https://example.org/partner-set|v1')): + for user in (self.both, self.admin): + self.assertEqual(self.keys(self.search( + resource_type, base='/orgs/Partner', user=user, **{name: self.PRIVATE})), [key], (name, user)) + self.assertEqual(self.search(resource_type, user=user, **{name: self.PRIVATE})['total'], 0) + for user in (self.partner_only, None): + private = self.answer(resource_type, user, name, self.PRIVATE, '/orgs/Partner') + self.assertEqual(private, self.answer(resource_type, user, name, self.UNKNOWN, '/orgs/Partner'), + (name, user)) + self.assertEqual(private[0], 0, (name, user)) + # the caller's own rights are a subquery of the source lookup, not a query of their own + with self.assertNumQueries(5): # the owner, the caller's membership of it, the sources, the matches, the page + self.search('ConceptMap', base='/orgs/Partner', user=self.both, **{'target-system': self.PRIVATE}) + + def test_the_source_filter_agrees_with_is_visible(self): + """viewable_rows is is_visible's queryset form, over every source row, HEAD and versions alike.""" + self.with_partner() + for namespace in (None, '/orgs/Partner/', '/orgs/MOH/', '/orgs/FHIR-Fixtures/'): + for user in (AnonymousUser(), self.outsider, self.member, self.moh_member, self.partner_only, self.both, + self.admin): + expected = {row.id for row in Source.objects.all() if fhir_resolve.is_visible(row, namespace, user)} + shown = set(fhir_resolve.viewable_rows(Source, namespace, user).values_list('id', flat=True)) + self.assertEqual(shown, expected, (namespace, user)) + self.assertIn(self.private.id, fhir_resolve.viewable_rows(Source, '/orgs/Partner/', self.both).values_list( + 'id', flat=True)) + + +class SearchCostTest(SearchCorpusMixin, OCLTestCase): + """ + A search's cost is bounded: the canonical URLs a parameter names are looked up together, in one query, and a + parameter takes at most MAX_ALTERNATIVES values (its alternatives and repeats together), a search at most + MAX_SEARCH_ALTERNATIVES; past either the search is too costly, a 400 naming the parameter. + """ + def setUp(self): + super().setUp() + ConceptFactory(parent=self.ciel_source, mnemonic='c19', sources=[self.ciel_v2]) + MappingFactory(parent=self.ciel_source, from_source=self.ciel_source, to_source=self.sehat_icd, + sources=[self.ciel_v2]) + + @staticmethod + def alternatives(count, last): + return ','.join([f'https://example.org/unknown-{index}' for index in range(count - 1)] + [last]) + + def test_canonicals_are_looked_up_in_one_query_per_parameter(self): + for resource_type, name, url, total in ( + ('CodeSystem', 'url', CIEL, 2), ('CodeSystem', 'system', ICD10, 2), ('ValueSet', 'url', CIEL_VS, 1), + ('ValueSet', 'reference', CIEL, 1), ('ConceptMap', 'source-system', CIEL, 1), + ('ConceptMap', 'target-uri', ICD10, 1)): + for count in (1, 20): + with self.assertNumQueries(3): # the canonicals' repositories or sources, the matches' ids, the page + response = self.client.get(f'/fhir/{resource_type}/', {name: self.alternatives(count, url)}) + self.assertEqual(response.data['total'], total, (resource_type, name, count)) + with self.assertNumQueries(4): # one lookup for each parameter + response = self.client.get('/fhir/CodeSystem/', { + 'url': self.alternatives(20, CIEL), 'code': ','.join(f'{CIEL}|c{index}' for index in range(20))}) + self.assertEqual(self.keys(response.data), [f'{CIEL}|v2']) + + def test_a_parameter_takes_at_most_20_values(self): + for params, expression in ( + ({'url': self.alternatives(21, CIEL)}, 'url'), + ({'url': [self.alternatives(15, CIEL), self.alternatives(6, ICD10)]}, 'url'), # repeats count too + ({'_id': ','.join(['1'] * 21)}, '_id'), + ({'name': ','.join(['a'] * 11), 'name:exact': ','.join(['b'] * 10)}, 'name:exact')): + with self.assertNumQueries(0): + response = self.client.get('/fhir/CodeSystem/', params) + issue = self.assertOutcome(response, 400, 'too-costly', 'too-costly') + self.assertEqual(issue['expression'], [expression], params) + self.assertEqual(self.search('CodeSystem', url=self.alternatives(20, CIEL))['total'], 2) + + def test_a_search_takes_at_most_50_values(self): + params = {'url': self.alternatives(20, CIEL), 'system': self.alternatives(20, CIEL), + 'version': ','.join(f'v{index}' for index in range(10))} + self.assertEqual(self.search('CodeSystem', **params)['total'], 2) + params['_id'] = '1' + issue = self.assertOutcome(self.client.get('/fhir/CodeSystem/', params), 400, 'too-costly', 'too-costly') + self.assertEqual(issue['expression'], ['_id']) + + +class VersionIdTest(SearchCorpusMixin, OCLTestCase): + """ + Every released version has a stable meta.versionId, its opaque id, so an owner-base Bundle with several releases + of one repository (one mnemonic id, one fullUrl) still identifies each entry (R4 bdl-7). + """ + def test_owner_base_entries_are_told_apart_by_version_id(self): + Collection.objects.filter(id=self.ciel_vs_v2.id).update(released=True) + for resource_type, versions in (('CodeSystem', (self.ciel_v1, self.ciel_v2)), + ('ConceptMap', (self.ciel_v1, self.ciel_v2)), + ('ValueSet', (self.ciel_vs_v1, self.ciel_vs_v2))): + bundle = self.search(resource_type, base='/orgs/CIEL') + identities = [(entry['fullUrl'], entry['resource']['meta']['versionId']) for entry in bundle['entry']] + self.assertEqual(len(identities), 2, resource_type) + self.assertEqual(len(set(identities)), 2, resource_type) + self.assertEqual(len({fullurl for fullurl, _ in identities}), 1, resource_type) # ids stay mnemonics + self.assertEqual(sorted(version_id for _, version_id in identities), + sorted(str(version.id) for version in versions), resource_type) + for entry in bundle['entry']: + meta = entry['resource']['meta'] + self.assertIn('lastUpdated', meta) + self.assertIn(SUBSETTED, meta['tag']) + + def test_reads_and_global_entries_carry_it_too(self): + for resource_type, version, owner_path in ( + ('CodeSystem', self.ciel_v2, '/orgs/CIEL/CodeSystem/CIEL/'), + ('ConceptMap', self.ciel_v2, '/orgs/CIEL/ConceptMap/CIEL/'), + ('ValueSet', self.ciel_vs_v1, '/orgs/CIEL/ValueSet/ciel-set/')): + for path in (f'/fhir/{resource_type}/{version.id}', owner_path): + meta = self.get(path).data['meta'] + self.assertEqual(meta['versionId'], str(version.id), path) + self.assertIn('lastUpdated', meta) + entry = next(entry for entry in self.search(resource_type)['entry'] + if entry['resource']['id'] == str(version.id)) + self.assertEqual(entry['resource']['meta']['versionId'], entry['resource']['id']) + + +class ReadTest(SearchCorpusMixin, OCLTestCase): + def test_global_read_by_opaque_id(self): + for resource_type, version in (('CodeSystem', self.who_icd_2019), ('CodeSystem', self.ciel_v1), + ('ConceptMap', self.ciel_v2), ('ValueSet', self.icd_vs_v1)): + resource = self.read(f'/fhir/{resource_type}/{version.id}') + self.assertEqual( + (resource['resourceType'], resource['id'], resource['version']), + (resource_type, str(version.id), version.version)) + self.assertEqual(self.read(f'/fhir/{resource_type}/{version.id}/')['id'], str(version.id)) + + self.assertEqual(self.read(f'/fhir/CodeSystem/{self.who_icd_2019.id}')['url'], ICD10) # the repository's + self.assertEqual(self.read(f'/fhir/ValueSet/{self.icd_vs_v1.id}')['url'], ICD10_VS) + self.assertEqual(self.read(f'/fhir/CodeSystem/{self.ciel_v1.id}')['status'], 'retired') + + def test_global_read_finds_only_what_global_search_lists(self): + for version in (self.who_icd, self.sehat_icd_v2, self.draft_v1, self.gone_v1, self.flagged, self.fixture_v1): + for user in (None, self.member, self.admin): + self.assertNotFound(f'/fhir/CodeSystem/{version.id}', user) + self.assertNotFound(f'/fhir/ValueSet/{self.ciel_vs_v2.id}') + for missing in ('CIEL', 'ICD-10', '0', f'0{self.ciel_v2.id}', '99999999', '12345678901234567890123'): + self.assertNotFound(f'/fhir/CodeSystem/{missing}') + self.assertNotFound(f'/fhir/ConceptMap/{missing}') + + def test_owner_read_is_the_latest_release_or_the_version_named(self): + Source.objects.filter(id=self.ciel_v1.id).update(retired=False) + newest = source(self.ciel, 'CIEL', 'v3', canonical_url=CIEL) # unreleased + resource = self.read('/orgs/CIEL/CodeSystem/CIEL/') + self.assertEqual((resource['id'], resource['version']), ('CIEL', 'v2')) + self.assertEqual(self.read('/orgs/CIEL/CodeSystem/CIEL', version='v1')['version'], 'v1') + self.assertEqual(self.read('/orgs/CIEL/ConceptMap/CIEL/')['version'], 'v2') + self.assertEqual(self.read('/orgs/sehatapp/ValueSet/icd10-all/')['url'], ICD10_VS) + self.assertEqual(self.read('/users/jdoe/CodeSystem/my-codes/')['version'], 'v1') + for version in (newest.version, HEAD, 'no-such-version'): + self.assertNotFound('/orgs/CIEL/CodeSystem/CIEL/', version=version) + for path in ('/orgs/misc/CodeSystem/draft/', '/orgs/misc/CodeSystem/gone/', '/orgs/misc/CodeSystem/flagged/', + '/orgs/nobody/CodeSystem/CIEL/', '/orgs/WHO/CodeSystem/CIEL/', '/orgs/CIEL/ValueSet/CIEL/', + '/orgs/CIEL/ConceptMap/nothing/'): + self.assertNotFound(path) + + def test_a_retired_release_is_the_default_only_when_named(self): + self.assertEqual(self.read('/orgs/CIEL/CodeSystem/CIEL/')['version'], 'v2') + self.assertEqual(self.read('/orgs/CIEL/CodeSystem/CIEL/', version='v1')['status'], 'retired') + + def test_private_owner_read(self): + for user in (self.member, self.admin): + resource = self.read('/orgs/FHIR-Fixtures/CodeSystem/fixture-codes/', user) + self.assertEqual((resource['id'], resource['version']), ('fixture-codes', '2.0.0')) + for user in (None, self.outsider): + self.assertNotFound('/orgs/FHIR-Fixtures/CodeSystem/fixture-codes/', user) + + def test_code_system_content_is_complete_up_to_the_limit(self): + names = [ConceptNameFactory.build(name='Malaria', locale='en', locale_preferred=True), + ConceptNameFactory.build(name='Paludisme', locale='fr')] + malaria = ConceptFactory(parent=self.ciel_source, mnemonic='1427', names=names, sources=[self.ciel_v2], + descriptions=[ConceptDescriptionFactory.build(name='A mosquito-borne disease')]) + retired = ConceptFactory(parent=self.ciel_source, mnemonic='9999', retired=True, sources=[self.ciel_v2]) + ConceptFactory(parent=self.ciel_source, mnemonic='gone', is_active=False, sources=[self.ciel_v2]) + + resource = self.read(f'/fhir/CodeSystem/{self.ciel_v2.id}') + + self.assertEqual((resource['content'], resource['count']), ('complete', 2)) + concepts = {concept['code']: concept for concept in resource['concept']} + self.assertEqual(set(concepts), {malaria.mnemonic, retired.mnemonic}) + self.assertEqual(concepts['1427']['display'], 'Malaria') + self.assertEqual(concepts['1427']['definition'], 'A mosquito-borne disease') + self.assertEqual( + sorted((name['language'], name['value']) for name in concepts['1427']['designation']), + [('en', 'Malaria'), ('fr', 'Paludisme')]) + self.assertIn({'code': 'inactive', 'valueBoolean': True}, concepts['9999']['property']) + self.assertNotIn('tag', resource['meta']) + + # above the limit the count is the version's concept links, an index-only count: a deleted concept still + # linked would count (production has none on a released version) + with patch('core.fhir.search.CONCEPT_LIMIT', 1): + resource = self.read(f'/fhir/CodeSystem/{self.ciel_v2.id}') + self.assertEqual((resource['content'], resource['count']), ('not-present', 3)) + self.assertNotIn('concept', resource) + + def test_reads_run_no_query_per_concept_or_mapping(self): + def read_queries(path): + with self.assertNumQueries(5): # the version, its concept count, the concepts, their names, descriptions + self.read(path) + + ConceptFactory(parent=self.ciel_source, sources=[self.ciel_v2], names=1, descriptions=1) + read_queries(f'/fhir/CodeSystem/{self.ciel_v2.id}') + for _ in range(3): + ConceptFactory(parent=self.ciel_source, sources=[self.ciel_v2], names=2, descriptions=1) + read_queries(f'/fhir/CodeSystem/{self.ciel_v2.id}') + read_queries('/orgs/CIEL/CodeSystem/CIEL/') + + for _ in range(3): + MappingFactory(parent=self.ciel_source, from_source=self.ciel_source, to_source=self.sehat_icd, + sources=[self.ciel_v2]) + with self.assertNumQueries(2): # the version, its mappings + self.read(f'/fhir/ConceptMap/{self.ciel_v2.id}') + + def test_writes_keep_the_owner_check(self): + client = APIClient() + client.force_authenticate(self.admin) + response = client.put(f'/fhir/CodeSystem/{self.ciel_v2.id}/', {}, format='json') + self.assertEqual(response.status_code, 404) + + +class RepositoryAccessTest(SearchCorpusMixin, OCLTestCase): + """ + A version is shown only where its repository is too: its HEAD, which governs access to its concepts and mappings, + must be viewable as well as the version. MOH made hiv-codes and hiv-set private after releasing v1 public, so v1 + is shown at MOH's base to MOH's members and staff, and nowhere else. A retired release counts only beside a live + one the caller is shown, and an owner-base read without a version serves the newest release the caller is shown. + """ + def setUp(self): + super().setUp() + ConceptFactory(parent=self.hiv, mnemonic='H1', sources=[self.hiv_v1]) + MappingFactory(parent=self.hiv, from_source=self.hiv, to_source=self.ciel_source, sources=[self.hiv_v1]) + self.users = (None, self.outsider, self.member, self.moh_member, self.admin) + self.viewers = (self.moh_member, self.admin) + + def test_search_shows_a_release_where_its_repository_is_shown(self): + hiv, hiv_set = [f'{HIV}|v1'], [f'{HIV_VS}|v1'] + for user in self.users: + for base, shown in (('/fhir', False), ('/orgs/MOH', user in self.viewers)): + for resource_type, params, keys in ( + ('CodeSystem', {}, hiv), ('CodeSystem', {'url': HIV}, hiv), ('CodeSystem', {'code': 'H1'}, hiv), + ('CodeSystem', {'code': f'{HIV}|'}, hiv), ('ConceptMap', {}, hiv), + ('ConceptMap', {'url': HIV}, hiv), ('ValueSet', {}, hiv_set), + ('ValueSet', {'url': HIV_VS}, hiv_set), ('ValueSet', {'reference': CIEL}, hiv_set)): + bundle = self.search(resource_type, base=base, user=user, **params) + found = [key for key in self.keys(bundle) if key in keys] + self.assertEqual(found, keys if shown else [], (user, base, resource_type, params)) + if base == '/orgs/MOH': + self.assertEqual(bundle['total'], len(found), (user, resource_type, params)) + + def test_read_serves_a_release_where_its_repository_is_shown(self): + for user in self.users: + for resource_type, version in ( + ('CodeSystem', self.hiv_v1), ('ConceptMap', self.hiv_v1), ('ValueSet', self.hiv_vs_v1)): + self.assertNotFound(f'/fhir/{resource_type}/{version.id}', user) + for path, url in (('/orgs/MOH/CodeSystem/hiv-codes/', HIV), ('/orgs/MOH/ConceptMap/hiv-codes/', HIV), + ('/orgs/MOH/ValueSet/hiv-set/', HIV_VS)): + if user in self.viewers: + resource = self.read(path, user) + self.assertEqual((resource['version'], resource['url']), ('v1', url), (path, user)) + else: + self.assertNotFound(path, user) + self.assertNotFound(path, user, version='v1') + self.assertEqual([concept['code'] for concept in self.read( + '/orgs/MOH/CodeSystem/hiv-codes/', self.moh_member)['concept']], ['H1']) + + def test_a_retired_release_counts_beside_a_live_one_the_caller_is_shown(self): + url = 'http://moh.example/fhir/CodeSystem/mixed' + source(self.moh, 'mixed', canonical_url=url) + retired = source(self.moh, 'mixed', 'v1', released=True, retired=True, canonical_url=url) + source(self.moh, 'mixed', 'v2', released=True, canonical_url=url, public_access=ACCESS_TYPE_NONE) + for user in self.users: + shown = user in self.viewers + self.assertEqual(self.search('CodeSystem', user=user, url=url)['total'], 0, user) + self.assertNotFound(f'/fhir/CodeSystem/{retired.id}', user) + self.assertEqual(self.keys(self.search('CodeSystem', base='/orgs/MOH', user=user, url=url)), + [f'{url}|v1', f'{url}|v2'] if shown else [], user) + if shown: + self.assertEqual(self.read('/orgs/MOH/CodeSystem/mixed/', user, version='v1')['status'], 'retired') + else: + self.assertNotFound('/orgs/MOH/CodeSystem/mixed/', user, version='v1') + + def test_an_owner_read_serves_the_newest_release_the_caller_is_shown(self): + url = 'http://moh.example/fhir/CodeSystem/newest' + source(self.moh, 'newest', canonical_url=url) + source(self.moh, 'newest', 'v1', released=True, canonical_url=url) + source(self.moh, 'newest', 'v2', released=True, canonical_url=url, public_access=ACCESS_TYPE_NONE) + for user in self.users: + newest = 'v2' if user in self.viewers else 'v1' + self.assertEqual(self.read('/orgs/MOH/CodeSystem/newest/', user)['version'], newest, user) + listed = self.keys(self.search('CodeSystem', base='/orgs/MOH', user=user, _id='newest')) + self.assertIn(f'{url}|{newest}', listed, user) + + +class VisibleVersionsTest(SearchCorpusMixin, OCLTestCase): + """visible_versions is the queryset form of is_visible and has_release: they agree on every row.""" + def test_the_queryset_agrees_with_is_visible_and_has_release(self): + # a retired public release beside a private live one; a private newest release; a deleted repository (HEAD) + source(self.moh, 'mixed', canonical_url='urn:mixed') + source(self.moh, 'mixed', 'v1', released=True, retired=True, canonical_url='urn:mixed') + source(self.moh, 'mixed', 'v2', released=True, canonical_url='urn:mixed', public_access=ACCESS_TYPE_NONE) + source(self.moh, 'newest', canonical_url='urn:newest') + source(self.moh, 'newest', 'v1', released=True, canonical_url='urn:newest') + source(self.moh, 'newest', 'v2', released=True, canonical_url='urn:newest', public_access=ACCESS_TYPE_NONE) + Source.objects.filter(id=self.jdoe_codes.id).update(is_active=False) + users = (AnonymousUser(), self.outsider, self.member, self.moh_member, self.admin) + cases = [({}, user) for user in users] + [ + ({'org': org}, user) for org in ('FHIR-Fixtures', 'MOH') for user in users] + [ + ({'org': 'CIEL'}, AnonymousUser()), ({'org': 'misc'}, self.admin), ({'user': 'jdoe'}, AnonymousUser()), + ({'user': 'jdoe'}, self.jdoe)] + for kwargs, user in cases: + for resource_type in ('CodeSystem', 'ValueSet'): + model = fhir_resolve.REPO_TYPES[resource_type] + namespace = fhir_resolve.namespace_of(kwargs) + expected = { + row.id for row in model.objects.filter(is_active=True, released=True).exclude(version=HEAD) + if (namespace is None or row.parent.uri == namespace) and + fhir_resolve.is_visible(row, namespace, user) and fhir_resolve.has_release(row, namespace, user)} + visible = set(fhir_resolve.visible_versions(resource_type, kwargs, user).values_list('id', flat=True)) + self.assertEqual(visible, expected, (kwargs, user, resource_type)) + self.assertIn(self.hiv_v1.id, fhir_resolve.visible_versions( + 'CodeSystem', {'org': 'MOH'}, self.moh_member).values_list('id', flat=True)) + + +class SearchTimeoutTest(TransactionTestCase): + """A search the database cancels at SEARCH_TIMEOUT_MS is too costly (a 422 outcome), never a 5xx.""" + def test_a_cancelled_query_is_too_costly(self): + def slow(): + with connection.cursor() as cursor: + cursor.execute('SELECT pg_sleep(1)') + + with patch('core.fhir.search.SEARCH_TIMEOUT_MS', 50): + with self.assertRaises(TooCostlyError) as context: + fhir_search.within_search_timeout(slow) + self.assertEqual((context.exception.status_code, context.exception.issue_code), (422, 'too-costly')) + + self.assertEqual(fhir_search.within_search_timeout(lambda: 'answered'), 'answered') + with connection.cursor() as cursor: # the timeout ended with the search + cursor.execute('SHOW statement_timeout') + self.assertEqual(cursor.fetchone()[0], '0') + + def test_the_timeout_bounds_the_search_as_a_whole(self): + """Each query may take only the time the search has left, so queries that fit alone can still add up past it.""" + def slow(): + with connection.cursor() as cursor: + for _ in range(4): + cursor.execute('SELECT pg_sleep(0.04)') + + with patch('core.fhir.search.SEARCH_TIMEOUT_MS', 100): + with self.assertRaises(TooCostlyError): + fhir_search.within_search_timeout(slow) + self.assertEqual(fhir_search.within_search_timeout(lambda: 'answered'), 'answered') + + def test_a_query_with_no_time_left_is_too_costly(self): + """A query that starts after the search's time is up isn't sent: the search is too costly already.""" + def late(): + time.sleep(0.06) + with connection.cursor() as cursor: + cursor.execute('SELECT 1') + + with patch('core.fhir.search.SEARCH_TIMEOUT_MS', 50): + with self.assertRaises(TooCostlyError): + fhir_search.within_search_timeout(late) + + def test_other_database_errors_are_raised(self): + def broken(): + raise OperationalError('connection lost') + + with self.assertRaises(OperationalError): + fhir_search.within_search_timeout(broken) diff --git a/core/integration_tests/tests_code_systems.py b/core/integration_tests/tests_code_systems.py index 763493947..1bc701e0e 100644 --- a/core/integration_tests/tests_code_systems.py +++ b/core/integration_tests/tests_code_systems.py @@ -220,6 +220,7 @@ def test_posting_code_systems_specific_json(self): 'type': 'Coding' }], 'meta': { + 'versionId': ANY, 'lastUpdated': ANY }, 'version': '07/14/2020', diff --git a/core/integration_tests/tests_fhir_owner_access.py b/core/integration_tests/tests_fhir_owner_access.py index 6cb924d5c..a868f94f9 100644 --- a/core/integration_tests/tests_fhir_owner_access.py +++ b/core/integration_tests/tests_fhir_owner_access.py @@ -3,6 +3,7 @@ from rest_framework.exceptions import ValidationError from core.collections.models import Collection +from core.collections.tests.factories import OrganizationCollectionFactory from core.common.tests import OCLAPITestCase from core.importers.importer import ResourceImporter from core.importers.models import CREATED, UPDATED, PERMISSION_DENIED @@ -107,6 +108,71 @@ def test_outsider_cannot_create_under_org(self): self.assertEqual(response.status_code, 403) +@patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) +@patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) +class FHIRUpdateOwnerTest(OCLAPITestCase): + """ + A FHIR PUT or PATCH needs what a POST needs: staff, the owner, or a member of the owning org. A ConceptMap PUT to an + id that doesn't exist creates it, so it's refused to anyone else, and the answer for a missing id is the answer for + an existing one. + """ + + def setUp(self): + super().setUp() + self.org = OrganizationFactory(mnemonic='OtherOrg') + self.member = UserProfileFactory(username='othermember') + self.org.members.add(self.member) + self.outsider = UserProfileFactory(username='otheroutsider') + for mnemonic in ('existing-cs', 'existing-cm'): + OrganizationSourceFactory(organization=self.org, mnemonic=mnemonic, public_access='None') + OrganizationSourceFactory( + organization=self.org, mnemonic=mnemonic, version='1.0', released=True, public_access='None') + OrganizationCollectionFactory(organization=self.org, mnemonic='existing-vs', public_access='None') + OrganizationCollectionFactory( + organization=self.org, mnemonic='existing-vs', version='1.0', released=True, public_access='None') + + def write(self, method, user, resource_type, mnemonic): + resource = fhir_resource(resource_type, mnemonic, f'http://example.com/{mnemonic}') + resource['version'] = '2.0' + auth = {'HTTP_AUTHORIZATION': 'Token ' + user.get_token()} if user else {} + return getattr(self.client, method)( + f'/orgs/{self.org.mnemonic}/{resource_type}/{mnemonic}/', resource, format='json', **auth) + + def repositories(self): + return sorted(f'{repo.mnemonic}|{repo.version}' for model in (Source, Collection) + for repo in model.objects.filter(organization=self.org)) + + def test_others_create_and_change_nothing(self): + before = self.repositories() + for resource_type, existing in (('CodeSystem', 'existing-cs'), ('ValueSet', 'existing-vs'), + ('ConceptMap', 'existing-cm')): + for method in ('put', 'patch'): + for user in (None, self.outsider): + answers = [self.write(method, user, resource_type, mnemonic).status_code + for mnemonic in (existing, 'missing')] + self.assertIn(answers[0], (401, 403), (resource_type, method, user)) + self.assertEqual(answers[1], answers[0], (resource_type, method, user)) + self.assertEqual(self.repositories(), before) + + def test_a_concept_map_put_checks_ownership_before_it_creates(self): + """The create itself checks ownership, whatever looked the owner up.""" + with patch('core.common.mixins.SubResourceMixin.get_object_for_path', Mock(return_value=self.org)): + for method in ('put', 'patch'): + self.assertEqual(self.write(method, self.outsider, 'ConceptMap', 'missing').status_code, 403, method) + self.assertFalse(Source.objects.filter(mnemonic='missing').exists()) + + def test_a_member_may_create_a_concept_map_by_put(self): + response = self.write('put', self.member, 'ConceptMap', 'created') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual( + [repo for repo in self.repositories() if repo.startswith('created|')], ['created|2.0', 'created|HEAD']) + self.assertEqual(self.write('put', self.member, 'CodeSystem', 'missing').status_code, 404) + self.assertEqual(self.client.delete( + f'/orgs/{self.org.mnemonic}/ConceptMap/missing/', + HTTP_AUTHORIZATION='Token ' + self.member.get_token()).status_code, 404) + + class BulkImportOwnerAccessTest(OCLAPITestCase): """The requester must own, or be a member of, the owner they import into.""" diff --git a/core/value_sets/serializers.py b/core/value_sets/serializers.py index f01a7722a..f26676d05 100644 --- a/core/value_sets/serializers.py +++ b/core/value_sets/serializers.py @@ -11,6 +11,7 @@ from core.common.constants import HEAD from core.common.fhir_helpers import delete_empty_fields from core.common.serializers import StatusField, IdentifierSerializer, ReadSerializerMixin +from core.fhir.search import CanonicalUrlField, FhirRepositorySerializerMixin from core.parameters.serializers import ParametersSerializer from core.value_sets.constants import RESOURCE_TYPE @@ -144,10 +145,11 @@ def to_representation(self, value): return None -class ValueSetDetailSerializer(serializers.ModelSerializer): +class ValueSetDetailSerializer(FhirRepositorySerializerMixin, serializers.ModelSerializer): + content_field = 'compose' resourceType = SerializerMethodField(method_name='get_resource_type') id = CharField(source='mnemonic') - url = CharField(source='canonical_url', required=False) + url = CanonicalUrlField(source='canonical_url', required=False) title = CharField(source='full_name', required=False) status = StatusField(source='*') meta = SerializerMethodField() @@ -255,6 +257,7 @@ def to_representation(self, instance): rep = super().to_representation(instance) delete_empty_fields(rep) IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep) + self.with_fhir_view(instance, rep) except (Exception, ): msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}' logger.exception(msg) diff --git a/core/value_sets/tests.py b/core/value_sets/tests.py index f68e35f21..3aed58614 100644 --- a/core/value_sets/tests.py +++ b/core/value_sets/tests.py @@ -41,7 +41,8 @@ def setUp(self): self.collection = OrganizationCollectionFactory( organization=self.org, mnemonic='c1', canonical_url='http://c1.com', version='HEAD') self.collection_v1 = OrganizationCollectionFactory( - mnemonic='c1', canonical_url='http://c1.com', version='v1', organization=self.collection.organization) + mnemonic='c1', canonical_url='http://c1.com', version='v1', organization=self.collection.organization, + released=True) expansion = ExpansionFactory(mnemonic='e1', collection_version=self.collection) self.collection.expansion_uri = expansion.uri self.collection.save() @@ -75,7 +76,8 @@ def test_public_can_find_globally(self): response = self.client.get('/fhir/ValueSet/?url=http://c1.com') self.assertEqual(len(response.data['entry']), 1) - resource = response.data['entry'][0]['resource'] + self.assertNotIn('compose', response.data['entry'][0]['resource']) # a search entry is a summary + resource = self.client.get(response.data['entry'][0]['fullUrl']).data self.assertEqual( resource['identifier'][0]['value'], f'/orgs/{self.org.mnemonic}/ValueSet/{self.collection.mnemonic}/') self.assertEqual(len(resource['compose']['include']), 1) diff --git a/core/value_sets/views.py b/core/value_sets/views.py index 810d63b8c..84def3e0a 100644 --- a/core/value_sets/views.py +++ b/core/value_sets/views.py @@ -4,15 +4,14 @@ from rest_framework import status from rest_framework.response import Response -from core.bundles.serializers import FHIRBundleSerializer -from core.code_systems.views import CodeSystemValidateCodeView, get_viewable_repo, FhirParameterRequiredError +from core.code_systems.views import CodeSystemValidateCodeView, get_viewable_repo from core.collections.models import Collection from core.collections.views import CollectionListView, CollectionRetrieveUpdateDestroyView, \ CollectionVersionExpansionsView from core.common.constants import HEAD, LATEST -from core.common.fhir_helpers import translate_fhir_query from core.concepts.models import Concept from core.fhir.base import FhirViewMixin +from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.sources.models import Source from core.value_sets.serializers import ValueSetDetailSerializer, \ ValueSetExpansionParametersSerializer, ValueSetExpansionSerializer @@ -20,30 +19,10 @@ logger = logging.getLogger('oclapi') -class ValueSetListView(FhirViewMixin, CollectionListView): +class ValueSetListView(FhirViewMixin, FhirSearchMixin, CollectionListView): + resource_type = 'ValueSet' serializer_class = ValueSetDetailSerializer - @staticmethod - def bundle_response(data, paginator): - bundle = FHIRBundleSerializer( - {'meta': {}, 'total': len(data), 'type': 'searchset', 'entry': FHIRBundleSerializer.convert_to_entry(data)}, - context={'paginator': paginator} - ) - return bundle.data - - def get_filter_params(self, default_version_to_head=True): - return super().get_filter_params(False) - - def apply_query_filters(self, queryset): - query_fields = list(self.serializer_class.Meta.fields) - queryset = translate_fhir_query(query_fields, self.request.query_params, queryset) - - return queryset - - def apply_filters(self, queryset): - queryset = queryset.exclude(version=HEAD).filter(is_latest_version=True) - return self.apply_query_filters(queryset) - def get_serializer_class(self): return self.serializer_class @@ -131,7 +110,8 @@ def get_queryset(self): return queryset -class ValueSetRetrieveUpdateView(FhirViewMixin, CollectionRetrieveUpdateDestroyView): +class ValueSetRetrieveUpdateView(FhirViewMixin, FhirReadMixin, CollectionRetrieveUpdateDestroyView): + resource_type = 'ValueSet' serializer_class = ValueSetDetailSerializer def get_filter_params(self, default_version_to_head=True): From d007d554e638ee0e637cf9ae707e2965b09553df Mon Sep 17 00:00:00 2001 From: Jonathan Payne Date: Thu, 22 Oct 2026 12:23:24 -0400 Subject: [PATCH 04/18] OpenConceptLab/ocl_online#451 | Missing FHIR operation inputs get consistent, typed error outcomes Required-input errors in $validate-code and $translate now use the shared FHIR InvalidError. A missing required input is a 400 OperationOutcome naming it, with terminology issue coding. CodeSystem $validate-code for an unknown or unavailable system still answers result=false, with or without a code. core/fhir/tests_operation_bounds.py checks FHIR JSON POSTs and the issue coding, alongside bounded concept and mapping reads and unchanged answers for list-only options. It replaces master's copy of these bounds tests (core/code_systems/tests_operation_bounds.py, which came from this work) with core/fhir/tests_operation_bounds.py. Its $translate cases run by GET here; the POST cases come back with the $translate work later in the series. The operation input checks of OpenConceptLab/ocl_online#451. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R8hw7w8753DDSCezrmrDA8 --- core/code_systems/views.py | 7 +++---- core/concept_maps/views.py | 14 +++++++------- .../tests_operation_bounds.py | 11 ++++++----- core/value_sets/views.py | 7 ++++--- 4 files changed, 20 insertions(+), 19 deletions(-) rename core/{code_systems => fhir}/tests_operation_bounds.py (97%) diff --git a/core/code_systems/views.py b/core/code_systems/views.py index 99518162b..0666d76c0 100644 --- a/core/code_systems/views.py +++ b/core/code_systems/views.py @@ -10,7 +10,7 @@ from core.concepts.permissions import CanViewParentDictionaryAsGuest from core.concepts.views import ConceptRetrieveUpdateDestroyView from core.fhir.base import FhirViewMixin -from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM, InvalidError from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.parameters.serializers import ParametersSerializer from core.sources.models import Source @@ -157,10 +157,9 @@ def get_queryset(self): return queryset.none() if not code: - raise FhirParameterRequiredError('A code to validate is required.', expression='code') + raise InvalidError('A code to validate is required.', expression='code') if not source: - raise FhirParameterRequiredError( - 'url is required with a code: the code system to validate it in.', expression='url') + raise InvalidError('url is required with a code: the code system to validate it in.', expression='url') queryset = queryset.filter(sources=source, mnemonic=code) diff --git a/core/concept_maps/views.py b/core/concept_maps/views.py index 9ce92b260..58c217eb0 100644 --- a/core/concept_maps/views.py +++ b/core/concept_maps/views.py @@ -12,6 +12,7 @@ from core.concept_maps.serializers import ConceptMapDetailSerializer, ConceptMapParametersSerializer from core.concepts.permissions import CanAccessParentDictionary from core.fhir.base import FhirViewMixin +from core.fhir.outcomes import InvalidError from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.mappings.constants import SAME_AS from core.mappings.views import MappingListView @@ -72,10 +73,9 @@ def get_permissions(self): def get_serializer_class(self): return ConceptMapParametersSerializer - # $translate answers with the translation of the mappings apply_filters names, and nothing else. The mapping - # list's shortcuts (a search, a sort or a faceted filter, facets, search stats, a CSV export) would answer in its - # place, and the search ones skip apply_filters, so each is off. Their parameters are ignored like any other this - # operation doesn't define. + # $translate answers from the mappings apply_filters names, and from nothing else. The mapping list's shortcuts + # (a search, a sort or a faceted filter, facets, search stats, a CSV export) would skip apply_filters, so each is + # off, and their parameters are ignored like any other this operation doesn't define. allow_csv_export = False def should_perform_es_search(self): @@ -115,10 +115,10 @@ def apply_filters(self, queryset): targetsystem = params.get('targetsystem') # TODO: implement 'source' and 'target' if not code: - raise FhirParameterRequiredError('A code to translate is required.', expression='code') + raise InvalidError('A code to translate is required.', expression='code') if not system: - raise FhirParameterRequiredError( - 'system is required with a code: the code system the code is from.', expression='system') + raise InvalidError('system is required with a code: the code system the code is from.', + expression='system') if url: queryset = queryset.filter(parent__canonical_url=url) diff --git a/core/code_systems/tests_operation_bounds.py b/core/fhir/tests_operation_bounds.py similarity index 97% rename from core/code_systems/tests_operation_bounds.py rename to core/fhir/tests_operation_bounds.py index f000b9ffe..7f9d32cca 100644 --- a/core/code_systems/tests_operation_bounds.py +++ b/core/fhir/tests_operation_bounds.py @@ -23,6 +23,7 @@ from core.common.tests import OCLTestCase from core.common.views import BaseAPIView from core.concepts.tests.factories import ConceptFactory +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM from core.mappings.models import Mapping from core.mappings.tests.factories import MappingFactory from core.orgs.tests.factories import OrganizationFactory @@ -103,7 +104,7 @@ def request(self, path, method, headers=None, **inputs): parameters = [{'name': name, ('valueString' if 'valueString' in method else VALUE_TYPES[name]): value} for name, value in inputs.items()] response = self.client.post(path, json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), - content_type='application/json') + content_type='application/fhir+json') return response, context.captured_queries def assert_required(self, response, name, msg): @@ -112,6 +113,7 @@ def assert_required(self, response, name, msg): self.assertEqual(outcome['resourceType'], 'OperationOutcome', msg) self.assertEqual(outcome['issue'][0]['code'], 'invalid', msg) self.assertEqual(outcome['issue'][0]['expression'], [name], msg) + self.assertEqual(outcome['issue'][0]['details']['coding'][0]['system'], TX_ISSUE_TYPE_SYSTEM, msg) def assert_result(self, response, expected, msg): self.assertEqual(response.status_code, 200, msg) @@ -171,12 +173,11 @@ def test_value_set_validate_code_reads_only_the_named_code(self): self.assert_result(response, display is None, msg) def test_translate_reads_only_the_named_code(self): - for method, url, code, system in product( - ('GET', 'POST', 'POST valueString'), (None, SYSTEM), (None, 'c1'), (None, SYSTEM)): - msg = (method, url, code, system) + for url, code, system in product((None, SYSTEM), (None, 'c1'), (None, SYSTEM)): + msg = (url, code, system) with self.subTest(msg): response, queries = self.request( - '/fhir/ConceptMap/$translate', method, url=url, code=code, system=system) + '/fhir/ConceptMap/$translate', 'GET', url=url, code=code, system=system) self.assertEqual(unbounded_reads(queries), [], msg) if not code: diff --git a/core/value_sets/views.py b/core/value_sets/views.py index 84def3e0a..3ede02898 100644 --- a/core/value_sets/views.py +++ b/core/value_sets/views.py @@ -11,6 +11,7 @@ from core.common.constants import HEAD, LATEST from core.concepts.models import Concept from core.fhir.base import FhirViewMixin +from core.fhir.outcomes import InvalidError from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.sources.models import Source from core.value_sets.serializers import ValueSetDetailSerializer, \ @@ -87,10 +88,10 @@ def get_queryset(self): queryset = queryset.filter(expansion_set__collection_version__in=version_ids) if not code: - raise FhirParameterRequiredError('A code to validate is required.', expression='code') + raise InvalidError('A code to validate is required.', expression='code') if not system: - raise FhirParameterRequiredError( - 'system is required with a code: the code system the code is from.', expression='system') + raise InvalidError('system is required with a code: the code system the code is from.', + expression='system') concept_source = Source.objects.filter(canonical_url=system) if system_version: From ca20fa52eb2d8e10da0a41ec27945643788190c0 Mon Sep 17 00:00:00 2001 From: Jonathan Payne Date: Thu, 22 Oct 2026 12:23:24 -0400 Subject: [PATCH 05/18] OpenConceptLab/ocl_online#451 | FHIR lookup and validation answer for the requested version and value set $lookup and CodeSystem and ValueSet $validate-code now use shared operations in core/fhir/ops and an R4 response adapter. GET and POST accept separate inputs or a Coding, with instance routes for lookup and validation. CodeableConcept validation accepts one coding. Operations resolve the newest release the caller may view unless a version is named. Without the URL registry, a canonical at the global base resolves only among repositories the caller is shown, so a private copy never hides a public one; the shared resolver in core/common/models.py gains optional heads and head_shown arguments for this, and other callers keep their defaults. Owner instance routes accept HEAD when explicitly named. Conflicting systems or versions are reported, deleted concepts are absent, and unavailable repositories receive the same answers as missing ones. $lookup returns the resolved version, display, designations and inactive property. Validation returns the code, system, version, display and issues; display checks accept any concept name regardless of case, and retired codes remain valid with a warning. ValueSet membership comes from its default expansion, reporting a visible holding release: the explicit pin, then the evaluated pin, then the newest. Tests cover GET and POST, instance routes, encoded codes, display languages, version conflicts, expansion membership and access with both registry settings. Query checks keep reads focused on the named code as repositories grow. Work package WP4 + WP5 of OpenConceptLab/ocl_online#451. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R8hw7w8753DDSCezrmrDA8 --- core/code_systems/tests.py | 165 ++++++---- core/code_systems/urls.py | 10 +- core/code_systems/views.py | 203 +----------- core/common/models.py | 22 +- core/fhir/adapters/__init__.py | 0 core/fhir/adapters/r4.py | 57 ++++ core/fhir/base.py | 24 ++ core/fhir/ops/__init__.py | 0 core/fhir/ops/concepts.py | 197 +++++++++++ core/fhir/ops/lookup.py | 61 ++++ core/fhir/ops/validate_code.py | 265 +++++++++++++++ core/fhir/resolve.py | 88 ++++- core/fhir/search.py | 27 +- core/fhir/tests_operation_bounds.py | 38 +++ core/fhir/tests_ops_lookup.py | 449 +++++++++++++++++++++++++ core/fhir/tests_ops_validate_code.py | 474 +++++++++++++++++++++++++++ core/fhir/tests_protocol.py | 11 +- core/fhir/tests_resolve.py | 50 ++- core/parameters/serializers.py | 24 -- core/parameters/tests.py | 19 -- core/value_sets/tests.py | 115 ++++--- core/value_sets/views.py | 94 +----- 22 files changed, 1929 insertions(+), 464 deletions(-) create mode 100644 core/fhir/adapters/__init__.py create mode 100644 core/fhir/adapters/r4.py create mode 100644 core/fhir/ops/__init__.py create mode 100644 core/fhir/ops/concepts.py create mode 100644 core/fhir/ops/lookup.py create mode 100644 core/fhir/ops/validate_code.py create mode 100644 core/fhir/tests_ops_lookup.py create mode 100644 core/fhir/tests_ops_validate_code.py diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index d1fa73a8c..517ec5d39 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -6,7 +6,6 @@ from core.code_systems.converter import CodeSystemConverter from core.code_systems.serializers import CodeSystemDetailSerializer -from core.code_systems.views import CodeSystemLookupNotFoundError from core.common.constants import ACCESS_TYPE_NONE from core.common.tests import OCLTestCase from core.concepts.models import Concept @@ -14,17 +13,23 @@ from core.orgs.tests.factories import OrganizationFactory from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory +from core.url_registry.tests_resolution import set_registry_toggle from core.users.tests.factories import UserProfileFactory +SYSTEM = 'http://some/url' +PRIVATE = 'http://private.example/url' +SHARED = 'http://shared.example/url' + class CodeSystemTest(OCLTestCase): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def setUp(self): + set_registry_toggle(False) # the operations resolve through core.fhir.resolve: no registry, as in production super().setUp() self.org = OrganizationFactory() - self.org_source = OrganizationSourceFactory(organization=self.org, canonical_url='/some/url', + self.org_source = OrganizationSourceFactory(organization=self.org, canonical_url=SYSTEM, full_name='source name') self.org_source_v1 = OrganizationSourceFactory.build( version='v1', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) @@ -37,7 +42,7 @@ def setUp(self): self.user = UserProfileFactory() self.user_token = self.user.get_token() - self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url='/some/url') + self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url=SYSTEM) self.user_source_v1 = UserSourceFactory.build( version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent, released=True) Source.persist_new_version(self.user_source_v1, self.user_source.created_by) @@ -45,7 +50,7 @@ def setUp(self): self.client = APIClient() def test_public_can_view(self): - response = self.client.get('/fhir/CodeSystem/?url=/some/url') + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}') self.assertEqual(len(response.data['entry']), 1) self.assertEqual(response.data['total'], 1) @@ -65,7 +70,7 @@ def test_public_can_view(self): def test_private_can_view(self): # the global base shows public repositories only, whoever asks; the owner base shows the owner its own - response = self.client.get('/fhir/CodeSystem/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}', HTTP_AUTHORIZATION='Token ' + self.user_token) self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] @@ -76,7 +81,7 @@ def test_private_can_view(self): self.assertEqual(resource['version'], 'v2') response = self.client.get( - f'/users/{self.user.mnemonic}/CodeSystem/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + f'/users/{self.user.mnemonic}/CodeSystem/?url={SYSTEM}', HTTP_AUTHORIZATION='Token ' + self.user_token) self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] @@ -85,7 +90,7 @@ def test_private_can_view(self): '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v1') - self.assertEqual(self.client.get(f'/users/{self.user.mnemonic}/CodeSystem/?url=/some/url').data['total'], 0) + self.assertEqual(self.client.get(f'/users/{self.user.mnemonic}/CodeSystem/?url={SYSTEM}').data['total'], 0) def test_public_can_list(self): response = self.client.get('/fhir/CodeSystem/') @@ -166,13 +171,29 @@ def test_get_code_system(self): self.assertEqual(response.data['id'], self.org_source.mnemonic) self.assertEqual(response.data['version'], 'v2') + def valid(self, concept): + """$validate-code's answer for a valid code of the org source, at the version resolved (v2).""" + return {'resourceType': 'Parameters', 'parameter': [ + {'name': 'result', 'valueBoolean': True}, + {'name': 'display', 'valueString': concept.display_name}, + {'name': 'code', 'valueCode': concept.mnemonic}, + {'name': 'system', 'valueUri': SYSTEM}, + {'name': 'version', 'valueString': 'v2'}]} + + def assert_issues(self, response, *issues): + """The answer is result false, with these issues (code, tx-issue-type, expression).""" + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['parameter'][0], {'name': 'result', 'valueBoolean': False}) + outcome = next(item['resource'] for item in response.data['parameter'] if item['name'] == 'issues') + self.assertEqual([(item['code'], item['details']['coding'][0]['code'], item['expression']) + for item in outcome['issue']], list(issues)) + def test_validate_code_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url={self.org_source.canonical_url}&code={self.concept_1.mnemonic}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_for_code_system_via_post(self): response = self.client.post( @@ -188,28 +209,19 @@ def test_validate_code_for_code_system_via_post(self): ) self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_for_code_system_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url={self.org_source.canonical_url}&code=non_existing_code') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('code-invalid', 'invalid-code', ['code'])) def test_validate_code_for_code_system_with_invalid_url_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url=non_existing_url&code=${self.concept_1.mnemonic}') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('not-found', 'not-found', ['system'])) def test_validate_code_with_display_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' @@ -218,8 +230,7 @@ def test_validate_code_with_display_for_code_system(self): f'&display={self.concept_1.display_name}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_with_display_for_code_system_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' @@ -227,11 +238,7 @@ def test_validate_code_with_display_for_code_system_negative(self): f'&code={self.concept_1.mnemonic}' f'&display=wrong_display') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('invalid', 'invalid-display', ['display'])) def test_lookup_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$lookup/' @@ -241,97 +248,115 @@ def test_lookup_for_code_system(self): self.assertEqual(response.status_code, 200) self.assertJSONEqual(json.dumps(response.data), json.dumps( {'resourceType': 'Parameters', 'parameter': [ - {'name': 'name', 'valueString': self.org_source.mnemonic}, + {'name': 'name', 'valueString': Source.objects.get(id=self.org_source_v2.id).name}, {'name': 'version', 'valueString': self.org_source_v2.version}, - {'name': 'display', 'valueString': self.concept_1.display_name}]})) + {'name': 'display', 'valueString': self.concept_1.display_name}, + {'name': 'code', 'valueCode': self.concept_1.mnemonic}, + {'name': 'system', 'valueUri': SYSTEM}, + {'name': 'abstract', 'valueBoolean': False}, + {'name': 'designation', 'part': [ + {'name': 'language', 'valueCode': 'en'}, + {'name': 'use', 'valueCoding': {'code': 'FULLY_SPECIFIED'}}, + {'name': 'value', 'valueString': 'concept_1_name'}]}, + {'name': 'property', 'part': [ + {'name': 'code', 'valueCode': 'inactive'}, {'name': 'value', 'valueBoolean': False}]}]})) def test_lookup_for_empty_code_system(self): response = self.client.get('/fhir/CodeSystem/$lookup/?system=&code=') self.assertEqual(response.status_code, 400) - self.assertJSONEqual(json.dumps(response.data), json.dumps(CodeSystemLookupNotFoundError().detail)) + self.assertEqual(response.data['issue'][0]['expression'], ['code']) def test_lookup_for_code_system_with_new_canonical(self): - self.org_source.canonical_url = '/new/url' + """A code system is found by its repository's current canonical URL, not by one a version recorded.""" + self.org_source.canonical_url = 'http://new/url' Source.save(self.org_source) response = self.client.get(f'/fhir/CodeSystem/$lookup/' - f'?system={self.org_source.canonical_url}' + f'?system={self.org_source_v1.canonical_url}' f'&code={self.concept_1.mnemonic}') - self.assertEqual(response.status_code, 400) - self.assertJSONEqual(json.dumps(response.data), json.dumps(CodeSystemLookupNotFoundError().detail)) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.data['issue'][0]['code'], 'not-found') - response = self.client.get(f'/fhir/CodeSystem/$lookup/' - f'?system={self.org_source_v1.canonical_url}' - f'&code={self.concept_1.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$lookup/?system=http://new/url&code={self.concept_1.mnemonic}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'name', 'valueString': self.org_source.mnemonic}, - {'name': 'version', 'valueString': self.org_source_v2.version}, - {'name': 'display', 'valueString': self.concept_1.display_name}]})) + self.assertEqual(response.data['parameter'][1], {'name': 'version', 'valueString': self.org_source_v2.version}) + self.assertEqual(response.data['parameter'][2], {'name': 'display', 'valueString': self.concept_1.display_name}) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def test_private_code_system_answers_as_missing_for_non_viewers(self): # pylint: disable=too-many-locals + """ + A code system the requester can't view answers exactly as one that doesn't exist: at the global base, which + shows public code systems only, and at its owner's base to anyone but the owner's members and staff. + """ private_org = OrganizationFactory() private_source = OrganizationSourceFactory( - organization=private_org, canonical_url='/private/url', public_access=ACCESS_TYPE_NONE) + organization=private_org, canonical_url=PRIVATE, public_access=ACCESS_TYPE_NONE) concept = ConceptFactory(parent=private_source, names=1, names__name='private_concept_name') Source.persist_new_version(OrganizationSourceFactory.build( version='v1', mnemonic=private_source.mnemonic, organization=private_org, - public_access=ACCESS_TYPE_NONE), private_source.created_by) + public_access=ACCESS_TYPE_NONE, released=True), private_source.created_by) member = UserProfileFactory() private_org.members.add(member) outsider = {'HTTP_AUTHORIZATION': 'Token ' + self.user_token} - validate_url = f'/fhir/CodeSystem/$validate-code/?url=/private/url&code={concept.mnemonic}' - lookup_url = '/fhir/CodeSystem/$lookup/?system={}&code={}' - not_found = self.client.get(lookup_url.format('/unknown/url', concept.mnemonic)) - self.assertEqual(not_found.status_code, 400) + member_headers = {'HTTP_AUTHORIZATION': 'Token ' + member.get_token()} + owner_base = f'/orgs/{private_org.mnemonic}' + validate_url = '{}/CodeSystem/$validate-code/?url={}&code={}' + lookup_url = '{}/CodeSystem/$lookup/?system={}&code={}' public_concept = ConceptFactory() selector = (f'&org={public_concept.parent.organization.mnemonic}&source={public_concept.parent.mnemonic}' f'&concept={public_concept.mnemonic}') - for headers in [{}, outsider]: + hidden = [] + for base, headers in [('/fhir', {}), ('/fhir', outsider), ('/fhir', member_headers), (owner_base, {}), + (owner_base, outsider)]: # without a code, a code system the requester can't view still answers as one that doesn't exist - for system_url in ['/private/url', '/unknown/url']: - response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url={system_url}{selector}', **headers) - self.assertFalse(response.data['parameter'][0]['valueBoolean'], (system_url, headers)) - for url in [validate_url, f'{validate_url}&display={concept.display_name}']: - response = self.client.get(url, **headers) - self.assertEqual(response.status_code, 200, (url, headers)) - self.assertFalse(response.data['parameter'][0]['valueBoolean'], (url, headers)) + response = self.client.get(f'{base}/CodeSystem/$validate-code/?url={PRIVATE}{selector}', **headers) + self.assertFalse(response.data['parameter'][0]['valueBoolean'], (base, headers)) + hidden.append((f'{base}/CodeSystem/$validate-code/?url={PRIVATE}{selector}', headers, response)) for code in [concept.mnemonic, 'missing']: - response = self.client.get(lookup_url.format('/private/url', code), **headers) - self.assertEqual(response.status_code, 400, (code, headers)) - self.assertEqual(response.content, not_found.content, (code, headers)) - - member_headers = {'HTTP_AUTHORIZATION': 'Token ' + member.get_token()} - response = self.client.get(validate_url, **member_headers) + for url in [validate_url.format(base, PRIVATE, code), + validate_url.format(base, PRIVATE, code) + f'&display={concept.display_name}']: + response = self.client.get(url, **headers) + self.assertEqual(response.status_code, 200, (url, headers)) + self.assertFalse(response.data['parameter'][0]['valueBoolean'], (url, headers)) + hidden.append((url, headers, response)) + response = self.client.get(lookup_url.format(base, PRIVATE, code), **headers) + self.assertEqual(response.status_code, 422, (code, headers)) + hidden.append((lookup_url.format(base, PRIVATE, code), headers, response)) + + response = self.client.get(validate_url.format(owner_base, PRIVATE, concept.mnemonic), **member_headers) self.assertTrue(response.data['parameter'][0]['valueBoolean']) - response = self.client.get(lookup_url.format('/private/url', concept.mnemonic), **member_headers) + response = self.client.get(lookup_url.format(owner_base, PRIVATE, concept.mnemonic), **member_headers) self.assertEqual(response.status_code, 200) self.assertEqual(response.data['parameter'][2]['valueString'], concept.display_name) + private_source.is_active = False # now it doesn't exist + private_source.save() + for url, headers, response in hidden: # each against the same request, at the same base, once it's missing + not_found = self.client.get(url, **headers) + self.assertEqual((response.status_code, response.content), (not_found.status_code, not_found.content)) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def test_private_code_system_does_not_hide_public_one_with_same_url(self): - private_source = UserSourceFactory(user=self.user, public_access=ACCESS_TYPE_NONE, canonical_url='/shared/url') + private_source = UserSourceFactory(user=self.user, public_access=ACCESS_TYPE_NONE, canonical_url=SHARED) Source.persist_new_version(UserSourceFactory.build( - version='v1', mnemonic=private_source.mnemonic, user=self.user), private_source.created_by) - public_source = OrganizationSourceFactory(canonical_url='/shared/url') + version='v1', mnemonic=private_source.mnemonic, user=self.user, released=True), private_source.created_by) + public_source = OrganizationSourceFactory(canonical_url=SHARED) concept = ConceptFactory(parent=public_source) Source.persist_new_version(OrganizationSourceFactory.build( - version='v1', mnemonic=public_source.mnemonic, organization=public_source.organization), + version='v1', mnemonic=public_source.mnemonic, organization=public_source.organization, released=True), public_source.created_by) - response = self.client.get(f'/fhir/CodeSystem/$lookup/?system=/shared/url&code={concept.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$lookup/?system={SHARED}&code={concept.mnemonic}') self.assertEqual(response.status_code, 200) self.assertEqual(response.data['parameter'][0]['valueString'], public_source.mnemonic) - response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url=/shared/url&code={concept.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url={SHARED}&code={concept.mnemonic}') self.assertTrue(response.data['parameter'][0]['valueBoolean']) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) diff --git a/core/code_systems/urls.py b/core/code_systems/urls.py index b290d6b2b..5e72b41ef 100644 --- a/core/code_systems/urls.py +++ b/core/code_systems/urls.py @@ -4,10 +4,16 @@ urlpatterns = [ path('', views.CodeSystemListView.as_view(), name='code-system-list'), - path('$lookup/', views.CodeSystemListLookupView.as_view(), name='code-system-list-lookup'), - path('$lookup', views.CodeSystemListLookupView.as_view(), name='code-system-list-lookup_no_slash'), + path('$lookup/', views.CodeSystemLookupView.as_view(), name='code-system-list-lookup'), + path('$lookup', views.CodeSystemLookupView.as_view(), name='code-system-list-lookup_no_slash'), path('$validate-code/', views.CodeSystemValidateCodeView.as_view(), name='code-system-validate-code'), path('$validate-code', views.CodeSystemValidateCodeView.as_view(), name='code-system-validate-code_no_slash'), + path("/$lookup/", views.CodeSystemLookupView.as_view(), name='code-system-lookup'), + path("/$lookup", views.CodeSystemLookupView.as_view(), name='code-system-lookup_no_slash'), + path("/$validate-code/", views.CodeSystemValidateCodeView.as_view(), + name='code-system-instance-validate-code'), + path("/$validate-code", views.CodeSystemValidateCodeView.as_view(), + name='code-system-instance-validate-code_no_slash'), path("/", views.CodeSystemRetrieveUpdateView.as_view(), name='code-system-detail'), path("", views.CodeSystemRetrieveUpdateView.as_view(), name='code-system-detail_no_slash'), ] diff --git a/core/code_systems/views.py b/core/code_systems/views.py index 0666d76c0..1c7d32a34 100644 --- a/core/code_systems/views.py +++ b/core/code_systems/views.py @@ -1,34 +1,17 @@ import logging -from django.db.models import F -from rest_framework.exceptions import ValidationError, NotAuthenticated, PermissionDenied - -from core.code_systems.serializers import CodeSystemDetailSerializer, \ - ValidateCodeParametersSerializer +from core.code_systems.serializers import CodeSystemDetailSerializer from core.common.constants import HEAD -from core.common.serializers import IdentifierSerializer -from core.concepts.permissions import CanViewParentDictionaryAsGuest -from core.concepts.views import ConceptRetrieveUpdateDestroyView -from core.fhir.base import FhirViewMixin -from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM, InvalidError +from core.fhir.adapters import r4 +from core.fhir.base import FhirViewMixin, FhirOperationView +from core.fhir.ops.lookup import lookup +from core.fhir.ops.validate_code import validate_code_system from core.fhir.search import FhirSearchMixin, FhirReadMixin -from core.parameters.serializers import ParametersSerializer -from core.sources.models import Source from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView logger = logging.getLogger('oclapi') -def get_viewable_repo(repos, user): - """The first of these repo versions the user can view, skipping the rest as if they didn't exist. Its HEAD, - which governs access to its concepts, must be viewable too.""" - for repo in repos.order_by('id'): - head = repo.head - if head and repo.has_view_access(user) and head.has_view_access(user): - return repo - return None - - class CodeSystemListView(FhirViewMixin, FhirSearchMixin, SourceListView): resource_type = 'CodeSystem' serializer_class = CodeSystemDetailSerializer @@ -40,174 +23,18 @@ def get_detail_serializer(self, obj): return self.serializer_class(obj) -class CodeSystemLookupNotFoundError(ValidationError): - def __init__(self, code: str = None): - message = f"Code '{code}' not found" if code else "Code not found" - super().__init__( - detail={ - "resourceType": "OperationOutcome", - "id": "exception", - "text": { - "status": "additional", - "div": f'
{message}
', - }, - "issue": [ - { - "severity": "error", - "code": "not-found", - "details": {"coding": [{"system": TX_ISSUE_TYPE_SYSTEM, "code": "not-found"}], "text": message}, - } - ], - } - ) - - -class CodeSystemListLookupView(FhirViewMixin, ConceptRetrieveUpdateDestroyView): - serializer_class = ParametersSerializer - - def is_container_version_specified(self): - return True - - def verify_scope(self): - pass - - def get_queryset(self): - queryset = super().get_queryset() - code = self.request.query_params.get('code') - system = self.request.query_params.get('system') - version = self.request.query_params.get('version') - if code and system: - source = Source.objects.filter(canonical_url=system) - if not source: - system = IdentifierSerializer.convert_fhir_url_to_ocl_uri(system, 'sources') - source = Source.objects.filter(uri=system) - if source: - if version: - source = source.filter(version=version) - else: - source = source.filter(is_latest_version=True).exclude(version=HEAD) - - source = get_viewable_repo(source, self.request.user) - if source: - queryset = queryset.filter(sources=source, mnemonic=code) - if not queryset.exists(): - raise CodeSystemLookupNotFoundError(code) - return queryset - - raise CodeSystemLookupNotFoundError() - - def get_object(self, queryset=None): - """A code the requester can't view gives the same answer as a code system that doesn't exist.""" - try: - return super().get_object(queryset) - except (NotAuthenticated, PermissionDenied) as ex: - raise CodeSystemLookupNotFoundError() from ex - - def get_serializer(self, instance=None): # pylint: disable=arguments-differ - if instance: - return ParametersSerializer.from_concept(instance) - return None - - -class CodeSystemValidateCodeView(FhirViewMixin, ConceptRetrieveUpdateDestroyView): - serializer_class = ValidateCodeParametersSerializer - - def verify_scope(self): - pass - - def post(self, request, *args, **kwargs): - return self.retrieve(request, *args, **kwargs) - - def update(self, request, *args, **kwargs): - return self.retrieve(request, *args, **kwargs) - - def delete(self, request, *args, **kwargs): - # Not supported - pass - - def is_container_version_specified(self): - return True - - def get_permissions(self): - return [CanViewParentDictionaryAsGuest(), ] - - def get_queryset(self): - """The concept with the code in the code system `url` names. A code system that doesn't resolve, or that the - requester can't view, reads nothing; without a code, or with a code but no url, the request is a 400.""" - queryset = super().get_queryset() - - parameters = self.get_parameters() - url = parameters.get('url') - code = parameters.get('code') - version = parameters.get('version') - display = parameters.get('display') - source = None - if url: - source = Source.objects.filter(canonical_url=url) - if not source: - url = IdentifierSerializer.convert_fhir_url_to_ocl_uri(url, 'sources') - source = Source.objects.filter(uri=url) - - if version: - source = source.filter(version=version) - else: - source = source.filter(is_latest_version=True).exclude(version=HEAD) - source = get_viewable_repo(source, self.request.user) - if not source: - return queryset.none() - - if not code: - raise InvalidError('A code to validate is required.', expression='code') - if not source: - raise InvalidError('url is required with a code: the code system to validate it in.', expression='url') - - queryset = queryset.filter(sources=source, mnemonic=code) - - if display: - instance = queryset.first() - if not instance or display not in (instance.name, instance.display_name): - return queryset.none() - - return queryset - - def get_parameters(self): - if self.request.method in ['POST', 'PUT']: - parameters = self.get_serializer(data=self.request.data, instance=None) - else: - parameters = self.get_serializer_class().parse_query_params(self.request.query_params) - if not parameters.is_valid(): - raise ValidationError(parameters.errors) - params = parameters.validated_data - params = params.get('parameters', {}) - return params - - def get_object(self, queryset=None): - queryset = self.get_queryset() - if not self.is_container_version_specified(): - queryset = queryset.filter(id=F('versioned_object_id')) - instance = queryset.first() - if instance: - try: - self.check_object_permissions(self.request, instance) - except (NotAuthenticated, PermissionDenied): - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ] - } +class CodeSystemLookupView(FhirOperationView): + """CodeSystem/$lookup and CodeSystem/{id}/$lookup: core.fhir.ops.lookup.""" + instance = 'source' + operation = staticmethod(lookup) + adapter = staticmethod(r4.lookup_parameters) - if instance: - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': True} - ] - } - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ] - } +class CodeSystemValidateCodeView(FhirOperationView): + """CodeSystem/$validate-code and CodeSystem/{id}/$validate-code: core.fhir.ops.validate_code.""" + instance = 'source' + operation = staticmethod(validate_code_system) + adapter = staticmethod(r4.validate_code_parameters) class CodeSystemRetrieveUpdateView(FhirViewMixin, FhirReadMixin, SourceRetrieveUpdateDestroyView): diff --git a/core/common/models.py b/core/common/models.py index abb1b1eab..e61e71111 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -1539,8 +1539,8 @@ def resolve_expression_to_version(cls, expression, namespace=None): return instance, resolved_registry_entry @classmethod - def resolve_reference_expression( # pylint: disable=too-many-arguments - cls, url, namespace=None, version=None, repo_type=None, allow_head=True): + def resolve_reference_expression( # pylint: disable=too-many-arguments,too-many-locals + cls, url, namespace=None, version=None, repo_type=None, allow_head=True, heads=None, head_shown=None): """ resolves to repository version according to this process: @@ -1574,6 +1574,10 @@ def resolve_reference_expression( # pylint: disable=too-many-arguments active. Unresolved is an unsaved repo_type. - allow_head=False: with no version named, the latest released version other than HEAD (even a HEAD flagged released) or nothing. A version named HEAD is still served. + - heads (a Q on HEAD rows, with repo_type): without the registry, a canonical URL matches only the HEADs it + holds, so a repository the caller isn't shown never stands in for one they are (FHIR's global base). + - head_shown (a callable, with repo_type): a HEAD it refuses resolves to nothing before any version of it is + read, so a repository the caller isn't shown costs what one that doesn't exist does (FHIR). """ resolution_url, version, is_canonical = cls.__get_resolution_url(url, version) @@ -1599,11 +1603,13 @@ def resolve_reference_expression( # pylint: disable=too-many-arguments instance = registry_entry.find_repo(repo_type) return cls.resolve_repo( - instance, version, is_canonical, resolution_url, repo_type, allow_head), registry_entry + instance, version, is_canonical, resolution_url, repo_type, allow_head, head_shown), registry_entry criteria &= models.Q(canonical_url=resolution_url) if repo_type: criteria &= models.Q(version=HEAD) + if heads is not None: + criteria &= heads if namespace: criteria &= models.Q(models.Q(user__uri=namespace) | models.Q(organization__uri=namespace)) else: @@ -1611,15 +1617,17 @@ def resolve_reference_expression( # pylint: disable=too-many-arguments from core.repos.models import Repository return cls.resolve_repo( - Repository.get(criteria, repo_type), version, is_canonical, resolution_url, repo_type, allow_head - ), registry_entry + Repository.get(criteria, repo_type), version, is_canonical, resolution_url, repo_type, allow_head, + head_shown), registry_entry @classmethod def resolve_repo( # pylint: disable=too-many-arguments - cls, instance, version, is_canonical, resolution_url, repo_type=None, allow_head=True): + cls, instance, version, is_canonical, resolution_url, repo_type=None, allow_head=True, head_shown=None): if instance: head = instance if repo_type and instance.is_head else None - if version: + if head and head_shown and not head_shown(head): + instance = None + elif version: versions = instance.active_versions if repo_type else instance.versions instance = versions.filter(version=decode_string(version)).first() elif instance.is_head: diff --git a/core/fhir/adapters/__init__.py b/core/fhir/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/adapters/r4.py b/core/fhir/adapters/r4.py new file mode 100644 index 000000000..cf4027b99 --- /dev/null +++ b/core/fhir/adapters/r4.py @@ -0,0 +1,57 @@ +""" +The FHIR R4 wire form of the version-neutral answers core.fhir.ops build, as R4 Parameters resources. + +- $lookup: R4's outputs in R4's order (name, version, display, designation, property), with code, system and abstract + after display, as FHIRsmith and the terminology ecosystem IG return them. inactive is a property. +- $validate-code: result first, then message, display, code, system, version, inactive, the unknown system, and the + issues as an OperationOutcome. +""" + + +def parameter(name, value_type, value): + return {'name': name, value_type: value} + + +def parameters(items): + """A Parameters resource of (name, value type, value) items, leaving out those without a value.""" + return {'resourceType': 'Parameters', 'parameter': [ + item if isinstance(item, dict) else parameter(*item) for item in items + if isinstance(item, dict) or item[2] is not None]} + + +def part_parameter(name, parts): + return {'name': name, 'part': [parameter(*part) for part in parts if part[2] is not None]} + + +def lookup_parameters(answer): + """$lookup's answer (core.fhir.ops.lookup) as R4 Parameters.""" + return parameters([ + ('name', 'valueString', answer['name']), + ('version', 'valueString', answer['version']), + ('display', 'valueString', answer['display']), + ('code', 'valueCode', answer['code']), + ('system', 'valueUri', answer['system']), + ('abstract', 'valueBoolean', answer['abstract']), + *(part_parameter('designation', [ + ('language', 'valueCode', designation['language']), + ('use', 'valueCoding', designation['use']), + ('value', 'valueString', designation['value'])]) for designation in answer['designations']), + part_parameter('property', [('code', 'valueCode', 'inactive'), ('value', 'valueBoolean', answer['inactive'])]), + ]) + + +def validate_code_parameters(answer): + """$validate-code's answer (core.fhir.ops.validate_code) as R4 Parameters.""" + unknown_name, unknown_system = answer.get('unknown') or (None, None) + issues = answer['issues'] + return parameters([ + ('result', 'valueBoolean', answer['result']), + ('message', 'valueString', answer['message']), + ('display', 'valueString', answer.get('display')), + ('code', 'valueCode', answer['code']), + ('system', 'valueUri', answer['system']), + ('version', 'valueString', answer.get('version')), + ('inactive', 'valueBoolean', answer.get('inactive')), + (unknown_name, 'valueCanonical', unknown_system), + ('issues', 'resource', {'resourceType': 'OperationOutcome', 'issue': issues} if issues else None), + ]) diff --git a/core/fhir/base.py b/core/fhir/base.py index fc17e72d1..36c7fe759 100644 --- a/core/fhir/base.py +++ b/core/fhir/base.py @@ -27,12 +27,15 @@ from rest_framework.exceptions import NotAcceptable, ParseError, UnsupportedMediaType from rest_framework.negotiation import DefaultContentNegotiation from rest_framework.parsers import BaseParser, FormParser, JSONParser, MultiPartParser +from rest_framework.permissions import AllowAny from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response from rest_framework.utils.encoders import JSONEncoder from rest_framework.utils.mediatypes import _MediaType from rest_framework.versioning import AcceptHeaderVersioning +from rest_framework.views import APIView +from core.common.throttling import ThrottleUtil from core.fhir import outcomes logger = logging.getLogger('oclapi') @@ -564,3 +567,24 @@ def get_fhir_parameters(self): def get_fhir_base_url(self): return fhir_base_url(self.request, self.kwargs) + + +class FhirOperationView(FhirViewMixin, APIView): + """ + A FHIR operation that only reads, GET and POST alike, at the global base and at owner bases: `operation`'s answer + for the request's FhirParameters, base and user (and the resource the route kwarg `instance` names, at the instance + level), written by `adapter`. Throttled as OCL's other views are. + """ + permission_classes = (AllowAny,) # what an operation shows the caller is core.fhir.resolve's rule + operation = adapter = None + instance = None + + def get_throttles(self): + return ThrottleUtil.get_throttles_by_user_plan(self.request.user) + + def get(self, request, *args, **kwargs): # pylint: disable=unused-argument + answer = self.operation( # pylint: disable=not-callable + self.get_fhir_parameters(), self.kwargs, request.user, kwargs.get(self.instance)) + return Response(self.adapter(answer)) # pylint: disable=not-callable + + post = get diff --git a/core/fhir/ops/__init__.py b/core/fhir/ops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/ops/concepts.py b/core/fhir/ops/concepts.py new file mode 100644 index 000000000..1fecc3359 --- /dev/null +++ b/core/fhir/ops/concepts.py @@ -0,0 +1,197 @@ +""" +What $lookup and $validate-code share (core.fhir.ops.lookup, core.fhir.ops.validate_code): reading the code they are +asked about, finding it in a code system version, and the concept's display and names. + +- Input: a code with its system, version and display as their own parameters, or as a Coding (`coding`, or a + codeableConcept holding one coding). Text inputs may come as any FHIR string type (valueString, valueCode, + valueUri); anything else is a 400 naming the parameter. +- A code is found among the version's own concepts, retired ones included, as FHIR spells it or percent-encoded, as + OCL stores a code with reserved characters. One query for the concept (with its repository), one for its names. +- Display: with a language, the concept's preferred name in that language when it has one; else its own display (OCL's + display_name: the preferred name in its code system's default locale). +""" +from collections import namedtuple +from urllib.parse import quote, unquote + +from django.db.models import F, Q + +from core.common.constants import HEAD +from core.common.utils import to_parent_uri +from core.concepts.models import Concept +from core.fhir import outcomes +from core.fhir.resolve import REPO_PATHS, UnresolvedError, namespace_of, resolve, to_ocl_path +from core.fhir.search import read_version + +# A code, as an operation is given it; `parameter` is the Coding parameter it came in, None for separate parameters +Coding = namedtuple('Coding', 'system code version display parameter') +PATHS = {None: '', 'coding': 'Coding.', 'codeableConcept': 'CodeableConcept.coding[0].'} + + +def text(parameters, name): + """A text input (a string, code or uri), or None when it's absent or empty; any other value is a 400 naming it.""" + value = parameters.get(name) + if value is None or value == '': + return None + if not isinstance(value, str): + raise outcomes.InvalidError(f"'{name}' must be text.", expression=name) + return value + + +def coding_of(value, name): + """A Coding parameter as a Coding: a dict, or in a GET, a system|code token. Its parts must be text.""" + if isinstance(value, str): + system, _, code = value.rpartition('|') + value = {'system': system, 'code': code} + parts = {key: value.get(key) for key in ('system', 'code', 'version', 'display')} if isinstance(value, dict) else {} + if not parts or not all(part is None or isinstance(part, str) for part in parts.values()): + raise outcomes.InvalidError(f"'{name}' must be a Coding.", expression=name) + return Coding(*(part or None for part in parts.values()), parameter=name) + + +def coded_input(parameters, system, version): + """ + The code an operation is asked about: `coding`, a codeableConcept with one coding, or else code and display with + the system and version from the parameters named `system` and `version`. + """ + if 'coding' in parameters: + return coding_of(parameters.get('coding'), 'coding') + if 'codeableConcept' in parameters: + concept = parameters.get('codeableConcept') + codings = concept.get('coding') if isinstance(concept, dict) else None + if not isinstance(codings, list) or len(codings) != 1: + raise outcomes.NotSupportedError( + 'A codeableConcept is supported with one coding: send each coding on its own.', + expression='codeableConcept') + return coding_of(codings[0], 'codeableConcept') + return Coding(text(parameters, system), text(parameters, 'code'), text(parameters, version), + text(parameters, 'display'), None) + + +def path(coding, part): + """The FHIRPath of a part of the code in an issue: the parameter itself, or the part of the Coding it came in.""" + return PATHS[coding.parameter] + part + + +def require(coding, part, message): + """The part of the code an operation needs, or a 400 naming the input it's missing from (`part`, or the Coding).""" + value = getattr(coding, part) + if not value: + raise outcomes.InvalidError(message, expression=coding.parameter or part) + return value + + +def stored_codes(code): + """The forms a code may be stored in: as it is, and percent-encoded as OCL encodes reserved characters.""" + return list(dict.fromkeys([code, quote(code, safe=''), quote(code)])) + + +def code_q(field, code): + """Q for rows whose `field` is the code in a form OCL stores it in: an equality when there's only one.""" + forms = stored_codes(code) + return Q(**{field: forms[0]}) if len(forms) == 1 else Q(**{f'{field}__in': forms}) + + +def fhir_code(code): + """A code as FHIR spells it: OCL stores one with reserved characters percent-encoded (ICD-11's ND53.Y&XK70).""" + return unquote(code) if code and '%' in code else code + + +def system_of(version, named=None): + """A code system version's canonical URL (else its repository's), or else the URL it was named by.""" + return version.canonical_url or getattr(version, 'repository_canonical_url', None) or named + + +def instance_version(resource_type, instance_id, version, kwargs, user): + """ + The repository version an instance-level operation is on: as a read finds it (a release: at the global base the + opaque id's, at an owner base the mnemonic's `version`, else the newest shown); or, at an owner base with HEAD + named, HEAD through core.fhir.resolve, with its visibility rule (design D6). A global id names a release, so a + HEAD named there is left to the caller's version check. Not found is a read's 404. + """ + namespace = namespace_of(kwargs) + if namespace is None or version != HEAD: + return read_version(resource_type, instance_id, version, kwargs, user) + try: + return resolve(resource_type, f'{namespace}{REPO_PATHS[resource_type]}/{instance_id}/', HEAD, kwargs, user) + except UnresolvedError as ex: + raise outcomes.NotFoundError(f'{resource_type}/{instance_id} not found') from ex + + +def is_system_of(system, version, named=None): + """ + Whether a coding's system (or a value set url) names a repository version: its canonical URL (or its + repository's), the URL it was resolved by (`named`), or an OCL path of its repository or of the version. + """ + url = system.partition('|')[0] + canonicals = {version.canonical_url, getattr(version, 'repository_canonical_url', None), + (named or '').partition('|')[0]} + if url in canonicals - {None, ''}: + return True + repository = to_parent_uri(to_ocl_path(url)) + repository = repository if repository.endswith('/') else f'{repository}/' + return repository in {version.uri, '/'.join(version.uri.split('/')[:5]) + '/'} + + +def mismatch(parameters, coding, version, system_name, instance_id): + """ + (expression, message) of an input naming another code system or version than the one resolved, else None. At the + instance level, the id names it: the system parameter (`system_name`, url or system), the coding's system and any + version named (the version parameter, the coding's, a url|version) must all be its. At the type level, a Coding's + system and version must be those of the code system it is validated in, which url and version may name. + """ + if instance_id: + systems = [(text(parameters, system_name), system_name), (coding.system, path(coding, 'system'))] + versions = [(text(parameters, 'version'), 'version'), (coding.version, path(coding, 'version'))] + named = None + elif coding.parameter: + systems, versions = [(coding.system, path(coding, 'system'))], [(coding.version, path(coding, 'version'))] + named = text(parameters, system_name) or coding.system + else: + return None + for system, expression in systems: + if system and not is_system_of(system, version, named): + return expression, f"'{system}' is not the code system '{system_of(version, named)}' it is about" + versions.append((system and system.partition('|')[2], expression)) + for named_version, expression in versions: + if named_version and named_version != version.version: + return expression, f"Version '{named_version}' is not the version '{version.version}' it is about" + return None + + +def find_concept(version, code): + """ + The code system version's concept with the code, retired or not (a deleted one, is_active false, is gone), with its + repository and names; else None. + """ + concepts = version.concepts_set.filter(id=F('versioned_object_id')) if version.is_head else \ + Concept.objects.filter(sources=version) + return concepts.filter(code_q('mnemonic', code), is_active=True).select_related('parent').prefetch_related( + 'names').order_by('id').first() + + +def display_of(concept, language=None): + """ + The concept's display: with a language, its preferred name in that language, newest first, when it has one; else + its own display. None for a concept without names. + """ + if language: + names = sorted((name for name in concept.names.all() if name.locale == language and not name.retired), + key=lambda name: (name.locale_preferred, name.created_at), reverse=True) + if names: + return names[0].name + return concept.display_name + + +def has_name(concept, display): + """ + Whether a display is right for the concept: one of its names, in any language, ignoring case (FHIRsmith's default + check). A concept without names has nothing to check a display against. + """ + names = [name.name.casefold() for name in concept.names.all()] + return not names or display.casefold() in names + + +def designations(concept): + """Each of the concept's names as a designation: its language, its OCL name type as `use`, and its text.""" + return [{'language': name.locale, 'use': {'code': name.type} if name.type else None, 'value': name.name} + for name in concept.names.all()] diff --git a/core/fhir/ops/lookup.py b/core/fhir/ops/lookup.py new file mode 100644 index 000000000..5106621b5 --- /dev/null +++ b/core/fhir/ops/lookup.py @@ -0,0 +1,61 @@ +""" +CodeSystem $lookup (SVCM ITI-98, FHIR R4's $lookup), version-neutral: what OCL holds about a code of the code system +version the request names. core.fhir.adapters.r4 writes the answer as R4 Parameters. + +- The code system: `system` and `version` (or a `coding`'s), resolved by core.fhir.resolve at the base's namespace: a + released version, the newest the caller is shown unless one is named (HEAD only when named); public only at /fhir/; + at an owner base, private ones to those who may view them. CodeSystem/{id}/$lookup names it as a read does. One the + caller isn't shown answers exactly as one that doesn't exist. +- The code: that version's concept, retired ones too, which are flagged inactive (core.fhir.ops.concepts). +- The answer: name, version (the one resolved), display, code, system, abstract (OCL has no abstract concepts), each + of the concept's names as a designation, and the inactive property. `displayLanguage` picks the display among the + concept's names in that language; a concept without names is displayed by its code. +- Errors, as FHIRsmith answers them: no code, or no system at the type level, is a 400; a code system that doesn't + resolve is a 422 not-found, a code it doesn't hold a 404 (tx-issue-type invalid-code). +- Cost: a fixed number of queries whatever the size of the code system: resolution, the concept, its names. +""" +from rest_framework import status + +from core.fhir import outcomes +from core.fhir.ops.concepts import ( + coded_input, require, find_concept, display_of, designations, fhir_code, instance_version, mismatch, system_of, + text) +from core.fhir.resolve import UnresolvedError, resolve + + +def code_system_version(coding, kwargs, user, instance_id=None): + """The code system version a $lookup or $validate-code names: the instance, else the coding's system and version.""" + if instance_id: + return instance_version('CodeSystem', instance_id, coding.version, kwargs, user) + return resolve('CodeSystem', coding.system, coding.version, kwargs, user) + + +def lookup(parameters, kwargs, user, instance_id=None): + """ + $lookup's answer for its FhirParameters at the base `kwargs` names (CodeSystem/{instance_id}/$lookup at the instance + level), for `user`: {name, version, display, code, system, abstract, inactive, designations}. + """ + coding = coded_input(parameters, 'system', 'version') + target = coding._replace(version=text(parameters, 'version') or coding.version) + code = require(coding, 'code', '$lookup needs the code to look up.') + if not instance_id: + require(coding, 'system', '$lookup needs the system the code is from.') + try: + version = code_system_version(target, kwargs, user, instance_id) + except UnresolvedError as ex: + raise outcomes.NotFoundError( + str(ex.detail), tx_issue_type='not-found', status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) from ex + conflict = mismatch(parameters, coding, version, 'system', instance_id) + if conflict: + raise outcomes.InvalidError(f'{conflict[1]}: this $lookup is on {instance_id}.', expression=conflict[0]) + system = system_of(version, coding.system) + concept = find_concept(version, code) + if not concept: + raise outcomes.NotFoundError( + f"Unable to find code '{code}' in {system} version {version.version}", tx_issue_type='invalid-code') + return { + 'name': version.name, 'version': version.version, + 'display': display_of(concept, text(parameters, 'displayLanguage')) or fhir_code(concept.mnemonic), + 'code': fhir_code(concept.mnemonic), 'system': system, 'abstract': False, 'inactive': bool(concept.retired), + 'designations': designations(concept), + } diff --git a/core/fhir/ops/validate_code.py b/core/fhir/ops/validate_code.py new file mode 100644 index 000000000..3f85a8687 --- /dev/null +++ b/core/fhir/ops/validate_code.py @@ -0,0 +1,265 @@ +""" +CodeSystem and ValueSet $validate-code (SVCM ITI-99, FHIR R4's $validate-code), version-neutral: whether a code is +valid, with what OCL knows about it. core.fhir.adapters.r4 writes the answer as R4 Parameters. + +- Resolution and access are core.fhir.resolve's (a released version, the newest the caller is shown unless one is + named; public only at /fhir/; at an owner base, private ones to those who may view them); CodeSystem/{id} and + ValueSet/{id} name theirs as a read does. A code system, value set or content the caller isn't shown answers exactly + as one that doesn't exist. +- CodeSystem: the code in the version `url` (and `version`) names: result, and the code, system, version used and + display; `display` must be one of the concept's names (ignoring case); a retired concept is valid but flagged + inactive, with a warning. The code system is the instance, else url (and version), else a coding's system; a + coding of another system is result false (invalid-data). A code system that doesn't resolve answers result false + with a not-found issue and x-caused-by-unknown-system, as FHIRsmith does, with or without a code. Then a code is + required (400), and a url. +- ValueSet: membership is the resolved version's default expansion: the code, from a concept whose code system is + `system` (and `systemVersion`), in a code system version the caller is shown, released or not. The version + answered is a release the caller is shown that holds the concept: the expansion's explicit version, else the one + it evaluated, else the newest; none without one (never HEAD or an unreleased version). Not a member: result false + (not-in-vs), with what the code system itself says of the code (unknown system, unknown code, or its display). A + value set that doesn't resolve is a 422 not-found OperationOutcome. Without url (and not at the instance level), + the code is checked in its code system alone, as OCL has always answered. +- Issues carry FHIRsmith's message ids (operationoutcome-message-id) and expressions, so clients and the parity + suite compare them by kind; `message` joins their texts. +- Cost: a fixed number of queries whatever the size of the code system or value set. +""" +from django.db.models import Case, Exists, IntegerField, OuterRef, Q, Value, When +from rest_framework import status + +from core.collections.models import Expansion +from core.common.constants import HEAD +from core.concepts.models import Concept +from core.fhir import outcomes +from core.fhir.ops.concepts import ( + coded_input, path, require, text, find_concept, display_of, has_name, fhir_code, code_q, instance_version, + is_system_of, mismatch, system_of) +from core.fhir.ops.lookup import code_system_version +from core.fhir.resolve import UnresolvedError, namespace_of, resolve, to_ocl_path, viewable_rows +from core.sources.models import Source + +UNKNOWN_CODE_SYSTEM = 'x-caused-by-unknown-system' # how CodeSystem $validate-code names an unknown system +UNKNOWN_VALUE_SET_SYSTEM = 'x-unknown-system' # and ValueSet $validate-code + + +def issue(coding, part, kind, message_id, message): + """An issue about one part of the code (code, system or display), of a kind in KINDS.""" + severity, code, tx_issue_type = KINDS[kind] + return outcomes.issue(code, message, severity=severity, expression=path(coding, part), + tx_issue_type=tx_issue_type, message_id=message_id) + + +# kind: (severity, issue type, tx-issue-type), as FHIRsmith reports each +KINDS = { + 'unknown-system': ('error', 'not-found', 'not-found'), + 'unknown-code': ('error', 'code-invalid', 'invalid-code'), + 'wrong-display': ('error', 'invalid', 'invalid-display'), + 'inactive': ('warning', 'business-rule', 'code-comment'), + 'not-in-value-set': ('error', 'code-invalid', 'not-in-vs'), + 'other-system': ('error', 'invalid', 'invalid-data'), +} + + +def answer(coding, issues=(), **values): + """ + The neutral answer: result (no error issue), the code and system asked about, what was found (version, display, + inactive, the unknown system), the issues and the message their texts make. + """ + issues = list(issues) + return { + 'result': not any(item['severity'] in ('error', 'fatal') for item in issues), + 'message': '; '.join(item['details']['text'] for item in issues) or None, + 'code': fhir_code(coding.code), 'system': coding.system, **values, 'issues': issues, + } + + +def unknown_system(coding, parameter, *more, **extra): + """ + The answer for a code system that doesn't resolve, or that the caller isn't shown: the same for both, as it depends + only on what was asked (the system, and the version named or url|version). `more` are issues after its own. + """ + system, _, piped = coding.system.partition('|') + version = coding.version or piped or None + if version: + message_id = 'UNKNOWN_CODESYSTEM_VERSION' + message = f"A definition for CodeSystem '{system}' version '{version}' could not be found, " \ + "so the code cannot be validated" + else: + message_id = 'UNKNOWN_CODESYSTEM' + message = f"A definition for CodeSystem '{system}' could not be found, so the code cannot be validated" + named = f'{system}|{version}' if version else system + return answer(coding, [issue(coding, 'system', 'unknown-system', message_id, message), *more], + unknown=(parameter, named), **extra) + + +def unknown_code(coding, system, version): + return issue(coding, 'code', 'unknown-code', 'Unknown_Code_in_Version', + f"Unknown code '{fhir_code(coding.code)}' in the CodeSystem '{system}' version '{version}'") + + +def found(coding, concept, version, language): + """ + The answer for a code its code system (or value set) holds: its version, display and inactive flag, with the issues + of a display that isn't one of its names and of a retired concept. + """ + display = display_of(concept, language) + issues = [] + if coding.display and not has_name(concept, coding.display): + issues.append(issue( + coding, 'display', 'wrong-display', 'Display_Name_for__should_be_one_of__instead_of', + f"Wrong Display Name '{coding.display}' for {coding.system}#{fhir_code(coding.code)}: it should be one of " + f"the concept's names, such as '{display}'")) + if concept.retired: + issues.append(issue(coding, 'code', 'inactive', 'INACTIVE_CONCEPT_FOUND', + f"The concept '{fhir_code(coding.code)}' has a status of inactive and its use should be " + "reviewed")) + return answer(coding, issues, version=version, display=display, inactive=concept.retired or None) + + +def in_code_system(coding, version, language): + """The answer for a code in a code system version it resolved to.""" + system = system_of(version, coding.system) + coding = coding._replace(system=system) + concept = find_concept(version, coding.code) + if not concept: + return answer(coding, [unknown_code(coding, system, version.version)], version=version.version) + return found(coding, concept, version.version, language) + + +def validate_code_system(parameters, kwargs, user, instance_id=None): + """ + CodeSystem $validate-code's answer for its FhirParameters at the base `kwargs` names (CodeSystem/{instance_id} at + the instance level), for `user`. A url that doesn't resolve answers before the code is required, as it always has. + """ + coding = coded_input(parameters, 'url', 'version') + # the code system validated against: the instance, else url and version, else the coding's + target = coding._replace( + system=text(parameters, 'url') or coding.system, version=text(parameters, 'version') or coding.version) + version = None + if instance_id or target.system: + try: + version = code_system_version(target, kwargs, user, instance_id) + except UnresolvedError: + return unknown_system(target, UNKNOWN_CODE_SYSTEM, version=target.version) + require(coding, 'code', 'A code to validate is required.') + if version is None: + raise outcomes.InvalidError('url is required with a code: the code system to validate it in.', + expression=coding.parameter or 'url') + conflict = mismatch(parameters, coding, version, 'url', instance_id) + if conflict: + severity, code, tx_issue_type = KINDS['other-system'] + return answer(coding, [outcomes.issue( + code, f'{conflict[1]}: the code is not validated', severity=severity, expression=conflict[0], + tx_issue_type=tx_issue_type)]) + return in_code_system(coding, version, text(parameters, 'displayLanguage')) + + +def system_q(field, system): + """Q for concepts (through `field`) whose repository is the system: by canonical URL, or by an OCL path.""" + reference = to_ocl_path(system) + if reference.startswith('/'): + return Q(**{f'{field}__uri': reference if reference.endswith('/') else f'{reference}/'}) + return Q(**{f'{field}__canonical_url': system}) + + +def member(value_set, coding, namespace, user): + """ + (the concept, the version of its code system) of the code in the value set version's default expansion, or None: + a concept with the code (not deleted), of the system (in systemVersion, if named), in a code system version the + caller is shown (not deleted), released or not: the expansion defines the value set. The version answered is a + release the caller is shown that holds the concept: the one the expansion pins (explicit), else the one it + evaluated, else the newest; None without one, never HEAD or an unreleased version. + The code's concepts are read first, by their code and only in code systems the caller is shown (so what hidden ones + hold costs nothing), then looked for in the expansion by its (expansion, concept) index, so the cost doesn't grow + with the size of the value set. + """ + if not value_set.expansion_uri: + return None + concept_ids = list(Concept.objects.filter( + code_q('mnemonic', coding.code), system_q('parent', coding.system), + Exists(viewable_rows(Source, namespace, user).filter(id=OuterRef('parent_id'))), is_active=True).values_list( + 'id', flat=True)) + if concept_ids: + concept_ids = list(Expansion.concepts.through.objects.filter( + expansion__uri=value_set.expansion_uri, concept_id__in=concept_ids).values_list('concept_id', flat=True)) + if not concept_ids: + return None + rows = Concept.sources.through.objects.filter( + Exists(viewable_rows(Source, namespace, user).filter(id=OuterRef('source_id'))), concept_id__in=concept_ids, + source__is_active=True) + if coding.version: + rows = rows.filter(source__version=coding.version) + def pinned(versions): + return Exists(versions.through.objects.filter( + expansion__uri=value_set.expansion_uri, source_id=OuterRef('source_id'))) + release = Q(source__released=True, source__retired=False) & ~Q(source__version=HEAD) + row = rows.annotate( + release=Case(When(release, then=Value(1)), default=Value(0), output_field=IntegerField()), + explicit=pinned(Expansion.explicit_source_versions), evaluated=pinned(Expansion.evaluated_source_versions), + ).order_by('-release', '-explicit', '-evaluated', '-source__created_at', 'concept_id').values_list( + 'concept_id', 'source__version', 'release').first() + if not row: + return None + concept = Concept.objects.filter(id=row[0]).select_related('parent').prefetch_related('names').first() + return concept, row[1] if row[2] else None + + +def value_set_version(parameters, kwargs, user, instance_id): + """ + The value set version the request names; a url that doesn't resolve is a 422 not-found. At the instance level the id + names it, and a url or valueSetVersion naming another one is a 400. + """ + version, url = text(parameters, 'valueSetVersion'), text(parameters, 'url') + if instance_id: + value_set = instance_version('ValueSet', instance_id, version, kwargs, user) + if url and not is_system_of(url, value_set): + raise outcomes.InvalidError(f"'{url}' is not the value set {instance_id} this $validate-code is on.", + expression='url') + for named, expression in ((version, 'valueSetVersion'), ((url or '').partition('|')[2], 'url')): + if named and named != value_set.version: + raise outcomes.InvalidError( + f"Version '{named}' is not the version '{value_set.version}' of the value set {instance_id}.", + expression=expression) + return value_set + try: + return resolve('ValueSet', url, version, kwargs, user) + except UnresolvedError as ex: + raise outcomes.NotFoundError( + f"A definition for the value Set '{url}' could not be found", tx_issue_type='not-found', + message_id='Unable_to_resolve_value_Set_', status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) from ex + + +def validate_value_set(parameters, kwargs, user, instance_id=None): + """ + ValueSet $validate-code's answer for its FhirParameters at the base `kwargs` names (ValueSet/{instance_id} at the + instance level), for `user`. A code and its system are required first. + """ + coding = coded_input(parameters, 'system', 'systemVersion') + require(coding, 'code', 'A code to validate is required.') + require(coding, 'system', 'system is required with a code: the code system the code is from.') + language = text(parameters, 'displayLanguage') + if not instance_id and not text(parameters, 'url'): + try: + return in_code_system(coding, resolve('CodeSystem', coding.system, coding.version, kwargs, user), language) + except UnresolvedError: + return unknown_system(coding, UNKNOWN_CODE_SYSTEM, version=coding.version) + + value_set = value_set_version(parameters, kwargs, user, instance_id) + hit = member(value_set, coding, namespace_of(kwargs), user) + return found(coding, *hit, language) if hit else not_a_member(coding, value_set, kwargs, user, language) + + +def not_a_member(coding, value_set, kwargs, user, language): + """A code outside the value set: not-in-vs, with what its code system says of it.""" + name = f"{system_of(value_set, value_set.uri)}|{value_set.version}" + outside = issue(coding, 'code', 'not-in-value-set', 'None_of_the_provided_codes_are_in_the_value_set_one', + f"The provided code '{coding.system}#{fhir_code(coding.code)}' was not found in the value set " + f"'{name}'") + try: + version = resolve('CodeSystem', coding.system, coding.version, kwargs, user) + except UnresolvedError: + return unknown_system(coding, UNKNOWN_VALUE_SET_SYSTEM, outside) + system = system_of(version, coding.system) + concept = find_concept(version, coding.code) + if not concept: + return answer(coding, [unknown_code(coding, system, version.version), outside]) + return answer(coding, [outside], version=version.version, display=display_of(concept, language)) diff --git a/core/fhir/resolve.py b/core/fhir/resolve.py index 075446c20..6a743addf 100644 --- a/core/fhir/resolve.py +++ b/core/fhir/resolve.py @@ -5,11 +5,15 @@ - Resource type: CodeSystem and ConceptMap resolve to a Source, ValueSet to a Collection. - Namespace: the base URL's. /fhir/ is global; /orgs/{org}/ and /users/{user}/ are that owner's. It comes from the route, never from the request. -- Version: the one named (the operation's version parameter, else url|version), HEAD included. With none named, the - latest released version, never HEAD. A repository with nothing released isn't visible, whatever version is named. +- Version: the one named (the operation's version parameter, else url|version, else the version an OCL path names; + a path naming one version with another named names none), HEAD included. With none named, the newest released + version the caller is shown, never HEAD. A repository with nothing released isn't visible, whatever version is + named. - Access, after resolving: at the global base public repositories only, at an owner base what the caller may view. - The version and its repository (HEAD) must both be viewable. A repository the caller can't see is not found, and - no other repository is tried. + The version and its repository (HEAD) must both be viewable. Without the URL registry, the global base's canonical + match considers only the repositories it shows, so a private copy of a canonical never hides a public one. Otherwise + (an owner base, a registry entry) a repository the caller can't see is not found, and no other repository is tried; + none of its versions is read. Every failure is the same UnresolvedError, so it never tells a caller that a repository they can't see exists. """ @@ -19,7 +23,7 @@ from core.collections.models import Collection from core.common.constants import HEAD, ACCESS_TYPE_VIEW, RETIRED_ACCESS_TYPE_EDIT from core.common.mixins import SourceContainerMixin -from core.common.utils import normalize_public_access +from core.common.utils import normalize_public_access, to_parent_uri from core.fhir.outcomes import NotFoundError from core.sources.models import Source @@ -60,21 +64,81 @@ def resolve(resource_type, url, version=None, kwargs=None, user=None, expression """ url = url or '' lookup_url, _, piped_version = url.partition('|') - error = UnresolvedError(resource_type, lookup_url, version or piped_version or None, expression) + in_path, elsewhere = path_version(lookup_url), version or piped_version or None + error = UnresolvedError(resource_type, lookup_url, elsewhere or in_path, expression) namespace = namespace_of(kwargs) - if not lookup_url or '|' in piped_version or ( - namespace and not SourceContainerMixin.get_object_from_namespace(namespace)): + # a second pipe, or a path naming one version with another named, names no version OCL holds + conflicting = '|' in piped_version or bool(in_path and elsewhere and in_path != elsewhere) + if not lookup_url or conflicting or (namespace and not SourceContainerMixin.get_object_from_namespace(namespace)): raise error - repo_type = REPO_TYPES[resource_type] - instance, _ = repo_type.resolve_reference_expression( - to_ocl_path(url), namespace, version, repo_type=repo_type, allow_head=False) + instance = shared_resolution(REPO_TYPES[resource_type], url, version, namespace, user) + if instance.id and not (elsewhere or in_path) and not is_shown(instance, namespace, user): + instance = latest_public_release(instance) or instance # the newest release is private: the newest public one - if not instance.id or not is_visible(instance, namespace, user) or not has_release(instance, namespace, user): + if not instance.id or not is_shown(instance, namespace, user): raise error return instance +def shared_resolution(repo_type, url, version, namespace, user): + """ + The shared resolver's answer, with FHIR's flags: no HEAD by default; at the global base only the HEADs it shows; + and a HEAD the caller isn't shown resolves to nothing before any version is read. The version found shares the + HEAD's check of the caller's right to the owner's private repositories, so it isn't asked twice. + """ + heads = [] + + def head_shown(head): + heads.append(head) + return is_visible(head, namespace, user) + instance, _ = repo_type.resolve_reference_expression( + to_ocl_path(url), namespace, version, repo_type=repo_type, allow_head=False, + heads=None if namespace else shown_heads(repo_type), head_shown=head_shown) + if heads and hasattr(heads[0], 'fhir_may_view_private'): + instance.fhir_may_view_private = heads[0].fhir_may_view_private + return instance + + +def path_version(url): + """The version an OCL path names (/orgs/{org}/sources/{id}/{version}/, or as a CodeSystem path), or None.""" + parts = to_parent_uri(to_ocl_path(url)).split('/') + if len(parts) > 5 and not parts[0] and parts[1] in ('orgs', 'users') and parts[3] in REPO_PATHS.values(): + return parts[5] or None + return None + + +def is_shown(instance, namespace, user): + """Whether a base shows the caller a repository version: it is visible there, and so is a release of it.""" + return is_visible(instance, namespace, user) and has_release(instance, namespace, user) + + +def shown_heads(model): + """ + Q for the HEAD rows the global base shows: public, with a live (released, not retired) public release. The + shared resolver's canonical match takes the first of them, without the URL registry. One EXISTS with the owner + matched inside it, as with_head, so the planner runs it as a semi-join on the few rows the canonical URL matches. + """ + return Q(public_access__in=PUBLIC_ACCESS) & Exists(model.objects.filter( + mnemonic=OuterRef('mnemonic'), is_active=True, released=True, retired=False, + public_access__in=PUBLIC_ACCESS).exclude(version=HEAD).filter( + Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id')))) + + +def latest_public_release(instance): + """The newest live public release of a version's repository, resolved as the version was, or None.""" + owner = {'organization_id': instance.organization_id} if instance.organization_id else {'user_id': instance.user_id} + latest = type(instance).objects.filter( + mnemonic=instance.mnemonic, is_active=True, released=True, retired=False, public_access__in=PUBLIC_ACCESS, + **owner).exclude(version=HEAD).order_by('-created_at').first() + if latest: + latest.repository_public_access = repository_access(instance) + latest.is_fqdn, latest.resolution_url = instance.is_fqdn, instance.resolution_url + if instance.is_fqdn and not latest.canonical_url: + latest.canonical_url = instance.resolution_url + return latest + + def to_ocl_path(url): """ An OCL FHIR path (/orgs/{org}/CodeSystem/{id}/...) as the OCL path of the repository it names, by its own diff --git a/core/fhir/search.py b/core/fhir/search.py index cf697229c..0c5f4dce5 100644 --- a/core/fhir/search.py +++ b/core/fhir/search.py @@ -52,7 +52,7 @@ from core.fhir import outcomes from core.fhir.base import CONTROL_PARAMETERS, fhir_base_url from core.fhir.resolve import ( - PUBLIC_ACCESS, REPO_TYPES, namespace_of, owner_access, visible_versions, viewable_rows, is_visible, has_release, + PUBLIC_ACCESS, REPO_TYPES, namespace_of, owner_access, visible_versions, viewable_rows, is_public, is_shown, repository_rows, with_repository_access) from core.mappings.models import Mapping from core.sources.models import Source @@ -564,27 +564,34 @@ def read(view): mnemonic, the version ?version= names (an OCL extension) or else the latest release the requester is shown. It must be one search lists: released, never HEAD, visible to the requester there. Raises NotFoundError. """ - resource_type, resource_id = view.resource_type, view.kwargs[view.lookup_field] - namespace, user = namespace_of(view.kwargs), view.request.user + return read_version(view.resource_type, view.kwargs[view.lookup_field], view.request.query_params.get('version'), + view.kwargs, view.request.user) + + +def read_version(resource_type, resource_id, version, kwargs, user): + """ + read's rule for a resource `resource_id` names at the base `kwargs` names, for `user`: the id at the global base, + the mnemonic and `version` (else the latest release shown) at an owner base. For the instance-level operations too. + """ + namespace = namespace_of(kwargs) versions = REPO_TYPES[resource_type].objects.filter(is_active=True, released=True).exclude(version=HEAD) latest = False if namespace is None: versions = versions.filter(id=int(resource_id)) if OPAQUE_ID.fullmatch(resource_id) else versions.none() else: - versions = view.filter_queryset_by_owner(versions.filter(mnemonic=resource_id)) - version = view.request.query_params.get('version') + owner = {'user__username': kwargs['user']} if kwargs.get('user') else {'organization__mnemonic': kwargs['org']} + versions = versions.filter(mnemonic=resource_id, **owner) latest = not version versions = versions.filter(version=version) if version else versions.filter( retired=False).order_by('-created_at') - def shown(instance): - return instance and is_visible(instance, namespace, user) and has_release(instance, namespace, user) - instance = with_repository_access(with_repository_canonical(versions)).first() - if latest and instance and not shown(instance): # the latest is private: the latest public one, if shown + if latest and instance and not is_shown(instance, namespace, user) and is_public( + instance.repository_public_access or ''): + # the latest is private, in a public repository: the latest public one instance = with_repository_access(with_repository_canonical( versions.filter(public_access__in=PUBLIC_ACCESS))).first() - if not shown(instance): + if not instance or not is_shown(instance, namespace, user): raise outcomes.NotFoundError(f'{resource_type}/{resource_id} not found') return instance diff --git a/core/fhir/tests_operation_bounds.py b/core/fhir/tests_operation_bounds.py index 7f9d32cca..b780322ea 100644 --- a/core/fhir/tests_operation_bounds.py +++ b/core/fhir/tests_operation_bounds.py @@ -22,6 +22,7 @@ from core.common.mixins import ListWithHeadersMixin from core.common.tests import OCLTestCase from core.common.views import BaseAPIView +from core.concepts.models import Concept from core.concepts.tests.factories import ConceptFactory from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM from core.mappings.models import Mapping @@ -29,6 +30,7 @@ from core.orgs.tests.factories import OrganizationFactory from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory +from core.url_registry.tests_resolution import set_registry_toggle SYSTEM = 'http://example.org/fhir/CodeSystem/bounds' VALUE_SET = 'http://example.org/fhir/ValueSet/bounds' @@ -70,6 +72,7 @@ class FhirOperationBoundsTest(OCLTestCase): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def setUp(self): + set_registry_toggle(False) # the operations resolve through core.fhir.resolve: no registry, as in production super().setUp() self.org = OrganizationFactory(mnemonic='BoundsOrg') self.source = OrganizationSourceFactory(organization=self.org, mnemonic='bounds', canonical_url=SYSTEM) @@ -91,6 +94,8 @@ def setUp(self): self.collection.add_references([CollectionReference( expression=self.concept.uri, collection=self.collection, code='c1', system=self.source.uri, version='v1')]) self.collection_v1.seed_references() + self.collection_v1.expansion.concepts.add( # as the release's expansion holds it: the code system release's c1 + *Concept.objects.filter(sources=self.source_v1, mnemonic='c1')) self.client = APIClient() def request(self, path, method, headers=None, **inputs): @@ -196,6 +201,39 @@ def test_lookup_reads_only_the_named_code(self): self.assertEqual(unbounded_reads(queries), [], (system, code)) self.assertEqual(response.status_code, 200 if system and code else 400, (system, code)) + def test_codings_and_instance_routes_read_only_the_named_code(self): + """The Coding inputs and the instance-level routes read only the code they name too; without one, a 400.""" + operations = [ + ('/fhir/CodeSystem/$lookup', None), ('/fhir/CodeSystem/$validate-code', None), + ('/fhir/ValueSet/$validate-code', VALUE_SET), + (f'/fhir/CodeSystem/{self.source_v1.id}/$lookup', None), + (f'/fhir/CodeSystem/{self.source_v1.id}/$validate-code', None), + (f'/orgs/{self.org.mnemonic}/ValueSet/boundsvs/$validate-code', None), + ] + inputs = [('coding', 'valueCoding', lambda coding: coding), + ('codeableConcept', 'valueCodeableConcept', lambda coding: {'coding': [coding]})] + for (path, url), (name, value_type, wrap), code in product(operations, inputs, (None, 'c1')): + msg = (path, name, code) + with self.subTest(msg): + coding = {'system': SYSTEM, **({'code': code} if code else {})} + parameters = [{'name': name, value_type: wrap(coding)}] + if url: + parameters.append({'name': 'url', 'valueUri': url}) + with CaptureQueriesContext(connection) as context: + response = self.client.post( + path, json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), + content_type='application/fhir+json') + + self.assertEqual(unbounded_reads(context.captured_queries), [], msg) + if code: + self.assertEqual(response.status_code, 200, msg) + else: + self.assert_required(response, name, msg) + for path in ('/fhir/CodeSystem/$lookup', '/fhir/CodeSystem/$validate-code'): + response, queries = self.request(path, 'GET', coding=f'{SYSTEM}|c1') # a GET's Coding: system|code + self.assertEqual(unbounded_reads(queries), [], path) + self.assertEqual(response.status_code, 200, path) + def test_operations_take_no_list_shortcut(self): """A search, sort, facets, search stats or CSV export never stands in for an operation's own checks.""" operations = [ diff --git a/core/fhir/tests_ops_lookup.py b/core/fhir/tests_ops_lookup.py new file mode 100644 index 000000000..70225493f --- /dev/null +++ b/core/fhir/tests_ops_lookup.py @@ -0,0 +1,449 @@ +""" +CodeSystem/$lookup (SVCM ITI-98, FHIR R4 $lookup) at the global base /fhir/ and at owner bases, GET and POST, at the +type and instance levels: the code system resolved as core.fhir.resolve does (released versions only, the newest the +caller is shown unless one is named, public only at /fhir/), with the URL registry toggle off and on; name, version +(the one resolved), display, code, system, abstract, designations and the inactive property; retired codes found and +flagged; displayLanguage; FHIRsmith's errors; a code system the caller isn't shown answers as a missing one; a fixed +number of queries. OpenConceptLab/ocl_online#451. +""" +import json + +from django.db import connection +from django.test.utils import CaptureQueriesContext +from rest_framework.test import APIClient + +from core.collections.tests.factories import ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.tests_search import source, collection +from core.orgs.tests.factories import OrganizationFactory +from core.url_registry.factories import GlobalURLRegistryFactory +from core.url_registry.tests_resolution import set_registry_toggle +from core.users.models import UserProfile +from core.users.tests.factories import UserProfileFactory + +CIEL = 'https://CIELterminology.org' +ICD10 = 'http://hl7.org/fhir/sid/icd-10' +HIV = 'http://moh.example/fhir/CodeSystem/hiv' +REST = 'http://hl7.org/fhir/restful-interaction' +SHARED = 'http://example.org/fhir/CodeSystem/shared' +NEWEST_PRIVATE = 'http://example.org/fhir/CodeSystem/newest-private' +UNKNOWN = 'http://example.org/ocl-fhir-conformance/unknown-system' +FEVERS = 'https://example.org/fhir/ValueSet/fevers' +INACTIVE = {'name': 'property', 'part': [ + {'name': 'code', 'valueCode': 'inactive'}, {'name': 'value', 'valueBoolean': False}]} + + +def name(text, locale='en', preferred=False, name_type='Fully-Specified'): + return ConceptNameFactory.build(name=text, locale=locale, locale_preferred=preferred, type=name_type) + + +class OpsCorpusMixin: + """ + CIEL with two releases: 117053 in both (names in three languages), retired 103 in the newer, 999 in the older only. + WHO's ICD-10, whose release has no canonical of its own; HL7's restful-interaction, whose 'batch' has no names. + MOH's private code system (members see it at MOH's base). A canonical held privately (lower id) and publicly; a + code system whose newest release is private. Acme's public value set 'fevers', whose release's default expansion + holds CIEL 117053 and 103, MOH's P1 and HL7's batch. `registry` is the URL registry toggle: off as in production, + or on with the curated registry's global entries (none for private repositories). + """ + registry = False + + def setUp(self): # pylint: disable=too-many-statements + set_registry_toggle(self.registry) + super().setUp() + self.admin = UserProfile.objects.get(username='ocladmin') + self.ciel_org = OrganizationFactory(mnemonic='CIEL') + self.who = OrganizationFactory(mnemonic='WHO') + self.hl7 = OrganizationFactory(mnemonic='HL7') + self.moh = OrganizationFactory(mnemonic='MOH') + self.acme = OrganizationFactory(mnemonic='Acme') + self.moh_member = UserProfileFactory(username='moh-member') + self.moh.members.add(self.moh_member) + self.outsider = UserProfileFactory(username='outsider') + self.jdoe = UserProfileFactory(username='jdoe') + + self.ciel = source(self.ciel_org, 'CIEL', canonical_url=CIEL, default_locale='en', supported_locales=['en']) + self.ciel_v1 = source(self.ciel_org, 'CIEL', 'v1', released=True, canonical_url=CIEL) + self.ciel_v2 = source(self.ciel_org, 'CIEL', 'v2', released=True, canonical_url=CIEL) + self.infectious = ConceptFactory( + parent=self.ciel, mnemonic='117053', sources=[self.ciel_v1, self.ciel_v2], names=[ + name('Infectious enteritis', preferred=True), name('Entérite infectieuse', 'fr', True), + name('enteritis infecciosa', 'es'), name('Enteritis of infectious origin', name_type='Synonym')]) + self.retired = ConceptFactory(parent=self.ciel, mnemonic='103', retired=True, sources=[self.ciel_v2], + names=[name('HIV antibody', preferred=True)]) + self.older = ConceptFactory(parent=self.ciel, mnemonic='999', sources=[self.ciel_v1], + names=[name('Older code', preferred=True)]) + + self.icd = source(self.who, 'ICD-10-WHO', canonical_url=ICD10) + self.icd_2019 = source(self.who, 'ICD-10-WHO', 'ICD102019', released=True, canonical_url=None) + ConceptFactory(parent=self.icd, mnemonic='S20.4', sources=[self.icd_2019], + names=[name('Other superficial injuries of back wall of thorax', preferred=True)]) + self.rest = source(self.hl7, 'restful-interaction', canonical_url=REST) + self.rest_v1 = source(self.hl7, 'restful-interaction', '4.0.1', released=True, canonical_url=REST) + self.batch = ConceptFactory(parent=self.rest, mnemonic='batch', sources=[self.rest_v1]) + + self.hiv = source(self.moh, 'hiv', canonical_url=HIV, public_access=ACCESS_TYPE_NONE) + self.hiv_v1 = source(self.moh, 'hiv', 'v1', released=True, canonical_url=HIV, public_access=ACCESS_TYPE_NONE) + self.private_code = ConceptFactory(parent=self.hiv, mnemonic='P1', sources=[self.hiv_v1], + names=[name('Private code', preferred=True)]) + + private_shared = source(self.jdoe, 'shared', canonical_url=SHARED, public_access=ACCESS_TYPE_NONE) + private_shared_v1 = source(self.jdoe, 'shared', 'v1', released=True, canonical_url=SHARED, + public_access=ACCESS_TYPE_NONE) + ConceptFactory(parent=private_shared, mnemonic='S1', sources=[private_shared_v1], + names=[name('Private copy', preferred=True)]) + self.public_shared = source(self.hl7, 'shared', canonical_url=SHARED) + self.public_shared_v1 = source(self.hl7, 'shared', 'v1', released=True, canonical_url=SHARED) + ConceptFactory(parent=self.public_shared, mnemonic='S1', sources=[self.public_shared_v1], + names=[name('Public copy', preferred=True)]) + + self.newest_private = source(self.hl7, 'newest', canonical_url=NEWEST_PRIVATE) + self.newest_public_v1 = source(self.hl7, 'newest', 'v1', released=True, canonical_url=NEWEST_PRIVATE) + self.newest_private_v2 = source(self.hl7, 'newest', 'v2', released=True, canonical_url=NEWEST_PRIVATE, + public_access=ACCESS_TYPE_NONE) + ConceptFactory(parent=self.newest_private, mnemonic='N1', + sources=[self.newest_public_v1, self.newest_private_v2], names=[name('N one', preferred=True)]) + + self.fevers = collection(self.acme, 'fevers', canonical_url=FEVERS) + self.fevers_v1 = collection(self.acme, 'fevers', 'v1', released=True, canonical_url=FEVERS) + expansion = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=self.fevers_v1) + expansion.concepts.add(self.infectious, self.retired, self.private_code, self.batch) + expansion.evaluated_source_versions.add(self.ciel_v2) + self.fevers_v1.expansion_uri = expansion.uri + self.fevers_v1.save() + + if self.registry: + for url, owner in ((CIEL, self.ciel_org), (ICD10, self.who), (REST, self.hl7), (SHARED, self.hl7), + (NEWEST_PRIVATE, self.hl7), (FEVERS, self.acme)): + GlobalURLRegistryFactory(url=url, namespace=owner.uri) + + @staticmethod + def client_for(user=None): + client = APIClient() + if user: + client.force_authenticate(user) + return client + + def get(self, operation, base='/fhir', user=None, resource='CodeSystem', **params): + return self.client_for(user).get(f'{base}/{resource}/{operation}', params) + + def post( # pylint: disable=too-many-arguments + self, operation, parameters, base='/fhir', user=None, resource='CodeSystem'): + return self.client_for(user).post( + f'{base}/{resource}/{operation}', json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), + content_type='application/fhir+json') + + @staticmethod + def by_name(response): + """A Parameters response's parameters: {name: parameter}, each designation in `designations`.""" + parameters = {} + for parameter in response.data['parameter']: + if parameter['name'] == 'designation': + parameters.setdefault('designations', []).append(parameter) + else: + parameters[parameter['name']] = parameter + return parameters + + def assert_outcome(self, response, status_code, issue, tx_issue_type): + self.assertEqual(response.status_code, status_code, response.data) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + self.assertEqual(response.data['issue'][0]['code'], issue) + self.assertEqual(response.data['issue'][0]['details']['coding'], + [{'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type}]) + + +class LookupCases(OpsCorpusMixin): + def lookup(self, **params): + return self.get('$lookup', **params) + + def test_answers_name_version_display_code_system_abstract_designations_and_inactive(self): + response = self.lookup(system=CIEL, code='117053') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['parameter'][:6], [ + {'name': 'name', 'valueString': 'CIEL'}, + {'name': 'version', 'valueString': 'v2'}, + {'name': 'display', 'valueString': 'Infectious enteritis'}, + {'name': 'code', 'valueCode': '117053'}, + {'name': 'system', 'valueUri': CIEL}, + {'name': 'abstract', 'valueBoolean': False}, + ]) + self.assertEqual(response.data['parameter'][-1], INACTIVE) + designations = sorted( + [{part['name']: next(value for key, value in part.items() if key.startswith('value')) + for part in designation['part']} for designation in self.by_name(response)['designations']], + key=lambda designation: designation['value']) + self.assertEqual(designations, [ + {'language': 'en', 'use': {'code': 'Synonym'}, 'value': 'Enteritis of infectious origin'}, + {'language': 'fr', 'use': {'code': 'Fully-Specified'}, 'value': 'Entérite infectieuse'}, + {'language': 'en', 'use': {'code': 'Fully-Specified'}, 'value': 'Infectious enteritis'}, + {'language': 'es', 'use': {'code': 'Fully-Specified'}, 'value': 'enteritis infecciosa'}, + ]) + + def test_answers_the_version_it_resolved(self): + """The version named, else the newest release; never the repository's latest when another was named.""" + self.assertEqual(self.by_name(self.lookup(system=CIEL, code='117053', version='v1'))['version']['valueString'], + 'v1') + older = self.lookup(system=CIEL, code='999', version='v1') + self.assertEqual(older.status_code, 200, older.data) + self.assertEqual((self.by_name(older)['version']['valueString'], self.by_name(older)['display']['valueString']), + ('v1', 'Older code')) + self.assert_outcome(self.lookup(system=CIEL, code='999'), 404, 'not-found', 'invalid-code') + self.assertEqual(self.by_name(self.lookup(system=f'{CIEL}|v1', code='999'))['version']['valueString'], 'v1') + + def test_a_release_without_a_canonical_of_its_own_resolves_by_its_repository(self): + response = self.lookup(system=ICD10, code='S20.4', version='ICD102019') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'ICD102019') + self.assertEqual(self.by_name(response)['system']['valueUri'], ICD10) + + def test_a_retired_code_is_found_and_flagged_inactive(self): + response = self.lookup(system=CIEL, code='103') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['property']['part'][1], {'name': 'value', 'valueBoolean': True}) + self.assertEqual(self.by_name(response)['display']['valueString'], 'HIV antibody') + + def test_display_language(self): + response = self.lookup(system=CIEL, code='117053', displayLanguage='fr') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Entérite infectieuse') + # a language the concept has no name in: its own display + response = self.lookup(system=CIEL, code='117053', displayLanguage='ru') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Infectious enteritis') + + def test_a_concept_without_names_is_displayed_by_its_code(self): + response = self.lookup(system=REST, code='batch') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['display'], {'name': 'display', 'valueString': 'batch'}) + self.assertNotIn('designations', self.by_name(response)) + + def test_errors(self): + self.assert_outcome(self.lookup(system=CIEL, code='nope'), 404, 'not-found', 'invalid-code') + self.assertNotIn('expression', self.lookup(system=CIEL, code='nope').data['issue'][0]) + for params in ({'system': UNKNOWN, 'code': '1'}, {'system': CIEL, 'code': '117053', 'version': 'nope'}): + response = self.lookup(**params) + self.assert_outcome(response, 422, 'not-found', 'not-found') + self.assertNotIn('expression', response.data['issue'][0]) + for params, missing in (({'system': CIEL}, 'code'), ({'code': '117053'}, 'system'), ({}, 'code')): + response = self.lookup(**params) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [missing]) + + def test_post(self): + for value_types in (('valueUri', 'valueCode', 'valueString'), ('valueString', 'valueString', 'valueString')): + response = self.post('$lookup', [ + {'name': 'system', value_types[0]: CIEL}, {'name': 'code', value_types[1]: '117053'}, + {'name': 'version', value_types[2]: 'v1'}]) + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + self.assertEqual(self.by_name(response)['code']['valueCode'], '117053') + response = self.client_for().post( + '/fhir/CodeSystem/$lookup', + '', + content_type='application/fhir+xml', HTTP_ACCEPT='application/fhir+xml') + self.assertEqual(response.status_code, 200) + self.assertIn(b'', response.content) + + def test_a_coding(self): + response = self.post('$lookup', [ + {'name': 'coding', 'valueCoding': {'system': CIEL, 'code': '999', 'version': 'v1'}}]) + self.assertEqual(self.by_name(response)['display']['valueString'], 'Older code') + response = self.post('$lookup', [{'name': 'coding', 'valueCoding': {'system': CIEL}}]) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['coding']) + + def test_the_instance_level(self): + response = self.get(f'{self.ciel_v1.id}/$lookup', code='999') + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.get(f'{self.icd_2019.id}/$lookup', code='S20.4') # a release with no canonical of its own + self.assertEqual(self.by_name(response)['system']['valueUri'], ICD10) + + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='117053') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v2') + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='999', version='v1') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Older code') + response = self.post('CIEL/$lookup/', [{'name': 'code', 'valueCode': '103'}], base='/orgs/CIEL') + self.assertEqual(self.by_name(response)['display']['valueString'], 'HIV antibody') + + self.assert_outcome(self.get('CIEL/$lookup', code='117053'), 404, 'not-found', 'not-found') + self.assert_outcome(self.get(f'{self.hiv_v1.id}/$lookup', code='P1', user=self.moh_member), + 404, 'not-found', 'not-found') + response = self.get('hiv/$lookup', base='/orgs/MOH', code='P1', user=self.moh_member) + self.assertEqual(self.by_name(response)['display']['valueString'], 'Private code') + + def test_a_code_system_the_caller_is_not_shown_answers_as_a_missing_one(self): + member = self.get('$lookup', base='/orgs/MOH', user=self.moh_member, system=HIV, code='P1') + self.assertEqual(member.status_code, 200, member.data) + self.assertEqual(self.by_name(member)['display']['valueString'], 'Private code') + + hidden = [self.get('$lookup', base=base, user=user, system=HIV, code=code) + for base, user in (('/fhir', None), ('/fhir', self.outsider), ('/fhir', self.moh_member), + ('/orgs/MOH', None), ('/orgs/MOH', self.outsider)) + for code in ('P1', 'nope')] + self.hiv.is_active = False # now it doesn't exist + self.hiv.save() + missing = self.get('$lookup', system=HIV, code='P1') + self.assertEqual(missing.status_code, 422) + for response in hidden: + self.assertEqual((response.status_code, response.content), (missing.status_code, missing.content)) + + def test_a_private_copy_of_a_canonical_does_not_hide_the_public_one(self): + response = self.lookup(system=SHARED, code='S1') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Public copy') + + def test_the_default_version_is_the_newest_release_the_caller_is_shown(self): + response = self.lookup(system=NEWEST_PRIVATE, code='N1') + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.get('$lookup', base='/orgs/HL7', user=self.admin, system=NEWEST_PRIVATE, code='N1') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v2') + + def test_a_hidden_version_in_a_path_answers_as_a_missing_one(self): + """A private version named in an OCL path is not found, exactly as one that doesn't exist (Codex F1.1).""" + path = '/orgs/HL7/CodeSystem/newest/v2/' + hidden = self.lookup(system=path, code='N1') + self.assertEqual(hidden.status_code, 422, hidden.data) + self.assertEqual(self.by_name(self.lookup(system='/orgs/HL7/CodeSystem/newest/v1/', code='N1'))['version'], + {'name': 'version', 'valueString': 'v1'}) + self.newest_private_v2.is_active = False + self.newest_private_v2.save() + self.assertEqual(self.lookup(system=path, code='N1').content, hidden.content) + + def test_a_path_naming_a_version_and_another_version_named(self): + """ + An OCL path naming one version with another named (version, url|version, HEAD, a coding's) names no version + OCL holds: a private version in the path answers byte for byte as a missing one, at /fhir/ and at an owner base + for a non-member (adversarial A.1). + """ + path = '/orgs/HL7/CodeSystem/newest/v2/' + requests = [ + ('get', {'system': path, 'version': 'v1', 'code': 'N1'}), ('get', {'system': f'{path}|v1', 'code': 'N1'}), + ('get', {'system': path, 'version': 'HEAD', 'code': 'N1'}), + ('post', [{'name': 'coding', 'valueCoding': {'system': path, 'version': 'v1', 'code': 'N1'}}])] + def ask(base, user, method, inputs): + if method == 'get': + return self.get('$lookup', base=base, user=user, **inputs) + return self.post('$lookup', inputs, base=base, user=user) + bases = [('/fhir', None), ('/orgs/HL7', None), ('/orgs/HL7', self.outsider)] + hidden = [(base, user, method, inputs, ask(base, user, method, inputs)) + for base, user in bases for method, inputs in requests] + self.newest_private_v2.is_active = False + self.newest_private_v2.save() + for base, user, method, inputs, response in hidden: + missing = ask(base, user, method, inputs) + self.assertEqual((response.status_code, response.content), (missing.status_code, missing.content), + (base, method, inputs)) + self.assertEqual(response.status_code, 422, (base, method, inputs)) + + def test_an_owner_base_costs_the_same_for_a_hidden_and_an_unknown_code_system(self): + """Anonymous, at an owner base: a private code system costs the queries an unknown one does (adversarial A.3). + For a signed-in non-member the membership check remains, and with the registry an owner-base match stops the + fall-through to the global entry: both are decision A.3's.""" + if self.registry: + self.skipTest('with the registry, an owner-base match stops the fall-through (decision A.3)') + counts = {} + for name_, operation, params, status_code in ( + ('hidden', '$lookup', {'system': HIV}, 422), + ('unknown', '$lookup', {'system': 'http://moh.example/fhir/CodeSystem/nope'}, 422), + ('hidden instance', 'hiv/$lookup', {}, 404), ('unknown instance', 'nope/$lookup', {}, 404)): + with CaptureQueriesContext(connection) as context: + response = self.get(operation, base='/orgs/MOH', code='P1', **params) + self.assertEqual(response.status_code, status_code, name_) + counts[name_] = len(context.captured_queries) + self.assertEqual(counts['hidden'], counts['unknown']) + self.assertEqual(counts['hidden instance'], counts['unknown instance']) + + def test_an_instance_and_a_coding_of_another_system(self): + """At the instance level, a coding names the code system it is from: another one's code isn't looked up.""" + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': {'system': ICD10, 'code': '117053'}}], + base='/orgs/CIEL') + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['Coding.system']) + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': {'system': CIEL, 'code': '117053'}}], + base='/orgs/CIEL') + self.assertEqual(response.status_code, 200, response.data) + + def test_an_instance_and_another_version_or_system_named(self): + """ + At the instance level, the id names the code system version: a version (the version parameter, or a coding's) + or a system parameter naming another one is a 400, never answered from the instance (Codex F2.1, F2.2). + """ + global_ciel_v1 = f'{self.ciel_v1.id}/$lookup' + for base, operation, params, expression in ( + ('/fhir', global_ciel_v1, {'code': '999', 'version': 'v2'}, 'version'), + ('/fhir', global_ciel_v1, {'code': '999', 'system': ICD10}, 'system'), + ('/orgs/CIEL', 'CIEL/$lookup', {'code': '117053', 'system': ICD10}, 'system')): + response = self.get(operation, base=base, **params) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression], (base, params)) + for base, operation, extra, expression in ( + ('/fhir', global_ciel_v1, [], 'Coding.version'), + ('/orgs/CIEL', 'CIEL/$lookup', [{'name': 'version', 'valueString': 'v1'}], 'Coding.version'), + ('/orgs/CIEL', 'CIEL/$lookup', [{'name': 'system', 'valueUri': ICD10}], 'system')): + coding = {'system': CIEL, 'code': '999', 'version': 'v2' if expression.endswith('version') else 'v1'} + response = self.post(operation, [{'name': 'coding', 'valueCoding': coding}, *extra], base=base) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression], (base, extra)) + # the instance's own version, named any of these ways, is answered + response = self.get(global_ciel_v1, code='999', version='v1', system=f'{CIEL}|v1') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': { + 'system': CIEL, 'code': '999', 'version': 'v1'}}], base='/orgs/CIEL') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + + def test_an_owner_instance_serves_head_when_named(self): + """ + At an owner base, CodeSystem/{id}/$lookup serves HEAD when version=HEAD names it, as the type level does + (design D6), with HEAD's visibility rule; a global id names a release, so HEAD there is a conflict (Codex F3.1). + """ + ConceptFactory(parent=self.ciel, mnemonic='H1', names=[name('In HEAD only', preferred=True)]) + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='H1', version='HEAD') + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual((answer['version']['valueString'], answer['display']['valueString']), ('HEAD', 'In HEAD only')) + self.assert_outcome(self.get('CIEL/$lookup', base='/orgs/CIEL', code='H1'), 404, 'not-found', 'invalid-code') + response = self.get('hiv/$lookup', base='/orgs/MOH', user=self.moh_member, code='P1', version='HEAD') + self.assertEqual(self.by_name(response)['version']['valueString'], 'HEAD') + hidden = self.get('hiv/$lookup', base='/orgs/MOH', user=self.outsider, code='P1', version='HEAD') + self.assertEqual(hidden.status_code, 404) + self.hiv.is_active = False + self.hiv.save() + self.assertEqual(self.get('hiv/$lookup', base='/orgs/MOH', code='P1', version='HEAD').content, hidden.content) + response = self.get(f'{self.ciel_v1.id}/$lookup', code='999', version='HEAD') + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['version']) + + def test_a_deleted_concept_version_is_not_found(self): + """A concept version deleted (is_active false) is gone, a retired one stays (Codex F1.4).""" + ConceptFactory(parent=self.ciel, mnemonic='555', sources=[self.ciel_v2], is_active=False, + names=[name('Deleted', preferred=True)]) + self.assert_outcome(self.lookup(system=CIEL, code='555'), 404, 'not-found', 'invalid-code') + + def test_a_fixed_number_of_queries(self): + """Resolution (the toggle, the repository, its release; the registry adds its entry and namespace owner), the + concept and its names, however many concepts the code system holds.""" + count = 7 if self.registry else 5 + with self.assertNumQueries(count): + self.lookup(system=CIEL, code='117053') + for number in range(30): + ConceptFactory(parent=self.ciel, mnemonic=f'x{number}', sources=[self.ciel_v2], names=[name(f'x {number}')]) + with self.assertNumQueries(count): + self.lookup(system=CIEL, code='117053') + with self.assertNumQueries(count - 1): # no names to read + self.lookup(system=CIEL, code='nope') + + +class LookupWithoutRegistryTest(LookupCases, OCLTestCase): + registry = False + + +class LookupWithRegistryTest(LookupCases, OCLTestCase): + registry = True diff --git a/core/fhir/tests_ops_validate_code.py b/core/fhir/tests_ops_validate_code.py new file mode 100644 index 000000000..b7d3860e0 --- /dev/null +++ b/core/fhir/tests_ops_validate_code.py @@ -0,0 +1,474 @@ +""" +CodeSystem/$validate-code and ValueSet/$validate-code (SVCM ITI-99, FHIR R4 $validate-code) at the global base /fhir/ +and at owner bases, GET and POST, at the type and instance levels, with the URL registry toggle off and on: the +answer's code, system, version used and display; FHIRsmith's issues and message ids (unknown system or version, unknown +code, wrong display, inactive, not in the value set); value set membership from the resolved release's default +expansion; a value set that doesn't resolve is a 422 OperationOutcome; whatever the caller isn't shown answers as +missing; a fixed number of queries. OpenConceptLab/ocl_online#451. +""" +from django.db import connection +from django.test.utils import CaptureQueriesContext + +from core.collections.models import Collection +from core.collections.tests.factories import ExpansionFactory +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory +from core.fhir.outcomes import MESSAGE_ID_EXTENSION +from core.fhir.tests_ops_lookup import ( + OpsCorpusMixin, name, CIEL, ICD10, HIV, REST, UNKNOWN, FEVERS, SHARED) +from core.fhir.tests_search import source, collection +from core.orgs.tests.factories import OrganizationFactory +from core.url_registry.factories import GlobalURLRegistryFactory + +DRAFT_ONLY = 'http://example.org/fhir/CodeSystem/head-only' +GONE = 'http://example.org/fhir/CodeSystem/gone' +DRAFT_UNRELEASED = 'http://example.org/fhir/CodeSystem/unreleased' + + +def issues_of(parameters): + """The `issues` OperationOutcome's issues as severity|code|tx-issue-type|expression|message id.""" + outcome = parameters.get('issues', {}).get('resource', {'issue': []}) + return sorted( + '|'.join([item['severity'], item['code'], item['details']['coding'][0]['code'], + ','.join(item.get('expression', [])), + ','.join(e['valueString'] for e in item.get('extension', []) if e['url'] == MESSAGE_ID_EXTENSION)]) + for item in outcome['issue']) + + +class ValidateCodeCases(OpsCorpusMixin): + def validate(self, resource='CodeSystem', base='/fhir', user=None, **params): + return self.get('$validate-code', base=base, user=user, resource=resource, **params) + + def answer(self, response): + """{name: value} of a Parameters answer, the issues as issues_of has them.""" + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['parameter'][0]['name'], 'result') + parameters = self.by_name(response) + values = {key: next(value for name_, value in parameter.items() if name_ not in ('name',)) + for key, parameter in parameters.items() if key != 'issues'} + if 'issues' in parameters: + values['issues'] = issues_of(parameters) + self.assertIn('message', values) + return values + + # CodeSystem + + def test_a_valid_code_answers_its_code_system_version_and_display(self): + self.assertEqual(self.answer(self.validate(url=CIEL, code='117053')), { + 'result': True, 'system': CIEL, 'code': '117053', 'version': 'v2', 'display': 'Infectious enteritis'}) + self.assertEqual(self.answer(self.validate(url=CIEL, code='999', version='v1'))['version'], 'v1') + self.assertEqual(self.answer(self.validate(url=ICD10, code='S20.4', version='ICD102019'))['result'], True) + + def test_a_display_is_one_of_the_concepts_names_in_any_case(self): + for display in ('Infectious enteritis', 'INFECTIOUS ENTERITIS', 'Entérite infectieuse', + 'Enteritis of infectious origin'): + self.assertTrue(self.answer(self.validate(url=CIEL, code='117053', display=display))['result'], display) + self.assertEqual(self.answer(self.validate(url=CIEL, code='117053', display='Not it')), { + 'result': False, 'system': CIEL, 'code': '117053', 'version': 'v2', 'display': 'Infectious enteritis', + 'message': self.answer(self.validate(url=CIEL, code='117053', display='Not it'))['message'], + 'issues': ['error|invalid|invalid-display|display|Display_Name_for__should_be_one_of__instead_of']}) + # a concept without names has none to check a display against + self.assertEqual(self.answer(self.validate(url=REST, code='batch', display='Anything')), { + 'result': True, 'system': REST, 'code': 'batch', 'version': '4.0.1'}) + + def test_a_retired_code_is_valid_and_inactive_with_a_warning(self): + answer = self.answer(self.validate(url=CIEL, code='103')) + self.assertEqual((answer['result'], answer['inactive'], answer['issues']), + (True, True, ['warning|business-rule|code-comment|code|INACTIVE_CONCEPT_FOUND'])) + + def test_an_unknown_code(self): + answer = self.answer(self.validate(url=CIEL, code='nope')) + self.assertEqual((answer['result'], answer['version'], answer['issues']), + (False, 'v2', ['error|code-invalid|invalid-code|code|Unknown_Code_in_Version'])) + + def test_an_unknown_system_or_version_with_or_without_a_code(self): + self.assertEqual(self.answer(self.validate(url=UNKNOWN, code='1')), { + 'result': False, 'system': UNKNOWN, 'code': '1', 'x-caused-by-unknown-system': UNKNOWN, + 'message': f"A definition for CodeSystem '{UNKNOWN}' could not be found, so the code cannot be validated", + 'issues': ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM']}) + answer = self.answer(self.validate(url=CIEL, code='117053', version='nope')) + self.assertEqual((answer['version'], answer['x-caused-by-unknown-system'], answer['issues']), + ('nope', f'{CIEL}|nope', ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM_VERSION'])) + self.assertNotIn('code', self.answer(self.validate(url=UNKNOWN))) + answer = self.answer(self.validate(url=f'{CIEL}|nope', code='117053')) + self.assertEqual((answer['x-caused-by-unknown-system'], answer['issues']), + (f'{CIEL}|nope', ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM_VERSION'])) + + def test_a_code_system_the_caller_is_not_shown_answers_as_a_missing_one(self): + member = self.validate(base='/orgs/MOH', user=self.moh_member, url=HIV, code='P1') + self.assertTrue(self.answer(member)['result']) + hidden = [self.validate(base=base, user=user, url=HIV, **code) + for base, user in (('/fhir', None), ('/fhir', self.moh_member), ('/orgs/MOH', None), + ('/orgs/MOH', self.outsider)) + for code in ({'code': 'P1'}, {'code': 'nope'}, {})] + self.hiv.is_active = False + self.hiv.save() + missing = {code: self.validate(url=HIV, **({'code': code} if code else {})).content + for code in ('P1', 'nope', None)} + for response, code in zip(hidden, ['P1', 'nope', None] * 4): + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, missing[code]) + + def test_a_private_copy_of_a_canonical_does_not_hide_the_public_one(self): + self.assertEqual(self.answer(self.validate(url=SHARED, code='S1'))['display'], 'Public copy') + + def test_post_and_a_coding(self): + response = self.post('$validate-code', [ + {'name': 'url', 'valueUri': CIEL}, {'name': 'code', 'valueString': '117053'}, + {'name': 'display', 'valueString': 'Infectious enteritis'}]) + self.assertTrue(self.answer(response)['result']) + response = self.post('$validate-code', [{'name': 'coding', 'valueCoding': { + 'system': CIEL, 'code': '117053', 'version': 'v1', 'display': 'Wrong'}}]) + answer = self.answer(response) + self.assertEqual((answer['version'], answer['issues']), ('v1', [ + 'error|invalid|invalid-display|Coding.display|Display_Name_for__should_be_one_of__instead_of'])) + response = self.validate(coding=f'{CIEL}|103') + self.assertTrue(self.answer(response)['inactive']) + + def test_a_codeable_concept_and_inputs_that_are_not_text(self): + response = self.post('$validate-code', [{'name': 'codeableConcept', 'valueCodeableConcept': { + 'coding': [{'system': CIEL, 'code': '117053', 'display': 'Infectious enteritis'}]}}]) + self.assertTrue(self.answer(response)['result']) + response = self.post('$validate-code', [{'name': 'codeableConcept', 'valueCodeableConcept': { + 'coding': [{'system': CIEL, 'code': '117053'}, {'system': ICD10, 'code': 'S20.4'}]}}]) + self.assert_outcome(response, 400, 'not-supported', 'not-supported') + self.assertEqual(response.data['issue'][0]['expression'], ['codeableConcept']) + for parameter, expression in (({'name': 'url', 'valueBoolean': True}, 'url'), + ({'name': 'coding', 'valueCoding': {'system': CIEL, 'code': 7}}, 'coding')): + response = self.post('$validate-code', [parameter, {'name': 'code', 'valueCode': '117053'}]) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression]) + + def test_a_code_stored_percent_encoded(self): + """OCL stores a code with reserved characters percent-encoded (ICD-11's ND53.Y&XK70); FHIR names it as it is.""" + ConceptFactory(parent=self.ciel, mnemonic='ND53.Y%26XK70', sources=[self.ciel_v2], + names=[name('Encoded', preferred=True)]) + self.assertEqual(self.answer(self.validate(url=CIEL, code='ND53.Y&XK70')), { + 'result': True, 'system': CIEL, 'code': 'ND53.Y&XK70', 'version': 'v2', 'display': 'Encoded'}) + + def test_the_instance_level(self): + response = self.get(f'{self.ciel_v1.id}/$validate-code', code='999') + self.assertEqual(self.answer(response)['version'], 'v1') + response = self.get('CIEL/$validate-code', base='/orgs/CIEL', code='117053', display='Nope') + self.assertFalse(self.answer(response)['result']) + + def test_xml(self): + response = self.client_for().get( + f'/fhir/CodeSystem/$validate-code?url={UNKNOWN}&code=1&_format=xml') + self.assertEqual(response.status_code, 200) + self.assertIn(b'', response.content) + self.assertIn(b'