diff --git a/.env.example b/.env.example index d0e096b39..f726c2c8a 100644 --- a/.env.example +++ b/.env.example @@ -82,7 +82,6 @@ FLOWER_PASSWORD=Root123 # ----------------------------------------------------------------------------- # FHIR Validator # ----------------------------------------------------------------------------- -FHIR_VALIDATOR_URL=http://fhir_validator:3500 # ----------------------------------------------------------------------------- # Error tracking (optional) diff --git a/.pylintrc b/.pylintrc index cc0e2c9ff..de482fed9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,6 +1,7 @@ [MESSAGES CONTROL] disable=missing-docstring,too-few-public-methods,no-member,too-many-ancestors,attribute-defined-outside-init,fixme,broad-except,import-outside-toplevel,too-many-instance-attributes,unsupported-membership-test,too-many-public-methods,too-many-lines,unused-private-member,consider-using-with,duplicate-code,unspecified-encoding,invalid-name,cyclic-import,missing-timeout,too-many-positional-arguments [MASTER] +extension-pkg-allow-list=lxml ignore=migrations,fixtures,scripts,commands,documents.py,password_validation.py,v1_dump [FORMAT] max-line-length=120 diff --git a/core/bundles/serializers.py b/core/bundles/serializers.py index 8fbd307cd..be9a7ee24 100644 --- a/core/bundles/serializers.py +++ b/core/bundles/serializers.py @@ -21,7 +21,7 @@ class BundleLinkSerializer(CommonSerializer): class BundleMetaSerializer(CommonSerializer): - lastUpdated = DateTimeField(read_only=True, default=datetime.now()) + lastUpdated = DateTimeField(read_only=True, default=datetime.now) class BundleExtensionSerializer(CommonSerializer): @@ -33,7 +33,9 @@ class BundleExtensionSerializer(CommonSerializer): class BundleEntrySerializer(CommonSerializer): + fullUrl = CharField(required=False) resource = JSONField() + search = JSONField(required=False) class FHIRBundleSerializer(CommonSerializer): @@ -53,17 +55,29 @@ def get_resource_type(cls, _obj): return 'Bundle' @classmethod - def convert_to_entry(cls, resources): + def convert_to_entry(cls, resources, base_url): + """ + Searchset entries: each resource a match with its fullUrl (base_url is the base and resource type), and a + resource that couldn't be represented (an OperationOutcome) an outcome. + """ entry = [] for resource in resources: - entry.append({'resource': resource}) + if resource.get('resourceType') == 'OperationOutcome': + entry.append({'resource': resource, 'search': {'mode': 'outcome'}}) + else: + entry.append( + {'fullUrl': f"{base_url}/{resource['id']}", 'resource': resource, 'search': {'mode': 'match'}}) return entry def get_link(self, _): + """self, previous and next, for the page the paginator holds; only self without one (_count=0)""" paginator = self.context.get('paginator') + if paginator is None: + return BundleLinkSerializer( + [{'relation': 'self', 'url': self.context['request'].build_absolute_uri()}], many=True).data links = [{'relation': 'self', 'url': paginator.get_current_page_url()}] if paginator.has_previous(): - links.append({'relation': 'prev', 'url': paginator.get_previous_page_url()}) + links.append({'relation': 'previous', 'url': paginator.get_previous_page_url()}) if paginator.has_next(): links.append({'relation': 'next', 'url': paginator.get_next_page_url()}) return BundleLinkSerializer(links, many=True).data diff --git a/core/code_systems/converter.py b/core/code_systems/converter.py index c3ca2b2e0..d9cbe2872 100644 --- a/core/code_systems/converter.py +++ b/core/code_systems/converter.py @@ -1,4 +1,4 @@ -from fhir.resources.codesystem import CodeSystem +from fhir.resources.R4B.codesystem import CodeSystem # R4B models are the R4 wire models from core.settings import DEFAULT_LOCALE from core.sources.models import Source @@ -32,7 +32,7 @@ def to_fhir(source): 'language': source.default_locale, 'count': source.active_concepts, } - return CodeSystem(**code_system).dict() + return CodeSystem(**code_system).model_dump() @staticmethod def from_fhir(code_system): diff --git a/core/code_systems/serializers.py b/core/code_systems/serializers.py index 635d15261..c9164adea 100644 --- a/core/code_systems/serializers.py +++ b/core/code_systems/serializers.py @@ -13,23 +13,14 @@ from core.common.serializers import ReadSerializerMixin, StatusField, IdentifierSerializer from core.concepts.models import Concept, ConceptName from core.concepts.serializers import ConceptDetailSerializer -from core.parameters.serializers import ParametersSerializer +from core.fhir import search as fhir_search +from core.fhir.search import CanonicalUrlField, FhirRepositorySerializerMixin, TitleField from core.sources.models import Source from core.sources.serializers import SourceCreateOrUpdateSerializer logger = logging.getLogger('oclapi') -class ValidateCodeParametersSerializer(ParametersSerializer): - allowed_input_parameters = { - 'url': 'valueUri', - 'code': 'valueCode', - 'display': 'valueString', - 'version': 'valueString', - 'systemVersion': 'valueString', - 'system': 'valueUri' - } - class CodeSystemConceptDesignationUseSerializer(serializers.Field): def to_internal_value(self, data): if 'code' in data: @@ -89,10 +80,10 @@ def to_internal_value(self, data): return {'descriptions': [{'description': data, 'locale': settings.DEFAULT_LOCALE, 'locale_preferred': True}]} def to_representation(self, value): - descriptions = value.descriptions_for_default_locale - if descriptions and len(descriptions) > 0: - return value.descriptions_for_default_locale[0] - return None + """The first description in the default locale, from the descriptions a read prefetches.""" + descriptions = [description.name for description in value.descriptions.all() + if description.locale == settings.DEFAULT_LOCALE and not description.retired] + return descriptions[0] if descriptions else None class CodeSystemConceptSerializer(ReadSerializerMixin, serializers.Serializer): @@ -149,13 +140,9 @@ def to_internal_value(self, data): return {'concepts': concepts.validated_data} def to_representation(self, value): - # limit to 1000 concepts by default - # TODO: support graphQL to go around the limit - if self.context.get('has_many', False): - limit = 25 - else: - limit = 1000 - return CodeSystemConceptSerializer(value.concepts.order_by('id')[:limit], many=True).data + """Every concept, retired ones included, up to the limit; none above it (content not-present).""" + concepts = fhir_search.code_system_concepts(value)[1] + return CodeSystemConceptSerializer(concepts, many=True).data if concepts else [] class TextField(ReadSerializerMixin, serializers.Serializer): @@ -178,11 +165,12 @@ def to_representation(self, instance): return super().to_representation(obj) -class CodeSystemDetailSerializer(serializers.ModelSerializer): +class CodeSystemDetailSerializer(FhirRepositorySerializerMixin, serializers.ModelSerializer): + content_field = 'concept' resourceType = SerializerMethodField(method_name='get_resource_type') id = CharField(source='mnemonic') - url = CharField(source='canonical_url', required=False) - title = CharField(source='full_name', required=False) + url = CanonicalUrlField(source='canonical_url', required=False) + title = TitleField(source='full_name', required=False) status = StatusField(source='*') language = CharField(source='default_locale', required=False) count = SerializerMethodField(method_name='get_total_count') @@ -210,21 +198,20 @@ class Meta: 'copyright', 'revisionDate', 'experimental', 'caseSensitive', 'compositional', 'versionNeeded', 'collectionReference', 'hierarchyMeaning', 'concept', 'text', 'filters') - def __new__(cls, *args, **kwargs): - if kwargs.get('many', False): - context = kwargs.get('context', {}) - context.update({'has_many': True}) - kwargs.update({'context': context}) - - return super().__new__(cls, *args, **kwargs) - @staticmethod def get_resource_type(_): return RESOURCE_TYPE + def get_total_count(self, obj): + """A read's concept count; a search's summary has none (it only checks the limit).""" + return None if self.context.get('has_many') else fhir_search.code_system_concepts(obj)[0] + @staticmethod - def get_total_count(obj): - return obj.concepts.count() + def get_content(obj): + """complete (or the content mode the CodeSystem was saved with) up to the concept limit, else not-present""" + if fhir_search.concepts_over_limit(obj): + return 'not-present' + return obj.content_type if obj.content_type in ('example', 'fragment', 'supplement') else 'complete' @staticmethod def get_property(obj): @@ -264,11 +251,11 @@ def get_meta(obj): def to_representation(self, instance): try: - if not instance.content_type: - instance.content_type = 'complete' rep = super().to_representation(instance) + rep['content'] = self.get_content(instance) delete_empty_fields(rep) IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep) + self.with_fhir_view(instance, rep) except (Exception, ): msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}' logger.exception(msg) diff --git a/core/code_systems/tests.py b/core/code_systems/tests.py index ff5384f11..517ec5d39 100644 --- a/core/code_systems/tests.py +++ b/core/code_systems/tests.py @@ -1,12 +1,11 @@ import json -from fhir.resources.codesystem import CodeSystem +from fhir.resources.R4B.codesystem import CodeSystem from mock.mock import patch, Mock from rest_framework.test import APIClient from core.code_systems.converter import CodeSystemConverter from core.code_systems.serializers import CodeSystemDetailSerializer -from core.code_systems.views import CodeSystemLookupNotFoundError from core.common.constants import ACCESS_TYPE_NONE from core.common.tests import OCLTestCase from core.concepts.models import Concept @@ -14,17 +13,23 @@ from core.orgs.tests.factories import OrganizationFactory from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory +from core.url_registry.tests_resolution import set_registry_toggle from core.users.tests.factories import UserProfileFactory +SYSTEM = 'http://some/url' +PRIVATE = 'http://private.example/url' +SHARED = 'http://shared.example/url' + class CodeSystemTest(OCLTestCase): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def setUp(self): + set_registry_toggle(False) # the operations resolve through core.fhir.resolve: no registry, as in production super().setUp() self.org = OrganizationFactory() - self.org_source = OrganizationSourceFactory(organization=self.org, canonical_url='/some/url', + self.org_source = OrganizationSourceFactory(organization=self.org, canonical_url=SYSTEM, full_name='source name') self.org_source_v1 = OrganizationSourceFactory.build( version='v1', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) @@ -32,20 +37,20 @@ def setUp(self): self.concept_1 = ConceptFactory(parent=self.org_source, names=1, names__name="concept_1_name") self.concept_2 = ConceptFactory(parent=self.org_source) self.org_source_v2 = OrganizationSourceFactory.build( - version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) + version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent, released=True) Source.persist_new_version(self.org_source_v2, self.org_source.created_by) self.user = UserProfileFactory() self.user_token = self.user.get_token() - self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url='/some/url') + self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url=SYSTEM) self.user_source_v1 = UserSourceFactory.build( - version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent) + version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent, released=True) Source.persist_new_version(self.user_source_v1, self.user_source.created_by) self.client = APIClient() def test_public_can_view(self): - response = self.client.get('/fhir/CodeSystem/?url=/some/url') + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}') self.assertEqual(len(response.data['entry']), 1) self.assertEqual(response.data['total'], 1) @@ -56,28 +61,36 @@ def test_public_can_view(self): '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v2') + self.assertNotIn('concept', resource) # a search entry is a summary; the read has the concepts + + resource = self.client.get(response.data['entry'][0]['fullUrl']).data self.assertEqual(len(resource['concept']), 2) self.assertEqual(resource['concept'][0]['code'], self.concept_1.mnemonic) self.assertEqual(resource['concept'][1]['code'], self.concept_2.mnemonic) def test_private_can_view(self): - response = self.client.get('/fhir/CodeSystem/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + # the global base shows public repositories only, whoever asks; the owner base shows the owner its own + response = self.client.get(f'/fhir/CodeSystem/?url={SYSTEM}', HTTP_AUTHORIZATION='Token ' + self.user_token) + + self.assertEqual(response.data['total'], 1) + resource = response.data['entry'][0]['resource'] + self.assertEqual( + resource['identifier'][0]['value'], + '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' + ) + self.assertEqual(resource['version'], 'v2') - self.assertEqual(len(response.data['entry']), 2) - self.assertEqual(response.data['total'], 2) + response = self.client.get( + f'/users/{self.user.mnemonic}/CodeSystem/?url={SYSTEM}', HTTP_AUTHORIZATION='Token ' + self.user_token) + self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v1') - resource_2 = response.data['entry'][1]['resource'] - self.assertEqual( - resource_2['identifier'][0]['value'], - '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' - ) - self.assertEqual(resource_2['version'], 'v2') + self.assertEqual(self.client.get(f'/users/{self.user.mnemonic}/CodeSystem/?url={SYSTEM}').data['total'], 0) def test_public_can_list(self): response = self.client.get('/fhir/CodeSystem/') @@ -132,19 +145,24 @@ def test_find_by_non_supported_param(self): def test_private_can_list(self): response = self.client.get('/fhir/CodeSystem/', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 2) + self.assertEqual(len(response.data['entry']), 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], - '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' + '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' ) - self.assertEqual(resource['version'], 'v1') - resource_2 = response.data['entry'][1]['resource'] + self.assertEqual(resource['version'], 'v2') + + response = self.client.get( + f'/users/{self.user.mnemonic}/CodeSystem/', HTTP_AUTHORIZATION='Token ' + self.user_token) + + self.assertEqual(len(response.data['entry']), 1) + resource = response.data['entry'][0]['resource'] self.assertEqual( - resource_2['identifier'][0]['value'], - '/orgs/' + self.org.mnemonic + '/CodeSystem/' + self.org_source.mnemonic + '/' + resource['identifier'][0]['value'], + '/users/' + self.user.mnemonic + '/CodeSystem/' + self.user_source.mnemonic + '/' ) - self.assertEqual(resource_2['version'], 'v2') + self.assertEqual(resource['version'], 'v1') def test_get_code_system(self): response = self.client.get(f'/orgs/{self.org.mnemonic}/CodeSystem/{self.org_source.mnemonic}/') @@ -153,13 +171,29 @@ def test_get_code_system(self): self.assertEqual(response.data['id'], self.org_source.mnemonic) self.assertEqual(response.data['version'], 'v2') + def valid(self, concept): + """$validate-code's answer for a valid code of the org source, at the version resolved (v2).""" + return {'resourceType': 'Parameters', 'parameter': [ + {'name': 'result', 'valueBoolean': True}, + {'name': 'display', 'valueString': concept.display_name}, + {'name': 'code', 'valueCode': concept.mnemonic}, + {'name': 'system', 'valueUri': SYSTEM}, + {'name': 'version', 'valueString': 'v2'}]} + + def assert_issues(self, response, *issues): + """The answer is result false, with these issues (code, tx-issue-type, expression).""" + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['parameter'][0], {'name': 'result', 'valueBoolean': False}) + outcome = next(item['resource'] for item in response.data['parameter'] if item['name'] == 'issues') + self.assertEqual([(item['code'], item['details']['coding'][0]['code'], item['expression']) + for item in outcome['issue']], list(issues)) + def test_validate_code_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url={self.org_source.canonical_url}&code={self.concept_1.mnemonic}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_for_code_system_via_post(self): response = self.client.post( @@ -175,28 +209,19 @@ def test_validate_code_for_code_system_via_post(self): ) self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_for_code_system_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url={self.org_source.canonical_url}&code=non_existing_code') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('code-invalid', 'invalid-code', ['code'])) def test_validate_code_for_code_system_with_invalid_url_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' f'?url=non_existing_url&code=${self.concept_1.mnemonic}') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('not-found', 'not-found', ['system'])) def test_validate_code_with_display_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' @@ -205,8 +230,7 @@ def test_validate_code_with_display_for_code_system(self): f'&display={self.concept_1.display_name}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [{'name': 'result', 'valueBoolean': True}]})) + self.assertJSONEqual(json.dumps(response.data), json.dumps(self.valid(self.concept_1))) def test_validate_code_with_display_for_code_system_negative(self): response = self.client.get(f'/fhir/CodeSystem/$validate-code/' @@ -214,11 +238,7 @@ def test_validate_code_with_display_for_code_system_negative(self): f'&code={self.concept_1.mnemonic}' f'&display=wrong_display') - self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ]})) + self.assert_issues(response, ('invalid', 'invalid-display', ['display'])) def test_lookup_for_code_system(self): response = self.client.get(f'/fhir/CodeSystem/$lookup/' @@ -228,97 +248,115 @@ def test_lookup_for_code_system(self): self.assertEqual(response.status_code, 200) self.assertJSONEqual(json.dumps(response.data), json.dumps( {'resourceType': 'Parameters', 'parameter': [ - {'name': 'name', 'valueString': self.org_source.mnemonic}, + {'name': 'name', 'valueString': Source.objects.get(id=self.org_source_v2.id).name}, {'name': 'version', 'valueString': self.org_source_v2.version}, - {'name': 'display', 'valueString': self.concept_1.display_name}]})) + {'name': 'display', 'valueString': self.concept_1.display_name}, + {'name': 'code', 'valueCode': self.concept_1.mnemonic}, + {'name': 'system', 'valueUri': SYSTEM}, + {'name': 'abstract', 'valueBoolean': False}, + {'name': 'designation', 'part': [ + {'name': 'language', 'valueCode': 'en'}, + {'name': 'use', 'valueCoding': {'code': 'FULLY_SPECIFIED'}}, + {'name': 'value', 'valueString': 'concept_1_name'}]}, + {'name': 'property', 'part': [ + {'name': 'code', 'valueCode': 'inactive'}, {'name': 'value', 'valueBoolean': False}]}]})) def test_lookup_for_empty_code_system(self): response = self.client.get('/fhir/CodeSystem/$lookup/?system=&code=') self.assertEqual(response.status_code, 400) - self.assertJSONEqual(json.dumps(response.data), json.dumps(CodeSystemLookupNotFoundError().detail)) + self.assertEqual(response.data['issue'][0]['expression'], ['code']) def test_lookup_for_code_system_with_new_canonical(self): - self.org_source.canonical_url = '/new/url' + """A code system is found by its repository's current canonical URL, not by one a version recorded.""" + self.org_source.canonical_url = 'http://new/url' Source.save(self.org_source) response = self.client.get(f'/fhir/CodeSystem/$lookup/' - f'?system={self.org_source.canonical_url}' + f'?system={self.org_source_v1.canonical_url}' f'&code={self.concept_1.mnemonic}') - self.assertEqual(response.status_code, 400) - self.assertJSONEqual(json.dumps(response.data), json.dumps(CodeSystemLookupNotFoundError().detail)) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.data['issue'][0]['code'], 'not-found') - response = self.client.get(f'/fhir/CodeSystem/$lookup/' - f'?system={self.org_source_v1.canonical_url}' - f'&code={self.concept_1.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$lookup/?system=http://new/url&code={self.concept_1.mnemonic}') self.assertEqual(response.status_code, 200) - self.assertJSONEqual(json.dumps(response.data), json.dumps( - {'resourceType': 'Parameters', 'parameter': [ - {'name': 'name', 'valueString': self.org_source.mnemonic}, - {'name': 'version', 'valueString': self.org_source_v2.version}, - {'name': 'display', 'valueString': self.concept_1.display_name}]})) + self.assertEqual(response.data['parameter'][1], {'name': 'version', 'valueString': self.org_source_v2.version}) + self.assertEqual(response.data['parameter'][2], {'name': 'display', 'valueString': self.concept_1.display_name}) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def test_private_code_system_answers_as_missing_for_non_viewers(self): # pylint: disable=too-many-locals + """ + A code system the requester can't view answers exactly as one that doesn't exist: at the global base, which + shows public code systems only, and at its owner's base to anyone but the owner's members and staff. + """ private_org = OrganizationFactory() private_source = OrganizationSourceFactory( - organization=private_org, canonical_url='/private/url', public_access=ACCESS_TYPE_NONE) + organization=private_org, canonical_url=PRIVATE, public_access=ACCESS_TYPE_NONE) concept = ConceptFactory(parent=private_source, names=1, names__name='private_concept_name') Source.persist_new_version(OrganizationSourceFactory.build( version='v1', mnemonic=private_source.mnemonic, organization=private_org, - public_access=ACCESS_TYPE_NONE), private_source.created_by) + public_access=ACCESS_TYPE_NONE, released=True), private_source.created_by) member = UserProfileFactory() private_org.members.add(member) outsider = {'HTTP_AUTHORIZATION': 'Token ' + self.user_token} - validate_url = f'/fhir/CodeSystem/$validate-code/?url=/private/url&code={concept.mnemonic}' - lookup_url = '/fhir/CodeSystem/$lookup/?system={}&code={}' - not_found = self.client.get(lookup_url.format('/unknown/url', concept.mnemonic)) - self.assertEqual(not_found.status_code, 400) + member_headers = {'HTTP_AUTHORIZATION': 'Token ' + member.get_token()} + owner_base = f'/orgs/{private_org.mnemonic}' + validate_url = '{}/CodeSystem/$validate-code/?url={}&code={}' + lookup_url = '{}/CodeSystem/$lookup/?system={}&code={}' public_concept = ConceptFactory() selector = (f'&org={public_concept.parent.organization.mnemonic}&source={public_concept.parent.mnemonic}' f'&concept={public_concept.mnemonic}') - for headers in [{}, outsider]: + hidden = [] + for base, headers in [('/fhir', {}), ('/fhir', outsider), ('/fhir', member_headers), (owner_base, {}), + (owner_base, outsider)]: # without a code, a code system the requester can't view still answers as one that doesn't exist - for system_url in ['/private/url', '/unknown/url']: - response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url={system_url}{selector}', **headers) - self.assertFalse(response.data['parameter'][0]['valueBoolean'], (system_url, headers)) - for url in [validate_url, f'{validate_url}&display={concept.display_name}']: - response = self.client.get(url, **headers) - self.assertEqual(response.status_code, 200, (url, headers)) - self.assertFalse(response.data['parameter'][0]['valueBoolean'], (url, headers)) + response = self.client.get(f'{base}/CodeSystem/$validate-code/?url={PRIVATE}{selector}', **headers) + self.assertFalse(response.data['parameter'][0]['valueBoolean'], (base, headers)) + hidden.append((f'{base}/CodeSystem/$validate-code/?url={PRIVATE}{selector}', headers, response)) for code in [concept.mnemonic, 'missing']: - response = self.client.get(lookup_url.format('/private/url', code), **headers) - self.assertEqual(response.status_code, 400, (code, headers)) - self.assertEqual(response.content, not_found.content, (code, headers)) - - member_headers = {'HTTP_AUTHORIZATION': 'Token ' + member.get_token()} - response = self.client.get(validate_url, **member_headers) + for url in [validate_url.format(base, PRIVATE, code), + validate_url.format(base, PRIVATE, code) + f'&display={concept.display_name}']: + response = self.client.get(url, **headers) + self.assertEqual(response.status_code, 200, (url, headers)) + self.assertFalse(response.data['parameter'][0]['valueBoolean'], (url, headers)) + hidden.append((url, headers, response)) + response = self.client.get(lookup_url.format(base, PRIVATE, code), **headers) + self.assertEqual(response.status_code, 422, (code, headers)) + hidden.append((lookup_url.format(base, PRIVATE, code), headers, response)) + + response = self.client.get(validate_url.format(owner_base, PRIVATE, concept.mnemonic), **member_headers) self.assertTrue(response.data['parameter'][0]['valueBoolean']) - response = self.client.get(lookup_url.format('/private/url', concept.mnemonic), **member_headers) + response = self.client.get(lookup_url.format(owner_base, PRIVATE, concept.mnemonic), **member_headers) self.assertEqual(response.status_code, 200) self.assertEqual(response.data['parameter'][2]['valueString'], concept.display_name) + private_source.is_active = False # now it doesn't exist + private_source.save() + for url, headers, response in hidden: # each against the same request, at the same base, once it's missing + not_found = self.client.get(url, **headers) + self.assertEqual((response.status_code, response.content), (not_found.status_code, not_found.content)) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def test_private_code_system_does_not_hide_public_one_with_same_url(self): - private_source = UserSourceFactory(user=self.user, public_access=ACCESS_TYPE_NONE, canonical_url='/shared/url') + private_source = UserSourceFactory(user=self.user, public_access=ACCESS_TYPE_NONE, canonical_url=SHARED) Source.persist_new_version(UserSourceFactory.build( - version='v1', mnemonic=private_source.mnemonic, user=self.user), private_source.created_by) - public_source = OrganizationSourceFactory(canonical_url='/shared/url') + version='v1', mnemonic=private_source.mnemonic, user=self.user, released=True), private_source.created_by) + public_source = OrganizationSourceFactory(canonical_url=SHARED) concept = ConceptFactory(parent=public_source) Source.persist_new_version(OrganizationSourceFactory.build( - version='v1', mnemonic=public_source.mnemonic, organization=public_source.organization), + version='v1', mnemonic=public_source.mnemonic, organization=public_source.organization, released=True), public_source.created_by) - response = self.client.get(f'/fhir/CodeSystem/$lookup/?system=/shared/url&code={concept.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$lookup/?system={SHARED}&code={concept.mnemonic}') self.assertEqual(response.status_code, 200) self.assertEqual(response.data['parameter'][0]['valueString'], public_source.mnemonic) - response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url=/shared/url&code={concept.mnemonic}') + response = self.client.get(f'/fhir/CodeSystem/$validate-code/?url={SHARED}&code={concept.mnemonic}') self.assertTrue(response.data['parameter'][0]['valueBoolean']) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) diff --git a/core/code_systems/tests_converter.py b/core/code_systems/tests_converter.py new file mode 100644 index 000000000..5fdbfd346 --- /dev/null +++ b/core/code_systems/tests_converter.py @@ -0,0 +1,34 @@ +import warnings + +from django.test import SimpleTestCase +from fhir.resources.R4B.codesystem import CodeSystem + +from core.code_systems.converter import CodeSystemConverter +from core.sources.tests.factories import OrganizationSourceFactory + + +class CodeSystemConverterR4Test(SimpleTestCase): + """The converter speaks R4 through fhir.resources 8's R4B models, without its deprecated API.""" + + def test_to_fhir_builds_an_r4_code_system(self): + source = OrganizationSourceFactory.build( + mnemonic='test', canonical_url='http://example.org/cs', name='Test', default_locale='en', released=True) + + with warnings.catch_warnings(): + warnings.simplefilter('error') + code_system = CodeSystemConverter.to_fhir(source) + + self.assertEqual(code_system['resourceType'], 'CodeSystem') + self.assertEqual(code_system['url'], 'http://example.org/cs') + self.assertEqual(code_system['status'], 'active') + CodeSystem.model_validate(code_system) + + def test_from_fhir_reads_an_r4_code_system(self): + code_system = CodeSystem(id='test', status='retired', content='complete', url='http://example.org/cs') + + self.assertTrue(CodeSystemConverter.can_convert_from_fhir(code_system)) + source = CodeSystemConverter.from_fhir(code_system.model_dump()) + + self.assertEqual(source['mnemonic'], 'test') + self.assertEqual(source['canonical_url'], 'http://example.org/cs') + self.assertTrue(source['retired']) diff --git a/core/code_systems/urls.py b/core/code_systems/urls.py index b290d6b2b..5e72b41ef 100644 --- a/core/code_systems/urls.py +++ b/core/code_systems/urls.py @@ -4,10 +4,16 @@ urlpatterns = [ path('', views.CodeSystemListView.as_view(), name='code-system-list'), - path('$lookup/', views.CodeSystemListLookupView.as_view(), name='code-system-list-lookup'), - path('$lookup', views.CodeSystemListLookupView.as_view(), name='code-system-list-lookup_no_slash'), + path('$lookup/', views.CodeSystemLookupView.as_view(), name='code-system-list-lookup'), + path('$lookup', views.CodeSystemLookupView.as_view(), name='code-system-list-lookup_no_slash'), path('$validate-code/', views.CodeSystemValidateCodeView.as_view(), name='code-system-validate-code'), path('$validate-code', views.CodeSystemValidateCodeView.as_view(), name='code-system-validate-code_no_slash'), + path("/$lookup/", views.CodeSystemLookupView.as_view(), name='code-system-lookup'), + path("/$lookup", views.CodeSystemLookupView.as_view(), name='code-system-lookup_no_slash'), + path("/$validate-code/", views.CodeSystemValidateCodeView.as_view(), + name='code-system-instance-validate-code'), + path("/$validate-code", views.CodeSystemValidateCodeView.as_view(), + name='code-system-instance-validate-code_no_slash'), path("/", views.CodeSystemRetrieveUpdateView.as_view(), name='code-system-detail'), path("", views.CodeSystemRetrieveUpdateView.as_view(), name='code-system-detail_no_slash'), ] diff --git a/core/code_systems/views.py b/core/code_systems/views.py index b216160d4..1c7d32a34 100644 --- a/core/code_systems/views.py +++ b/core/code_systems/views.py @@ -1,57 +1,21 @@ import logging -from django.db.models import F -from rest_framework.exceptions import ValidationError, NotAuthenticated, PermissionDenied - -from core.bundles.serializers import FHIRBundleSerializer -from core.code_systems.serializers import CodeSystemDetailSerializer, \ - ValidateCodeParametersSerializer +from core.code_systems.serializers import CodeSystemDetailSerializer from core.common.constants import HEAD -from core.common.fhir_helpers import translate_fhir_query -from core.common.serializers import IdentifierSerializer -from core.concepts.permissions import CanViewParentDictionaryAsGuest -from core.concepts.views import ConceptRetrieveUpdateDestroyView -from core.parameters.serializers import ParametersSerializer -from core.sources.models import Source +from core.fhir.adapters import r4 +from core.fhir.base import FhirViewMixin, FhirOperationView +from core.fhir.ops.lookup import lookup +from core.fhir.ops.validate_code import validate_code_system +from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView logger = logging.getLogger('oclapi') -def get_viewable_repo(repos, user): - """The first of these repo versions the user can view, skipping the rest as if they didn't exist. Its HEAD, - which governs access to its concepts, must be viewable too.""" - for repo in repos.order_by('id'): - head = repo.head - if head and repo.has_view_access(user) and head.has_view_access(user): - return repo - return None - - -class CodeSystemListView(SourceListView): +class CodeSystemListView(FhirViewMixin, FhirSearchMixin, SourceListView): + resource_type = 'CodeSystem' serializer_class = CodeSystemDetailSerializer - @staticmethod - def bundle_response(data, paginator): - bundle = FHIRBundleSerializer( - {'meta': {}, 'total': len(data), 'type': 'searchset', 'entry': FHIRBundleSerializer.convert_to_entry(data)}, - context={'paginator': paginator} - ) - return bundle.data - - def get_filter_params(self, default_version_to_head=True): - return super().get_filter_params(False) - - def apply_query_filters(self, queryset): - query_fields = list(self.serializer_class.Meta.fields) - query_fields.extend(['code']) - queryset = translate_fhir_query(query_fields, self.request.query_params, queryset) - return queryset - - def apply_filters(self, queryset): - queryset = queryset.exclude(version=HEAD).filter(is_latest_version=True) - return self.apply_query_filters(queryset) - def get_serializer_class(self): return self.serializer_class @@ -59,201 +23,22 @@ def get_detail_serializer(self, obj): return self.serializer_class(obj) -class CodeSystemLookupNotFoundError(ValidationError): - def __init__(self, code: str = None): - message = f"Code '{code}' not found" if code else "Code not found" - super().__init__( - detail={ - "resourceType": "OperationOutcome", - "id": "exception", - "text": { - "status": "additional", - "div": f'
{message}
', - }, - "issue": [ - { - "severity": "error", - "code": "not-found", - "details": {"text": message}, - } - ], - } - ) - - -class FhirParameterRequiredError(ValidationError): - """A 400 OperationOutcome for an operation input that is required, naming it in `expression`.""" - def __init__(self, message: str, expression: str): - super().__init__( - detail={ - "resourceType": "OperationOutcome", - "id": "exception", - "text": { - "status": "additional", - "div": f'
{message}
', - }, - "issue": [ - { - "severity": "error", - "code": "invalid", - "details": {"text": message}, - "expression": [expression], - } - ], - } - ) - - -class CodeSystemListLookupView(ConceptRetrieveUpdateDestroyView): - serializer_class = ParametersSerializer - - def is_container_version_specified(self): - return True - - def verify_scope(self): - pass - - def get_queryset(self): - queryset = super().get_queryset() - code = self.request.query_params.get('code') - system = self.request.query_params.get('system') - version = self.request.query_params.get('version') - if code and system: - source = Source.objects.filter(canonical_url=system) - if not source: - system = IdentifierSerializer.convert_fhir_url_to_ocl_uri(system, 'sources') - source = Source.objects.filter(uri=system) - if source: - if version: - source = source.filter(version=version) - else: - source = source.filter(is_latest_version=True).exclude(version=HEAD) - - source = get_viewable_repo(source, self.request.user) - if source: - queryset = queryset.filter(sources=source, mnemonic=code) - if not queryset.exists(): - raise CodeSystemLookupNotFoundError(code) - return queryset - - raise CodeSystemLookupNotFoundError() - - def get_object(self, queryset=None): - """A code the requester can't view gives the same answer as a code system that doesn't exist.""" - try: - return super().get_object(queryset) - except (NotAuthenticated, PermissionDenied) as ex: - raise CodeSystemLookupNotFoundError() from ex - - def get_serializer(self, instance=None): # pylint: disable=arguments-differ - if instance: - return ParametersSerializer.from_concept(instance) - return None - - -class CodeSystemValidateCodeView(ConceptRetrieveUpdateDestroyView): - serializer_class = ValidateCodeParametersSerializer - - def verify_scope(self): - pass - - def post(self, request, *args, **kwargs): - return self.retrieve(request, *args, **kwargs) - - def update(self, request, *args, **kwargs): - return self.retrieve(request, *args, **kwargs) - - def delete(self, request, *args, **kwargs): - # Not supported - pass - - def is_container_version_specified(self): - return True - - def get_permissions(self): - return [CanViewParentDictionaryAsGuest(), ] - - def get_queryset(self): - """The concept with the code in the code system `url` names. A code system that doesn't resolve, or that the - requester can't view, reads nothing; without a code, or with a code but no url, the request is a 400.""" - queryset = super().get_queryset() - - parameters = self.get_parameters() - url = parameters.get('url') - code = parameters.get('code') - version = parameters.get('version') - display = parameters.get('display') - source = None - if url: - source = Source.objects.filter(canonical_url=url) - if not source: - url = IdentifierSerializer.convert_fhir_url_to_ocl_uri(url, 'sources') - source = Source.objects.filter(uri=url) - - if version: - source = source.filter(version=version) - else: - source = source.filter(is_latest_version=True).exclude(version=HEAD) - source = get_viewable_repo(source, self.request.user) - if not source: - return queryset.none() - - if not code: - raise FhirParameterRequiredError('A code to validate is required.', expression='code') - if not source: - raise FhirParameterRequiredError( - 'url is required with a code: the code system to validate it in.', expression='url') - - queryset = queryset.filter(sources=source, mnemonic=code) - - if display: - instance = queryset.first() - if not instance or display not in (instance.name, instance.display_name): - return queryset.none() - - return queryset - - def get_parameters(self): - if self.request.method in ['POST', 'PUT']: - parameters = self.get_serializer(data=self.request.data, instance=None) - else: - parameters = self.get_serializer_class().parse_query_params(self.request.query_params) - if not parameters.is_valid(): - raise ValidationError(parameters.errors) - params = parameters.validated_data - params = params.get('parameters', {}) - return params - - def get_object(self, queryset=None): - queryset = self.get_queryset() - if not self.is_container_version_specified(): - queryset = queryset.filter(id=F('versioned_object_id')) - instance = queryset.first() - if instance: - try: - self.check_object_permissions(self.request, instance) - except (NotAuthenticated, PermissionDenied): - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ] - } +class CodeSystemLookupView(FhirOperationView): + """CodeSystem/$lookup and CodeSystem/{id}/$lookup: core.fhir.ops.lookup.""" + instance = 'source' + operation = staticmethod(lookup) + adapter = staticmethod(r4.lookup_parameters) - if instance: - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': True} - ] - } - return { - 'parameter': [ - {'name': 'result', 'valueBoolean': False} - ] - } +class CodeSystemValidateCodeView(FhirOperationView): + """CodeSystem/$validate-code and CodeSystem/{id}/$validate-code: core.fhir.ops.validate_code.""" + instance = 'source' + operation = staticmethod(validate_code_system) + adapter = staticmethod(r4.validate_code_parameters) -class CodeSystemRetrieveUpdateView(SourceRetrieveUpdateDestroyView): +class CodeSystemRetrieveUpdateView(FhirViewMixin, FhirReadMixin, SourceRetrieveUpdateDestroyView): + resource_type = 'CodeSystem' serializer_class = CodeSystemDetailSerializer def get_filter_params(self, default_version_to_head=True): diff --git a/core/collections/models.py b/core/collections/models.py index ffb013656..74b880de5 100644 --- a/core/collections/models.py +++ b/core/collections/models.py @@ -167,8 +167,11 @@ def save( super().save(*args, force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields) - if self.id and 'canonical_url' in dirty_fields and self.active_url_registry_entries.exists(): - resolve_url_registry_entries.apply_async((self.id, self.resource_type), queue='default', permanent=False) + if self.id and 'canonical_url' in dirty_fields: + if self.active_url_registry_entries.exists(): + resolve_url_registry_entries.apply_async( + (self.id, self.resource_type), queue='default', permanent=False) + self.link_url_registry_entries() def post_create_actions(self): if self.should_auto_expand: diff --git a/core/collections/views.py b/core/collections/views.py index e45e6b8dc..395b64217 100644 --- a/core/collections/views.py +++ b/core/collections/views.py @@ -71,6 +71,7 @@ from core.tasks.mixins import TaskMixin, IndexingTaskMixin from core.tasks.models import Task from core.tasks.serializers import TaskBriefSerializer +from core.toggles.models import Toggle logger = logging.getLogger('oclapi') @@ -1298,15 +1299,26 @@ class ReferenceExpressionResolveView(APIView): def get_throttles(self): return ThrottleUtil.get_throttles_by_user_plan(self.request.user) + def get_namespace(self): + """ + ?namespace= is the namespace of every expression that doesn't name its own. Clients such as the Mapper send + their project's owner to choose which URL registries apply, so it is used only while the resolver uses the + registries. Without them a namespace limits a canonical URL to that owner's repositories, and such a client + would stop resolving every canonical URL its owner doesn't hold. + """ + namespace = self.request.query_params.get('namespace') + return namespace if namespace and Toggle.get('URL_REGISTRY_IN_RESOLVE_REFERENCE_TOGGLE') else None + def get_results(self): data = self.request.data if not isinstance(data, list): data = [data] from core.common.models import ConceptContainerModel + namespace = self.get_namespace() results = [] for expression in data: - instance, registry_entry = ConceptContainerModel.resolve_expression_to_version(expression) + instance, registry_entry = ConceptContainerModel.resolve_expression_to_version(expression, namespace) if instance: result = { **ReferenceExpressionResolveSerializer(instance).data, @@ -1333,6 +1345,12 @@ def get_results(self): responses={ 200: ReferenceExpressionResolveSerializer(many=True) }, + manual_parameters=[ + openapi.Parameter( + 'namespace', openapi.IN_QUERY, type=openapi.TYPE_STRING, + description='Namespace (/orgs/{org}/ or /users/{user}/) for the expressions that name none. ' + 'Used only while canonical URLs resolve through the URL registries') + ], request_body=openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema( diff --git a/core/common/fhir_helpers.py b/core/common/fhir_helpers.py index 892092b34..70d405db4 100644 --- a/core/common/fhir_helpers.py +++ b/core/common/fhir_helpers.py @@ -1,54 +1,3 @@ -import json - -from core.common.exceptions import Http400 - - -def translate_fhir_query(fhir_query_fields, query_params, queryset): - remaining_query_params = query_params.copy() - query_fields = fhir_query_fields.copy() - url = remaining_query_params.get('url') - if url and 'url' in query_fields: - queryset = queryset.filter(canonical_url=url) - remaining_query_params.pop('url') - query_fields.remove('url') - language = remaining_query_params.get('language') - if language and 'language' in query_fields: - queryset = queryset.filter(default_locale=language) - remaining_query_params.pop('language') - query_fields.remove('language') - status = remaining_query_params.get('status') - if status and 'status' in query_fields: - query_fields.remove('status') - remaining_query_params.pop('status') - if status == 'retired': - queryset = queryset.filter(retired=True) - elif status == 'active': - queryset = queryset.filter(released=True) - elif status == 'draft': - queryset = queryset.filter(released=False) - title = remaining_query_params.get('title') - if title and 'title' in query_fields: - remaining_query_params.pop('title') - query_fields.remove('title') - queryset = queryset.filter(full_name=title) - code = remaining_query_params.get('code') - if code and 'code' in query_fields: - remaining_query_params.pop('code') - query_fields.remove('code') - queryset = queryset.filter(concepts__mnemonic=code) - for query_field in query_fields: - query_value = remaining_query_params.get(query_field) - if query_value: - remaining_query_params.pop(query_field, None) - kwargs = {query_field: query_value} - queryset = queryset.filter(**kwargs) - - if remaining_query_params: - raise Http400('The following query params are not supported: ' + json.dumps(remaining_query_params)) - - return queryset - - def delete_empty_fields(obj): if isinstance(obj, dict): for field in list(obj.keys()): diff --git a/core/common/mixins.py b/core/common/mixins.py index 57ea324e5..206739f7c 100644 --- a/core/common/mixins.py +++ b/core/common/mixins.py @@ -562,8 +562,17 @@ def update(self, request, *args, **kwargs): # pylint: disable=unused-argument class SourceContainerMixin: - def find_repo_by_canonical_url(self, canonical_url): + def find_repo_by_canonical_url(self, canonical_url, repo_type=None): + """ + This owner's lowest-id repository row with the canonical URL, a Source before a Collection; `url|version` + names a version. With repo_type (Source or Collection), only that type's active, non-retired rows, and the + HEAD unless a version is named. + """ url, version = canonical_url_to_url_and_version(canonical_url) + if repo_type: + repos = {'Source': self.source_set, 'Collection': self.collection_set}[repo_type.OBJECT_TYPE] + queryset = repos.filter(canonical_url=url, is_active=True, retired=False) + return queryset.filter(version=decode_string(version) if version else HEAD).first() queryset = self.source_set.filter(canonical_url=url) if not queryset.exists(): queryset = self.collection_set.filter(canonical_url=url) diff --git a/core/common/models.py b/core/common/models.py index 743f2f586..c8348e20a 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -6,6 +6,7 @@ from celery_once import AlreadyQueued from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import ArrayField from django.core.cache import cache from django.core.exceptions import ValidationError @@ -712,8 +713,12 @@ def get_latest_version(self): def get_last_version(self): return self.active_versions.order_by('-created_at').first() - def get_latest_released_version(self): - return self.released_versions.filter(retired=False).order_by('-created_at').first() + def get_latest_released_version(self, exclude_head=False): + """The newest released version that isn't retired; exclude_head skips a HEAD flagged released""" + versions = self.released_versions.filter(retired=False) + if exclude_head: + versions = versions.exclude(version=HEAD) + return versions.order_by('-created_at').first() def get_prev_released_version(self): return self.released_versions.exclude(id=self.id).order_by('-created_at').first() @@ -922,13 +927,16 @@ def versions(self): def delete(self, using=None, keep_parents=False, force=False, sync=False): # pylint: disable=arguments-differ export_paths = self.get_export_paths_to_delete() deleted_urls = [self.url] + deleted_ids = [self.id] + canonical_urls = {self.canonical_url} if self.is_head: other_versions = list(self.versions.exclude(id=self.id)) for other_version in other_versions: other_version.clear_cache() export_paths += other_version.get_export_paths_to_delete() deleted_urls.append(other_version.url) - self.versions.exclude(id=self.id).delete() + deleted_ids.append(other_version.id) + canonical_urls.add(other_version.canonical_url) elif self.is_latest_version: prev_version = self.prev_version if not force and not prev_version: @@ -937,6 +945,10 @@ def delete(self, using=None, keep_parents=False, force=False, sync=False): # py prev_version.is_latest_version = True prev_version.save() + registry_entry_ids = self.unlink_url_registry_entries(deleted_ids) + if self.is_head: + self.versions.exclude(id=self.id).delete() + self.delete_pins() self.delete_following() self.delete_version_changelogs(deleted_urls) @@ -945,6 +957,32 @@ def delete(self, using=None, keep_parents=False, force=False, sync=False): # py super().delete(using=using, keep_parents=keep_parents) self.delete_export_paths(export_paths, sync) self.post_delete_actions() + self.relink_url_registry_entries(registry_entry_ids, canonical_urls - {None, ''}) + + def unlink_url_registry_entries(self, repo_ids): + """ + Unlinks (saved, so reindexed) the registry entries linked to these rows of this repository before the rows + are deleted: url_registry_entries is a GenericRelation, which would delete them with the rows. Returns + their ids for relink_url_registry_entries. + """ + from core.url_registry.models import URLRegistry + entries = list(URLRegistry.objects.filter( + repo_type=ContentType.objects.get_for_model(self), repo_id__in=repo_ids)) + for entry in entries: + entry.unlink() + return [entry.id for entry in entries] + + @staticmethod + def relink_url_registry_entries(entry_ids, canonical_urls): + """ + After a delete: every active entry that was linked to a deleted row, and every active entry naming one of the + deleted rows' canonical URLs, linked or not, is re-resolved (lookup_entry). A retry of a delete that failed + part-way so finds, by URL, the entries the failed attempt unlinked. The link is a cache that resolution never + reads, so a failure here only leaves it stale until the next delete, retry or write that relinks. + """ + from core.url_registry.models import URLRegistry + for entry in URLRegistry.objects.filter(Q(id__in=entry_ids) | Q(url__in=canonical_urls), is_active=True): + entry.lookup_entry() @staticmethod def delete_version_changelogs(urls): @@ -1399,6 +1437,20 @@ def is_exporting(self): def active_url_registry_entries(self): return self.url_registry_entries.filter(is_active=True) + def link_url_registry_entries(self): + """ + For a repository row (HEAD or version) created with, or given, a canonical URL: the active registry entries + that name it in this repository's namespace are pointed at what they resolve to now (lookup_entry: the + namespace's lowest-id row with the URL, as the registry has always resolved), so an entry made before the row + that answers it gets it. The entries already linked to this row are re-resolved by the + resolve_url_registry_entries task. + """ + if not self.canonical_url: + return + from core.url_registry.models import URLRegistry + for entry in URLRegistry.objects.filter(is_active=True, url=self.canonical_url, namespace=self.parent_url): + entry.lookup_entry() + @cached_property def version_export_path(self): last_update = self.last_child_update.strftime('%Y-%m-%d_%H%M%S') @@ -1472,22 +1524,23 @@ def can_view_all_content(self, user): ) or self.public_can_view or self.user_id == user.id or self.organization.members.filter(id=user.id).exists() @classmethod - def resolve_expression_to_version(cls, expression): + def resolve_expression_to_version(cls, expression, namespace=None): + """`namespace` is the namespace of an expression that doesn't name its own (a dict without the key, or a url)""" url = expression - namespace = None version = None instance = None resolved_registry_entry = None if isinstance(expression, dict) and get(expression, 'url'): url = expression['url'] - namespace = expression.get('namespace', None) + namespace = expression.get('namespace', namespace) version = expression.get('version', None) if url: instance, resolved_registry_entry = cls.resolve_reference_expression(url, namespace, version) return instance, resolved_registry_entry @classmethod - def resolve_reference_expression(cls, url, namespace=None, version=None): + def resolve_reference_expression( # pylint: disable=too-many-arguments,too-many-locals + cls, url, namespace=None, version=None, repo_type=None, allow_head=True, heads=None, head_shown=None): """ resolves to repository version according to this process: @@ -1511,6 +1564,20 @@ def resolve_reference_expression(cls, url, namespace=None, version=None): is defined somewhere else in OCL) 2. Else if relative URL provided: - Return the repository directly using the relative URL, or return 404 if not found + + Resolving reads only: it never writes a registry entry. + + Two flags make the answer stricter, for FHIR (core.fhir.resolve); their defaults keep the behaviour above: + - repo_type (Source or Collection): resolve only to a repository of that class. A canonical URL matches the + active, non-retired HEAD of a repository (its current canonical URL) in every step, so a registry entry whose + namespace holds no such repository resolves to nothing, and nothing else is tried. A named version must be + active. Unresolved is an unsaved repo_type. + - allow_head=False: with no version named, the latest released version other than HEAD (even a HEAD flagged + released) or nothing. A version named HEAD is still served. + - heads (a Q on HEAD rows, with repo_type): without the registry, a canonical URL matches only the HEADs it + holds, so a repository the caller isn't shown never stands in for one they are (FHIR's global base). + - head_shown (a callable, with repo_type): a HEAD it refuses resolves to nothing before any version of it is + read, so a repository the caller isn't shown costs what one that doesn't exist does (FHIR). """ resolution_url, version, is_canonical = cls.__get_resolution_url(url, version) @@ -1527,36 +1594,52 @@ def resolve_reference_expression(cls, url, namespace=None, version=None): owner = SourceContainerMixin.get_object_from_namespace(namespace) if owner: registry_entry = owner.url_registry_entries.filter(is_active=True, url=resolution_url).first() - instance = registry_entry.lookup_entry() if registry_entry else owner.find_repo_by_canonical_url( # pylint: disable=line-too-long - resolution_url) + instance = registry_entry.find_repo(repo_type) if registry_entry else \ + owner.find_repo_by_canonical_url(resolution_url, repo_type) if is_global_namespace or (not registry_entry and not instance): registry_entry = URLRegistry.get_active_global_entries().filter(url=resolution_url).first() if registry_entry: - instance = registry_entry.lookup_entry() + instance = registry_entry.find_repo(repo_type) - return cls.resolve_repo(instance, version, is_canonical, resolution_url), registry_entry + return cls.resolve_repo( + instance, version, is_canonical, resolution_url, repo_type, allow_head, head_shown), registry_entry criteria &= models.Q(canonical_url=resolution_url) + if repo_type: + criteria &= models.Q(version=HEAD) + if heads is not None: + criteria &= heads if namespace: criteria &= models.Q(models.Q(user__uri=namespace) | models.Q(organization__uri=namespace)) else: criteria &= models.Q(uri=(resolution_url if resolution_url.endswith('/') else resolution_url + '/')) from core.repos.models import Repository - return cls.resolve_repo(Repository.get(criteria), version, is_canonical, resolution_url), registry_entry + return cls.resolve_repo( + Repository.get(criteria, repo_type), version, is_canonical, resolution_url, repo_type, allow_head, + head_shown), registry_entry @classmethod - def resolve_repo(cls, instance, version, is_canonical, resolution_url): + def resolve_repo( # pylint: disable=too-many-arguments + cls, instance, version, is_canonical, resolution_url, repo_type=None, allow_head=True, head_shown=None): if instance: - if version: - instance = instance.versions.filter(version=decode_string(version)).first() + head = instance if repo_type and instance.is_head else None + if head and head_shown and not head_shown(head): + instance = None + elif version: + versions = instance.active_versions if repo_type else instance.versions + instance = versions.filter(version=decode_string(version)).first() elif instance.is_head: - instance = instance.get_latest_released_version() or instance + instance = (instance.get_latest_released_version() or instance) if allow_head else \ + instance.get_latest_released_version(exclude_head=True) + if instance and head: # the repository's (its HEAD's) access and canonical URL, which FHIR reads too + instance.repository_public_access = head.public_access + instance.repository_canonical_url = head.canonical_url if not instance: from core.sources.models import Source - instance = Source() + instance = (repo_type or Source)() instance.is_fqdn = is_canonical instance.resolution_url = resolution_url diff --git a/core/common/serializers.py b/core/common/serializers.py index 868b6c876..c2bfdecb2 100644 --- a/core/common/serializers.py +++ b/core/common/serializers.py @@ -114,16 +114,19 @@ def parse_identifier(accession_id): @staticmethod def convert_ocl_uri_to_fhir_url(uri, resource_type): - resource_type_uri = f"/{resource_type}/" + """ + An OCL API relative URI (/{owner_type}/{owner_id}/{sources|collections}/{id}/...) as its repository's FHIR one, + /{owner_type}/{owner_id}/{resource_type}/{id}/: only the repository type's segment is renamed, so an owner or a + repository named sources or collections keeps its name. Any other URI is returned as it is. + """ if uri and (uri.startswith('/orgs/') or uri.startswith('/users/')): - # Recognize OCL API relative URI - uri = uri.replace('/sources/', resource_type_uri).replace('/collections/', resource_type_uri) - uri = uri.strip('/') - parts = uri.split('/') + parts = uri.strip('/').split('/') + if len(parts) > 2 and parts[2] in ('sources', 'collections'): + parts[2] = resource_type if len(parts) < 4: raise ValidationError( 'Identifier must be in a format: /{owner_type}/{owner_id}/{resourceType}/{resource_id}/, given: ' - + uri) + + '/'.join(parts)) uri = '/' + '/'.join(parts[:4]) + '/' return uri diff --git a/core/common/swagger.py b/core/common/swagger.py index 75e6172a3..d43579e4e 100644 --- a/core/common/swagger.py +++ b/core/common/swagger.py @@ -1,7 +1,13 @@ """Swagger helpers for OCL-specific OpenAPI generation.""" +from copy import deepcopy + +from django.conf import settings from drf_yasg.generators import EndpointEnumerator, OpenAPISchemaGenerator +from core.fhir.base import FhirViewMixin, is_fhir_path +from core.fhir.swagger import FhirSwaggerSchemaGenerator, fhir_paths, fhir_swagger_url, on_fhir_host + class OCLSwaggerEndpointEnumerator(EndpointEnumerator): """Preserve literal dollar signs when deriving OpenAPI paths from Django URL regexes.""" @@ -18,3 +24,26 @@ class OCLSwaggerSchemaGenerator(OpenAPISchemaGenerator): """Use the OCL endpoint enumerator so Swagger paths match the published URLs.""" endpoint_enumerator_class = OCLSwaggerEndpointEnumerator + + def get_endpoints(self, request): + """Replace every FHIR view entry, including malformed paths from no-slash includes.""" + return {path: endpoint for path, endpoint in super().get_endpoints(request).items() + if not is_fhir_path(path) and not issubclass(endpoint[0], FhirViewMixin)} + + def get_paths(self, endpoints, components, request, public): + """Keep the other API operations intact, adding FHIR only when both share a host.""" + paths, prefix = super().get_paths(endpoints, components, request, public) + if not settings.FHIR_SUBDOMAIN: + paths.update(fhir_paths(components)) + return paths, prefix + + def get_schema(self, request=None, public=False): + """Select the host's schema and link the API documentation to the separate FHIR service.""" + if on_fhir_host(request): + return FhirSwaggerSchemaGenerator(self.info).get_schema(request, public) + schema = super().get_schema(request, public) + if settings.FHIR_SUBDOMAIN: + schema.info = deepcopy(schema.info) + link = fhir_swagger_url(request, self.url or settings.API_BASE_URL) + schema.info.description = schema.info.get('description', '') + f'\n\n[FHIR terminology Swagger]({link}).' + return schema diff --git a/core/common/tests.py b/core/common/tests.py index af3a169d7..f4ec11a45 100644 --- a/core/common/tests.py +++ b/core/common/tests.py @@ -75,10 +75,8 @@ from .backends import OCLOIDCAuthenticationBackend from .exceptions import SSOIdentityMismatchRefused from .checksums import Checksum -from .fhir_helpers import translate_fhir_query from .serializers import IdentifierSerializer from .validators import URIValidator -from ..code_systems.serializers import CodeSystemDetailSerializer from ..concepts.serializers import ConceptDetailSerializer from ..concepts.tests.factories import ConceptFactory, ConceptNameFactory from ..mappings.serializers import MappingDetailSerializer @@ -325,57 +323,6 @@ def setUp(self): self.maxDiff = None -class FhirHelpersTest(OCLTestCase): - def test_language_to_default_locale(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'language': 'eng'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('"sources"."default_locale" = eng' in str(query_set.query)) - - def test_status_retired(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'retired'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."retired"' in str(query_set.query)) - - def test_status_active(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'active'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."released"' in str(query_set.query)) - - def test_status_draft(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'status': 'draft'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE NOT "sources"."released"' in str(query_set.query)) - - def test_title_to_full_name(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'title': 'some title'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('WHERE "sources"."full_name" = some title' in str(query_set.query)) - - def test_other_fields(self): - query_fields = list(CodeSystemDetailSerializer.Meta.fields) - query_params = {'version': 'v1', 'id': '2'} - query_set = Source.objects.all() - - query_set = translate_fhir_query(query_fields, query_params, query_set) - self.assertTrue('"sources"."version" = v1' in str(query_set.query)) - self.assertTrue('"sources"."id" = 2' in str(query_set.query)) - - class IdentifierSerializerTest(OCLTestCase): def test_deserialize(self): data = {'system': '/org/OCL/test', diff --git a/core/concept_maps/constants.py b/core/concept_maps/constants.py index 68cc367f2..a9b51bd6e 100644 --- a/core/concept_maps/constants.py +++ b/core/concept_maps/constants.py @@ -1 +1,43 @@ RESOURCE_TYPE = 'ConceptMap' + +# ConceptMap relationships as FHIR R5 names them (http://hl7.org/fhir/concept-map-relationship): the version-neutral +# form, which an adapter writes as its version's codes (core.fhir.adapters.r4: R4's equivalence) +EQUIVALENT = 'equivalent' +RELATED_TO = 'related-to' +SOURCE_IS_NARROWER = 'source-is-narrower-than-target' +SOURCE_IS_BROADER = 'source-is-broader-than-target' +NOT_RELATED_TO = 'not-related-to' + +# The relationship of each OCL map type, the map type spelt as core.fhir.ops.translate.map_type_key spells it (upper +# case, hyphens for spaces, trimmed), so SAME-AS, SAME AS and Same As are one (design D27). Any other map type is +# RELATED_TO. A mapping says its from concept is NARROWER-THAN (or BROADER-THAN) its to concept. +MAP_TYPE_RELATIONSHIPS = { + 'SAME-AS': EQUIVALENT, + 'NARROWER-THAN': SOURCE_IS_NARROWER, + 'BROADER-THAN': SOURCE_IS_BROADER, + 'NOT-SAME-AS': NOT_RELATED_TO, + # Before WP6 a FHIR ConceptMap write stored an R4 equivalence other than equivalent as the map type, verbatim: + # each is the relationship HL7's R4/R5 conversion gives it (unmatched, R5's noMap, is not-related-to; relatedto + # and inexact are the default). R5's relationship codes are themselves, related-to the default. + 'EQUIVALENT': EQUIVALENT, + 'EQUAL': EQUIVALENT, + 'WIDER': SOURCE_IS_NARROWER, + 'SUBSUMES': SOURCE_IS_NARROWER, + 'NARROWER': SOURCE_IS_BROADER, + 'SPECIALIZES': SOURCE_IS_BROADER, + 'DISJOINT': NOT_RELATED_TO, + 'UNMATCHED': NOT_RELATED_TO, + 'SOURCE-IS-NARROWER-THAN-TARGET': SOURCE_IS_NARROWER, + 'SOURCE-IS-BROADER-THAN-TARGET': SOURCE_IS_BROADER, + 'NOT-RELATED-TO': NOT_RELATED_TO, +} +# A relationship read the other way, from target to source +REVERSED_RELATIONSHIPS = {SOURCE_IS_NARROWER: SOURCE_IS_BROADER, SOURCE_IS_BROADER: SOURCE_IS_NARROWER} +# The OCL map type a relationship is stored as, when a ConceptMap is written; related-to has none (the equivalence +# written is kept) +RELATIONSHIP_MAP_TYPES = { + EQUIVALENT: 'SAME-AS', + SOURCE_IS_NARROWER: 'NARROWER-THAN', + SOURCE_IS_BROADER: 'BROADER-THAN', + NOT_RELATED_TO: 'NOT-SAME-AS', +} diff --git a/core/concept_maps/serializers.py b/core/concept_maps/serializers.py index 28f126fa1..93868bf47 100644 --- a/core/concept_maps/serializers.py +++ b/core/concept_maps/serializers.py @@ -6,28 +6,36 @@ DateTimeField from core.common.fhir_helpers import delete_empty_fields -from core.concept_maps.constants import RESOURCE_TYPE +from core.concept_maps.constants import RESOURCE_TYPE, RELATIONSHIP_MAP_TYPES from core.common.constants import HEAD from core.common.serializers import StatusField, IdentifierSerializer -from core.mappings.constants import SAME_AS +from core.fhir.adapters import r4 +from core.fhir.ops.concepts import fhir_code +from core.fhir.ops.translate import relationship, side_system, with_side +from core.fhir.resolve import namespace_of +from core.fhir.search import CanonicalUrlField, FhirRepositorySerializerMixin from core.mappings.models import Mapping from core.mappings.serializers import MappingDetailSerializer -from core.parameters.serializers import ParametersSerializer from core.sources.models import Source from core.sources.serializers import SourceCreateOrUpdateSerializer logger = logging.getLogger('oclapi') +MAPPING_LIMIT = 1000 class ConceptMapGroupField(serializers.Field): + """ + A ConceptMap's groups, one per source and target system, each mapping an element target, with core.fhir.ops. + translate's systems (for the caller, at the read's base) and relationships, in R4 (equivalence). Writes store an + equivalence as the map type it is. + """ + def to_internal_value(self, data): mappings = [] for group in data: for element in group.get('element', []): for target in element.get('target', []): - map_type = target.get('equivalence') - if map_type == 'equivalent': - map_type = SAME_AS + equivalence = target.get('equivalence') mapping = { 'from_source_url': IdentifierSerializer.convert_fhir_url_to_ocl_uri(group.get('source'), 'sources'), @@ -35,74 +43,53 @@ def to_internal_value(self, data): 'sources'), 'from_concept_code': element.get('code'), 'to_concept_code': target.get('code'), - 'map_type': map_type + 'map_type': RELATIONSHIP_MAP_TYPES.get(relationship(equivalence), equivalence) } mappings.append(mapping) return {'mappings': mappings} def to_representation(self, value): - # limit to 1000 mappings by default + """ + At most 1,000 live mappings, grouped by the systems they are shown as: each side named as $translate names it + (side_system), its source's canonical URL when the caller is shown that source at the read's base, else the URL + the mapping stores. Whether a source is shown is a subquery of the one statement. + """ # TODO: support graphQL to go around the limit - limit = self.get_limit() - mappings = value.get_mappings_queryset().filter(retired=False).order_by('id')[:limit] - groups = {} - for mapping in mappings: - key = (mapping.from_source_url or '') + (mapping.to_source_url or '') - group = groups.get(key) - if not group: - if mapping.from_source and mapping.from_source.canonical_url: - from_url = mapping.from_source.canonical_url - else: - from_url = IdentifierSerializer.convert_ocl_uri_to_fhir_url(mapping.from_source_url, RESOURCE_TYPE) - - if mapping.to_source and mapping.to_source.canonical_url: - to_url = mapping.to_source.canonical_url - else: - to_url = IdentifierSerializer.convert_ocl_uri_to_fhir_url(mapping.to_source_url, RESOURCE_TYPE) - - group = { - 'source': from_url, - 'target': to_url - } - groups.update({key: group}) - elements = group.get('element') - if not elements: - elements = [] - group.update({'element': elements}) - element = None - for candidate in elements: - if candidate.get('code') is mapping.from_concept_code: - element = candidate - break - if not element: - element = { - 'code': mapping.from_concept_code, - 'target': [] - } - elements.append(element) - targets = element.get('target') - relationship = mapping.map_type - if mapping.map_type == SAME_AS: - relationship = 'equivalent' - targets.append({ - 'code': mapping.to_concept_code, - 'equivalence': relationship + request, view = self.context.get('request'), self.context.get('view') + namespace, user = namespace_of(getattr(view, 'kwargs', None)), getattr(request, 'user', None) + mappings = value.get_active_mappings() # neither retired nor deleted + for side in ('from', 'to'): + mappings = with_side(mappings, side, namespace, user) + rows = list(mappings.order_by('id').values( + 'map_type', 'from_concept_code', 'to_concept_code', *( + f'{side}_{field}' for side in ('from', 'to') + for field in ('source_url', 'source__uri', 'shown', 'canonical')))[:MAPPING_LIMIT + 1]) + self.subsetted = len(rows) > MAPPING_LIMIT + groups, elements = {}, {} + for row in rows[:MAPPING_LIMIT]: + key = tuple(side_system(row[f'{side}_canonical'], row[f'{side}_source_url'], + row[f'{side}_source__uri'] if row[f'{side}_shown'] else None) + for side in ('from', 'to')) + if key not in groups: + groups[key] = {'source': key[0], 'target': key[1], 'element': []} + code = fhir_code(row['from_concept_code']) + element = elements.get((key, code)) + if element is None: + element = elements[(key, code)] = {'code': code, 'target': []} + groups[key]['element'].append(element) + element['target'].append({ + 'code': fhir_code(row['to_concept_code']), + 'equivalence': r4.equivalence(relationship(row['map_type'])) }) return [*groups.values()] - def get_limit(self): - if self.context.get('has_many', False): - limit = 25 - else: - limit = 1000 - return limit - -class ConceptMapDetailSerializer(serializers.ModelSerializer): +class ConceptMapDetailSerializer(FhirRepositorySerializerMixin, serializers.ModelSerializer): + content_field = 'group' resourceType = SerializerMethodField(method_name='get_resource_type') id = CharField(source='mnemonic') - url = CharField(source='canonical_url', required=False) + url = CanonicalUrlField(source='canonical_url', required=False) title = CharField(source='full_name', required=False) status = StatusField(source='*') language = CharField(source='default_locale', required=False) @@ -117,14 +104,6 @@ class Meta: 'version', 'identifier', 'contact', 'jurisdiction', 'name', 'description', 'publisher', 'purpose', 'copyright', 'date', 'experimental', 'group') - def __new__(cls, *args, **kwargs): - if kwargs.get('many', False): - context = kwargs.get('context', {}) - context.update({'has_many': True}) - kwargs.update({'context': context}) - - return super().__new__(cls, *args, **kwargs) - @staticmethod def get_resource_type(_): return RESOURCE_TYPE @@ -138,6 +117,7 @@ def to_representation(self, instance): rep = super().to_representation(instance) delete_empty_fields(rep) IdentifierSerializer.include_ocl_identifier(instance.uri, RESOURCE_TYPE, rep) + self.with_fhir_view(instance, rep) except (Exception, ): msg = f'Failed to represent "{instance.uri}" as {RESOURCE_TYPE}' logger.exception(msg) @@ -259,27 +239,3 @@ def update(self, instance, validated_data): self._errors.update(errors) return source - - -class ConceptMapParametersSerializer(ParametersSerializer): - - def update(self, instance, validated_data): - pass - - def create(self, validated_data): - pass - - allowed_input_parameters = { - 'url': 'valueUri', - 'conceptMapVersion': 'valueString', - 'code': 'valueCode', - 'system': 'valueUri', - 'version': 'valueString', - 'source': 'valueUri', - 'coding': 'valueCoding', - 'codeableConcept': 'valueCodeableConcept', - 'target': 'valueUri', - 'targetsystem': 'valueUri', - # TODO: dependency? - 'reverse': 'valueBoolean' - } diff --git a/core/concept_maps/tests.py b/core/concept_maps/tests.py index 013a3858b..49c9ff107 100644 --- a/core/concept_maps/tests.py +++ b/core/concept_maps/tests.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from mock.mock import patch, Mock from rest_framework.test import APIClient @@ -35,7 +33,7 @@ def setUp(self): self.user_token = self.user.get_token() self.user_source = UserSourceFactory(user=self.user, public_access='None', canonical_url='/some/url') self.user_source_v1 = UserSourceFactory.build( - version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent) + version='v1', mnemonic=self.user_source.mnemonic, user=self.user_source.parent, released=True) Source.persist_new_version(self.user_source_v1, self.user_source.created_by) self.org_source_B = OrganizationSourceFactory(organization=self.org, canonical_url='/some/url/B', @@ -44,7 +42,7 @@ def setUp(self): names=[ConceptNameFactory.build(name="concept_B_1_name")]) self.concept_B_2 = ConceptFactory(parent=self.org_source_B, mnemonic='concept_B_2') self.org_source_B_v1 = OrganizationSourceFactory.build( - version='v1', mnemonic=self.org_source_B.mnemonic, organization=self.org_source_B.parent) + version='v1', mnemonic=self.org_source_B.mnemonic, organization=self.org_source_B.parent, released=True) Source.persist_new_version(self.org_source_B_v1, self.org_source_B.created_by) self.mapping_1 = MappingFactory(parent=self.org_source, to_concept=self.concept_1, @@ -61,7 +59,7 @@ def setUp(self): self.mapping_3.save() self.org_source_v2 = OrganizationSourceFactory.build( - version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent) + version='v2', mnemonic=self.org_source.mnemonic, organization=self.org_source.parent, released=True) Source.persist_new_version(self.org_source_v2, self.org_source.created_by) for mapping in self.org_source_v2.get_mappings_queryset().all(): @@ -82,31 +80,36 @@ def test_public_can_view(self): '/orgs/' + self.org.mnemonic + '/ConceptMap/' + self.org_source.mnemonic + '/' ) self.assertEqual(resource['version'], 'v2') + self.assertNotIn('group', resource) # a search entry is a summary; the read has the groups + + resource = self.client.get(response.data['entry'][0]['fullUrl']).data self.assertEqual(len(resource['group']), 1) self.assertEqual(resource['group'], [{'source': self.org_source_B_v1.canonical_url, 'target': self.org_source.canonical_url, 'element': [ {'code': 'concept_B_1', 'target': [{'code': 'concept_1', 'equivalence': 'equivalent'}]}, - {'code': 'concept_B_2', - 'target': [{'code': 'concept_2', 'equivalence': 'equivalent'}]}, - {'code': 'concept_B_2', - 'target': [{'code': 'concept_1', 'equivalence': 'equivalent'}]}]}]) + {'code': 'concept_B_2', # one element per code, each of its targets + 'target': [{'code': 'concept_2', 'equivalence': 'equivalent'}, + {'code': 'concept_1', 'equivalence': 'equivalent'}]}]}]) def test_private_can_view(self): + # the global base shows public repositories only, whoever asks; the owner base shows the owner its own response = self.client.get('/fhir/ConceptMap/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 2) - self.assertEqual(response.data['total'], 2) + self.assertEqual(len(response.data['entry']), 1) + self.assertEqual(response.data['total'], 1) resource = response.data['entry'][0]['resource'] self.assertEqual( resource['identifier'][0]['value'], '/orgs/' + self.org.mnemonic + '/ConceptMap/' + self.org_source.mnemonic + '/' ) - self.assertEqual(resource['version'], 'v2') - resource2 = response.data['entry'][1]['resource'] + + response = self.client.get( + f'/users/{self.user.mnemonic}/ConceptMap/?url=/some/url', HTTP_AUTHORIZATION='Token ' + self.user_token) + resource2 = response.data['entry'][0]['resource'] self.assertEqual( resource2['identifier'][0]['value'], '/users/' + self.user.mnemonic + '/ConceptMap/' + self.user_source.mnemonic + '/' @@ -132,7 +135,7 @@ def test_public_can_list(self): self.assertEqual(resource2['version'], 'v1') def test_find_by_title(self): - response = self.client.get('/fhir/ConceptMap/?title=source name') + response = self.client.get('/fhir/ConceptMap/?title:exact=source name') # title alone is 'starts with' self.assertEqual(len(response.data['entry']), 1) resource = response.data['entry'][0]['resource'] @@ -145,7 +148,10 @@ def test_find_by_title(self): def test_private_can_list(self): response = self.client.get('/fhir/ConceptMap/', HTTP_AUTHORIZATION='Token ' + self.user_token) - self.assertEqual(len(response.data['entry']), 3) + self.assertEqual(len(response.data['entry']), 2) # public only at the global base + response = self.client.get( + f'/users/{self.user.mnemonic}/ConceptMap/', HTTP_AUTHORIZATION='Token ' + self.user_token) + self.assertEqual(len(response.data['entry']), 1) def test_get_concept_map(self): response = self.client.get(f'/orgs/{self.org.mnemonic}/ConceptMap/{self.org_source.mnemonic}/') @@ -510,77 +516,90 @@ def test_put_concept_map_overwriting_version(self): # check if bad request self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, {'version': 'Version 1.0 already exists for CodeSystem test2.'}) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + self.assertEqual( + [(issue['code'], issue['expression'], issue['details']['text']) for issue in response.data['issue']], + [('invalid', ['version'], 'Version 1.0 already exists for CodeSystem test2.')]) + + def translate(self, base, token=True, **params): + """$translate at a base, as the user (or anonymously), after the user's PUT of their ConceptMap, released.""" + if not Source.objects.filter(mnemonic=self.user_source.mnemonic, user=self.user, version='1.0').exists(): + self.putConceptMap(status='active') + headers = {'HTTP_AUTHORIZATION': 'Token ' + self.user_token} if token else {} + return self.client.get(f'{base}/ConceptMap/$translate', params, **headers) + + def assertTranslation(self, response, matches): # pylint: disable=invalid-name + """result, a message only when there is no match, and each match: (code, system, equivalence).""" + self.assertEqual(response.status_code, 200, response.data) + parameters = response.data['parameter'] + self.assertEqual(parameters[0], {'name': 'result', 'valueBoolean': bool(matches)}) + self.assertEqual([item['name'] for item in parameters[1:]], ['match'] * len(matches) or ['message']) + found = [item['part'] for item in parameters[1:] if item['name'] == 'match'] + self.assertEqual( + [(part[1]['valueCoding']['code'], part[1]['valueCoding']['system'], part[0]['valueCode'], + part[2]['valueUri']) for part in found], + [(*match, 'http://localhost/url|1.0') for match in matches]) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def test_translate_positive(self): - self.putConceptMap() + """The ConceptMap the user PUT, read back by $translate: the mapping its group wrote.""" + self.assertTranslation( + self.translate(f'/users/{self.user.mnemonic}', url='http://localhost/url', + system=self.org_source_B_v1.canonical_url, code='concept_B_1'), + [('concept_1', '/some/url', 'equivalent')]) - response = self.client.get( - f'/users/{self.user.mnemonic}/ConceptMap/$translate?' - f'system={self.org_source_B_v1.canonical_url}&code=concept_B_1', - HTTP_AUTHORIZATION='Token ' + self.user_token) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_public_translate_negative(self): + """The user's ConceptMap is private: anonymous, it is not found, and no code system has the mapping.""" + response = self.translate(f'/users/{self.user.mnemonic}', token=False, url='http://localhost/url', + system=self.org_source_B_v1.canonical_url, code='concept_B_1') + self.assertEqual(response.status_code, 404) + self.assertEqual(response.data['issue'][0]['expression'], ['url']) + self.assertTranslation(self.translate(f'/users/{self.user.mnemonic}', token=False, + system=self.org_source_B_v1.canonical_url, code='concept_B_1'), []) - self.assertEqual(response.status_code, 200) - self.assertDictEqual( - response.data, - { - 'resourceType': 'Parameters', - 'parameter': [ - OrderedDict([ - ('name', 'result'), - ('valueBoolean', True) - ]), - OrderedDict([ - ('name', 'match'), - ( - 'part', - [ - OrderedDict([ - ('name', 'equivalence'), - ('valueCode', 'equivalent') - ]), - OrderedDict([ - ('name', 'concept'), - ('valueCoding', OrderedDict([('system', '/some/url'), ('code', 'concept_1')])) - ]) - ] - ) - ]), - OrderedDict([ - ('name', 'match'), - ( - 'part', - [ - OrderedDict([ - ('name', 'equivalence'), - ('valueCode', 'equivalent') - ]), - OrderedDict([ - ('name', 'concept'), - ('valueCoding', OrderedDict([('system', '/some/url'), ('code', 'concept_1')])) - ]) - ] - ) - ]) - ] - }) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_translate_negative(self): + self.assertTranslation( + self.translate(f'/users/{self.user.mnemonic}', url='http://localhost/url', + system=self.org_source_B_v1.canonical_url, code='concept_1'), []) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_public_translate_negative(self): - self.putConceptMap() + def test_translate_with_url(self): + """The global base shows public repositories only: the user's private ConceptMap isn't found there.""" + response = self.translate('/fhir', url='http://localhost/url', system=self.org_source_B_v1.canonical_url, + code='concept_B_1') + self.assertEqual(response.status_code, 404) + self.assertEqual(response.data['issue'][0]['code'], 'not-found') - response = self.client.get( - f'/users/{self.user.mnemonic}/ConceptMap/$translate?' - f'system={self.org_source_B_v1.canonical_url}&code=concept_1') + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_translate_with_url_negative(self): + response = self.translate(f'/users/{self.user.mnemonic}', url='http://unknown-concept-map/url', + system=self.org_source_B_v1.canonical_url, code='concept_B_1') + self.assertEqual(response.status_code, 404) + self.assertEqual(response.data['issue'][0]['expression'], ['url']) - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.data, { - 'resourceType': 'Parameters', - 'parameter': [OrderedDict( - [('name', 'result'), ('valueBoolean', False)])]}) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_translate_with_target(self): + self.assertTranslation( + self.translate(f'/users/{self.user.mnemonic}', url='http://localhost/url', + system=self.org_source_B_v1.canonical_url, code='concept_B_1', + targetsystem=self.org_source.canonical_url), + [('concept_1', '/some/url', 'equivalent')]) + + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_translate_with_target_negative(self): + self.assertTranslation( + self.translate(f'/users/{self.user.mnemonic}', url='http://localhost/url', + system=self.org_source_B_v1.canonical_url, code='concept_B_1', + targetsystem=self.org_source_B_v1.canonical_url), []) def newConceptMap(self, owner, identifier, version, group): if not owner: @@ -606,7 +625,7 @@ def newConceptMap(self, owner, identifier, version, group): } - def putConceptMap(self): + def putConceptMap(self, status='draft'): response = self.client.put( f'/users/{self.user.mnemonic}/ConceptMap/{self.user_source.mnemonic}/', HTTP_AUTHORIZATION='Token ' + self.user_token, @@ -626,7 +645,7 @@ def putConceptMap(self): 'version': '1.0', 'name': 'test', 'id': self.user_source.mnemonic, - 'status': 'draft', + 'status': status, 'group': [{'source': self.org_source_B_v1.canonical_url, 'target': self.org_source.canonical_url, 'element': [ @@ -642,151 +661,36 @@ def putConceptMap(self): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_translate_negative(self): - self.putConceptMap() - - response = self.client.get( - f'/users/{self.user.mnemonic}/ConceptMap/$translate?' - f'system={self.org_source_B_v1.canonical_url}&code=concept_1', - HTTP_AUTHORIZATION='Token ' + self.user_token) - - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.data, { - 'resourceType': 'Parameters', - 'parameter': [OrderedDict( - [('name', 'result'), ('valueBoolean', False)] - )]}) - - @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) - @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_translate_with_url(self): - self.putConceptMap() - - response = self.client.get( - '/fhir/ConceptMap/$translate?' - f'url=http://localhost/url&system={self.org_source_B_v1.canonical_url}&code=concept_B_1', - HTTP_AUTHORIZATION='Token ' + self.user_token) - - self.assertEqual(response.status_code, 200) - self.assertDictEqual( - response.data, - { - 'resourceType': 'Parameters', - 'parameter': [ - OrderedDict([ - ('name', 'result'), - ('valueBoolean', True) - ]), - OrderedDict([ - ('name', 'match'), - ( - 'part', - [ - OrderedDict([ - ('name', 'equivalence'), - ('valueCode', 'equivalent') - ]), - OrderedDict([ - ('name', 'concept'), - ('valueCoding', OrderedDict([('system', '/some/url'), ('code', 'concept_1')])) - ]) - ] - ) - ]) - ] - }) - - @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) - @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_translate_with_url_negative(self): - # system/code alone would match mapping_1 (owned by org_source), but the ConceptMap - # identified by 'url' has no such mapping, so no match should be returned. - self.putConceptMap() - - response = self.client.get( - '/fhir/ConceptMap/$translate?' - f'url=http://unknown-concept-map/url&system={self.org_source_B_v1.canonical_url}&code=concept_B_1', - HTTP_AUTHORIZATION='Token ' + self.user_token) - - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.data, { - 'resourceType': 'Parameters', - 'parameter': [OrderedDict( - [('name', 'result'), ('valueBoolean', False)] - )]}) - - @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) - @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_translate_with_target(self): - self.putConceptMap() - - response = self.client.get( - f'/users/{self.user.mnemonic}/ConceptMap/$translate?' - f'system={self.org_source_B_v1.canonical_url}&code=concept_B_1' - f'&targetsystem={self.org_source.canonical_url}', HTTP_AUTHORIZATION='Token ' + self.user_token) - - self.assertEqual(response.status_code, 200) - self.assertDictEqual( - response.data, - { - 'resourceType': 'Parameters', - 'parameter': [ - OrderedDict([ - ('name', 'result'), - ('valueBoolean', True) - ]), - OrderedDict([ - ('name', 'match'), - ( - 'part', - [ - OrderedDict([ - ('name', 'equivalence'), - ('valueCode', 'equivalent') - ]), - OrderedDict([ - ('name', 'concept'), - ('valueCoding', OrderedDict([('system', '/some/url'), ('code', 'concept_1')])) - ]) - ] - ) - ]), - OrderedDict([ - ('name', 'match'), - ( - 'part', - [ - OrderedDict([ - ('name', 'equivalence'), - ('valueCode', 'equivalent') - ]), - OrderedDict([ - ('name', 'concept'), - ('valueCoding', OrderedDict([('system', '/some/url'), ('code', 'concept_1')])) - ]) - ] - ) - ]) - ] - } - ) - - @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) - @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_translate_with_target_negative(self): - self.putConceptMap() - - response = self.client.get( - f'/users/{self.user.mnemonic}/ConceptMap/$translate?' - f'system={self.org_source_B_v1.canonical_url}&code=concept_B_1' - f'&targetsystem={self.org_source_B_v1.canonical_url}', - HTTP_AUTHORIZATION='Token ' + self.user_token) - - self.assertEqual(response.status_code, 200) - self.assertDictEqual(response.data, { - 'resourceType': 'Parameters', - 'parameter': [OrderedDict( - [('name', 'result'), ('valueBoolean', False)])]}) + def test_group_equivalence_is_the_map_type_relationship(self): + """ + A group target's equivalence is its map type's relationship in R4 ($translate's table), and a written + equivalence is stored as the map type it is, so it reads back the same; one with no map type (relatedto, + inexact) is kept as it is. + One element per code, whatever the number of its targets. + """ + written = ['equivalent', 'wider', 'narrower', 'disjoint', 'relatedto', 'equal', 'subsumes', 'specializes', + 'inexact', 'unmatched'] + stored = ['SAME-AS', 'NARROWER-THAN', 'BROADER-THAN', 'NOT-SAME-AS', 'relatedto', 'SAME-AS', 'NARROWER-THAN', + 'BROADER-THAN', 'inexact', 'NOT-SAME-AS'] + read = ['equivalent', 'wider', 'narrower', 'disjoint', 'relatedto', 'equivalent', 'wider', 'narrower', + 'relatedto', 'disjoint'] + group = [{'source': self.org_source_B_v1.canonical_url, 'target': self.org_source.canonical_url, + 'element': [{'code': 'concept_B_1', 'target': [ + {'code': f'concept_{index}', 'equivalence': equivalence} + for index, equivalence in enumerate(written)]}]}] + response = self.client.post( + f'/users/{self.user.mnemonic}/ConceptMap/', HTTP_AUTHORIZATION='Token ' + self.user_token, + data={**self.newConceptMap(None, 'equivalences', '1.0', group), 'status': 'active'}, format='json') + self.assertEqual(response.status_code, 201, response.data) + version = Source.objects.get(mnemonic='equivalences', version='1.0', user=self.user) + self.assertEqual(list(version.get_mappings_queryset().order_by('id').values_list('map_type', flat=True)), + stored) + + resource = self.client.get(f'/users/{self.user.mnemonic}/ConceptMap/equivalences/', + HTTP_AUTHORIZATION='Token ' + self.user_token).data + [element] = resource['group'][0]['element'] + self.assertEqual(element['code'], 'concept_B_1') + self.assertEqual([target['equivalence'] for target in element['target']], read) def test_unable_to_represent_as_fhir(self): instance = Source(id='1', uri='/invalid/uri') diff --git a/core/concept_maps/views.py b/core/concept_maps/views.py index e1aa117fa..acdb8fb8f 100644 --- a/core/concept_maps/views.py +++ b/core/concept_maps/views.py @@ -1,57 +1,32 @@ import logging -from django.core.exceptions import ValidationError -from django.db.models import Q from django.http import Http404 -from pydash import get -from core.bundles.serializers import FHIRBundleSerializer -from core.code_systems.views import FhirParameterRequiredError from core.common.constants import HEAD -from core.common.fhir_helpers import translate_fhir_query -from core.common.permissions import CanViewConceptDictionary -from core.common.serializers import IdentifierSerializer -from core.concept_maps.constants import RESOURCE_TYPE -from core.concept_maps.serializers import ConceptMapDetailSerializer, ConceptMapParametersSerializer -from core.concepts.permissions import CanAccessParentDictionary -from core.mappings.constants import SAME_AS -from core.mappings.views import MappingListView +from core.common.permissions import HasOwnership +from core.concept_maps.serializers import ConceptMapDetailSerializer +from core.fhir.adapters import r4 +from core.fhir.base import FhirViewMixin, FhirOperationView +from core.fhir.ops.translate import translate +from core.fhir.search import FhirSearchMixin, FhirReadMixin from core.sources.views import SourceListView, SourceRetrieveUpdateDestroyView logger = logging.getLogger('oclapi') -class ConceptMapListView(SourceListView): +class ConceptMapListView(FhirViewMixin, FhirSearchMixin, SourceListView): + resource_type = 'ConceptMap' serializer_class = ConceptMapDetailSerializer - @staticmethod - def bundle_response(data, paginator): - bundle = FHIRBundleSerializer( - {'meta': {}, 'total': len(data), 'type': 'searchset', 'entry': FHIRBundleSerializer.convert_to_entry(data)}, - context={'paginator': paginator} - ) - return bundle.data - - def get_filter_params(self, default_version_to_head=True): - return super().get_filter_params(False) - - def apply_query_filters(self, queryset): - query_fields = list(self.serializer_class.Meta.fields) - queryset = translate_fhir_query(query_fields, self.request.query_params, queryset) - return queryset - - def apply_filters(self, queryset): - queryset = queryset.exclude(version=HEAD).filter(is_latest_version=True) - return self.apply_query_filters(queryset) - def get_serializer_class(self): return self.serializer_class def get_detail_serializer(self, obj): - return self.serializer_class(obj) + return self.serializer_class(obj, context=self.get_serializer_context()) -class ConceptMapRetrieveUpdateView(SourceRetrieveUpdateDestroyView): +class ConceptMapRetrieveUpdateView(FhirViewMixin, FhirReadMixin, SourceRetrieveUpdateDestroyView): + resource_type = 'ConceptMap' serializer_class = ConceptMapDetailSerializer def get_filter_params(self, default_version_to_head=True): @@ -64,145 +39,25 @@ def get_queryset(self): return queryset.exclude(version=HEAD) def get_object(self, queryset=None): + """ + A PUT or PATCH to a ConceptMap that doesn't exist gets None, which creates it, so it needs what a POST needs: + staff, the owner, or a member of the owning org (HasOwnership on the owner). Anyone else is refused as they are + for an existing one. Any other request for one is a 404 (a read's from FhirReadMixin). + """ try: return super().get_object(queryset) except Http404: + if self.request.method not in ('PUT', 'PATCH'): + raise + if not HasOwnership().has_object_permission(self.request, self, self.parent_resource): + self.permission_denied(self.request) return None def get_detail_serializer(self, obj): - return ConceptMapDetailSerializer(obj) - - -class ConceptMapTranslateView(MappingListView): - serializer_class = ConceptMapParametersSerializer - - def get_permissions(self): - return [CanAccessParentDictionary(), CanViewConceptDictionary()] - - def get_serializer_class(self): - return ConceptMapParametersSerializer - - # $translate answers with the translation of the mappings apply_filters names, and nothing else. The mapping - # list's shortcuts (a search, a sort or a faceted filter, facets, search stats, a CSV export) would answer in its - # place, and the search ones skip apply_filters, so each is off. Their parameters are ignored like any other this - # operation doesn't define. - allow_csv_export = False - - def should_perform_es_search(self): - return False - - def only_facets(self): - return False + return ConceptMapDetailSerializer(obj, context=self.get_serializer_context()) - def only_search_stats(self): - return False - def should_include_facets(self): - return False - - def verify_scope(self): - """ - Override to not verify scope and allow POSTing for everyone - :return: None - """ - - def apply_filters(self, queryset): - # get_serializer renders the answer, so a POSTed Parameters resource is read by the serializer class itself - if self.request.method in ['POST', 'PUT']: - parameters = self.get_serializer_class()(data=self.request.data) - else: - parameters = self.get_serializer_class().parse_query_params(self.request.query_params) - - if not parameters.is_valid(): - raise ValidationError(message=parameters.errors) - - params = parameters.validated_data - params = params.get('parameters', {}) - - url = params.get('url') - code = params.get('code') - system = params.get('system') - targetsystem = params.get('targetsystem') - # TODO: implement 'source' and 'target' - if not code: - raise FhirParameterRequiredError('A code to translate is required.', expression='code') - if not system: - raise FhirParameterRequiredError( - 'system is required with a code: the code system the code is from.', expression='system') - - if url: - queryset = queryset.filter(parent__canonical_url=url) - queryset = queryset.filter(from_concept_code=code) - system_url = IdentifierSerializer.convert_fhir_url_to_ocl_uri(system, 'sources') - queryset = queryset.filter(Q(from_source__canonical_url=system) | - Q(from_source_url=system_url) | - Q(from_source__uri=system_url)) - if targetsystem: - target_url = IdentifierSerializer.convert_fhir_url_to_ocl_uri(targetsystem, 'sources') - queryset = queryset.filter(Q(to_source__canonical_url=targetsystem) | - Q(to_source_url=target_url) | - Q(to_source__uri=target_url)) - return queryset - - def get_serializer(self, *args, **kwargs): - instance = get(args, '0') - many = kwargs.get('many', False) - if many: - if isinstance(instance, list) and len(instance) != 0: - matches = [] - for mapping in instance: - equivalence = mapping.map_type - if mapping.map_type == SAME_AS: - equivalence = "equivalent" - - to_url = None - if mapping.to_source and mapping.to_source.canonical_url: - to_url = mapping.to_source.canonical_url - elif mapping.to_source_url: - to_url = IdentifierSerializer.convert_ocl_uri_to_fhir_url(mapping.to_source_url, RESOURCE_TYPE) - elif mapping.to_source: - to_url = IdentifierSerializer.convert_ocl_uri_to_fhir_url(mapping.to_source.uri, RESOURCE_TYPE) - - matches.append({ - 'name': 'match', - 'part': [ - { - 'name': 'equivalence', - 'valueCode': equivalence - }, - { - 'name': 'concept', - 'valueCoding': { - 'system': to_url, - 'code': mapping.to_concept_code, - 'userSelected': False - } - } - ] - }) - return ConceptMapParametersSerializer({'parameter': [ - { - 'name': 'result', - 'valueBoolean': True - }, - *matches - ]}) - - return ConceptMapParametersSerializer({'parameter': [ - { - 'name': 'result', - 'valueBoolean': False - } - ]}) - return super().get_serializer(instance, many, *args, **kwargs) - - # Change POST behavior to get - def post(self, request, *args, **kwargs): - """ - Change POST behavior to simply list - :param request: - :param args: - :param kwargs: - :return: parameters - """ - return self.get(request, *args, **kwargs) +class ConceptMapTranslateView(FhirOperationView): + """ConceptMap/$translate, GET and POST, at the global base and at owner bases: core.fhir.ops.translate, in R4.""" + operation = staticmethod(translate) + adapter = staticmethod(r4.translate_parameters) diff --git a/core/fhir/__init__.py b/core/fhir/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/adapters/__init__.py b/core/fhir/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/adapters/r4.py b/core/fhir/adapters/r4.py new file mode 100644 index 000000000..e353cf191 --- /dev/null +++ b/core/fhir/adapters/r4.py @@ -0,0 +1,203 @@ +""" +The FHIR R4 wire form of the version-neutral (R5-shaped) answers core.fhir.ops build, as R4 Parameters resources. + +- $lookup: R4's outputs in R4's order (name, version, display, designation, property), with code, system and abstract + after display, as FHIRsmith and the terminology ecosystem IG return them. inactive is a property. +- $validate-code: result first, then message, display, code, system, version, inactive, the unknown system, and the + issues as an OperationOutcome. +- $translate: result, message, and each match's equivalence (a ConceptMap relationship as R4's equivalence), concept + and source. +""" +from core.concept_maps.constants import ( + EQUIVALENT, NOT_RELATED_TO, RELATED_TO, SOURCE_IS_BROADER, SOURCE_IS_NARROWER) +from core.fhir.base import FHIR_RELEASE, prune + +# R5 relationship -> R4 equivalence, as HL7's R4/R5 ConceptMap conversion maps them: the target is wider than a source +# that is narrower than it; not-related-to is disjoint +EQUIVALENCES = { + EQUIVALENT: 'equivalent', + RELATED_TO: 'relatedto', + SOURCE_IS_NARROWER: 'wider', + SOURCE_IS_BROADER: 'narrower', + NOT_RELATED_TO: 'disjoint', +} + + +def parameter(name, value_type, value): + return {'name': name, value_type: value} + + +def parameters(items): + """A Parameters resource of (name, value type, value) items, leaving out those without a value.""" + return {'resourceType': 'Parameters', 'parameter': [ + item if isinstance(item, dict) else parameter(*item) for item in items + if isinstance(item, dict) or item[2] is not None]} + + +def part_parameter(name, parts): + return {'name': name, 'part': [parameter(*part) for part in parts if part[2] is not None]} + + +# The metadata (core.fhir.capabilities) in R4: the service, SVCM 1.5.1's Terminology Repository, which the +# CapabilityStatement instantiates, and the definitions of the operations and search parameters it lists +SERVICE_NAME, SERVICE_TITLE = 'OCLTerminologyService', 'Open Concept Lab FHIR terminology service' +PUBLISHER = 'Open Concept Lab' +SVCM_TERMINOLOGY_REPOSITORY = ( + 'https://profiles.ihe.net/ITI/SVCM/CapabilityStatement/IHE.SVCM.TerminologyRepository|1.5.1') +OPERATION_DEFINITION = 'http://hl7.org/fhir/OperationDefinition/' +SYSTEM_OPERATION_DEFINITIONS = {'versions': 'CapabilityStatement-versions'} +SEARCH_PARAMETER = 'http://hl7.org/fhir/SearchParameter/' +SEARCH_PARAMETER_TYPES = { + '_id': 'token', '_lastUpdated': 'date', 'url': 'uri', 'version': 'token', 'status': 'token', 'identifier': 'token', + 'name': 'string', 'title': 'string', 'description': 'string', 'publisher': 'string', 'system': 'uri', + 'code': 'token', 'language': 'token', 'reference': 'uri', 'source-system': 'uri', 'source-uri': 'reference', + 'target-system': 'uri', 'target-uri': 'reference'} +# R4 defines these once for every resource, or for every conformance resource; the others per resource type +SHARED_SEARCH_PARAMETERS = { + '_id': 'Resource-id', '_lastUpdated': 'Resource-lastUpdated', + **{name: f'conformance-{name}' for name in ( + 'url', 'version', 'status', 'identifier', 'name', 'title', 'description', 'publisher')}} +# The search parameters OCL reads otherwise than R4 defines them, and how (core.fhir.search's mapped) +SEARCH_PARAMETER_DOCUMENTATION = { + ('ConceptMap', f'{side}-uri'): ( + f'Read as `{side}-system`: OCL keeps no {side} value set on a ConceptMap, so this matches the system of a ' + f"group's {side}, not a value set.") for side in ('source', 'target')} + + +def search_parameter(resource_type, name): + """A search parameter as a CapabilityStatement lists it: its name, R4 definition and type, and how OCL reads it + when that isn't as R4 defines it.""" + definition = SHARED_SEARCH_PARAMETERS.get(name, f'{resource_type}-{name}') + documentation = SEARCH_PARAMETER_DOCUMENTATION.get((resource_type, name)) + return {'name': name, 'definition': SEARCH_PARAMETER + definition, 'type': SEARCH_PARAMETER_TYPES[name], + **({'documentation': documentation} if documentation else {})} + + +def operation(name, resource_type=None): + """An operation as a CapabilityStatement lists it: its name and R4 OperationDefinition.""" + definition = f'{resource_type}-{name}' if resource_type else SYSTEM_OPERATION_DEFINITIONS[name] + return {'name': name, 'definition': OPERATION_DEFINITION + definition} + + +def described(resource_type, metadata): + """What a CapabilityStatement and a TerminologyCapabilities both say of the service, in R4's element order.""" + return { + 'resourceType': resource_type, 'name': SERVICE_NAME, 'title': SERVICE_TITLE, 'status': 'active', + 'date': metadata['date'], 'publisher': PUBLISHER, 'kind': 'instance', 'software': metadata['software'], + 'implementation': {'description': SERVICE_TITLE, 'url': metadata['url']}, + } + + +def capability_statement(statement): + """The CapabilityStatement (core.fhir.capabilities.capability_statement) in R4: fhirVersion 4.0.1, SVCM.""" + resource = described('CapabilityStatement', statement) + software, implementation = resource.pop('software'), resource.pop('implementation') + rest = {'mode': 'server', 'resource': [{ + 'type': item['type'], + 'interaction': [{'code': code} for code in item['interactions']], + 'searchParam': [search_parameter(item['type'], name) for name in item['search_parameters']], + 'operation': [operation(name, item['type']) for name in item['operations']], + } for item in statement['resources']], 'operation': [operation(name) for name in statement['operations']]} + return { + **resource, 'instantiates': [SVCM_TERMINOLOGY_REPOSITORY], 'software': software, + 'implementation': implementation, 'fhirVersion': FHIR_RELEASE, 'format': statement['formats'], + 'rest': [prune(rest)], + } + + +def terminology_capabilities(capabilities): + """ + The TerminologyCapabilities (core.fhir.capabilities.terminology_capabilities) in R4: each code system's versions, + the default one marked isDefault, then the blocks of the operations routed. + """ + return prune({ + **described('TerminologyCapabilities', capabilities), + 'codeSystem': [{'uri': item['url'], 'version': [ + {'code': version, **({'isDefault': True} if version == item['default'] else {})} + for version in item['versions']]} for item in capabilities['code_systems']], + **{block: capabilities[block] for block in ('expansion', 'translation') if block in capabilities}, + }) + + +def versions_parameters(answer): + """$versions' answer (core.fhir.capabilities.versions) as R4 Parameters: each version, then the default.""" + return parameters([*(('version', 'valueCode', version) for version in answer['versions']), + ('default', 'valueCode', answer['default'])]) + + +def lookup_parameters(answer): + """$lookup's answer (core.fhir.ops.lookup) as R4 Parameters.""" + return parameters([ + ('name', 'valueString', answer['name']), + ('version', 'valueString', answer['version']), + ('display', 'valueString', answer['display']), + ('code', 'valueCode', answer['code']), + ('system', 'valueUri', answer['system']), + ('abstract', 'valueBoolean', answer['abstract']), + *(part_parameter('designation', [ + ('language', 'valueCode', designation['language']), + ('use', 'valueCoding', designation['use']), + ('value', 'valueString', designation['value'])]) for designation in answer['designations']), + part_parameter('property', [('code', 'valueCode', 'inactive'), ('value', 'valueBoolean', answer['inactive'])]), + ]) + + +def validate_code_parameters(answer): + """$validate-code's answer (core.fhir.ops.validate_code) as R4 Parameters.""" + unknown_name, unknown_system = answer.get('unknown') or (None, None) + issues = answer['issues'] + return parameters([ + ('result', 'valueBoolean', answer['result']), + ('message', 'valueString', answer['message']), + ('display', 'valueString', answer.get('display')), + ('code', 'valueCode', answer['code']), + ('system', 'valueUri', answer['system']), + ('version', 'valueString', answer.get('version')), + ('inactive', 'valueBoolean', answer.get('inactive')), + (unknown_name, 'valueCanonical', unknown_system), + ('issues', 'resource', {'resourceType': 'OperationOutcome', 'issue': issues} if issues else None), + ]) + + +def equivalence(relationship_code): + """A relationship's R4 equivalence.""" + return EQUIVALENCES[relationship_code] + + +def translate_parameters(answer): + """$translate's answer (core.fhir.ops.translate) as R4 Parameters: result, message, and each match's parts.""" + items = [{'name': 'result', 'valueBoolean': answer['result']}] + if answer.get('message'): + items.append({'name': 'message', 'valueString': answer['message']}) + for match in answer['matches']: + items.append({'name': 'match', 'part': [ + {'name': 'equivalence', 'valueCode': equivalence(match['relationship'])}, + {'name': 'concept', 'valueCoding': match['concept']}, + {'name': 'source', 'valueUri': match['source']}, + ]}) + return {'resourceType': 'Parameters', 'parameter': items} + + +def expansion_value_set(resource, answer): + """ + $expand's answer (core.fhir.ops.expand) as an R4 ValueSet: `resource`, the value set version as a read shows it + without its compose, with the expansion: the stored expansion's identifier and timestamp, total, offset, the inputs + echoed and each used-codesystem, then the page of members. + """ + def present(items): + return {key: value for key, value in items if value not in (None, [])} + + expansion = { + 'identifier': answer['identifier'], + 'timestamp': answer['timestamp'], + 'total': answer['total'], + 'offset': answer['offset'], + 'parameter': [parameter(*item) for item in answer['parameters']] + [ + parameter('used-codesystem', 'valueUri', system) for system in answer['used']], + 'contains': [present(( + ('system', member['system']), ('version', member['version']), ('code', member['code']), + ('display', member['display']), ('inactive', member['inactive']), + ('designation', [present(designation.items()) for designation in member['designations'] or []]), + )) for member in answer['contains']], + } + return {**resource, 'expansion': present(expansion.items())} diff --git a/core/fhir/base.py b/core/fhir/base.py new file mode 100644 index 000000000..c79137a71 --- /dev/null +++ b/core/fhir/base.py @@ -0,0 +1,601 @@ +""" +The FHIR view layer: what every FHIR view shares, whatever its operation. + +- Media types: FHIR JSON and XML in and out, with the aliases FHIR clients use, and `_format`. +- The `fhirVersion` MIME parameter: R4 (4.0) is served and any other version is refused. +- XML through fhir.resources' R4B models, which are wire-identical to R4 for the terminology resources, so + elements always come out in FHIR's order. +- One reader for operation parameters, from a GET query string or a POSTed Parameters resource. +- The FHIR base URL of a request, from its own host. +- Errors as OperationOutcome (core.fhir.outcomes), in views and in the middleware. +""" +import gzip +import json +import logging +import re +import types +import typing +import zlib +from functools import lru_cache + +from django.core.exceptions import BadRequest, SuspiciousOperation +from django.http import HttpResponse +from fhir.resources.R4B import get_fhir_model_class +from fhir_core.types import FhirBase +from lxml import etree +from pydantic import ValidationError +from rest_framework.exceptions import NotAcceptable, ParseError, UnsupportedMediaType +from rest_framework.negotiation import DefaultContentNegotiation +from rest_framework.parsers import BaseParser, FormParser, JSONParser, MultiPartParser +from rest_framework.permissions import AllowAny +from rest_framework.renderers import BaseRenderer, JSONRenderer +from rest_framework.response import Response +from rest_framework.utils.encoders import JSONEncoder +from rest_framework.utils.mediatypes import _MediaType +from rest_framework.versioning import AcceptHeaderVersioning +from rest_framework.views import APIView + +from core.common.throttling import ThrottleUtil +from core.fhir import outcomes + +logger = logging.getLogger('oclapi') + +FHIR_VERSION = '4.0' # the fhirVersion MIME parameter, and what $versions answers +FHIR_RELEASE = '4.0.1' # CapabilityStatement.fhirVersion +ACCEPTED_FHIR_VERSIONS = (FHIR_VERSION, FHIR_RELEASE) + +FHIR_JSON = 'application/fhir+json' +FHIR_XML = 'application/fhir+xml' +# Other names for the two formats: the short _format values, the DSTU2 types, and the generic XML types R4 says +# a server SHOULD accept. Plain application/json stays itself, so OCL's own clients keep getting it. +MEDIA_TYPE_ALIASES = { + 'json': FHIR_JSON, + 'application/json+fhir': FHIR_JSON, + 'xml': FHIR_XML, + 'application/xml+fhir': FHIR_XML, + 'application/xml': FHIR_XML, + 'text/xml': FHIR_XML, +} +FORMAT_PARAMETERS = ('_format', 'format') +# Protocol parameters: they choose the response format, and are never operation or search input +CONTROL_PARAMETERS = frozenset({'_format', 'format', '_pretty'}) +RESOURCE_TYPES = ('CodeSystem', 'ValueSet', 'ConceptMap') +OWNER_BASE_PATHS = ('metadata', '$versions') # what an owner base answers besides its resource types + +FHIR_NS = 'http://hl7.org/fhir' +XHTML_NS = 'http://www.w3.org/1999/xhtml' +EXTENSION_ELEMENTS = ('extension', 'modifierExtension') +EMPTY_VALUES = (None, '', {}, []) +# What XML 1.0 can't hold: C0 controls other than tab, LF and CR, lone surrogates, U+FFFE and U+FFFF +XML_FORBIDDEN_CHARACTERS = re.compile('[^\x09\x0a\x0d\x20-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]') +REPLACEMENT_CHARACTER = '\ufffd' +XML_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_.-]*$') +LOGGED_DROPPED_ELEMENTS = set() # what the XML has left out so far: each kind is logged once per process + + +def xml_parser(): + """A parser that never fetches or expands entities: a request body is untrusted.""" + return etree.XMLParser(resolve_entities=False, no_network=True, load_dtd=False, remove_comments=True) + + +def is_fhir_path(path): + """ + Whether a path is under a FHIR base: the global /fhir/, or at an owner base (/orgs/{org}/, /users/{user}/) a FHIR + resource type, its metadata or $versions. + """ + segments = path.strip('/').split('/') + return segments[0] == 'fhir' or (len(segments) >= 3 and segments[0] in ('orgs', 'users') and ( + segments[2] in RESOURCE_TYPES or (len(segments) == 3 and segments[2] in OWNER_BASE_PATHS))) + + +def fhir_base_url(request, kwargs=None): + """ + The FHIR base a request came in on, from its own scheme and host (the forwarded ones behind the load balancer), + so links work on every host OCL serves FHIR from: /fhir, or the owner base /orgs/{org} or /users/{user}. + """ + kwargs = kwargs or {} + if kwargs.get('org'): + path = f"/orgs/{kwargs['org']}" + elif kwargs.get('user'): + path = f"/users/{kwargs['user']}" + else: + path = '/fhir' + return request.build_absolute_uri(path) + + +def normalize_media_type(media_range): + """A media range from Accept, Content-Type or _format, with FHIR's other names replaced by the formal type.""" + main, separator, params = media_range.partition(';') + main = main.strip().lower().replace(' ', '+') # an unencoded + in _format arrives as a space + return MEDIA_TYPE_ALIASES.get(main, main) + separator + params + + +def parse_media_type(media_type): + """Treat undecodable MIME parameters as a bad request, including unknown RFC 2231 encodings.""" + try: + return _MediaType(media_type) + except (LookupError, ValueError) as exc: + raise BadRequest('The media type parameters could not be parsed.') from exc + + +def accepted_media_ranges(request): + """The media ranges a request accepts: its _format, else its Accept header.""" + ranges = [item.strip() for item in request.META.get('HTTP_ACCEPT', '').split(',') if item.strip()] + for media_range in ranges: + parse_media_type(media_range) # validate before _format and browser preference shortcuts + for name in FORMAT_PARAMETERS: + value = request.GET.get(name) + if value: + return [normalize_media_type(value)] + if any(item.startswith('text/html') for item in ranges) and not any('fhir' in item for item in ranges): + return ['application/json'] # a browser: JSON as before, not the application/xml it lists after text/html + return [normalize_media_type(item) for item in ranges] or ['*/*'] + + +def quality(media_range): + try: + return float(parse_media_type(media_range).params.get('q', 1)) + except ValueError: + return 0.0 + + +def media_type_with_params(media_type, media_range): + """The renderer's media type, carrying the parameters the client sent other than q (such as fhirVersion).""" + params = [param.strip() for param in media_range.split(';')[1:]] + params = [param for param in params if param and not param.lower().startswith('q=')] + return '; '.join([media_type, *params]) + + +class FhirContentNegotiation(DefaultContentNegotiation): + """ + Content negotiation for FHIR views: `_format` (or `format`) overrides Accept, FHIR's other media type names are + understood, and quality values are honoured. DRF's own `?format=` renderer filter is not used. + """ + + def select_parser(self, request, parsers): + content_type = parse_media_type(normalize_media_type(request.content_type or '')) + return next((parser for parser in parsers if parse_media_type(parser.media_type).match(content_type)), None) + + def select_renderer(self, request, renderers, format_suffix=None): + """Each renderer takes its quality from its most specific match, including q=0 exclusions.""" + # Reject a malformed Content-Type before choosing XML, so every parsing failure can use safe JSON. + parse_media_type(request.META.get('CONTENT_TYPE', request.META.get('HTTP_CONTENT_TYPE', ''))) + ranges = sorted(enumerate(accepted_media_ranges(request)), + key=lambda item: (-parse_media_type(item[1]).precedence, -quality(item[1]))) + candidates = [] + for renderer in renderers: + for position, media_range in ranges: + parsed_range = parse_media_type(media_range) + if parse_media_type(renderer.media_type).match(parsed_range): + preference = (quality(media_range), parsed_range.precedence, -position) + if preference[0] > 0: + candidates.append((preference, renderer, media_range)) + break # a less specific range cannot override this renderer's quality + if candidates: + _, renderer, media_range = max(candidates, key=lambda candidate: candidate[0]) + return renderer, media_type_with_params(renderer.media_type, media_range) + raise NotAcceptable(available_renderers=renderers) + + +class FhirVersioning(AcceptHeaderVersioning): + """ + The fhirVersion MIME parameter, on Accept (or _format) and on Content-Type. R4 is served: 4.0 (or 4.0.1, or no + parameter) is accepted, and any other version is refused, 406 on Accept and 415 on Content-Type. Django + lowercases parameter names, so the parameter is read without regard to case. + """ + version_param = 'fhirVersion' + default_version = FHIR_VERSION + allowed_versions = ACCEPTED_FHIR_VERSIONS + invalid_version_message = f'This server speaks FHIR R4: fhirVersion={FHIR_VERSION}.' + + def version_of(self, media_type): + return parse_media_type(media_type or '').params.get(self.version_param.lower()) + + def determine_version(self, request, *args, **kwargs): + content_version = self.version_of(request.content_type) + if content_version is not None and content_version not in self.allowed_versions: + raise UnsupportedMediaType(request.content_type, detail=self.invalid_version_message) + accept_version = self.version_of(request.accepted_media_type) + if accept_version is not None and accept_version not in self.allowed_versions: + raise NotAcceptable(self.invalid_version_message) + return self.default_version + + +def prune(value): + """FHIR has no empty values: drop None, '', {} and [] at every depth.""" + if isinstance(value, dict): + items = ((key, prune(item)) for key, item in value.items()) + return {key: item for key, item in items if item not in EMPTY_VALUES} + if isinstance(value, (list, tuple)): + return [item for item in (prune(item) for item in value) if item not in EMPTY_VALUES] + return value + + +def from_xml(content): + """FHIR XML -> FHIR JSON, through the R4B model of the root element. Raises ParseError.""" + try: + root = etree.fromstring(content, parser=xml_parser()) + name = etree.QName(root) + if name.namespace != FHIR_NS: + raise ValueError(f'the root element {root.tag} is not a FHIR resource') + resource = get_fhir_model_class(name.localname).model_validate_xml(content, xmlparser=xml_parser()) + except Exception as ex: + message = (str(ex).splitlines() or [type(ex).__name__])[0] + raise ParseError(f'The body is not a valid FHIR XML resource: {message}') from ex + return json.loads(resource.model_dump_json()) + + +def json_values(data): + """The data as the JSON response carries it: DRF's encoder writes datetimes, dates, decimals and UUIDs.""" + return json.loads(json.dumps(data, cls=JSONEncoder)) + + +def xml_safe(value): + """(value, how many characters were replaced): every character XML 1.0 forbids becomes U+FFFD.""" + if isinstance(value, str): + return XML_FORBIDDEN_CHARACTERS.subn(REPLACEMENT_CHARACTER, value) + if isinstance(value, dict): + items = [(xml_safe(key), xml_safe(item)) for key, item in value.items()] + return {key: item for (key, _), (item, _) in items}, sum(left + right for (_, left), (_, right) in items) + if isinstance(value, (list, tuple)): + items = [xml_safe(item) for item in value] + return [item for item, _ in items], sum(count for _, count in items) + return value, 0 + + +def element_type(annotation): + """(the R4B model of a field's values, or None for a primitive; whether the field repeats).""" + repeats = False + while True: + origin = typing.get_origin(annotation) + if origin in (typing.Union, types.UnionType): + annotation = next(arg for arg in typing.get_args(annotation) if arg is not type(None)) + elif origin is list: + repeats, annotation = True, typing.get_args(annotation)[0] + elif origin is typing.Annotated: + annotation = typing.get_args(annotation)[0] + else: + break + is_model = isinstance(annotation, type) and issubclass(annotation, FhirBase) + return (annotation.get_model_klass() if is_model else None), repeats + + +@lru_cache(maxsize=None) +def model_elements(model_class): + """An R4B model's elements by their JSON name: (field name, the model of its values or None, repeats).""" + return {field.alias or name: (name, *element_type(field.annotation)) + for name, field in model_class.model_fields.items() if name != 'fhir_comments'} + + +def resource_model(data): + """The R4B model of a FHIR JSON resource, or None when it isn't one.""" + if not isinstance(data, dict) or not isinstance(data.get('resourceType'), str): + return None + try: + return get_fhir_model_class(data['resourceType']) + except ValueError: + return None + + +def log_dropped(model_class, key, reason): + if (model_class.__name__, key, reason) not in LOGGED_DROPPED_ELEMENTS: + LOGGED_DROPPED_ELEMENTS.add((model_class.__name__, key, reason)) + logger.warning('FHIR XML: %s.%s %s', model_class.__name__, key, reason) + + +def xhtml_markup(text): + """A narrative div as XHTML markup: the text itself when it is an XHTML div, else the text inside one.""" + try: + if etree.fromstring(text, parser=xml_parser()).tag == f'{{{XHTML_NS}}}div': + return text + except etree.XMLSyntaxError: + pass + return etree.tostring(xhtml_div(text), encoding='unicode') + + +def conform_value(value_model, value): + """One value of an element, conformed, or None when it doesn't have the element's shape.""" + if value_model is None: + return None if isinstance(value, (dict, list)) else value + if not isinstance(value, dict): + return None + model_class = resource_model(value) if 'resourceType' in value else value_model # a resource names its type + return conform(model_class, value) if model_class else None + + +def conform(model_class, data): + """ + FHIR JSON reduced to what the R4 model defines, at every depth. A key R4 doesn't define (OCL's revisionDate) and + a value of the wrong shape are left out, each kind logged once: that is the content's problem, not the wire + format's. The JSON response itself is not changed. + """ + elements, result = model_elements(model_class), {} + for key, value in data.items(): + if key == 'resourceType': + result[key] = value + continue + if key not in elements: + log_dropped(model_class, key, 'is an element R4 does not define, so the XML leaves it out') + continue + _, value_model, repeats = elements[key] + if isinstance(value, list) and not repeats: # OCL's ConceptMap.identifier is a list; R4 allows one + if len(value) > 1: + log_dropped(model_class, key, 'repeats, but not in R4, so the XML keeps only the first') + value = value[:1] + items = value if isinstance(value, list) else [value] + kept = [item for item in (conform_value(value_model, item) for item in items) if item is not None] + if len(kept) < len(items): + log_dropped(model_class, key, 'has values of a shape R4 does not allow, so the XML leaves them out') + if key == 'div' and kept: + kept = [xhtml_markup(kept[0])] + if kept: + result[key] = kept if repeats else kept[0] + return result + + +def construct(model_class, data): + """ + A model instance from conformed FHIR JSON, nested models included, built without validation, so the model + still writes its elements in FHIR's order. For what the model refuses only on a pattern, such as an OCL id + with '_' or '@', or a code with a trailing space. + """ + elements, values = model_elements(model_class), {} + for key, value in data.items(): + if key == 'resourceType': + continue + name, value_model, repeats = elements[key] + items = [construct_value(value_model, item) for item in (value if repeats else [value])] + values[name] = items if repeats else items[0] + return model_class.model_construct(**values) + + +def construct_value(value_model, value): + if value_model is None: + return value if isinstance(value, bool) else str(value) # the writer takes text, and booleans as they are + return construct(resource_model(value) or value_model, value) + + +def to_xml(data): + """ + FHIR JSON -> FHIR XML, always through the R4B model, so in FHIR's element order. It starts from the values the + JSON response carries. Keys R4 doesn't define are left out; a resource the model refuses on a pattern is built + without validation; characters XML 1.0 forbids become U+FFFD. Data that isn't a FHIR resource is answered with + an OperationOutcome. + """ + if data is None: + return b'' + data = json_values(data) + model_class = resource_model(data) + if model_class is None: + data = outcomes.operation_outcome(outcomes.issue('not-supported', 'This response has no FHIR XML form.')) + model_class = resource_model(data) + data, replaced = xml_safe(data) + if replaced: + logger.warning('FHIR XML: %s character(s) XML cannot hold became U+FFFD in a %s', replaced, + data['resourceType']) + data = prune(conform(model_class, prune(data))) + try: + instance = model_class.model_validate(data) + except ValidationError: + instance = construct(model_class, data) + try: + return instance.model_dump_xml() + except Exception: + logger.exception('FHIR XML: the %s model could not write it; written element by element', model_class.__name__) + root = etree.Element(f"{{{FHIR_NS}}}{data['resourceType']}", nsmap={None: FHIR_NS}) + write_xml_children(root, data, resource=True) + return etree.tostring(root, xml_declaration=True, encoding='utf-8') + + +def write_xml_children(element, obj, resource=False, extension=False): + """The last resort writer: element by element, in the data's own order.""" + for key, value in obj.items(): + if key == 'resourceType' or not XML_NAME.match(key): + continue + if not resource and (key == 'id' or (extension and key == 'url')) and not isinstance(value, (dict, list)): + element.set(key, str(value)) # an element's id and an extension's url are XML attributes + elif key == 'div' and isinstance(value, str): + element.append(xhtml_div(value)) + else: + for item in value if isinstance(value, list) else [value]: + write_xml_element(element, key, item) + + +def write_xml_element(parent, name, value): + child = etree.SubElement(parent, f'{{{FHIR_NS}}}{name}') + if isinstance(value, dict) and value.get('resourceType'): + inner = etree.SubElement(child, f"{{{FHIR_NS}}}{value['resourceType']}") # a resource in a container + write_xml_children(inner, value, resource=True) + elif isinstance(value, dict): + write_xml_children(child, value, extension=name in EXTENSION_ELEMENTS) + else: + child.set('value', ('true' if value else 'false') if isinstance(value, bool) else str(value)) + + +def xhtml_div(text): + try: + div = etree.fromstring(text, parser=xml_parser()) + if div.tag == f'{{{XHTML_NS}}}div': + return div + except etree.XMLSyntaxError: + pass + div = etree.Element(f'{{{XHTML_NS}}}div', nsmap={None: XHTML_NS}) + div.text = text + return div + + +class FhirJsonParser(JSONParser): + media_type = FHIR_JSON + + +class FhirXmlParser(BaseParser): + media_type = FHIR_XML + + def parse(self, stream, media_type=None, parser_context=None): + return from_xml(stream.read()) + + +class FhirJsonRenderer(JSONRenderer): + media_type = FHIR_JSON + format = 'json' + + +class FhirXmlRenderer(BaseRenderer): + media_type = FHIR_XML + format = 'xml' + charset = 'utf-8' + + def render(self, data, accepted_media_type=None, renderer_context=None): + return to_xml(data) + + +def parameter_value(parameter): + """ + A Parameters.parameter's value: its value[x], resource, or parts (as FhirParameters). R4 allows it one of them, so + a parameter with more is invalid; a primitive's _value[x] (its id and extensions) is that same value[x]. + """ + given = sorted({key.lstrip('_') for key in parameter if key.lstrip('_').startswith('value')} | + ({'resource', 'part'} & parameter.keys())) + if len(given) > 1: + raise outcomes.InvalidError( + f"The parameter '{parameter.get('name')}' has more than one value: {', '.join(given)}.", + expression=parameter.get('name')) + for key, value in parameter.items(): + if key.startswith('value'): + return value + if 'resource' in parameter: + return parameter['resource'] + if 'part' in parameter: + return FhirParameters.from_resource({'parameter': parameter['part']}) + return None + + +class FhirParameters: + """ + An operation's input, read the same way from a GET query string and from a POSTed Parameters resource (FHIR JSON, + FHIR XML or a form). Names can repeat. Values keep their FHIR JSON form: a string, boolean or number, a dict for + a Coding, CodeableConcept or resource, and FhirParameters for parts. On a POST, the body's parameters come first, + then the query string's. Protocol parameters such as _format are left out. + """ + + def __init__(self, items=()): + self.items = list(items) + + @classmethod + def from_resource(cls, resource): + return cls((item.get('name'), parameter_value(item)) for item in resource.get('parameter') or []) + + @classmethod + def from_request(cls, request): + items = cls.body_items(request.data) if request.method in ('POST', 'PUT') else [] + return cls(items + cls.query_items(request.query_params)) + + @staticmethod + def query_items(query): + return [(name, value) for name, values in query.lists() if name not in CONTROL_PARAMETERS for value in values] + + @classmethod + def body_items(cls, data): + if not data: + return [] + if hasattr(data, 'lists'): # a form + return cls.query_items(data) + if isinstance(data, dict) and data.get('resourceType') == 'Parameters': + return cls.from_resource(data).items + raise outcomes.InvalidError('The request body must be a Parameters resource.') + + def get(self, name, default=None): + return next((value for key, value in self.items if key == name), default) + + def getlist(self, name): + return [value for key, value in self.items if key == name] + + def __contains__(self, name): + return any(key == name for key, _ in self.items) + + +def outcome_response(request, outcome, status_code, headers=None): + """A middleware outcome in the requested format, or safe JSON 400 for unparseable queries or MIME parameters.""" + renderer = FhirJsonRenderer() + try: + renderer, _ = FhirContentNegotiation().select_renderer( + request, [klass() for klass in FhirViewMixin.renderer_classes]) + except NotAcceptable: + pass + except (BadRequest, SuspiciousOperation) as exc: + status_code, outcome, _ = outcomes.outcome_for_exception(exc) + content_type = f'{renderer.media_type}; charset={renderer.charset}' if renderer.charset else renderer.media_type + response = HttpResponse( + renderer.render(outcome), status=outcomes.response_status(status_code), content_type=content_type) + for key, value in (headers or {}).items(): + response[key] = value + return response + + +def response_json(response): + """A response's JSON body, gunzipped if need be, or None.""" + if 'json' not in response.get('Content-Type', '') or getattr(response, 'streaming', False): + return None + content = response.content + try: + if response.get('Content-Encoding') == 'gzip': + content = gzip.decompress(content) + return json.loads(content) + except (OSError, EOFError, zlib.error, ValueError): + return None + + +def is_outcome_response(response): + """Whether a response is already FHIR: an error from a FHIR view, rather than Django's or another view's.""" + return 'fhir' in response.get('Content-Type', '') or outcomes.is_outcome(response_json(response)) + + +class FhirViewMixin: + """ + The first base of every FHIR view: FHIR JSON and XML in and out, `_format`, the fhirVersion parameter, and every + error as an OperationOutcome with a tx-issue-type, never a 5xx. + """ + parser_classes = (FhirJsonParser, JSONParser, FhirXmlParser, FormParser, MultiPartParser) + renderer_classes = (FhirJsonRenderer, JSONRenderer, FhirXmlRenderer) + content_negotiation_class = FhirContentNegotiation + versioning_class = FhirVersioning + + def get_exception_handler(self): + return outcomes.exception_handler + + def finalize_response(self, request, response, *args, **kwargs): + """Normalize returned outcomes at every status and depth, and wrap other error responses as outcomes.""" + if isinstance(response, Response): + if response.status_code >= 400: + if not outcomes.is_outcome(response.data): + response.data = outcomes.outcome_for_status(response.status_code, response.data) + response.status_code = outcomes.response_status(response.status_code) + response.data = outcomes.normalize_outcomes(response.data) + return super().finalize_response(request, response, *args, **kwargs) + + def get_fhir_parameters(self): + return FhirParameters.from_request(self.request) + + def get_fhir_base_url(self): + return fhir_base_url(self.request, self.kwargs) + + +class FhirOperationView(FhirViewMixin, APIView): + """ + A FHIR operation that only reads, GET and POST alike, at the global base and at owner bases: `operation`'s answer + for the request's FhirParameters, base and user (and the resource the route kwarg `instance` names, at the instance + level), written by `adapter`. Throttled as OCL's other views are. + """ + permission_classes = (AllowAny,) # what an operation shows the caller is core.fhir.resolve's rule + operation = adapter = None + instance = None + + def get_throttles(self): + return ThrottleUtil.get_throttles_by_user_plan(self.request.user) + + def get(self, request, *args, **kwargs): # pylint: disable=unused-argument + answer = self.operation( # pylint: disable=not-callable + self.get_fhir_parameters(), self.kwargs, request.user, kwargs.get(self.instance)) + return Response(self.adapter(answer)) # pylint: disable=not-callable + + post = get diff --git a/core/fhir/capabilities.py b/core/fhir/capabilities.py new file mode 100644 index 000000000..70afe33df --- /dev/null +++ b/core/fhir/capabilities.py @@ -0,0 +1,181 @@ +""" +What the FHIR metadata says, version-neutral, built from the code: core.fhir.adapters.r4 writes it as R4's +CapabilityStatement, TerminologyCapabilities and $versions Parameters. + +- The routes: what a base offers is what the URL configuration routes under it (/fhir/, /orgs/{org}/ or + /users/{user}/). A resource type's search route (a FhirSearchMixin view) is its search-type interaction, its + instance route (a FhirReadMixin view) its read, and each $operation route to a FhirOperationView, at the type or + instance level, one of its operations; a $operation route directly under the base is a system operation. Routing a + view lists it; nothing else does. +- The search parameters are the search's own (core.fhir.search's PARAMETERS and STRING_FIELDS). +- The FHIR versions are those the fhirVersion MIME parameter negotiates (core.fhir.base): R4, 4.0. +- The software is the OCL API and its version; the base URL is the request's. +- TerminologyCapabilities lists each repository a CodeSystem search at the base shows the caller, under the identifier + the resolver serves it at there: its canonical URL (its HEAD's) when the resolver picks it for that URL (with the URL + registry when the toggle is on), else its OCL path as a CodeSystem; as resolve() serves them were the repositories + the caller isn't shown missing (core.fhir.resolve's served_repositories), so a hidden repository changes nothing in + it. Each with its released versions as stored, newest first, and the default, the one served without a version. + Under a canonical URL, after the repository's own, the releases of the other repositories with it the base shows + that the operations serve by name there (a version named resolves to its holder). A fixed number of queries, + bounded as a search's are (SEARCH_TIMEOUT_MS; past it the list is too costly, a 422). +- The routed operations' contracts fill TerminologyCapabilities' blocks: $translate finds the map from the code's + system (core.fhir.ops.translate), so it needs none; $expand pages, with offset and count. +""" +import re +from functools import lru_cache + +from django.urls import URLResolver, get_resolver +from django.utils import timezone + +from core import __version__ +from core.fhir import outcomes +from core.fhir.base import ( + RESOURCE_TYPES, FhirOperationView, FhirVersioning, FhirViewMixin, fhir_base_url) +from core.fhir.resolve import served_repositories +from core.fhir.search import PARAMETERS, STRING_FIELDS, FhirReadMixin, FhirSearchMixin, within_search_timeout + +SOFTWARE = {'name': 'OCL API', 'version': __version__} +STARTED = timezone.now().replace(microsecond=0).isoformat() # what the statements describe was deployed then +FORMATS = [renderer.media_type for renderer in FhirViewMixin.renderer_classes if 'fhir' in renderer.media_type] +OWNER_BASES = {'orgs': 'org', 'users': 'user'} # the owner bases' first segment, and the route kwarg naming the owner +INTERACTIONS = {'read': FhirReadMixin, 'search-type': FhirSearchMixin} # in R4's order, by the view serving each +CONVERTER = re.compile(r'<(?:\w+:)?(\w+)>') # a route's segment +# What TerminologyCapabilities says of an operation the base routes, by its block: its contract +OPERATION_BLOCKS = { + ('ConceptMap', 'translate'): ('translation', {'needsMap': False}), + ('ValueSet', 'expand'): ('expansion', {'paging': True}), +} + + +def routes(resolver, prefix=''): + """(route, view class) for every route of a URL configuration, its includes followed.""" + for pattern in resolver.url_patterns: + route = prefix + str(pattern.pattern) + if isinstance(pattern, URLResolver): + yield from routes(pattern, route) + else: + yield route, getattr(pattern.callback, 'view_class', None) + + +def base_of(route): + """(the base, the route's segments under it) of a route under a FHIR base: None for /fhir/, else the owner kwarg + ('org' or 'user'). (False, None) for any other route.""" + segments = route.strip('/').split('/') + if segments[0] == 'fhir': + return None, segments[1:] + converter = CONVERTER.fullmatch(segments[1]) if len(segments) > 1 else None + if segments[0] in OWNER_BASES and converter and converter[1] == OWNER_BASES[segments[0]]: + return converter[1], segments[2:] + return False, None + + +def is_view(view, kind): + return isinstance(view, type) and issubclass(view, kind) + + +def offering(segments, view): + """ + What a route under a base offers, from its segments under the base and its view: ('$', name) for a system + operation, (resource type, 'interaction', code) or (resource type, 'operation', name), else None. + """ + resource_type, *rest = segments + if not rest and resource_type.startswith('$') and is_view(view, FhirOperationView): + return '$', resource_type[1:] + if resource_type not in RESOURCE_TYPES: + return None + instance = bool(rest) and bool(CONVERTER.fullmatch(rest[0])) + rest = rest[1:] if instance else rest + interaction = 'read' if instance else 'search-type' + if not rest and is_view(view, INTERACTIONS[interaction]): + return resource_type, 'interaction', interaction + if len(rest) == 1 and rest[0].startswith('$') and is_view(view, FhirOperationView): + return resource_type, 'operation', rest[0][1:] + return None + + +@lru_cache(maxsize=None) +def routed(resolver): + """ + {base: (resources, system operations)} for a URL configuration: each base's resource types in route order, each + with its interactions (in R4's order) and its operations (in route order), and the base's own operations. + """ + found = {} + for route, view in routes(resolver): + fhir_base, segments = base_of(route) + offer = offering(segments, view) if fhir_base is not False and segments and segments[0] else None + if offer: + resources, system = found.setdefault(fhir_base, ({}, [])) + if offer[0] == '$': + system.append(offer[1]) + else: + resources.setdefault(offer[0], {'interaction': [], 'operation': []})[offer[1]].append(offer[2]) + return {fhir_base: ( + [{'type': resource_type, 'interactions': [code for code in INTERACTIONS if code in resource['interaction']], + 'operations': unique(resource['operation'])} for resource_type, resource in resources.items()], + unique(system)) for fhir_base, (resources, system) in found.items()} + + +def unique(items): + return list(dict.fromkeys(items)) + + +def offered(kwargs): + """(resources, system operations) the base `kwargs` names routes.""" + kind = next((kind for kind in OWNER_BASES.values() if (kwargs or {}).get(kind)), None) + return routed(get_resolver()).get(kind, ([], [])) + + +def described(request, kwargs): + """What both statements say of the service: the base URL, when it was deployed, and the software.""" + return {'url': fhir_base_url(request, kwargs), 'date': STARTED, 'software': SOFTWARE} + + +def capability_statement(request, kwargs): + """The CapabilityStatement of the base `kwargs` names, as the routes under it make it.""" + resources, system = offered(kwargs) + return { + **described(request, kwargs), + 'formats': FORMATS, + 'resources': [{ + **resource, + 'search_parameters': [*PARAMETERS[resource['type']], *STRING_FIELDS] + if 'search-type' in resource['interactions'] else []} for resource in resources], + 'operations': system, + } + + +def versions(parameters, kwargs, user, instance=None): # pylint: disable=unused-argument + """ + $versions: the FHIR versions the fhirVersion MIME parameter accepts, as major.minor (4.0 and 4.0.1 are R4, 4.0), + and the one served when a request names none. + """ + return {'versions': unique('.'.join(version.split('.')[:2]) for version in FhirVersioning.allowed_versions), + 'default': FhirVersioning.default_version} + + +def code_systems(kwargs, user): + """ + [{url, versions, default}] for each repository a CodeSystem search at the base shows the caller, under the + identifier the resolver serves it at (core.fhir.resolve's served_repositories): its released versions, newest + first, as they are stored (each a business version, as R4 lists them), and the one served without a version, the + newest live one. + """ + listed = [] + for identifier, rows in sorted(served_repositories(kwargs, user), key=lambda item: item[0]): + default = next((version for version, retired in rows if not retired), None) + if default is not None: + listed.append({'url': identifier, 'versions': [version for version, _ in rows], 'default': default}) + return listed + + +def terminology_capabilities(request, kwargs): + """The TerminologyCapabilities of the base `kwargs` names, for the request's user.""" + resources, _ = offered(kwargs) + operations = {(resource['type'], name) for resource in resources for name in resource['operations']} + return { + **described(request, kwargs), + 'code_systems': within_search_timeout( + lambda: code_systems(kwargs, request.user), + lambda: outcomes.TooCostlyError('The list of code systems would take too long to answer.')), + **dict(block for operation, block in OPERATION_BLOCKS.items() if operation in operations), + } diff --git a/core/fhir/mapping_index.py b/core/fhir/mapping_index.py new file mode 100644 index 000000000..0fb8d25ce --- /dev/null +++ b/core/fhir/mapping_index.py @@ -0,0 +1,166 @@ +""" +The versions that may hold mappings from or to a source, as OCL's mappings index (core.mappings.documents) lists them: +the candidates ConceptMap source-system and target-system search (core.fhir.search) then decides in the database, so a +search doesn't join a heavily mapped source's mappings with every version's links. + +- A mapping's document names the source of each side by owner type, owner and mnemonic ({side}_concept_owner_type, + {side}_concept_owner, {side}_concept_source), and the versions of its own repository that hold it (owner_type, owner, + source, source_version; owner and source in lowercase). +- One request per search parameter aggregates the documents whose side is one of the sources named, by owner type, + owner, repository and version: the repositories holding such mappings, and their versions. What it reads and lists + is scoped to the ConceptMap versions the database shows the caller at the base (scope_of: their owner types, + owners, repositories and version names, and HEAD), never by a document's own copy of its access, and only what + the base shows is kept: a ConceptMap or a version the caller isn't shown can't change the answer, or exhaust the + aggregation's limits unless its names are those of one the caller is shown. +- The index names candidates; it never decides. The database decides each candidate version by its own mappings + (core.fhir.search.mapped), so it alone says what's live, deleted or in the version, and tells apart two owners whose + names differ only in case. +- Bounded: at most MAX_BUCKETS owners, repositories of an owner and versions of a repository; a whole answer only (no + partial results), within the time the search has left and without retries. Past the limit or the time the search is + too costly; a failed shard, a missing index or any other error of the index is IndexUnavailableError. +- The index is written after the database. A release created within LAG_WINDOW, of a repository the index lists, is a + candidate whether the index lists the release yet or not; past that window, a release whose indexing never + completed is missed until it's indexed again. +""" +import logging +import math +import time +from datetime import timedelta + +from elasticsearch import ApiError, ConnectionTimeout, TransportError +from elasticsearch_dsl.connections import connections + +from core.common.constants import HEAD +from core.fhir import outcomes +from core.mappings.documents import MappingDocument + +logger = logging.getLogger('oclapi') + +MAX_BUCKETS = 1000 # the most owners, repositories of an owner and versions of a repository an aggregation lists +OWNER_TYPES = 2 # Organization and User +LAG_WINDOW = timedelta(days=7) # how long after its creation a release may still be missing from the index +UNAVAILABLE = 'The search could not be completed: the mappings index did not answer in full. Try again later.' + + +class IndexUnavailableError(outcomes.FhirError): + """The mappings index answered only in part, or not at all: an error that isn't the request's (422 exception).""" + issue_code = 'exception' + + +def scope_of(shown): + """ + {field: sorted values} of the documents of the repositories and versions `shown` ({(owner type, owner, mnemonic): + {version names}}), as the index keeps them: owner and repository in lowercase; the version names with HEAD. + """ + return {'owner_type': sorted({owner_type for owner_type, _, _ in shown}), + 'owner': sorted({owner.lower() for _, owner, _ in shown}), + 'source': sorted({mnemonic.lower() for _, _, mnemonic in shown}), + 'source_version': sorted(set().union(*shown.values()) | {HEAD})} + + +def side_query(side, repositories): + """The documents whose `side` ('from' or 'to') source is one of `repositories`: (owner type, owner, mnemonic).""" + return {'bool': {'should': [{'bool': {'filter': [ + {'term': {f'{side}_concept_owner_type': owner_type}}, {'term': {f'{side}_concept_owner': owner}}, + {'term': {f'{side}_concept_source': mnemonic}}]}} for owner_type, owner, mnemonic in sorted(repositories)], + 'minimum_should_match': 1}} + + +def terms(field, size, aggs=None, include=None): + """A terms aggregation of `field`, listing only the values of `include` (if any) before its size is counted.""" + return {'terms': {'field': field, 'size': size, **({'include': include} if include else {})}, + **({'aggs': aggs} if aggs else {})} + + +def by_version(scope): + """ + Owner type, owner, repository and version (HEAD included). The request's filter keeps only documents whose owner + type, owner and repository are among `scope`'s, and those are single-valued, so only the versions (a document + lists each one holding it) are cut to `scope`'s. + """ + return {'owner_type': terms('owner_type', OWNER_TYPES, {'owner': terms('owner', MAX_BUCKETS, { + 'source': terms('source', MAX_BUCKETS, { + 'version': terms('source_version', MAX_BUCKETS, include=scope['source_version'])})})})} + + +def versions_of(aggregation, too_costly): + """ + {(owner type, owner, repository): {version names}} for an aggregation by_version(), owner and repository in + lowercase; too costly when any level left buckets out. + """ + def buckets(level): + if level['sum_other_doc_count']: + raise too_costly() + return level['buckets'] + + return {(kind['key'], owner['key'], repository['key']): { + version['key'] for version in buckets(repository['version'])} + for kind in buckets(aggregation['owner_type']) for owner in buckets(kind['owner']) + for repository in buckets(owner['source'])} + + +def whole(response, too_costly): + """A response's aggregations, if the index answered in full; else too costly (its time ran out) or unavailable.""" + if response.get('timed_out'): + raise too_costly() + shards = response.get('_shards') or {} + if not shards or shards.get('failed') or shards.get('successful') != shards.get('total'): + logger.warning('The mappings index answered a FHIR search in part: %s', shards) + raise IndexUnavailableError(UNAVAILABLE) + return response.get('aggregations') or {} + + +def request(side, alternatives, scope, left, too_costly): + """The index's response for each of `alternatives`' repositories among `scope`, within `left` seconds.""" + client = connections.get_connection().options(request_timeout=left, max_retries=0, retry_on_timeout=False) + try: + return client.search( + index=MappingDocument.Index.name, size=0, track_total_hits=False, allow_partial_search_results=False, + timeout=f'{max(1, math.floor(left * 900))}ms', + query={'bool': {'filter': [ + {'bool': {'should': [side_query(side, repositories) for repositories in alternatives], + 'minimum_should_match': 1}}, + *[{'terms': {field: values}} for field, values in scope.items()]]}}, + aggs={str(number): {'filter': side_query(side, repositories), 'aggs': by_version(scope)} + for number, repositories in enumerate(alternatives)}) + except ConnectionTimeout as ex: + raise too_costly() from ex + except (ApiError, TransportError) as ex: + logger.warning('The mappings index did not answer a FHIR search: %s', ex) + raise IndexUnavailableError(UNAVAILABLE) from ex + + +def kept(found, shown): + """ + The repositories and versions of `found` that `shown` lists, HEAD with them; twins in lowercase share theirs. A + repository the index lists only by versions `shown` doesn't is left out. + """ + listed = {} + for (owner_type, owner, mnemonic), versions in shown.items(): + listed.setdefault((owner_type, owner.lower(), mnemonic.lower()), {HEAD}).update(versions) + return {repository: versions & listed[repository] for repository, versions in found.items() + if repository in listed and versions & listed[repository]} + + +def holding(side, repositories, shown, deadline, too_costly): # pylint: disable=too-many-arguments + """ + {key: {(owner type, owner, repository): {version names}}}, owner and repository in lowercase: for each key's + repositories ((owner type, owner, mnemonic) of its sources), the repositories and versions among `shown` (the + ConceptMap versions the base shows the caller, {(owner type, owner, mnemonic): {version names}}) holding a mapping + whose `side` ('from' or 'to') source is one of them, with HEAD when the index lists it. One request, by `deadline` + (time.monotonic()), else too_costly(); none when nothing is named or shown. + """ + keys = sorted(key for key, value in repositories.items() if value) + if not keys or not shown: + return {} + left = deadline - time.monotonic() + if left <= 0: + raise too_costly() + aggregations = whole(request(side, [repositories[key] for key in keys], scope_of(shown), left, too_costly), + too_costly) + try: + return {key: kept(versions_of(aggregations[str(number)], too_costly), shown) + for number, key in enumerate(keys)} + except (KeyError, TypeError) as ex: + logger.warning('The mappings index answered a FHIR search without its aggregations') + raise IndexUnavailableError(UNAVAILABLE) from ex diff --git a/core/fhir/ops/__init__.py b/core/fhir/ops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/fhir/ops/concepts.py b/core/fhir/ops/concepts.py new file mode 100644 index 000000000..ed6438a71 --- /dev/null +++ b/core/fhir/ops/concepts.py @@ -0,0 +1,201 @@ +""" +What $lookup and $validate-code share (core.fhir.ops.lookup, core.fhir.ops.validate_code): reading the code they are +asked about, finding it in a code system version, and the concept's display and names. + +- Input: a code with its system, version and display as their own parameters, or as a Coding (`coding`, or a + codeableConcept holding one coding). Text inputs may come as any FHIR string type (valueString, valueCode, + valueUri); anything else is a 400 naming the parameter. +- A code is found among the version's own concepts, retired ones included, as FHIR spells it or percent-encoded, as + OCL stores a code with reserved characters. One query for the concept (with its repository), one for its names. +- Display: with a language, the concept's preferred name in that language when it has one; else its own display (OCL's + display_name: the preferred name in its code system's default locale). +""" +from collections import namedtuple +from urllib.parse import quote, unquote + +from django.db.models import F, Q + +from core.common.constants import HEAD +from core.common.utils import to_parent_uri +from core.concepts.models import Concept +from core.fhir import outcomes +from core.fhir.resolve import REPO_PATHS, UnresolvedError, namespace_of, resolve, to_ocl_path +from core.fhir.search import read_version + +# A code, as an operation is given it; `parameter` is the Coding parameter it came in, None for separate parameters +Coding = namedtuple('Coding', 'system code version display parameter') +PATHS = {None: '', 'coding': 'Coding.', 'codeableConcept': 'CodeableConcept.coding[0].'} + + +def text(parameters, name): + """A text input (a string, code or uri), or None when it's absent or empty; any other value is a 400 naming it.""" + value = parameters.get(name) + if value is None or value == '': + return None + if not isinstance(value, str): + raise outcomes.InvalidError(f"'{name}' must be text.", expression=name) + return value + + +def coding_of(value, name): + """A Coding parameter as a Coding: a dict, or in a GET, a system|code token. Its parts must be text.""" + if isinstance(value, str): + system, _, code = value.rpartition('|') + value = {'system': system, 'code': code} + parts = {key: value.get(key) for key in ('system', 'code', 'version', 'display')} if isinstance(value, dict) else {} + if not parts or not all(part is None or isinstance(part, str) for part in parts.values()): + raise outcomes.InvalidError(f"'{name}' must be a Coding.", expression=name) + return Coding(*(part or None for part in parts.values()), parameter=name) + + +def coded_input(parameters, system, version): + """ + The code an operation is asked about: `coding`, a codeableConcept with one coding, or else code and display with + the system and version from the parameters named `system` and `version`. + """ + if 'coding' in parameters: + return coding_of(parameters.get('coding'), 'coding') + if 'codeableConcept' in parameters: + concept = parameters.get('codeableConcept') + codings = concept.get('coding') if isinstance(concept, dict) else None + if not isinstance(codings, list) or len(codings) != 1: + raise outcomes.NotSupportedError( + 'A codeableConcept is supported with one coding: send each coding on its own.', + expression='codeableConcept') + return coding_of(codings[0], 'codeableConcept') + return Coding(text(parameters, system), text(parameters, 'code'), text(parameters, version), + text(parameters, 'display'), None) + + +def path(coding, part): + """The FHIRPath of a part of the code in an issue: the parameter itself, or the part of the Coding it came in.""" + return PATHS[coding.parameter] + part + + +def require(coding, part, message): + """The part of the code an operation needs, or a 400 naming the input it's missing from (`part`, or the Coding).""" + value = getattr(coding, part) + if not value: + raise outcomes.InvalidError(message, expression=coding.parameter or part) + return value + + +def stored_codes(code): + """The forms a code may be stored in: as it is, and percent-encoded as OCL encodes reserved characters.""" + return list(dict.fromkeys([code, quote(code, safe=''), quote(code)])) + + +def code_q(field, code): + """Q for rows whose `field` is the code in a form OCL stores it in: an equality when there's only one.""" + forms = stored_codes(code) + return Q(**{field: forms[0]}) if len(forms) == 1 else Q(**{f'{field}__in': forms}) + + +def fhir_code(code): + """A code as FHIR spells it: OCL stores one with reserved characters percent-encoded (ICD-11's ND53.Y&XK70).""" + return unquote(code) if code and '%' in code else code + + +def system_of(version, named=None): + """A code system version's canonical URL (else its repository's), or else the URL it was named by.""" + return version.canonical_url or getattr(version, 'repository_canonical_url', None) or named + + +def instance_version(resource_type, instance_id, version, kwargs, user): + """ + The repository version an instance-level operation is on: as a read finds it (a release: at the global base the + opaque id's, at an owner base the mnemonic's `version`, else the newest shown). At an owner base, HEAD and a named + version the read doesn't find go through core.fhir.resolve, with its visibility rule. A global id names a release, + so any version named there is left to the caller's version check. Not found is a read's 404. + """ + namespace = namespace_of(kwargs) + if namespace is None or version != HEAD: + try: + return read_version(resource_type, instance_id, version, kwargs, user) + except outcomes.NotFoundError: + if namespace is None or not version: + raise + try: + return resolve(resource_type, f'{namespace}{REPO_PATHS[resource_type]}/{instance_id}/', version, kwargs, user) + except UnresolvedError as ex: + raise outcomes.NotFoundError(f'{resource_type}/{instance_id} not found') from ex + + +def is_system_of(system, version, named=None): + """ + Whether a coding's system (or a value set url) names a repository version: its canonical URL (or its + repository's), the URL it was resolved by (`named`), or an OCL path of its repository or of the version. + """ + url = system.partition('|')[0] + canonicals = {version.canonical_url, getattr(version, 'repository_canonical_url', None), + (named or '').partition('|')[0]} + if url in canonicals - {None, ''}: + return True + repository = to_parent_uri(to_ocl_path(url)) + repository = repository if repository.endswith('/') else f'{repository}/' + return repository in {version.uri, '/'.join(version.uri.split('/')[:5]) + '/'} + + +def mismatch(parameters, coding, version, system_name, instance_id): + """ + (expression, message) of an input naming another code system or version than the one resolved, else None. At the + instance level, the id names it: the system parameter (`system_name`, url or system), the coding's system and any + version named (the version parameter, the coding's, a url|version) must all be its. At the type level, a Coding's + system and version must be those of the code system it is validated in, which url and version may name. + """ + if instance_id: + systems = [(text(parameters, system_name), system_name), (coding.system, path(coding, 'system'))] + versions = [(text(parameters, 'version'), 'version'), (coding.version, path(coding, 'version'))] + named = None + elif coding.parameter: + systems, versions = [(coding.system, path(coding, 'system'))], [(coding.version, path(coding, 'version'))] + named = text(parameters, system_name) or coding.system + else: + return None + for system, expression in systems: + if system and not is_system_of(system, version, named): + return expression, f"'{system}' is not the code system '{system_of(version, named)}' it is about" + versions.append((system and system.partition('|')[2], expression)) + for named_version, expression in versions: + if named_version and named_version != version.version: + return expression, f"Version '{named_version}' is not the version '{version.version}' it is about" + return None + + +def find_concept(version, code): + """ + The code system version's concept with the code, retired or not (a deleted one, is_active false, is gone), with its + repository and names; else None. + """ + concepts = version.concepts_set.filter(id=F('versioned_object_id')) if version.is_head else \ + Concept.objects.filter(sources=version) + return concepts.filter(code_q('mnemonic', code), is_active=True).select_related('parent').prefetch_related( + 'names').order_by('id').first() + + +def display_of(concept, language=None): + """ + The concept's display: with a language, its preferred name in that language, newest first, when it has one; else + its own display. None for a concept without names. + """ + if language: + names = sorted((name for name in concept.names.all() if name.locale == language and not name.retired), + key=lambda name: (name.locale_preferred, name.created_at), reverse=True) + if names: + return names[0].name + return concept.display_name + + +def has_name(concept, display): + """ + Whether a display is right for the concept: one of its names, in any language, ignoring case (FHIRsmith's default + check). A concept without names has nothing to check a display against. + """ + names = [name.name.casefold() for name in concept.names.all()] + return not names or display.casefold() in names + + +def designations(concept): + """Each of the concept's names as a designation: its language, its OCL name type as `use`, and its text.""" + return [{'language': name.locale, 'use': {'code': name.type} if name.type else None, 'value': name.name} + for name in concept.names.all()] diff --git a/core/fhir/ops/expand.py b/core/fhir/ops/expand.py new file mode 100644 index 000000000..19c72bac9 --- /dev/null +++ b/core/fhir/ops/expand.py @@ -0,0 +1,394 @@ +""" +ValueSet $expand (SVCM ITI-97, FHIR R4's $expand), version-neutral: one page of the stored expansion of the value set +version the request names. core.fhir.adapters.r4 writes the answer as an R4 ValueSet. + +- The value set: url (and valueSetVersion, or url|version) at the type level, or ValueSet/{id} at the instance level, + resolved as $validate-code resolves it (core.fhir.ops.validate_code.value_set_version): a released version, the + newest the caller is shown unless one is named (HEAD only when named); public only at /fhir/; at an owner base, + private ones to those who may view them. One that doesn't resolve, or that the caller isn't shown, is a 422 not-found, + the same for both. +- The expansion: the version's default stored expansion, read and never computed or saved, so a POST reads exactly as + a GET does. A version without one, or whose expansion is still being computed, can't be expanded (422); so can't one + whose references named a code system or value set OCL didn't hold when it was computed (422 not-found, as FHIR + answers a value set that draws on an unknown code system), because its stored expansion is incomplete. +- Members: the expansion's concepts (not deleted) whose code system the caller is shown (core.fhir.resolve's + viewable_rows), as ValueSet $validate-code counts them. A member of a code system the caller isn't shown leaves no + trace: not in contains, total or used-codesystem, and no other statement runs. +- Each member: its code system (the canonical URL of the release it is reported in, else its repository's, else the + repository's OCL path as a CodeSystem), its code as FHIR spells it, its display (with + displayLanguage, its preferred name in that language when it has one), inactive when retired, its names as + designations with includeDesignations, and the version: a live release the caller is shown that holds the member's + row, the expansion's explicit pin first, then its evaluated pin, then the newest. A versioned-object row carries + the concept's newest content, so it uses the newest holding release of another, not deleted, version, whatever the + expansion pins. None without one, never HEAD or an unreleased version (as $validate-code answers). +- Inputs: activeOnly leaves retired concepts out; filter keeps the concepts whose code (as FHIR spells it) or one of + whose names has a word starting with each of the filter's words, ignoring case; offset and count page the members in + a fixed order (count at most MAX_COUNT, which is also the default); count=0 answers the total alone. total counts + what activeOnly and filter keep. The inputs that shaped the answer are echoed in expansion.parameter, then each code + system the members come from, with each version they are reported in, as used-codesystem. Every other R4 $expand + input is one a stored expansion always satisfies (SATISFIED: OCL's expansions are flat, with no abstract codes), one + supported only when false (ONLY_FALSE: an expansion has no definitions, and doesn't record which of its codes are + post-coordinated), or a 400 not-supported naming it (NOT_SUPPORTED): an inline value set, a context, a date, code + system versions and designations would each need an expansion computed for the request, and OCL serves the ones it + stores. Names that aren't $expand inputs are ignored, as _format is. +- Cost: a fixed number of queries whatever the size of the expansion; for members whose rows no release holds, + links of each repository's newest live release the caller is shown are read as a set, then older holding releases + probed per member only for those still unresolved. Full concepts and names are read for the page alone. A filtered + expansion reads the ids of its matching members once, and its total and page come from them. All queries within + SEARCH_TIMEOUT_MS together, past which the expansion is too costly (422). +""" +import re + +from django.db.models import BooleanField, Case, Exists, ExpressionWrapper, F, Func, IntegerField, JSONField, \ + OuterRef, Q, Subquery, Value, When +from django.db.models.functions import Coalesce, JSONObject +from django.db.models.lookups import Regex + +from core.collections.models import Expansion +from core.common.constants import HEAD +from core.common.serializers import IdentifierSerializer +from core.concepts.models import Concept, ConceptName +from core.fhir import outcomes +from core.fhir.ops.concepts import designations, display_of, fhir_code, text +from core.fhir.ops.validate_code import value_set_version +from core.fhir.resolve import namespace_of, viewable_rows +from core.fhir.search import within_search_timeout +from core.sources.models import Source + +MAX_COUNT = 1000 # the most members a page holds, and how many it holds when count isn't given +WORD = re.compile(r'\w+') +# an escape of an ASCII character that is no word character (not a letter, digit or _), matched ignoring case +NON_WORD_ESCAPE = '%(?:[01][0-9a-f]|2[0-9a-f]|3[a-f]|40|5[b-e]|60|7[b-f])' +# the escapes of one non-ASCII character as a stored code writes them, matched ignoring case: a well-formed UTF-8 +# sequence of 2 to 4 bytes (Unicode's Table 3-7), else a single byte, which makes no character +NON_ASCII_CHARACTER = ('(?:%c[2-9a-f]|%d[0-9a-f])%[89ab][0-9a-f]' + '|(?:%e0%[ab][0-9a-f]|%e[1-9a-cef]%[89ab][0-9a-f]|%ed%[89][0-9a-f])%[89ab][0-9a-f]' + '|(?:%f0%[9ab][0-9a-f]|%f[1-3]%[89ab][0-9a-f]|%f4%8[0-9a-f])%[89ab][0-9a-f]%[89ab][0-9a-f]' + '|%[89a-f][0-9a-f]') +STORED = 'OCL serves the expansion it stores for the value set version' +VERSIONS = f'Choosing code system versions is not supported: {STORED}, made from the versions used-codesystem names.' +NOT_SUPPORTED = { # R4 $expand inputs that need an expansion computed for the request + 'valueSet': f'An inline valueSet is not supported: {STORED}, named by url.', + 'context': 'The context parameter is not supported.', + 'contextDirection': 'The contextDirection parameter is not supported.', + 'date': f'An expansion as of a date is not supported: {STORED}.', + 'designation': 'Choosing designations is not supported: includeDesignations includes every name.', + 'exclude-system': f'Excluding a code system is not supported: {STORED}.', + 'system-version': VERSIONS, + 'check-system-version': VERSIONS, + 'force-system-version': VERSIONS, +} +ONLY_FALSE = { # boolean R4 $expand inputs supported when false + 'includeDefinition': 'Definitions are not included in an expansion.', + 'excludePostCoordinated': f'Excluding post-coordinated codes is not supported: {STORED}, which does not record ' + 'which of its codes are post-coordinated.', +} +SATISFIED = ('excludeNested', 'excludeNotForUI') # booleans every stored expansion meets +# a reference an expansion couldn't resolve, by its type: (message id, message) without and with a version named +UNRESOLVED = { + 'reference.valueset': ( + ('VS_EXP_IMPORT_UNK', "Unable to find included value set '{url}'"), + ('VS_EXP_IMPORT_UNK_PINNED', "Unable to find included value set '{url}' version '{version}'")), + 'system': ( + ('UNKNOWN_CODESYSTEM_EXP', + "A definition for CodeSystem '{url}' could not be found, so the value set cannot be expanded"), + ('UNKNOWN_CODESYSTEM_VERSION_EXP', + "A definition for CodeSystem '{url}' version '{version}' could not be found, so the value set cannot be " + "expanded")), +} + + +def too_costly(): + return outcomes.TooCostlyError( + 'The expansion would take too long to answer. Narrow it, for example with filter, or page it with count.') + + +def whole_number(parameters, name): + """A whole-number input (an integer, or its digits as text), or None when it's absent; anything else is a 400.""" + value = parameters.get(name) + if value is None or value == '': + return None + if isinstance(value, str) and re.fullmatch(r'[0-9]{1,9}', value.strip()): + return int(value) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + raise outcomes.InvalidError(f"'{name}' must be a whole number, got '{value}'.", expression=name) + + +def flag(parameters, name): + """A boolean input (a boolean, or true or false as text), or None when it's absent; anything else is a 400.""" + value = parameters.get(name) + if value in (None, ''): + return None + if isinstance(value, bool): + return value + if isinstance(value, str) and (lowered := value.strip().lower()) in ('true', 'false'): + return lowered == 'true' + raise outcomes.InvalidError(f"'{name}' must be true or false, got '{value}'.", expression=name) + + +class Inputs: # pylint: disable=too-few-public-methods + """$expand's inputs other than the value set, read and checked before anything is: the echo is what was given.""" + + def __init__(self, parameters): + for name, message in NOT_SUPPORTED.items(): + if name in parameters: + raise outcomes.NotSupportedError(message, expression=name) + for name, message in ONLY_FALSE.items(): + if flag(parameters, name): + raise outcomes.NotSupportedError(message, expression=name) + for name in SATISFIED: + flag(parameters, name) # either value is met; one that isn't a boolean is still a 400 + self.filter = text(parameters, 'filter') + self.language = text(parameters, 'displayLanguage') + self.designations = flag(parameters, 'includeDesignations') + self.active_only = flag(parameters, 'activeOnly') + self.offset = whole_number(parameters, 'offset') + count = whole_number(parameters, 'count') + self.count = MAX_COUNT if count is None else min(count, MAX_COUNT) + self.echo = [item for item in ( + ('filter', 'valueString', self.filter), + ('displayLanguage', 'valueCode', self.language), + ('includeDesignations', 'valueBoolean', self.designations), + ('activeOnly', 'valueBoolean', self.active_only), + ('offset', 'valueInteger', self.offset), + ('count', 'valueInteger', None if count is None else self.count), + ) if item[2] is not None] + + +def escapes(char): + """A character percent-encoded, as OCL stores it in a code: %XX for each of its UTF-8 bytes.""" + return ''.join(f'%{byte:02X}' for byte in char.encode()) + + +def spelled(word): + """ + A regex, matched ignoring case, for `word` as a stored code may spell it: each character as itself, or as the + escapes of itself or of its other case (é: é, %C3%A9 or %C3%89). \\w characters need no escaping in a regex. + """ + spellings = (dict.fromkeys([char, *(escapes(case) for case in (char, char.lower(), char.upper()) + if len(case) == 1)]) for char in word) + return ''.join(f"(?:{'|'.join(spelling)})" for spelling in spellings) + + +class NonWordBefore(Func): # pylint: disable=abstract-method + """ + Whether, somewhere in the code, a non-ASCII character's escapes stand right before `spelling` (a word, spelled) and + that character is no word character: a well-formed UTF-8 sequence, decoded, for a character Postgres doesn't count + as a letter, digit or _; or a single byte, which makes no character (FHIR's decoding spells it U+FFFD). A regex + takes the longest match where it starts, so a character's escapes are matched whole. + """ + template = ( + "EXISTS (SELECT FROM regexp_matches(%(expressions)s, 'gi') AS characters (escapes) WHERE CASE WHEN " + "length(escapes[1]) = 3 THEN true ELSE convert_from(decode(regexp_replace(escapes[1], '[^0-9A-Fa-f]', '', " + "'g'), 'hex'), 'UTF8') COLLATE \"default\" !~ '[[:alnum:]_]' END)") + output_field = BooleanField() + + def __init__(self, spelling): + super().__init__(F('mnemonic'), Value(f'(?:{NON_ASCII_CHARACTER})(?={spelling})')) + + +def code_has_word(word): + """ + Whether a member's code, as FHIR spells it (core.fhir.ops.concepts.fhir_code decodes OCL's percent-encoding), has + a word starting with `word`, ignoring case, by the rule filter applies to names (Postgres's \\m: a word character + is a letter or digit of any script, or _). The stored code is matched a whole character at a time, never from inside + an escape. A word starts there: + - in a code with no escape, where \\m says; so too for a word that doesn't start with a word character (filter's + words are Python's \\w+, which counts a few characters more, ² for one): \\m starts no word with one; + - at its start, or after a character that is no word character: written as itself, as an ASCII escape, or a % that + starts no escape; + - after a non-ASCII character's escapes, when that character, decoded, is no word character (NonWordBefore). + Each branch is tried only when the ones before it haven't decided, so a code without the word in any spelling costs + one regex, and one without escapes the regex names get. + """ + spelling = spelled(word) + return Case( + When(~Q(mnemonic__contains='%') | ~Q(Regex(Value(word), '^[[:alnum:]_]')), + then=ExpressionWrapper(Q(mnemonic__iregex=rf'\m{word}'), output_field=BooleanField())), + When(~Q(mnemonic__iregex=spelling), then=False), + When(mnemonic__iregex=rf'(?:^|[^[:alnum:]_%]){spelling}', then=True), + When(mnemonic__iregex=rf'{NON_WORD_ESCAPE}{spelling}', then=True), + When(mnemonic__iregex=rf'%(?![0-9a-f]{{2}}){spelling}', then=True), + When(mnemonic__iregex=rf'%[89a-f][0-9a-f]{spelling}', then=NonWordBefore(spelling)), + default=False, output_field=BooleanField()) + + +def expand(parameters, kwargs, user, instance_id=None): + """ + $expand's answer for its FhirParameters at the base `kwargs` names (ValueSet/{instance_id}/$expand at the instance + level), for `user`: {value_set, identifier, timestamp, total, offset, parameters, used, contains}. + """ + inputs = Inputs(parameters) + if not instance_id and not text(parameters, 'url'): + raise outcomes.InvalidError('url is required: the value set to expand.', expression='url') + return within_search_timeout(lambda: Expander(parameters, inputs, kwargs, user, instance_id).answer(), too_costly) + + +def stored_expansion(value_set): + """The value set version's default stored expansion, when it is one that can be served; else the 422 why not.""" + name = f'{value_set.canonical_url or value_set.uri}|{value_set.version}' + expansion = Expansion.objects.filter(uri=value_set.expansion_uri).only( + 'id', 'uri', 'created_at', 'is_processing', 'unresolved_repo_versions').first() if \ + value_set.expansion_uri else None + if expansion is None: + raise outcomes.InvalidError( + f'The value set {name} has no stored expansion, so it cannot be expanded.', tx_issue_type='vs-invalid', + status_code=422) + if expansion.is_processing: + raise outcomes.ProcessingError(f'The expansion of the value set {name} is being computed: try again shortly.') + if expansion.unresolved_repo_versions: + raise unresolved(expansion.unresolved_repo_versions[0]) + return expansion + + +def unresolved(reference): + """The 422 not-found for the first reference an expansion couldn't resolve: a value set, else a code system.""" + url, version = reference.get('url') or '', reference.get('version') or None + message_id, message = UNRESOLVED.get(reference.get('type'), UNRESOLVED['system'])[bool(version)] + return outcomes.NotFoundError(message.format(url=url, version=version), tx_issue_type='not-found', + message_id=message_id, status_code=422) + + +def code_system(canonical_url, uri, release=None): + """ + (system, version) of a member of the repository with this canonical URL and OCL path, reported in `release` + ({version, url}, or None): the release's own canonical URL, else the repository's, else its OCL path as a + CodeSystem (/orgs/{org}/CodeSystem/{id}/), as a read names the release; and the release's version. + """ + release = release or {} + system = release.get('url') or canonical_url or IdentifierSerializer.convert_ocl_uri_to_fhir_url(uri, 'CodeSystem') + return system, release.get('version') + + +class Expander: + """One $expand: the value set version, its stored expansion, and the members the caller is shown.""" + + def __init__(self, parameters, inputs, kwargs, user, instance_id): # pylint: disable=too-many-arguments + self.inputs = inputs + self.value_set = value_set_version(parameters, kwargs, user, instance_id) + self.expansion = stored_expansion(self.value_set) + self.namespace, self.user = namespace_of(kwargs), user + + def members(self): + """The expansion's concepts that are its members for the caller: not deleted, of a code system it is shown.""" + return Concept.objects.filter( + Exists(viewable_rows(Source, self.namespace, self.user).filter(id=OuterRef('parent_id'))), + expansion_set=self.expansion.id, is_active=True) + + def matching(self): + """The members activeOnly and filter keep.""" + members = self.members() + if self.inputs.active_only: + members = members.filter(retired=False) + # \m: a word starts here; \w characters need no escaping in a regex + for word in WORD.findall(self.inputs.filter or ''): + members = members.filter(Q(code_has_word(word)) | Exists(ConceptName.objects.filter( + concept_id=OuterRef('id'), name__iregex=rf'\m{word}'))) + return members + + def live_releases(self): + """The active, non-retired releases the caller is shown, never HEAD.""" + return viewable_rows(Source, self.namespace, self.user).filter( + is_active=True, released=True, retired=False).exclude(version=HEAD) + + def release(self): + """ + The live release the caller is shown that holds the member's row, as {version, url}: explicit pin first, + then evaluated, then newest by source id; None without one. The newest lookup runs only when no pin holds it. + """ + links = Concept.sources.through.objects.filter( + source_id__in=self.live_releases().values('id'), concept_id=OuterRef('id')) + explicit, evaluated = ( + versions.through.objects.filter(expansion_id=self.expansion.id).values('source_id') + for versions in (Expansion.explicit_source_versions, Expansion.evaluated_source_versions)) + pinned = links.filter(Q(source_id__in=explicit) | Q(source_id__in=evaluated)).annotate(explicit=Case( + When(source_id__in=explicit, then=Value(1)), default=Value(0), output_field=IntegerField())).order_by( + '-explicit', '-source_id') + return Coalesce(*(Subquery(holding.annotate( + release=JSONObject(version='source__version', url='source__canonical_url')).values('release')[:1], + output_field=JSONField()) for holding in (pinned, links.order_by('-source_id'))), + output_field=JSONField()) + + def releases(self, members, rows): + """ + {member id: release} from (id, versioned-object id, parent id, direct release) rows of `members`. + Members held by releases keep their direct release. A versioned-object member carries its concept's newest + content, so it uses the newest live release the caller is shown that holds an active version, ignoring pins. + Only a repository's own releases hold its concepts' versions, so a match in its newest visible live release + is also the newest holding release. Those links are read as a set; any still unresolved use a correlated + newest lookup, restricted to their member ids. Every member remains a key, with None if no release holds it. + """ + rows = list(rows) + found = {member: release for member, _, _, release in rows} + missing = {member: (root, parent) for member, root, parent, release in rows if release is None} + if not missing: + return found + newest = self.live_releases().filter(mnemonic=OuterRef('mnemonic')).filter( + Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id'))).order_by('-id') + heads = Source.objects.filter(id__in={parent for _, parent in missing.values()}).annotate( + newest=Subquery(newest.values('id')[:1])).values('newest') + held = dict(Concept.sources.through.objects.filter( + source_id__in=heads, concept__is_active=True, + concept__versioned_object_id__in=members.order_by().values('versioned_object_id')).annotate( + release=JSONObject(version='source__version', url='source__canonical_url')).values_list( + 'concept__versioned_object_id', 'release')) + for member, (root, _) in list(missing.items()): + if root in held: + found[member] = held[root] + del missing[member] + if missing: + holding = Concept.sources.through.objects.filter( + source_id__in=self.live_releases().values('id'), concept__is_active=True, + concept__versioned_object_id=OuterRef('versioned_object_id')).order_by('-source_id') + found.update(Concept.objects.filter(id__in=list(missing)).annotate(release=Subquery(holding.annotate( + release=JSONObject(version='source__version', url='source__canonical_url')).values('release')[:1], + output_field=JSONField())).values_list('id', 'release')) + return found + + def used(self): + """Each code system and release of all members, including those held only in other versions: system|version.""" + members = self.members() + rows = list(members.annotate(release=self.release()).values_list( + 'id', 'versioned_object_id', 'parent_id', 'release', 'parent__canonical_url', 'parent__uri').order_by()) + releases = self.releases(members, (row[:4] for row in rows)) + used = {code_system(url, uri, releases[member]) for member, _, _, _, url, uri in rows} + return sorted(f'{system}|{version}' if version else system for system, version in used) + + def page(self, ids): + """ + The members `ids` names (the page offset and count chose), in a fixed order, with what each is shown with. + Releases, concepts and names are read for the page's members alone, however deep the offset. + """ + page = Concept.objects.filter(id__in=ids) + concepts = page.annotate(release=self.release()).select_related('parent').only( + 'id', 'mnemonic', 'retired', 'parent_id', 'versioned_object_id', 'parent__canonical_url', 'parent__uri', + 'parent__default_locale', 'parent__supported_locales').prefetch_related('names').order_by('id') + releases = self.releases(page, ( + (concept.id, concept.versioned_object_id, concept.parent_id, concept.release) for concept in concepts)) + members = [] + for concept in concepts: + parent = concept.parent + system, version = code_system(parent.canonical_url, parent.uri, releases[concept.id]) + members.append({ + 'system': system, 'version': version, 'code': fhir_code(concept.mnemonic), + 'display': display_of(concept, self.inputs.language), 'inactive': concept.retired or None, + 'designations': designations(concept) if self.inputs.designations else None, + }) + return members + + def answer(self): + matching = self.matching() + if self.inputs.filter: + ids = list(matching.order_by('id').values_list('id', flat=True)) + total = len(ids) + else: + total = matching.count() + ids = matching.order_by('id').values('id') + offset = self.inputs.offset or 0 + return { + 'value_set': self.value_set, 'identifier': self.expansion.uri, 'timestamp': self.expansion.created_at, + 'total': total, 'offset': offset, 'parameters': self.inputs.echo, + 'used': self.used(), + 'contains': self.page(ids[offset:offset + self.inputs.count]) if self.inputs.count else [], + } diff --git a/core/fhir/ops/lookup.py b/core/fhir/ops/lookup.py new file mode 100644 index 000000000..5106621b5 --- /dev/null +++ b/core/fhir/ops/lookup.py @@ -0,0 +1,61 @@ +""" +CodeSystem $lookup (SVCM ITI-98, FHIR R4's $lookup), version-neutral: what OCL holds about a code of the code system +version the request names. core.fhir.adapters.r4 writes the answer as R4 Parameters. + +- The code system: `system` and `version` (or a `coding`'s), resolved by core.fhir.resolve at the base's namespace: a + released version, the newest the caller is shown unless one is named (HEAD only when named); public only at /fhir/; + at an owner base, private ones to those who may view them. CodeSystem/{id}/$lookup names it as a read does. One the + caller isn't shown answers exactly as one that doesn't exist. +- The code: that version's concept, retired ones too, which are flagged inactive (core.fhir.ops.concepts). +- The answer: name, version (the one resolved), display, code, system, abstract (OCL has no abstract concepts), each + of the concept's names as a designation, and the inactive property. `displayLanguage` picks the display among the + concept's names in that language; a concept without names is displayed by its code. +- Errors, as FHIRsmith answers them: no code, or no system at the type level, is a 400; a code system that doesn't + resolve is a 422 not-found, a code it doesn't hold a 404 (tx-issue-type invalid-code). +- Cost: a fixed number of queries whatever the size of the code system: resolution, the concept, its names. +""" +from rest_framework import status + +from core.fhir import outcomes +from core.fhir.ops.concepts import ( + coded_input, require, find_concept, display_of, designations, fhir_code, instance_version, mismatch, system_of, + text) +from core.fhir.resolve import UnresolvedError, resolve + + +def code_system_version(coding, kwargs, user, instance_id=None): + """The code system version a $lookup or $validate-code names: the instance, else the coding's system and version.""" + if instance_id: + return instance_version('CodeSystem', instance_id, coding.version, kwargs, user) + return resolve('CodeSystem', coding.system, coding.version, kwargs, user) + + +def lookup(parameters, kwargs, user, instance_id=None): + """ + $lookup's answer for its FhirParameters at the base `kwargs` names (CodeSystem/{instance_id}/$lookup at the instance + level), for `user`: {name, version, display, code, system, abstract, inactive, designations}. + """ + coding = coded_input(parameters, 'system', 'version') + target = coding._replace(version=text(parameters, 'version') or coding.version) + code = require(coding, 'code', '$lookup needs the code to look up.') + if not instance_id: + require(coding, 'system', '$lookup needs the system the code is from.') + try: + version = code_system_version(target, kwargs, user, instance_id) + except UnresolvedError as ex: + raise outcomes.NotFoundError( + str(ex.detail), tx_issue_type='not-found', status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) from ex + conflict = mismatch(parameters, coding, version, 'system', instance_id) + if conflict: + raise outcomes.InvalidError(f'{conflict[1]}: this $lookup is on {instance_id}.', expression=conflict[0]) + system = system_of(version, coding.system) + concept = find_concept(version, code) + if not concept: + raise outcomes.NotFoundError( + f"Unable to find code '{code}' in {system} version {version.version}", tx_issue_type='invalid-code') + return { + 'name': version.name, 'version': version.version, + 'display': display_of(concept, text(parameters, 'displayLanguage')) or fhir_code(concept.mnemonic), + 'code': fhir_code(concept.mnemonic), 'system': system, 'abstract': False, 'inactive': bool(concept.retired), + 'designations': designations(concept), + } diff --git a/core/fhir/ops/translate.py b/core/fhir/ops/translate.py new file mode 100644 index 000000000..cb19f288e --- /dev/null +++ b/core/fhir/ops/translate.py @@ -0,0 +1,386 @@ +""" +ConceptMap $translate (SVCM ITI-101, FHIR R4's $translate), version-neutral: what a translation finds, with OCL map +types as R5-shaped relationships. core.fhir.adapters.r4 writes the answer as an R4 Parameters resource. + +- The ConceptMap: the one `url` (and `conceptMapVersion`) names; else the code system the code comes from, `system` + resolved as a CodeSystem (in reverse, `targetsystem`, the system of the codes sought), whose own mappings are its + map, at the release `conceptMapVersion` names, else (forward) the code's `version`. Resolution, versions and access + are core.fhir.resolve's: a released version, the newest the caller is shown unless one is named; public only at + /fhir/; at an owner base, private ones to those who may view them. +- The matches: that version's live (neither retired nor deleted) mappings from the code (in reverse, to it) whose from + side (in reverse, to side) is shown as the code's system; with `targetsystem`, whose other side is shown as that + system. Never another repository's mappings, HEAD's or another release's. +- A side of a mapping is shown as its source's canonical URL, when the caller is shown that source and it has one; else + as the URL the mapping stores (its source's OCL URL when it stores none), an OCL URL as a ConceptMap path + (/orgs/{org}/ConceptMap/{id}/). So a source the caller isn't shown is named as a source OCL doesn't hold would be, + and a system only such sources have answers as an unknown one. ConceptMapGroupField names groups the same way. + A system given as an OCL path of a source (/orgs/{org}/CodeSystem/{id}/, or the sources or ConceptMap form) names + that repository if the caller is shown it, whether it has a canonical URL or not; the answer names it as above. A + ValueSet or collections path names a value set, not a source. +- Map types (design D27): case, spaces, hyphens, underscores and surrounding whitespace don't matter, so SAME-AS, + SAME AS and Same As are one. The relationships are core.concept_maps.constants'; result is true when some match isn't + not-related-to. A match's relationship reads from the code named to the match, so in reverse, where the code named + is the mapping's target, narrower and broader swap. With no match, the answer says why (message). +- The code's scope, `source` (the value set the code was chosen from; in reverse, which reverses the meaning of + source and target, `target`), is a precondition: a code outside it has no translation, and one that resolves to no + value set (or code system) the caller is shown is not found. The other scope narrows nothing: OCL's ConceptMaps + declare no scope, so it selects no map (as FHIRsmith reads OCL's maps); `targetsystem` narrows. +- The code's `version` (or its coding's) keeps the mappings of that version of its system, and those of none + (from_source_version; in reverse, to_source_version). + `conceptMap` (an inline map) and `dependency` aren't supported. +- Cost: a fixed number of queries whatever the number of mappings (resolution, the sources the systems name, the ids + of the release's mappings of the code, then those mappings, with whether the caller is shown their sources); at + most MATCH_LIMIT matches. Its statements are bounded as a search's are (too long is too costly, a 422). +""" +import re +from collections import namedtuple +from functools import cached_property, reduce +from operator import or_ + +from django.core.exceptions import ValidationError +from django.db.models import Case, Exists, F, OuterRef, Q, When + +from core.collections.models import Expansion +from core.concepts.models import Concept +from core.common.serializers import IdentifierSerializer +from core.concept_maps.constants import ( + MAP_TYPE_RELATIONSHIPS, NOT_RELATED_TO, RELATED_TO, RESOURCE_TYPE, REVERSED_RELATIONSHIPS) +from core.fhir import outcomes +from core.fhir.ops.concepts import fhir_code, stored_codes, text +from core.fhir.resolve import UnresolvedError, namespace_of, resolve, viewable_rows +from core.fhir.search import within_search_timeout +from core.mappings.models import Mapping +from core.sources.models import Source + +MATCH_LIMIT = 1000 # the most matches an answer lists +UNSUPPORTED_PARAMETERS = ('conceptMap', 'dependency') +# An OCL path of a source: its own (sources), or as FHIR names it (CodeSystem; ConceptMap, how OCL shows a source with +# no canonical URL). A collections or ValueSet path names a value set, never a source's alias. +OCL_PATH = re.compile(r'/(orgs|users)/([^/]+)/(?:sources|CodeSystem|ConceptMap)/([^/]+)/?') +IMPLICIT_VALUE_SET = '?fhir_vs' # {code system}?fhir_vs: every code of the code system + +Coding = namedtuple('Coding', 'system code version') +Request = namedtuple('Request', 'coding url concept_map_version scope scope_name targetsystem reverse') + + +def map_type_key(map_type): + """A map type as MAP_TYPE_RELATIONSHIPS spells it: upper case, a run of spaces, hyphens and underscores a hyphen.""" + return re.sub(r'[\s_-]+', '-', (map_type or '').strip().upper()).strip('-') + + +def relationship(map_type, reverse=False): + """ + The relationship an OCL map type is (design D27): MAP_TYPE_RELATIONSHIPS', else related-to. reverse reads it from + the mapping's target to its source, as a reverse translation answers it, relative to the code named (its target). + """ + found = MAP_TYPE_RELATIONSHIPS.get(map_type_key(map_type), RELATED_TO) + return REVERSED_RELATIONSHIPS.get(found, found) if reverse else found + + +def side_system(canonical, url, uri=None): + """ + How a side of a mapping is shown (as a ConceptMap group's source or target, or a translation's system): `canonical`, + its source's canonical URL when the caller is shown one that has it; else `url`, the URL the mapping stores, or + `uri`, its source's OCL URL when it stores none (pass it only for a source the caller is shown); an OCL URL as a + ConceptMap path. + """ + if canonical: + return canonical + url = url or uri + try: + return IdentifierSerializer.convert_ocl_uri_to_fhir_url(url, RESOURCE_TYPE) + except ValidationError: # an OCL URL too short to name a repository + return url + + +def repository_paths(system, kinds=('sources',)): + """ + The OCL URL prefixes of the repository an OCL path names (/orgs/{org}/{kind}/{id}/ for each kind), or None for a + canonical URL. + """ + path = OCL_PATH.fullmatch(system) + return [f'/{path[1]}/{path[2]}/{kind}/{path[3]}/' for kind in kinds] if path else None + + +def named_sources(systems, namespace, user): + """ + For each system, the ids of the Source rows the base shows the caller that it names, in one query: those with it as + their canonical URL; for an OCL path, the repository's rows, whether it has a canonical URL or not (the path is an + alias: side_system still shows one that has as its canonical URL). A literal list, so the planner reads the mappings + by their own indexes. + """ + paths = {system: (repository_paths(system) or [None])[0] for system in systems} + rows = viewable_rows(Source, namespace, user).filter(reduce(or_, ( + Q(uri__startswith=path) if path else Q(canonical_url=system) for system, path in paths.items()))) + named = {system: [] for system in systems} + for source_id, canonical, uri in rows.values_list('id', 'canonical_url', 'uri'): + for system, path in paths.items(): + if uri.startswith(path) if path else canonical == system: + named[system].append(source_id) + return named + + +def shown_as(side, system, source_ids, namespace, user): + """ + Q for mappings whose `side` ('from' or 'to') `system` names: their source is one of `source_ids` (named_sources'); + or it isn't one the caller is shown by a canonical URL, and the URL they store names the system, as side_system + then shows it. The subquery runs only on the mappings whose URL does. + """ + source, url = f'{side}_source', f'{side}_source_url' + unnamed = ~Exists(viewable_rows(Source, namespace, user).filter( + id=OuterRef(f'{source}_id'), canonical_url__gt='')) + paths = repository_paths(system, ('sources', 'collections', 'ConceptMap')) + stored = reduce(or_, (Q(**{f'{url}__startswith': path}) for path in paths)) if paths else Q(**{url: system}) + return Q(**{f'{source}_id__in': source_ids}) | (stored & unnamed) + + +def scope_system(scope): + """The code system a scope names, if it is one: the scope itself, or the code system of an implicit value set.""" + return scope.removesuffix(IMPLICIT_VALUE_SET) if scope else scope + + +def holds(value_set, code, sources): + """ + Whether a value set's default expansion holds the code as an active concept of one of the `sources`: from the + code's concepts, each looked up in the expansion, rather than from the expansion's members. + """ + return bool(value_set.expansion_uri and sources) and Expansion.concepts.through.objects.filter( + expansion__uri=value_set.expansion_uri, concept_id__in=Concept.objects.filter( + mnemonic__in=stored_codes(code), parent_id__in=sources, is_active=True).values('id')).exists() + + +def too_costly(): + return outcomes.TooCostlyError( + 'The translation would take too long to answer. Narrow it, for example with url, version or targetsystem.') + + +def of_version(side, version): + """Q for mappings whose `side` is that version of its system, or names no version; None names no constraint.""" + if not version: + return Q() + field = f'{side}_source_version' + return Q(**{field: version}) | Q(**{f'{field}__isnull': True}) | Q(**{field: ''}) + + +def with_side(queryset, side, namespace, user): + """ + Each mapping annotated with whether the base shows the caller its `side`'s source ({side}_shown), and the canonical + URL that side is then shown by ({side}_canonical, None for none): subqueries run only on the mappings listed. + """ + queryset = queryset.annotate(**{f'{side}_shown': Exists(viewable_rows(Source, namespace, user).filter( + id=OuterRef(f'{side}_source_id')))}) + return queryset.annotate(**{f'{side}_canonical': Case(When( + Q(**{f'{side}_shown': True, f'{side}_source__canonical_url__gt': ''}), + then=F(f'{side}_source__canonical_url')), default=None)}) + + +def required_text(parameters, name): + """A text parameter the code is given with (code, system); absent or empty, a 400 naming it.""" + value = text(parameters, name) + if value is None: + raise outcomes.InvalidError(f"$translate needs '{name}' with the code it translates.", expression=name) + return value + + +def boolean(parameters, name): + value = parameters.get(name) + if value in (None, ''): + return False + if isinstance(value, bool): + return value + if isinstance(value, str) and value.lower() in ('true', 'false'): + return value.lower() == 'true' + raise outcomes.InvalidError(f"'{name}' must be true or false.", expression=name) + + +def coding_of(value, name): + """A Coding parameter (in a GET, a system|code token), as a Coding; one without a system and a code is a 400.""" + if isinstance(value, str): + system, _, code = value.partition('|') + value = {'system': system, 'code': code} + if not isinstance(value, dict) or not all(isinstance(value.get(key), str) and value[key] + for key in ('system', 'code')): + raise outcomes.InvalidError(f"'{name}' must be a Coding with a system and a code.", expression=name) + return Coding(value['system'], value['code'], value.get('version')) + + +def read_request(parameters): + """The request in FhirParameters: the code (code and system, coding, or a codeableConcept's first coding), etc.""" + for name in UNSUPPORTED_PARAMETERS: + if name in parameters: + raise outcomes.NotSupportedError(f"$translate doesn't support '{name}'.", expression=name) + if 'coding' in parameters: + coding = coding_of(parameters.get('coding'), 'coding') + elif 'codeableConcept' in parameters: + concept = parameters.get('codeableConcept') + codings = (concept.get('coding') if isinstance(concept, dict) else None) or [] + if not isinstance(codings, list) or not codings: + raise outcomes.InvalidError("'codeableConcept' must hold a coding.", expression='codeableConcept') + coding = coding_of(codings[0], 'codeableConcept') + elif 'code' in parameters: + code = required_text(parameters, 'code') + system, _, piped = required_text(parameters, 'system').partition('|') + coding = Coding(system, code, text(parameters, 'version') or piped or None) + else: + raise outcomes.InvalidError('$translate needs the code it translates: code and system, coding, or ' + 'codeableConcept.', expression='code') + reverse = boolean(parameters, 'reverse') + scope_name = 'target' if reverse else 'source' # R4: reverse reverses the meaning of source and target + return Request(coding, text(parameters, 'url'), text(parameters, 'conceptMapVersion'), text(parameters, scope_name), + scope_name, text(parameters, 'targetsystem'), reverse) + + +class Translation: + """One $translate request at the base `kwargs` names, for `user`.""" + + def __init__(self, parameters, kwargs, user): + self.request = read_request(parameters) + self.kwargs, self.user, self.namespace = kwargs, user, namespace_of(kwargs) + + def answer(self): + """ + {'result', 'message', 'matches'}, each match {'relationship', 'concept': {system, code}, 'source'}. In + FHIRsmith's order: the ConceptMap url names, then the code's scope, then the ConceptMap of the code's system. + """ + request, coding = self.request, self.request.coding + concept_map, named = self.named_concept_map() if request.url else (None, None) + if request.scope and not self.in_scope(): + return self.no_match(f"'{coding.system}#{coding.code}' is not in the value set '{request.scope}', so no " + "ConceptMap can translate it") + if not request.url: + concept_map, named = self.code_system_map() + if concept_map is None: + return self.no_match(self.no_concept_map()) + matches, more = self.matches(concept_map, named) + if not matches: + into = f" {'from' if request.reverse else 'to'} '{request.targetsystem}'" if request.targetsystem else '' + return self.no_match(f"ConceptMap '{named}' has no mapping {'to' if request.reverse else 'from'} " + f"'{coding.system}#{coding.code}'{into}") + message = f'Only the first {MATCH_LIMIT} matches are listed.' if more else None + return {'result': any(match['relationship'] != NOT_RELATED_TO for match in matches), 'message': message, + 'matches': matches} + + @staticmethod + def no_match(message): + return {'result': False, 'message': message, 'matches': []} + + def no_concept_map(self): + names = [self.request.coding.system, self.request.targetsystem or '(unspecified)'] + source, target = names[::-1] if self.request.reverse else names + return f"No ConceptMap is available to translate from '{source}' to '{target}'" + + @cached_property + def named_sources(self): + """named_sources of the code's system, targetsystem and the code system its scope names, looked up once.""" + request, system = self.request, self.request.coding.system + # in this order, whatever their URLs, so the statement's shape is too + systems = dict.fromkeys([system, request.targetsystem or system, scope_system(request.scope) or system]) + return named_sources(systems, self.namespace, self.user) + + def resolve(self, resource_type, url, version, expression): + return resolve(resource_type, url, version, self.kwargs, self.user, expression) + + def named_concept_map(self): + """(the ConceptMap version url and conceptMapVersion name, how a match's source names it); else a 404.""" + instance = self.resolve('ConceptMap', self.request.url, self.request.concept_map_version, 'url') + return instance, self.map_name(instance, self.request.url) + + def code_system_map(self): + """ + Without url: (the version of the code system the codes come from, how a match's source names it), whose own + mappings are its map: the release conceptMapVersion names, else (forward) the code's version, else the newest + the caller is shown; (None, None) when there is none. + """ + request = self.request + if request.reverse: + url, version = request.targetsystem, request.concept_map_version + else: # conceptMapVersion names the map's release, else the code's version + url, version = request.coding.system, request.concept_map_version or request.coding.version + try: # resolve() raises before any query for no url (reverse without targetsystem) + instance = self.resolve('CodeSystem', url, version, 'system') + except UnresolvedError: + return None, None + return instance, self.map_name(instance, url) + + @staticmethod + def map_name(instance, url): + """A ConceptMap version as a match's source names it: canonical URL (else the URL it was named by)|version.""" + return f"{instance.canonical_url or url.partition('|')[0]}|{instance.version}" + + def in_scope(self): + """ + Whether the code is in its scope (`source`; in reverse, `target`): its own code system, or that system's + implicit value set, named any way named_sources reads (its canonical URL or a source's OCL path, of a repository + the caller is shown); a value set (a ValueSet or collections path is one) it is a member of (its default + expansion), as a concept of a source the caller is shown as the code's system, as the matches are; not another + code system. Neither one the caller is shown, nor ?fhir_vs of no system, is a 404 naming the parameter, as + FHIRsmith answers a scope it can't find. + """ + scope, name, coding = self.request.scope, self.request.scope_name, self.request.coding + system = scope_system(scope) + if not system: # ?fhir_vs of no system + raise self.scope_not_found(scope, name) + if system == coding.system: + return True + named = self.named_sources + if set(named[coding.system]) & set(named[system]): # one repository the caller is shown, whatever names it + return True + if system == scope: # not an implicit value set: a ValueSet, if one has that URL + try: + value_set = self.resolve('ValueSet', scope, None, name) + except UnresolvedError: + pass + else: + return holds(value_set, coding.code, named[coding.system]) + try: + self.resolve('CodeSystem', system, None, name) + except UnresolvedError as ex: + raise self.scope_not_found(scope, name) from ex + return False + + @staticmethod + def scope_not_found(scope, name): + return outcomes.NotFoundError( + f"The value set '{scope}' could not be found, so the scope of the translation cannot be determined", + expression=name, tx_issue_type='not-found') + + def match_ids(self, concept_map, near, far): + """ + The ids of the release's live mappings of the code whose sides are shown as the systems, sorted: in no order and + with no limit, so the planner reads the code's mappings by the code (and the source lists) and checks each + against the release; ordered and limited, it would walk the release in id order. + """ + request = self.request + sides = {near: request.coding.system, **({far: request.targetsystem} if request.targetsystem else {})} + mappings = concept_map.get_active_mappings().filter( # neither retired nor deleted + of_version(near, request.coding.version), **{f'{near}_concept_code__in': stored_codes(request.coding.code)}) + for side, system in sides.items(): + mappings = mappings.filter(shown_as(side, system, self.named_sources[system], self.namespace, self.user)) + return sorted(mappings.values_list('id', flat=True)) + + def matches(self, concept_map, named): + """ + (the matches, whether there were more than MATCH_LIMIT), in mapping order, each match once: the sources the + systems name, the mappings' ids, then the first MATCH_LIMIT + 1 of those mappings. + """ + near, far = ('to', 'from') if self.request.reverse else ('from', 'to') + ids = self.match_ids(concept_map, near, far) + rows = with_side(Mapping.objects.filter(id__in=ids[:MATCH_LIMIT + 1]), far, self.namespace, self.user).order_by( + 'id').values('map_type', f'{far}_concept_code', f'{far}_source_url', f'{far}_source__uri', f'{far}_shown', + f'{far}_canonical') + matches = {} + for row in rows: + system = side_system(row[f'{far}_canonical'], row[f'{far}_source_url'], + row[f'{far}_source__uri'] if row[f'{far}_shown'] else None) + match = (system, fhir_code(row[f'{far}_concept_code']), relationship(row['map_type'], self.request.reverse)) + matches.setdefault(match, { + 'relationship': match[2], 'concept': {'system': match[0], 'code': match[1]}, 'source': named}) + return list(matches.values())[:MATCH_LIMIT], len(ids) > MATCH_LIMIT + + +def translate(parameters, kwargs, user, _instance_id=None): + """ + A $translate's answer (Translation.answer) for its FhirParameters, at the base `kwargs` names, for `user`, its + statements bounded as a search's are (a cancelled one is too costly). It has no instance level. + """ + return within_search_timeout(lambda: Translation(parameters, kwargs, user).answer(), too_costly=too_costly) diff --git a/core/fhir/ops/validate_code.py b/core/fhir/ops/validate_code.py new file mode 100644 index 000000000..62314dae9 --- /dev/null +++ b/core/fhir/ops/validate_code.py @@ -0,0 +1,298 @@ +""" +CodeSystem and ValueSet $validate-code (SVCM ITI-99, FHIR R4's $validate-code), version-neutral: whether a code is +valid, with what OCL knows about it. core.fhir.adapters.r4 writes the answer as R4 Parameters. + +- Resolution and access are core.fhir.resolve's (a released version, the newest the caller is shown unless one is + named; public only at /fhir/; at an owner base, private ones to those who may view them); CodeSystem/{id} and + ValueSet/{id} name theirs as a read does. A code system, value set or content the caller isn't shown answers exactly + as one that doesn't exist. +- CodeSystem: the code in the version `url` (and `version`) names: result, and the code, system, version used and + display; `display` must be one of the concept's names (ignoring case); a retired concept is valid but flagged + inactive, with a warning. The code system is the instance, else url (and version), else a coding's system; a + coding of another system is result false (invalid-data). A code system that doesn't resolve answers result false + with a not-found issue and x-caused-by-unknown-system, as FHIRsmith does, with or without a code. Then a code is + required (400), and a url. +- ValueSet: membership is the resolved version's default expansion: the code, from a concept whose code system is + `system` (and `systemVersion`), in a code system version the caller is shown, released or not. The version + answered is a live release the caller is shown that holds the member's row: the expansion's explicit pin, then + its evaluated pin, then the newest. A versioned-object row carries the concept's newest content, so it uses the + newest holding release of another, not deleted, version, whatever the expansion pins; none without one (never HEAD + or an unreleased version). Not a member: result false (not-in-vs), with what the code system itself says of the code + (unknown system, unknown code, or its display). A value set that doesn't resolve is a 422 not-found OperationOutcome. + Without url (and not at the instance level), the code is checked in its code system alone, as OCL has always answered. +- Issues carry FHIRsmith's message ids (operationoutcome-message-id) and expressions, so clients and the parity + suite compare them by kind; `message` joins their texts. +- Cost: a fixed number of queries whatever the size of the code system or value set. +""" +from django.db.models import Case, Exists, IntegerField, OuterRef, Q, Value, When +from rest_framework import status + +from core.collections.models import Expansion +from core.common.constants import HEAD +from core.concepts.models import Concept +from core.fhir import outcomes +from core.fhir.ops.concepts import ( + coded_input, path, require, text, find_concept, display_of, has_name, fhir_code, code_q, instance_version, + is_system_of, mismatch, system_of) +from core.fhir.ops.lookup import code_system_version +from core.fhir.resolve import UnresolvedError, namespace_of, resolve, to_ocl_path, viewable_rows +from core.sources.models import Source + +UNKNOWN_CODE_SYSTEM = 'x-caused-by-unknown-system' # how CodeSystem $validate-code names an unknown system +UNKNOWN_VALUE_SET_SYSTEM = 'x-unknown-system' # and ValueSet $validate-code + + +def issue(coding, part, kind, message_id, message): + """An issue about one part of the code (code, system or display), of a kind in KINDS.""" + severity, code, tx_issue_type = KINDS[kind] + return outcomes.issue(code, message, severity=severity, expression=path(coding, part), + tx_issue_type=tx_issue_type, message_id=message_id) + + +# kind: (severity, issue type, tx-issue-type), as FHIRsmith reports each +KINDS = { + 'unknown-system': ('error', 'not-found', 'not-found'), + 'unknown-code': ('error', 'code-invalid', 'invalid-code'), + 'wrong-display': ('error', 'invalid', 'invalid-display'), + 'inactive': ('warning', 'business-rule', 'code-comment'), + 'not-in-value-set': ('error', 'code-invalid', 'not-in-vs'), + 'other-system': ('error', 'invalid', 'invalid-data'), +} + + +def answer(coding, issues=(), **values): + """ + The neutral answer: result (no error issue), the code and system asked about, what was found (version, display, + inactive, the unknown system), the issues and the message their texts make. + """ + issues = list(issues) + return { + 'result': not any(item['severity'] in ('error', 'fatal') for item in issues), + 'message': '; '.join(item['details']['text'] for item in issues) or None, + 'code': fhir_code(coding.code), 'system': coding.system, **values, 'issues': issues, + } + + +def unknown_system(coding, parameter, *more, **extra): + """ + The answer for a code system that doesn't resolve, or that the caller isn't shown: the same for both, as it depends + only on what was asked (the system, and the version named or url|version). `more` are issues after its own. + """ + system, _, piped = coding.system.partition('|') + version = coding.version or piped or None + if version: + message_id = 'UNKNOWN_CODESYSTEM_VERSION' + message = f"A definition for CodeSystem '{system}' version '{version}' could not be found, " \ + "so the code cannot be validated" + else: + message_id = 'UNKNOWN_CODESYSTEM' + message = f"A definition for CodeSystem '{system}' could not be found, so the code cannot be validated" + named = f'{system}|{version}' if version else system + return answer(coding, [issue(coding, 'system', 'unknown-system', message_id, message), *more], + unknown=(parameter, named), **extra) + + +def unknown_code(coding, system, version): + return issue(coding, 'code', 'unknown-code', 'Unknown_Code_in_Version', + f"Unknown code '{fhir_code(coding.code)}' in the CodeSystem '{system}' version '{version}'") + + +def found(coding, concept, version, language): + """ + The answer for a code its code system (or value set) holds: its version, display and inactive flag, with the issues + of a display that isn't one of its names and of a retired concept. + """ + display = display_of(concept, language) + issues = [] + if coding.display and not has_name(concept, coding.display): + issues.append(issue( + coding, 'display', 'wrong-display', 'Display_Name_for__should_be_one_of__instead_of', + f"Wrong Display Name '{coding.display}' for {coding.system}#{fhir_code(coding.code)}: it should be one of " + f"the concept's names, such as '{display}'")) + if concept.retired: + issues.append(issue(coding, 'code', 'inactive', 'INACTIVE_CONCEPT_FOUND', + f"The concept '{fhir_code(coding.code)}' has a status of inactive and its use should be " + "reviewed")) + return answer(coding, issues, version=version, display=display, inactive=concept.retired or None) + + +def in_code_system(coding, version, language): + """The answer for a code in a code system version it resolved to.""" + system = system_of(version, coding.system) + coding = coding._replace(system=system) + concept = find_concept(version, coding.code) + if not concept: + return answer(coding, [unknown_code(coding, system, version.version)], version=version.version) + return found(coding, concept, version.version, language) + + +def validate_code_system(parameters, kwargs, user, instance_id=None): + """ + CodeSystem $validate-code's answer for its FhirParameters at the base `kwargs` names (CodeSystem/{instance_id} at + the instance level), for `user`. A url that doesn't resolve answers before the code is required, as it always has. + """ + coding = coded_input(parameters, 'url', 'version') + # the code system validated against: the instance, else url and version, else the coding's + target = coding._replace( + system=text(parameters, 'url') or coding.system, version=text(parameters, 'version') or coding.version) + version = None + if instance_id or target.system: + try: + version = code_system_version(target, kwargs, user, instance_id) + except UnresolvedError: + return unknown_system(target, UNKNOWN_CODE_SYSTEM, version=target.version) + require(coding, 'code', 'A code to validate is required.') + if version is None: + raise outcomes.InvalidError('url is required with a code: the code system to validate it in.', + expression=coding.parameter or 'url') + conflict = mismatch(parameters, coding, version, 'url', instance_id) + if conflict: + severity, code, tx_issue_type = KINDS['other-system'] + return answer(coding, [outcomes.issue( + code, f'{conflict[1]}: the code is not validated', severity=severity, expression=conflict[0], + tx_issue_type=tx_issue_type)]) + return in_code_system(coding, version, text(parameters, 'displayLanguage')) + + +def system_q(field, system): + """Q for concepts (through `field`) whose repository is the system: by canonical URL, or by an OCL path.""" + reference = to_ocl_path(system) + if reference.startswith('/'): + return Q(**{f'{field}__uri': reference if reference.endswith('/') else f'{reference}/'}) + return Q(**{f'{field}__canonical_url': system}) + + +def named_by(system, shown): + """ + (Q for concepts, Q for their concept-source links) the system names, as a read names a code system version: by the + version's own canonical URL, else its repository's (HEAD's); an OCL path names every version of its repository. + So a concept is the system's when its repository is, or has a version the caller is shown (`shown`) with that + canonical URL of its own; and only the versions the system names are answered (as $expand reports a member). + The repositories are found among sources alone, so the concepts are still read by their code first. + """ + if to_ocl_path(system).startswith('/'): + return system_q('parent', system), Q() + renamed = shown.filter(canonical_url=system, mnemonic=OuterRef('parent__mnemonic')).filter( + Q(organization_id=OuterRef('parent__organization_id')) | Q(user_id=OuterRef('parent__user_id'))) + inherited = Q(source__canonical_url__isnull=True) | Q(source__canonical_url='') + return system_q('parent', system) | Exists(renamed), Q(source__canonical_url=system) | ( + inherited & Q(concept__parent__canonical_url=system)) + + +def member(value_set, coding, namespace, user): + """ + (the concept, the version of its code system) of the code in the value set version's default expansion, or None: + a concept with the code (not deleted), of the system (in systemVersion, if named), in a code system version the + caller is shown (not deleted), released or not, that the system names (named_by): the expansion defines the value + set. The version answered is a live release the caller is shown that holds the member's row: the expansion's + explicit pin, then its evaluated pin, then the newest. A versioned-object row, held only by HEAD, carries the + concept's newest content, so it uses the newest holding release of another, not deleted, version, ignoring pins. + None without one, never HEAD or an unreleased version. The concept answered is the member itself, as $expand + shows it. + The code's concepts are read first, by their code and only in code systems the caller is shown (so what hidden ones + hold costs nothing), then looked for in the expansion by its (expansion, concept) index, so the cost doesn't grow + with the size of the value set. + """ + if not value_set.expansion_uri: + return None + concepts_named, versions_named = named_by(coding.system, viewable_rows(Source, namespace, user)) + concepts = dict(Concept.objects.filter( + code_q('mnemonic', coding.code), concepts_named, + Exists(viewable_rows(Source, namespace, user).filter(id=OuterRef('parent_id'))), is_active=True).values_list( + 'id', 'versioned_object_id')) + concept_ids = list(Expansion.concepts.through.objects.filter( + expansion__uri=value_set.expansion_uri, concept_id__in=concepts).values_list('concept_id', flat=True)) + if not concept_ids: + return None + release = Q(source__released=True, source__retired=False) & ~Q(source__version=HEAD) + links = Concept.sources.through.objects.filter( + Exists(viewable_rows(Source, namespace, user).filter(id=OuterRef('source_id'))), source__is_active=True) + # another version of a member's concept counts only while no release the caller is shown holds the member itself + rows = links.filter( + Q(concept_id__in=concept_ids) | ~Exists(links.filter( + release, concept_id__in=concept_ids, + concept__versioned_object_id=OuterRef('concept__versioned_object_id'))), + versions_named, + concept__versioned_object_id__in=sorted({concepts[concept_id] for concept_id in concept_ids}), + concept__is_active=True) + if coding.version: + rows = rows.filter(source__version=coding.version) + def pinned(versions): + """Whether the link holds the member's row itself and the expansion pins its source.""" + return Case(When(Q(concept_id__in=concept_ids) & Exists(versions.through.objects.filter( + expansion__uri=value_set.expansion_uri, source_id=OuterRef('source_id'))), then=Value(1)), + default=Value(0), output_field=IntegerField()) + row = rows.annotate( + release=Case(When(release, then=Value(1)), default=Value(0), output_field=IntegerField()), + held=Case(When(concept_id__in=concept_ids, then=Value(1)), default=Value(0), output_field=IntegerField()), + explicit=pinned(Expansion.explicit_source_versions), evaluated=pinned(Expansion.evaluated_source_versions), + ).order_by('-release', '-held', '-explicit', '-evaluated', '-source__created_at', 'concept_id').values_list( + 'concept_id', 'concept__versioned_object_id', 'source__version', 'release').first() + if not row: + return None + member_id = row[0] if row[0] in concept_ids else min( + concept_id for concept_id in concept_ids if concepts[concept_id] == row[1]) + concept = Concept.objects.filter(id=member_id).select_related('parent').prefetch_related('names').first() + return concept, row[2] if row[3] else None + + +def value_set_version(parameters, kwargs, user, instance_id): + """ + The value set version the request names; a url that doesn't resolve is a 422 not-found. At the instance level the id + names it, and a url or valueSetVersion naming another one is a 400. + """ + version, url = text(parameters, 'valueSetVersion'), text(parameters, 'url') + if instance_id: + value_set = instance_version('ValueSet', instance_id, version, kwargs, user) + if url and not is_system_of(url, value_set): + raise outcomes.InvalidError(f"'{url}' is not the value set {instance_id} this $validate-code is on.", + expression='url') + for named, expression in ((version, 'valueSetVersion'), ((url or '').partition('|')[2], 'url')): + if named and named != value_set.version: + raise outcomes.InvalidError( + f"Version '{named}' is not the version '{value_set.version}' of the value set {instance_id}.", + expression=expression) + return value_set + try: + return resolve('ValueSet', url, version, kwargs, user) + except UnresolvedError as ex: + raise outcomes.NotFoundError( + f"A definition for the value Set '{url}' could not be found", tx_issue_type='not-found', + message_id='Unable_to_resolve_value_Set_', status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) from ex + + +def validate_value_set(parameters, kwargs, user, instance_id=None): + """ + ValueSet $validate-code's answer for its FhirParameters at the base `kwargs` names (ValueSet/{instance_id} at the + instance level), for `user`. A code and its system are required first. + """ + coding = coded_input(parameters, 'system', 'systemVersion') + require(coding, 'code', 'A code to validate is required.') + require(coding, 'system', 'system is required with a code: the code system the code is from.') + language = text(parameters, 'displayLanguage') + if not instance_id and not text(parameters, 'url'): + try: + return in_code_system(coding, resolve('CodeSystem', coding.system, coding.version, kwargs, user), language) + except UnresolvedError: + return unknown_system(coding, UNKNOWN_CODE_SYSTEM, version=coding.version) + + value_set = value_set_version(parameters, kwargs, user, instance_id) + hit = member(value_set, coding, namespace_of(kwargs), user) + return found(coding, *hit, language) if hit else not_a_member(coding, value_set, kwargs, user, language) + + +def not_a_member(coding, value_set, kwargs, user, language): + """A code outside the value set: not-in-vs, with what its code system says of it.""" + name = f"{system_of(value_set, value_set.uri)}|{value_set.version}" + outside = issue(coding, 'code', 'not-in-value-set', 'None_of_the_provided_codes_are_in_the_value_set_one', + f"The provided code '{coding.system}#{fhir_code(coding.code)}' was not found in the value set " + f"'{name}'") + try: + version = resolve('CodeSystem', coding.system, coding.version, kwargs, user) + except UnresolvedError: + return unknown_system(coding, UNKNOWN_VALUE_SET_SYSTEM, outside) + system = system_of(version, coding.system) + concept = find_concept(version, coding.code) + if not concept: + return answer(coding, [unknown_code(coding, system, version.version), outside]) + return answer(coding, [outside], version=version.version, display=display_of(concept, language)) diff --git a/core/fhir/outcomes.py b/core/fhir/outcomes.py new file mode 100644 index 000000000..a77484893 --- /dev/null +++ b/core/fhir/outcomes.py @@ -0,0 +1,247 @@ +""" +OperationOutcome for the FHIR API: typed errors, and the exception handler that answers every error a FHIR view +raises with an OperationOutcome. + +Every issue carries a tx-issue-type coding, as FHIRsmith and tx.fhir.org do, so clients and the parity suite can +compare errors by kind rather than by wording. No FHIR response is a 5xx: an error OCL can't attribute to the +request is answered with a 422 and the issue code `exception`, and logged. +""" +import logging +from http import HTTPStatus + +from django.core.exceptions import BadRequest, SuspiciousOperation, ValidationError as DjangoValidationError, \ + PermissionDenied as DjangoPermissionDenied +from django.http import Http404 +from rest_framework import exceptions, status +from rest_framework.response import Response +from rest_framework.views import set_rollback + +logger = logging.getLogger('oclapi') + +TX_ISSUE_TYPE_SYSTEM = 'http://hl7.org/fhir/tools/CodeSystem/tx-issue-type' +MESSAGE_ID_EXTENSION = 'http://hl7.org/fhir/StructureDefinition/operationoutcome-message-id' + +# The tx-issue-type of an issue that names none, by its FHIR issue type. This is FHIRsmith's table +# (tx/library/operation-outcome.js), plus `throttled`; every other issue type gets `invalid-data`, as there. +DEFAULT_TX_ISSUE_TYPES = { + 'not-found': 'not-found', + 'code-invalid': 'invalid-code', + 'business-rule': 'business-rule', + 'too-costly': 'too-costly', + 'not-supported': 'not-supported', + 'throttled': 'too-costly', +} +FALLBACK_TX_ISSUE_TYPE = 'invalid-data' + +# The issue type of an error that has only an HTTP status +STATUS_ISSUE_CODES = { + 400: 'invalid', + 401: 'login', + 403: 'forbidden', + 404: 'not-found', + 405: 'not-supported', + 406: 'not-supported', + 410: 'deleted', + 415: 'not-supported', + 422: 'processing', + 429: 'throttled', +} +NON_FIELD_ERROR_KEYS = ('non_field_errors', '__all__') +UNEXPECTED_ERROR_STATUS = status.HTTP_422_UNPROCESSABLE_ENTITY +UNEXPECTED_ERROR_MESSAGE = 'An unexpected error occurred while processing the request.' + + +def tx_issue_type_for(issue_code): + """The tx-issue-type for an issue that doesn't name one.""" + return DEFAULT_TX_ISSUE_TYPES.get(issue_code, FALLBACK_TX_ISSUE_TYPE) + + +def issue( # pylint: disable=too-many-arguments + code, text, severity='error', expression=None, tx_issue_type=None, message_id=None, diagnostics=None): + """One OperationOutcome issue. `expression` is a FHIRPath (the parameter or element at fault) or a list of them.""" + result = { + 'severity': severity, + 'code': code, + 'details': { + 'coding': [{'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type or tx_issue_type_for(code)}], + 'text': text + } + } + if expression: + result['expression'] = [expression] if isinstance(expression, str) else list(expression) + if message_id: + result['extension'] = [{'url': MESSAGE_ID_EXTENSION, 'valueString': message_id}] + if diagnostics: + result['diagnostics'] = diagnostics + return result + + +def operation_outcome(*issues): + return {'resourceType': 'OperationOutcome', 'issue': list(issues)} + + +def is_outcome(data): + return isinstance(data, dict) and data.get('resourceType') == 'OperationOutcome' + + +def response_status(status_code): + """The status a FHIR error goes out with: a 5xx becomes a 422.""" + return UNEXPECTED_ERROR_STATUS if status_code >= 500 else status_code + + +def status_issue_code(status_code): + if status_code >= 500: + return 'exception' + return STATUS_ISSUE_CODES.get(status_code, 'invalid') + + +def status_text(status_code): + try: + return HTTPStatus(status_code).phrase + except ValueError: + return UNEXPECTED_ERROR_MESSAGE + + +def to_plain(value): + """DRF error details (ErrorDetail strings, nested) as plain JSON values.""" + if isinstance(value, dict): + return {str(key): to_plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [to_plain(item) for item in value] + return str(value) if isinstance(value, str) else value + + +def detail_issues(detail, code='invalid', field=None): + """Issues for a serializer-style error detail: one per message, naming the field it belongs to.""" + if isinstance(detail, dict): + issues = [] + for key, value in detail.items(): + issues += detail_issues(value, code, field or (None if key in NON_FIELD_ERROR_KEYS else str(key))) + return issues + if isinstance(detail, (list, tuple)): + return [item for value in detail for item in detail_issues(value, code, field)] + return [issue(code, str(detail), expression=field)] + + +def with_tx_issue_types(outcome): + """An OperationOutcome built elsewhere, with a tx-issue-type coding and a code on every issue.""" + outcome = to_plain(outcome) + for item in outcome.get('issue') or []: + item.setdefault('severity', 'error') + item.setdefault('code', 'processing') + details = item.get('details') + if not isinstance(details, dict): + details = item['details'] = {'text': str(details)} if details else {} + codings = details.setdefault('coding', []) + if not any(coding.get('system') == TX_ISSUE_TYPE_SYSTEM for coding in codings): + codings.append({'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type_for(item['code'])}) + return outcome + + +def normalize_outcomes(data): + """Normalize every returned OperationOutcome, including resources nested in Bundle entries.""" + if isinstance(data, dict): + data = {key: normalize_outcomes(value) for key, value in data.items()} + return with_tx_issue_types(data) if is_outcome(data) else data + if isinstance(data, (list, tuple)): + return [normalize_outcomes(value) for value in data] + return data + + +def outcome_for_status(status_code, data=None): + """An error response that isn't an OperationOutcome (an empty 404, a {'detail': ...} body) as one.""" + code = status_issue_code(status_code) + if isinstance(data, dict) and 'detail' in data: + return operation_outcome(issue(code, str(data['detail']))) + if data and isinstance(data, (dict, list)): + return operation_outcome(*detail_issues(data, code)) + return operation_outcome(issue(code, status_text(status_code))) + + +class FhirError(exceptions.APIException): + """An error a FHIR view raises on purpose. Each subclass names its issue type and HTTP status.""" + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'processing' + default_detail = 'The request could not be processed.' + + def __init__( # pylint: disable=too-many-arguments + self, message=None, expression=None, tx_issue_type=None, message_id=None, status_code=None): + super().__init__(message or self.default_detail) + self.expression = expression + self.tx_issue_type = tx_issue_type + self.message_id = message_id + if status_code: + self.status_code = status_code + + def outcome(self): + return operation_outcome(issue( + self.issue_code, str(self.detail), expression=self.expression, tx_issue_type=self.tx_issue_type, + message_id=self.message_id)) + + +class NotFoundError(FhirError): + status_code = status.HTTP_404_NOT_FOUND + issue_code = 'not-found' + default_detail = 'The resource was not found.' + + +class InvalidError(FhirError): + status_code = status.HTTP_400_BAD_REQUEST + issue_code = 'invalid' + default_detail = 'The request is not valid.' + + +class NotSupportedError(FhirError): + status_code = status.HTTP_400_BAD_REQUEST + issue_code = 'not-supported' + default_detail = 'The request uses something this server does not support.' + + +class TooCostlyError(FhirError): + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'too-costly' + default_detail = 'The request is too costly to process.' + + +class ProcessingError(FhirError): + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + issue_code = 'processing' + + +def outcome_for_exception(exc): + """(status, OperationOutcome, headers) for an exception raised while answering a FHIR request.""" + headers = {} + if isinstance(exc, FhirError): + status_code, outcome = exc.status_code, exc.outcome() + elif isinstance(exc, (Http404, exceptions.NotFound)): + status_code = 404 + outcome = operation_outcome(issue('not-found', str(getattr(exc, 'detail', exc)) or status_text(404))) + elif isinstance(exc, DjangoPermissionDenied): + status_code, outcome = 403, operation_outcome(issue('forbidden', str(exc) or status_text(403))) + elif isinstance(exc, (BadRequest, SuspiciousOperation)): + status_code, outcome = 400, operation_outcome(issue('invalid', 'The request could not be parsed.')) + elif isinstance(exc, DjangoValidationError): + detail = exc.message_dict if hasattr(exc, 'error_dict') else exc.messages + status_code, outcome = 400, operation_outcome(*detail_issues(detail)) + elif isinstance(exc, exceptions.ValidationError): + status_code = exc.status_code + outcome = with_tx_issue_types(exc.detail) if is_outcome(exc.detail) else operation_outcome( + *detail_issues(exc.detail)) + elif isinstance(exc, exceptions.APIException): + if getattr(exc, 'auth_header', None): + headers['WWW-Authenticate'] = exc.auth_header + if getattr(exc, 'wait', None): + headers['Retry-After'] = str(int(exc.wait)) + status_code = exc.status_code + outcome = operation_outcome(*detail_issues(exc.detail, status_issue_code(exc.status_code))) + else: + logger.exception('Unexpected %s answering a FHIR request', type(exc).__name__) + status_code, outcome = UNEXPECTED_ERROR_STATUS, operation_outcome(issue('exception', UNEXPECTED_ERROR_MESSAGE)) + return status_code, outcome, headers + + +def exception_handler(exc, context): # pylint: disable=unused-argument + """The exception handler of FHIR views (FhirViewMixin): every error is an OperationOutcome, never a 5xx.""" + status_code, outcome, headers = outcome_for_exception(exc) + set_rollback() + return Response(outcome, status=response_status(status_code), headers=headers) diff --git a/core/fhir/resolve.py b/core/fhir/resolve.py new file mode 100644 index 000000000..18aae9fe9 --- /dev/null +++ b/core/fhir/resolve.py @@ -0,0 +1,527 @@ +""" +FHIR's canonical resolution: the resolver $resolveReference uses (ConceptContainerModel.resolve_reference_expression, +with or without the URL registry), with FHIR's rules on top. + +- Resource type: CodeSystem and ConceptMap resolve to a Source, ValueSet to a Collection. +- Namespace: the base URL's. /fhir/ is global; /orgs/{org}/ and /users/{user}/ are that owner's. It comes from the + route, never from the request. +- Version: the one named (the operation's version parameter, else url|version, else the version an OCL path names; + a path naming one version with another named names none), HEAD included, as it is stored: a request's value is + already decoded, so 2.0+build names 2.0+build. With none named, the newest released + version the caller is shown, never HEAD. A repository with nothing released isn't visible, whatever version is + named. +- A version named, with a canonical URL (decision 9): when the repository the resolver picks doesn't serve the caller + a release of that name, the release of the lowest-id repository with that canonical URL the base shows the caller + that holds one; failing that, the version so named (unreleased, or HEAD) of the lowest-id one it shows, as before; + else not found. A repository or version the caller isn't shown never holds it. This applies where the repositories + with the canonical URL are the step that picks: without the URL registry, at /fhir/ (when it finds one) and at an + owner base (whether or not it finds one, so a hidden repository is answered as a missing one is); with it, at an + owner base whose own registry has no entry for the URL and that has a repository with it. A registry entry decides + (at /fhir/ only an entry resolves), and where resolution finds no repository it still finds none. + At an owner base a repository the base doesn't show the caller, and one whose HEAD it shows but whose releases are + all private, are both answered as a missing one is, byte for byte; the statement count can still differ by one, + the A.3 residual of the shared resolver's early stop rather than of this rule, which a change in the shared + resolver would close. tests_resolve_named_version.HiddenHolderCases measures both, and measures the same + differences on the code before this rule. +- Access, after resolving: at the global base public repositories only, at an owner base what the caller may view. + The version and its repository (HEAD) must both be viewable. Without the URL registry, the global base's canonical + match considers only the repositories it shows, so a private copy of a canonical never hides a public one. Otherwise + (an owner base, a registry entry) a repository the caller can't see is not found, and no other repository is tried; + none of its versions is read. + +Every failure is the same UnresolvedError, so it never tells a caller that a repository they can't see exists. + +served_repositories is the set form of the resolver's choices for code systems, for TerminologyCapabilities: the +identifier each repository a search shows is served at, and the versions of it a caller is served. +""" +from collections import namedtuple +from urllib.parse import quote_plus + +from django.db.models import Case, Exists, OuterRef, Q, Subquery, Value, When +from django.db.models.functions import Coalesce +from rest_framework.exceptions import ValidationError + +from core.collections.models import Collection +from core.common.constants import HEAD, ACCESS_TYPE_VIEW, RETIRED_ACCESS_TYPE_EDIT +from core.common.mixins import SourceContainerMixin +from core.common.serializers import IdentifierSerializer +from core.common.utils import decode_string, is_canonical_uri, normalize_public_access, to_parent_uri +from core.fhir.outcomes import NotFoundError +from core.sources.models import Source +from core.toggles.models import Toggle +from core.url_registry.models import URLRegistry + +REPO_TYPES = {'CodeSystem': Source, 'ConceptMap': Source, 'ValueSet': Collection} +REPO_PATHS = {'CodeSystem': 'sources', 'ConceptMap': 'sources', 'ValueSet': 'collections'} +PUBLIC_ACCESS = (ACCESS_TYPE_VIEW, RETIRED_ACCESS_TYPE_EDIT) # what public_can_view reads as public + + +class UnresolvedError(NotFoundError): + """ + No repository version of the resource type the caller may see for a canonical URL (and version). `url` is the + url without a |version, `version` the one asked for: the version parameter, else the url's. + """ + + def __init__(self, resource_type, url, version=None, expression=None): + self.resource_type, self.url, self.version = resource_type, url, version + named = f" version '{version}'" if version else '' + super().__init__( + f"A definition for {resource_type} '{url}'{named} could not be found", expression=expression, + tx_issue_type='not-found') + + +def namespace_of(kwargs): + """The namespace of the FHIR base a route's kwargs name: the owner's URI, or None for the global base.""" + kwargs = kwargs or {} + if kwargs.get('org'): + return f"/orgs/{kwargs['org']}/" + if kwargs.get('user'): + return f"/users/{kwargs['user']}/" + return None + + +def resolve(resource_type, url, version=None, kwargs=None, user=None, expression=None): # pylint: disable=too-many-arguments,too-many-locals + """ + The repository version a FHIR request names: `url` (a canonical URL, or an OCL path such as + /orgs/{org}/CodeSystem/{id}/) and `version`, at the base `kwargs` names, for `user`. A view passes its own + self.kwargs and self.request.user. Raises UnresolvedError, whose `expression` is the parameter the url came in. + """ + url = url or '' + lookup_url, _, piped_version = url.partition('|') + in_path, elsewhere = path_version(lookup_url), version or piped_version or None + error = UnresolvedError(resource_type, lookup_url, elsewhere or in_path, expression) + namespace = namespace_of(kwargs) + # a second pipe, or a path naming one version with another named, names no version OCL holds + conflicting = '|' in piped_version or bool(in_path and elsewhere and in_path != elsewhere) + if not lookup_url or conflicting or (namespace and not SourceContainerMixin.get_object_from_namespace(namespace)): + raise error + + # the version as named (HTTP has decoded the query already), so the shared resolver, which decodes it again, + # reads it back as it is stored: 2.0+build stays 2.0+build + repo_type = REPO_TYPES[resource_type] + instance, entry, picked = shared_resolution( + repo_type, lookup_url, as_stored(elsewhere) if elsewhere else None, namespace, user) + if instance.id and not (elsewhere or in_path) and not is_shown(instance, namespace, user): + instance = latest_public_release(instance) or instance # the newest release is private: the newest public one + shown = bool(instance.id) and is_shown(instance, namespace, user) + # a version named that the repository picked doesn't serve as a release: its holder's, where the repositories with + # the canonical URL are the step that picks (decision 9) + if elsewhere and holders_pick(instance, entry, picked, namespace) and not ( + shown and (instance.is_head or instance.released)): + held = held_version(repo_type, instance.resolution_url, elsewhere, namespace, user) + if held and held.id != instance.id: + if namespace and hasattr(picked, 'fhir_may_view_private'): # the same owner: the same right + held.fhir_may_view_private = picked.fhir_may_view_private + instance, shown = held, is_shown(held, namespace, user) + + if not shown: + raise error + return instance + + +def shared_resolution(repo_type, url, version, namespace, user): + """ + (the shared resolver's answer, the registry entry that decided it or None, the HEAD it picked or None), with FHIR's + flags: no HEAD by default; at the global base only the HEADs it shows; and a HEAD the caller isn't shown resolves + to nothing before any version is read. The version found shares the HEAD's check of the caller's right to the + owner's private repositories, so it isn't asked twice. + """ + heads = [] + + def head_shown(head): + heads.append(head) + return is_visible(head, namespace, user) + instance, entry = repo_type.resolve_reference_expression( + to_ocl_path(url), namespace, version, repo_type=repo_type, allow_head=False, + heads=None if namespace else shown_heads(repo_type), head_shown=head_shown) + if heads and hasattr(heads[0], 'fhir_may_view_private'): + instance.fhir_may_view_private = heads[0].fhir_may_view_private + return instance, entry, heads[0] if heads else None + + +def holders_pick(instance, entry, picked, namespace): + """ + Whether the repositories with the canonical URL a request names are the step that picked `instance` (the shared + resolver's answer, `entry` the registry entry that decided it, `picked` the HEAD it found): no entry decided, and + at /fhir/ a HEAD was found (with the URL registry on, at /fhir/ only an entry finds one), at an owner base whether + or not one was. An OCL path names its repository itself. + """ + return instance.is_fqdn and entry is None and bool(namespace or picked) + + +def held_version(repo_type, url, version, namespace, user): + """ + The version named `version` the base shows the caller among the repositories with the canonical URL `url` it + shows (their active HEAD has it, isn't retired, and has a live release the caller is shown; at an owner base, the + owner's), as resolve() serves a version named when the repository the resolver picks serves no release of it: a + release (retired or not), of the lowest-id repository holding one; else the version so named (unreleased, or HEAD) + of the lowest-id repository; None without either. Its repository's access and canonical URL are set on it, as the + resolver sets them. One query, whatever the number of repositories; what the caller isn't shown is left out in it. + """ + shown = shown_q(namespace, user) + owner = {} + if namespace: + owner = {'organization__uri' if namespace.startswith('/orgs/') else 'user__uri': namespace} + same_repository = Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id')) + live = repo_type.objects.filter( + shown, same_repository, mnemonic=OuterRef('mnemonic'), is_active=True, released=True, + retired=False).exclude(version=HEAD) + heads = repo_type.objects.filter( + shown, Exists(live), canonical_url=url, version=HEAD, is_active=True, retired=False, **owner) + head = heads.filter(same_repository, mnemonic=OuterRef('mnemonic')) + release = Q(released=True) & ~Q(version=HEAD) + held = repo_type.objects.filter(shown, version=version, is_active=True, **owner).annotate( + head_id=Subquery(head.values('id')[:1]), repository_public_access=Subquery(head.values('public_access')[:1]), + ).filter(release | Q(head_id=Subquery(heads.order_by('id').values('id')[:1])), head_id__isnull=False).order_by( + Case(When(release, then=Value(0)), default=Value(1)), 'head_id').first() + if held: + held.is_fqdn, held.resolution_url, held.repository_canonical_url = True, url, url + held.canonical_url = held.canonical_url or url + return held + + +def path_version(url): + """The version an OCL path names (/orgs/{org}/sources/{id}/{version}/, or as a CodeSystem path), or None.""" + parts = to_parent_uri(to_ocl_path(url)).split('/') + if len(parts) > 5 and not parts[0] and parts[1] in ('orgs', 'users') and parts[3] in REPO_PATHS.values(): + return parts[5] or None + return None + + +def is_shown(instance, namespace, user): + """Whether a base shows the caller a repository version: it is visible there, and so is a release of it.""" + return is_visible(instance, namespace, user) and has_release(instance, namespace, user) + + +def shown_heads(model): + """ + Q for the HEAD rows the global base shows: public, with a live (released, not retired) public release. The + shared resolver's canonical match takes the first of them, without the URL registry. One EXISTS with the owner + matched inside it, as with_head, so the planner runs it as a semi-join on the few rows the canonical URL matches. + """ + return Q(public_access__in=PUBLIC_ACCESS) & Exists(model.objects.filter( + mnemonic=OuterRef('mnemonic'), is_active=True, released=True, retired=False, + public_access__in=PUBLIC_ACCESS).exclude(version=HEAD).filter( + Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id')))) + + +def latest_public_release(instance): + """The newest live public release of a version's repository, resolved as the version was, or None.""" + owner = {'organization_id': instance.organization_id} if instance.organization_id else {'user_id': instance.user_id} + latest = type(instance).objects.filter( + mnemonic=instance.mnemonic, is_active=True, released=True, retired=False, public_access__in=PUBLIC_ACCESS, + **owner).exclude(version=HEAD).order_by('-created_at').first() + if latest: + latest.repository_public_access = repository_access(instance) + latest.repository_canonical_url = getattr(instance, 'repository_canonical_url', None) + latest.is_fqdn, latest.resolution_url = instance.is_fqdn, instance.resolution_url + if instance.is_fqdn and not latest.canonical_url: + latest.canonical_url = instance.resolution_url + return latest + + +def to_ocl_path(url): + """ + An OCL FHIR path (/orgs/{org}/CodeSystem/{id}/...) as the OCL path of the repository it names, by its own + resource type, so a reference to the other type finds nothing; anything else as it is. + """ + parts = url.split('/') + if len(parts) > 3 and not parts[0] and parts[1] in ('orgs', 'users') and parts[3] in REPO_PATHS: + parts[3] = REPO_PATHS[parts[3]] + return '/'.join(parts) + + +def is_visible(instance, namespace, user): + """ + Whether a base shows a repository version: the version and its repository (its active HEAD, which governs access + to the repository's concepts and mappings) must both be viewable there. At the global base both are public, to + anyone; at an owner base the same, or any of the owner's versions to those who may view its private repositories + (staff, the org's members, the user). + """ + access = repository_access(instance) + if access is None: + return False + return (instance.public_can_view and is_public(access)) or may_view_private(instance, namespace, user) + + +def has_release(instance, namespace=None, user=None): + """ + Whether the repository of a version has a released version that isn't retired, and that the base shows the + caller: a public one, or any to those who may view the owner's private repositories. HEAD's released flag isn't + one. (The repository's own access, HEAD's, is is_visible's to check.) + """ + if instance.released and not instance.retired and not instance.is_head and ( + instance.public_can_view or may_view_private(instance, namespace, user)): + return True + live = instance.released_versions.filter(retired=False).exclude(version=HEAD) + if not may_view_private(instance, namespace, user): + live = live.filter(public_access__in=PUBLIC_ACCESS) + return live.exists() + + +def is_public(public_access): + """A public_access value as public_can_view reads it.""" + return normalize_public_access(public_access).lower() == ACCESS_TYPE_VIEW.lower() + + +def repository_access(instance): + """ + The public_access of a version's repository, its active HEAD's, or None without one. A read (with_repository_access) + and the resolver (resolve_repo) set it on the version they find, so it isn't looked up again; with the canonical + URL of its repository, which the HEAD looked up here gives the version too (one an OCL path names itself). + """ + if instance.is_head: + return instance.public_access if instance.is_active else None + if hasattr(instance, 'repository_public_access'): + return instance.repository_public_access + head = instance.head + if head and not hasattr(instance, 'repository_canonical_url'): + instance.repository_canonical_url = head.canonical_url + return head.public_access if head else None + + +def may_view_private(instance, namespace, user): + """At an owner base, whether `user` may view the owner's private repositories; asked once per version.""" + if namespace is None or not (user and user.is_authenticated): + return False + if not hasattr(instance, 'fhir_may_view_private'): + instance.fhir_may_view_private = instance.has_parent_edit_access(user) + return instance.fhir_may_view_private + + +def repository_rows(model, **filters): + """ + For a subquery: the rows of the outer row's repository (same mnemonic and owner) matching `filters`, as two + querysets, one for an org-owned outer row and one for a user-owned one (an owner column is NULL on the other). + """ + rows = model.objects.filter(mnemonic=OuterRef('mnemonic'), **filters) + return (rows.filter(organization_id=OuterRef('organization_id'), user_id__isnull=True), + rows.filter(user_id=OuterRef('user_id'), organization_id__isnull=True)) + + +def with_head(model, *conditions, **filters): + """ + For a subquery: whether the outer row's repository has an active HEAD matching `conditions` and `filters`. One + EXISTS, the owner matched inside it (an owner column is NULL on the other kind of row, and NULL equals nothing), + which the planner runs as one semi-join; two EXISTS joined by OR it plans as an index scan costing far more. + """ + return Exists(model.objects.filter( + *conditions, mnemonic=OuterRef('mnemonic'), version=HEAD, is_active=True, **filters).filter( + Q(organization_id=OuterRef('organization_id')) | Q(user_id=OuterRef('user_id')))) + + +def with_repository_access(queryset): + """Each row annotated with repository_public_access, its repository's (active HEAD's) public_access.""" + org_heads, user_heads = repository_rows(queryset.model, version=HEAD, is_active=True) + return queryset.annotate(repository_public_access=Coalesce( + Subquery(org_heads.values('public_access')[:1]), Subquery(user_heads.values('public_access')[:1]))) + + +def owner_access(kwargs, user): + """ + (owner filter, may view private) of the base `kwargs` names, for `user`: ({}, False) at the global base; at an + owner base, the filter on the owner's rows (None when the owner doesn't exist) and whether `user` may view its + private repositories (staff, the org's members, the user). + """ + namespace = namespace_of(kwargs) + if namespace is None: + return {}, False + owner = SourceContainerMixin.get_object_from_namespace(namespace) + if not owner: + return None, False + is_org = bool(kwargs.get('org')) + return {'organization_id' if is_org else 'user_id': owner.id}, bool(user and user.is_authenticated) and ( + user.is_staff or (owner.is_member(user) if is_org else owner.id == user.id)) + + +def viewable_rows(model, namespace, user): + """ + The queryset form of is_visible(row, namespace, user) over every row of `model`, HEAD and versions alike: anywhere + a public row of a public repository (active HEAD); at an owner base also, whoever owns them, the rows `user` may + view by their own rights (all to staff, their orgs' and their own), with an active HEAD. For what a version refers + to (a mapping's source, the code systems an expansion drew on), so only repositories the caller is shown match. + + A row and its HEAD share their owner, so "both public, or the caller's own" is "the row public or the caller's own, + and its HEAD public or the caller's own": one EXISTS, which the planner runs as a semi-join. + """ + shown = shown_q(namespace, user) + return model.objects.filter(shown, with_head(model, shown)) + + +def shown_q(namespace, user): + """ + Q for the rows viewable_rows requires of a row and of its HEAD: public; at an owner base also, to those signed in, + their own by their rights (everything to staff, their orgs' and their own). + """ + public = Q(public_access__in=PUBLIC_ACCESS) + if namespace is None or not (user and user.is_authenticated): + return public + if user.is_staff: + return Q() + return public | Q(organization_id__in=user.organizations.values('id')) | Q(user_id=user.id) + + +def visible_versions(resource_type, kwargs=None, user=None, access=None): + """ + The queryset form of is_visible and has_release, for search: the active released versions, never HEAD, that the + base `kwargs` names shows `user`. A row is in it exactly when is_visible(row, namespace, user) and + has_release(row, namespace, user): at the global base public versions of public repositories (HEAD), whoever asks; + at an owner base the owner's, private ones only to those who may view them (staff, the org's members, the user). + An owner base whose owner doesn't exist shows nothing. `access`, owner_access's, saves looking it up again. + + A retired release counts only beside a live one the caller is shown; retired releases are so few that they're + checked on their own, which keeps the planner on a plain scan of the table. + """ + model = REPO_TYPES[resource_type] + owner, private = access or owner_access(kwargs, user) + if owner is None: + return model.objects.none() + access = {} if private else {'public_access__in': PUBLIC_ACCESS} + releases = model.objects.filter(is_active=True, released=True, **owner, **access).exclude(version=HEAD).filter( + with_head(model, **access)) + org_live, user_live = repository_rows(model, is_active=True, released=True, retired=False, **access) + retired_beside_live = releases.filter(retired=True).filter( + Exists(org_live.exclude(version=HEAD)) | Exists(user_live.exclude(version=HEAD))) + return releases.filter(Q(retired=False) | Q(id__in=retired_beside_live.values('id'))) + + +REGISTRY_TOGGLE = 'URL_REGISTRY_IN_RESOLVE_REFERENCE_TOGGLE' # whether the resolver consults the URL registry +ServedHead = namedtuple( + 'ServedHead', 'id mnemonic organization_id user_id canonical_url uri organization_uri user_uri') + + +def is_looked_up_as_written(url): + """ + Whether the resolver looks a canonical URL up as it is written (resolve_reference_expression's reading of a url): + one it takes for canonical (':'), with no |version or ?query part, that isn't an OCL path. + """ + return bool(url) and is_canonical_uri(url) and '|' not in url and '?' not in url and to_ocl_path(url) == url + + +def ocl_path(uri): + """ + A repository's OCL path as a CodeSystem (/orgs/{org}/CodeSystem/{id}/), as $expand names the system of a member + without a canonical URL, if the resolver reads it back to the repository (`uri`), naming no version; else None. + """ + try: + path = IdentifierSerializer.convert_ocl_uri_to_fhir_url(uri, 'CodeSystem') + except ValidationError: + return None + if is_canonical_uri(path): + return None + return path if to_parent_uri(to_ocl_path(path)) == uri else None + + +def in_namespace(head, namespace): + """Whether a HEAD is the repository of the owner a registry entry's namespace names (get_object_from_namespace).""" + if namespace and '/orgs/' in namespace: + return head.organization_uri == namespace + if namespace and '/users/' in namespace: + return head.user_uri == namespace + return False + + +def registry_entries(owner): + """(the owner's, the global registry's) active entries, each {url: namespace}, for an owner filter: one query.""" + scope = Q(organization__isnull=True, user__isnull=True) + if owner: + scope |= Q(**owner) + own, shared = {}, {} + for url, namespace, organization_id, user_id in URLRegistry.objects.filter(is_active=True).filter( + scope).values_list('url', 'namespace', 'organization_id', 'user_id'): + (shared if organization_id is None and user_id is None else own)[url] = namespace + return own, shared + + +def as_stored(version): + """A version as the shared resolver must be given it to look up `version` as it is stored: it decodes what it's + given (decode_string), so one that decoding would change is given encoded.""" + return version if decode_string(version) == version else quote_plus(version) + + +def served_repositories(kwargs, user): # pylint: disable=too-many-locals + """ + [(identifier, [(version, retired)], newest first)] for each repository a CodeSystem search at the base `kwargs` + names shows `user` (visible_versions), under the identifier resolve() serves it at there, as it would were the + repositories the caller isn't shown not there (so a hidden repository changes nothing here): its canonical URL (its + HEAD's) when the resolver picks it for that URL, else its OCL path as a CodeSystem. A repository whose HEAD the + resolver passes over (a retired one) isn't listed. The versions are those search shows: what resolve() serves by + name. Under a canonical URL, after the repository's own, those of the other repositories with it the base shows + that resolve() serves by name there (decision 9), the default still the repository's own. + + The resolver's pick for a canonical URL (one it looks up as written), among active HEADs that aren't retired, as + resolve_reference_expression makes it for Sources. Without the URL registry: at /fhir/ the lowest-id HEAD the global + base shows (shown_heads); at an owner base the owner's lowest-id one. With it: at /fhir/ the lowest-id HEAD in the + global registry entry's namespace, none without an entry; at an owner base the owner's entry's namespace's if the + owner has one for the URL, else the owner's lowest-id HEAD. + + A fixed number of queries: the owner and the caller's rights over it, the toggle, the registry's entries (with it + on), the versions, the HEADs. + """ + access = owner_access(kwargs, user) + owner = access[0] + if owner is None: + return [] + registry = Toggle.get(REGISTRY_TOGGLE) + own_entries, global_entries = registry_entries(owner) if registry else ({}, {}) + versions = {} + for *repository, version, retired in visible_versions('CodeSystem', access=access).order_by( + '-created_at', '-id').values_list('mnemonic', 'organization_id', 'user_id', 'version', 'retired'): + versions.setdefault(tuple(repository), []).append((version, retired)) + if not versions: + return [] + + heads = viewable_rows(Source, namespace_of(kwargs), user).filter(version=HEAD, is_active=True, retired=False) + if owner: # the owner's, and where its registry entries point + heads = heads.filter(Q(**owner) | Q(canonical_url__in=list(own_entries))) + elif registry: # the repositories searched, and where the global registry's entries point + heads = heads.filter(Q(mnemonic__in={mnemonic for mnemonic, _, _ in versions}) | Q( + canonical_url__in=list(global_entries))) + else: + heads = heads.filter(shown_heads(Source)) + by_url, by_repository = {}, {} # the HEADs with each canonical URL, lowest id first; each repository's + for row in heads.order_by('id').values_list( + 'id', 'mnemonic', 'organization_id', 'user_id', 'canonical_url', 'uri', 'organization__uri', 'user__uri'): + head = ServedHead(*row) + by_url.setdefault(head.canonical_url, []).append(head) + by_repository[(head.mnemonic, head.organization_id, head.user_id)] = head + + def pick(url): + """The HEAD the resolver picks for a canonical URL at this base.""" + candidates = by_url.get(url, []) + if registry: + entries = own_entries if owner else global_entries + if url in entries: + return next((head for head in candidates if in_namespace(head, entries[url])), None) + if not owner: + return None + return next((head for head in candidates if not owner or all( + getattr(head, field) == value for field, value in owner.items())), None) + + def held(url, head): + """ + The releases resolve() serves named under a canonical URL it picks `head` for: the head's; then, where the + repositories with the URL are the step that picks (no registry entry decides), each other one's the base shows + that no earlier one holds, lowest id first (decision 9). + """ + rows = list(versions[(head.mnemonic, head.organization_id, head.user_id)]) + if registry and (not owner or url in own_entries): + return rows + names = {version for version, _ in rows} + for other in by_url[url]: + for version, retired in versions.get((other.mnemonic, other.organization_id, other.user_id), []): + if version not in names: + names.add(version) + rows.append((version, retired)) + return rows + + served = [] + for repository, rows in versions.items(): + head = by_repository.get(repository) + if not head: + continue + url = head.canonical_url + identifier = url if is_looked_up_as_written(url) and pick(url) == head else ocl_path(head.uri) + if identifier: + served.append((identifier, held(url, head) if identifier == url else rows)) + return served diff --git a/core/fhir/search.py b/core/fhir/search.py new file mode 100644 index 000000000..9f02b177a --- /dev/null +++ b/core/fhir/search.py @@ -0,0 +1,751 @@ +""" +FHIR search and read of CodeSystem, ValueSet and ConceptMap: SVCM's Query transactions (ITI-95, ITI-96, ITI-100). + +- What a base shows is core.fhir.resolve's rule: released versions only, never HEAD; at the global base /fhir/ the + public ones of public repositories (HEAD), whoever asks; at an owner base the owner's, private ones to those who may + view them. Search uses its queryset form (visible_versions), a read checks is_visible and has_release on the version + it finds. +- Ids: at the global base every released version has an opaque, stable id, its database key; an owner base keeps the + repository's mnemonic (so its versions share one), and a read there serves the latest release the requester is + shown, or the `version` named. +- Search parameters: SVCM's, with :exact and :contains on the string ones (no modifier is starts-with; both it and + :contains ignore case and accents, :exact neither). + A comma separates alternatives, a repeated parameter must hold every time. Anything else is a 400 not-supported. +- Paging: _count (at most MAX_COUNT, which is also the default) and page; Bundle.total is the full count, and every + entry has a fullUrl and a search mode. The search runs once, for the ids of every match; the page's rows are then + read by id. +- A search's queries are bounded together by SEARCH_TIMEOUT_MS: one the database cancels makes the search too costly + (422). A parameter takes at most MAX_ALTERNATIVES values and a search MAX_SEARCH_ALTERNATIVES (else a 400 + too-costly), and the canonical URLs each parameter names are looked up in one query. ConceptMap's source and target + systems also read their candidate versions from the mappings index (core.fhir.mapping_index), in one request per + parameter within the same time. +- A search entry is the resource's summary, marked SUBSETTED: no concept, compose or group, which only a read returns. + A CodeSystem is `complete` (with every concept, retired ones flagged) up to CONCEPT_LIMIT concepts, else + `not-present`. +- A version released before its repository had a canonical URL has the repository's (HEAD's), in its `url` and for + the url parameter. +""" +import json +import math +import re +import time +from datetime import datetime, timedelta +from functools import reduce +from operator import or_ + +from dateutil.relativedelta import relativedelta +from django.conf import settings +from django.db import connection, transaction, OperationalError +from django.db.models import BooleanField, Case, F, Exists, Func, OuterRef, Q, Subquery, TextField, Value, When +from django.db.models.functions import Coalesce +from django.db.models.lookups import Exact, IContains, In, IStartsWith +from django.utils import timezone +from psycopg2 import errorcodes +from rest_framework.fields import CharField +from rest_framework.response import Response + +from core.bundles.serializers import FHIRBundleSerializer +from core.collections.models import Expansion +from core.common.constants import HEAD +from core.common.db_functions import ImmutableUnaccent +from core.common.mixins import CustomPaginator +from core.common.utils import to_int +from core.concepts.models import Concept +from core.fhir import mapping_index, outcomes +from core.fhir.base import CONTROL_PARAMETERS, fhir_base_url +from core.fhir.resolve import ( + PUBLIC_ACCESS, REPO_TYPES, namespace_of, owner_access, visible_versions, viewable_rows, is_public, is_shown, + repository_rows, with_repository_access) +from core.mappings.models import Mapping +from core.orgs.models import Organization +from core.sources.models import Source +from core.users.models import UserProfile + +MAX_COUNT = 100 # the most entries a search page holds, and how many it holds when _count isn't given +SEARCH_TIMEOUT_MS = 5000 # the database time a search may take; past it the search is too costly +CONCEPT_LIMIT = 1000 # a CodeSystem is `complete` up to this many concepts, else `not-present` (design D9) +SUBSETTED = {'system': 'http://terminology.hl7.org/CodeSystem/v3-ObservationValue', 'code': 'SUBSETTED'} +PAGING_PARAMETERS = frozenset({'_count', 'page'}) +OPAQUE_ID = re.compile(r'[1-9][0-9]{0,17}') # a database key, as str() writes it +NOTHING = Q(pk__in=[]) +EVERYTHING = Q(pk__isnull=False) # unlike Q(), it holds as an alternative too (Q() | x is x) + +# string parameters: the field each searches, and the lookup of each modifier (:exact is the field's own equality) +STRING_FIELDS = {'name': 'name', 'title': 'full_name', 'description': 'description', 'publisher': 'publisher'} +# the title a CodeSystem or ValueSet shows (TitleField): the full name, else the name when there is none (null, empty or +# only Unicode whitespace as defined by str.strip()) +SHOWN_TITLE = Case( + When(Q(full_name__isnull=True) | Q(full_name__regex=( + '^[\t\n\v\f\r\x1c\x1d\x1e\x1f \x85\xa0\u1680' + '\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a' + '\u2028\u2029\u202f\u205f\u3000]*$')), then=F('name')), + default=F('full_name'), output_field=TextField()) +STRING_LOOKUPS = {'': IStartsWith, 'exact': None, 'contains': IContains} +# the other parameters, by resource type: the Search method each one's values go to +COMMON_PARAMETERS = { + '_id': 'by_id', '_lastUpdated': 'by_last_updated', 'url': 'by_url', 'version': 'by_version', + 'status': 'by_status', 'identifier': 'by_identifier'} +PARAMETERS = { + 'CodeSystem': {**COMMON_PARAMETERS, 'system': 'by_url', 'code': 'by_code', 'language': 'by_language'}, + 'ValueSet': {**COMMON_PARAMETERS, 'reference': 'by_reference'}, + 'ConceptMap': { + **COMMON_PARAMETERS, 'source-system': 'by_source', 'source-uri': 'by_source', 'target-system': 'by_target', + 'target-uri': 'by_target'}, +} +# A stored OCL accession identifier, as IdentifierSerializer.find_ocl_identifier finds one in the serialized list: a +# value that isn't empty, and a type coding that is ACSN of v2-0203. IdentifierTypeCodingSerializer shows a missing code +# as ACSN and a missing system as v2-0203 (a null one as nothing). +STORED_ACCESSION = ( + 'lax $[*] ? (@.value.type() == "string" && @.value != "" && exists(@.type.coding[*] ? (' + '(@.code == "ACSN" || !exists(@.code)) && (@.system == "http://hl7.org/fhir/v2/0203" || !exists(@.system)))))') +# token parameters, by the system their codes belong to: None for none defined (|v is then v), CODE_SYSTEM for the +# CodeSystem itself (code=|). identifier is a token too, with its own rules (by_identifier). +CODE_SYSTEM = 'the code system searched' +TOKEN_SYSTEMS = { + '_id': None, 'version': None, 'status': 'http://hl7.org/fhir/publication-status', 'language': 'urn:ietf:bcp:47', + 'code': CODE_SYSTEM} +# what system| (any code of the system) means for a parameter with a system of its own +ANY_CODE = {'status': EVERYTHING, 'language': Q(default_locale__isnull=False) & ~Q(default_locale='')} +STATUSES = {'active': Q(released=True, retired=False), 'retired': Q(retired=True)} # search shows no draft +# the parameters whose values are canonical URLs, looked up before the search (Search.look_up), by what they're among +LOOKUPS = { + 'url': 'repositories', 'system': 'repositories', 'code': 'repositories', 'reference': 'referenced', + 'source-system': 'sources', 'source-uri': 'sources', 'target-system': 'sources', 'target-uri': 'sources'} +# the side of a mapping each ConceptMap system parameter reads, by its Search method +SIDES = {'by_source': 'from', 'by_target': 'to'} +MAX_ALTERNATIVES = 20 # the values a search parameter takes: its alternatives and repeats together +MAX_SEARCH_ALTERNATIVES = 50 # the values a search takes, every parameter's together + +# a FHIR date or dateTime search value: prefix, year, month, day, hour, minute, second, fraction, zone +DATE = re.compile( + r'(eq|ne|gt|lt|ge|le|sa|eb)?(\d{4})(?:-(\d{2})(?:-(\d{2})' + r'(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}:\d{2})?)?)?)?') + + +def split(value, separator): + """A search value split on the separators that aren't escaped; escapes are kept for unescape().""" + parts, start, index = [], 0, 0 + while index < len(value): + if value[index] == '\\': + index += 2 + continue + if value[index] == separator: + parts.append(value[start:index]) + start = index + 1 + index += 1 + return parts + [value[start:]] + + +def token(value): + """(system, code) of a token search value, unescaped: system None without a '|', '' for an explicit none.""" + system, *code = split(value, '|') + return (None, unescape(system)) if not code else (unescape(system), unescape('|'.join(code))) + + +def unescape(value): + """FHIR's search escapes: a backslash makes the next character (, | $ or \\) literal.""" + return re.sub(r'\\(.)', r'\1', value) + + +def date_range(value): + """ + (prefix, start, end): the instants [start, end) a date search value covers at its precision, UTC by default. end is + None when it would fall past year 9999, which datetime can't hold: the range runs on unbounded. + """ + match = DATE.fullmatch(value) + if not match: + raise outcomes.InvalidError(f"'{value}' is not a supported FHIR date search value.", expression='_lastUpdated') + prefix, year, month, day, hour, minute, second, fraction, zone = match.groups() + try: + start = datetime( + int(year), int(month or 1), int(day or 1), int(hour or 0), int(minute or 0), int(second or 0), + int((fraction or '0')[:6].ljust(6, '0')), tzinfo=datetime.strptime(zone or 'Z', '%z').tzinfo) + except ValueError as ex: + raise outcomes.InvalidError(f"'{value}' is not a valid date.", expression='_lastUpdated') from ex + unit = 'seconds' if second else 'minutes' if minute else 'days' if day else 'months' if month else 'years' + try: + end = start + ( + timedelta(microseconds=10 ** max(0, 6 - len(fraction))) if fraction else relativedelta(**{unit: 1})) + except (ValueError, OverflowError): + end = None + return prefix or 'eq', start, end + + +def canonical_q(url, repositories): + """ + Rows whose canonical URL is `url` as CanonicalUrlField shows it: their own, or for a row with none (a version + released before its repository had one), their repository's (HEAD's). `repositories` are the (mnemonic, + organization_id, user_id) of the repositories whose HEAD has it, looked up first (repositories_with), so the + planner sees plain conditions rather than a correlated subquery it can't estimate. + """ + if not repositories: + return Q(canonical_url=url) + return Q(canonical_url=url) | ((Q(canonical_url__isnull=True) | Q(canonical_url='')) & reduce(or_, ( + Q(mnemonic=mnemonic, organization_id=organization_id, user_id=user_id) + for mnemonic, organization_id, user_id in repositories))) + + +def repositories_with(rows, urls): + """ + For each of `urls`, the (mnemonic, organization_id, user_id) of the repositories among `rows` whose HEAD has that + canonical URL: one query for them all. + """ + found = {url: [] for url in urls} + for url, *repository in rows.filter(version=HEAD, canonical_url__in=urls).values_list( + 'canonical_url', 'mnemonic', 'organization_id', 'user_id'): + found[url].append(tuple(repository)) + return found + + +def sources_with(rows, urls, versions): + """ + ({url: [(id, (owner type, owner, mnemonic))]}, {(owner type, owner, mnemonic): {version names}}): for each of + `urls` the Source rows among `rows` with that canonical URL, and the repositories and versions of the rows + `versions` lists, in one query for them all. + """ + fields = ('canonical_url', 'id', 'mnemonic', 'organization__mnemonic', 'user__username', 'version') + named = rows.filter(canonical_url__in=urls).annotate(listed=Value(False)).values_list(*fields, 'listed') + found, shown = {url: [] for url in urls}, {} + for url, source_id, mnemonic, org, username, version, listed in named.union( + versions.annotate(listed=Value(True)).values_list(*fields, 'listed'), all=True): + repository = ('Organization', org, mnemonic) if org else ('User', username, mnemonic) + if listed: + shown.setdefault(repository, set()).add(version) + else: + found[url].append((source_id, repository)) + return found, shown + + +def with_repository_canonical(queryset): + """Each row annotated with its repository's (HEAD's) canonical URL, for a version that has none of its own.""" + org_heads, user_heads = repository_rows(queryset.model, version=HEAD) + return queryset.annotate(repository_canonical_url=Coalesce( + Subquery(org_heads.values('canonical_url')[:1]), Subquery(user_heads.values('canonical_url')[:1]))) + + +def code_system_concepts(source): + """ + (count, concepts) for a CodeSystem read, cached on the version: its concepts, retired ones included, when it has + at most CONCEPT_LIMIT, else (its number of concepts, None). The number is counted on the version's concept links + alone (an index-only count, cheap even for LOINC); within the limit it's the concepts listed. + """ + if not hasattr(source, 'fhir_concepts'): + count, concepts = Concept.sources.through.objects.filter(source_id=source.id).count(), None + if count <= CONCEPT_LIMIT: + concepts = list(source.concepts.filter(is_active=True).select_related('parent').prefetch_related( + 'names', 'descriptions').order_by('id')) + count = len(concepts) + source.fhir_concepts = count, concepts + return source.fhir_concepts + + +def concepts_over_limit(source): + """Whether a CodeSystem has more than CONCEPT_LIMIT concepts: a search annotates it, a read counts them.""" + over = getattr(source, 'concepts_over_limit', None) + return code_system_concepts(source)[1] is None if over is None else over + + +def with_concepts_over_limit(queryset): + """Each Source row annotated with concepts_over_limit, reading at most CONCEPT_LIMIT + 1 of its concept links.""" + links = Concept.sources.through.objects.filter(source_id=OuterRef('id')) + return queryset.annotate(concepts_over_limit=Exists(links[CONCEPT_LIMIT:CONCEPT_LIMIT + 1])) + + +def defines_concept(**filters): + """ + A CodeSystem version with a concept (matching `filters`), as its read lists them: linked to the version and not + deleted. A retired concept counts; the read shows it flagged inactive. + """ + return Exists(Concept.sources.through.objects.filter(source_id=OuterRef('id'), concept__is_active=True, **filters)) + + +class JsonPathExists(Func): # pylint: disable=abstract-method + """jsonb_path_exists(field, path, vars, silent): whether the path finds anything, an error being no match.""" + function = 'jsonb_path_exists' + output_field = BooleanField() + + def __init__(self, field, path, variables): + super().__init__(field, Value(path), Value(json.dumps(variables)), Value(True)) + + +class JsonbTypeof(Func): # pylint: disable=abstract-method + function = 'jsonb_typeof' + output_field = TextField() + + +def held_identifier(system, value): + """ + Rows holding an identifier shown as system|value, both checked on the same identifier: `system` None is any + system, '' none; `value` None is any value. IdentifierSerializer shows an identifier stored without a system key + under API_BASE_URL, and one with a null or empty system without one. Only a list of identifiers is shown; lax mode + reads a missing key as nothing (strict mode would make exists() unknown). + """ + conditions, variables = [], {} + if value is not None: + conditions.append('@.value == $value') + variables['value'] = value + if system == '': + conditions.append('exists(@.system) && (@.system == null || @.system == "")') + elif system is not None: + no_key = ' || !exists(@.system)' if system == settings.API_BASE_URL else '' + conditions.append(f'(@.system == $system{no_key})') + variables['system'] = system + return Exact(JsonbTypeof('identifier'), 'array') & JsonPathExists( + 'identifier', f'lax $[*] ? ({" && ".join(conditions)})', variables) + + +def indexed_repository(owner_type, owner, mnemonic): + """ + The rows of a repository as the mappings index names it (owner and mnemonic in lowercase), twins included. The + owner is an uncorrelated IN, so the version rows need no join. + """ + if owner_type == 'Organization': + return Q(mnemonic__iexact=mnemonic, organization_id__in=Organization.objects.filter( + mnemonic__iexact=owner).values('id')) + return Q(mnemonic__iexact=mnemonic, user_id__in=UserProfile.objects.filter(username__iexact=owner).values('id')) + + +def mapped(side, value, sources, holding, namespace, user): # pylint: disable=too-many-arguments + """ + A ConceptMap version holding a live mapping whose group source (side 'from') or target ('to') shows `value`, as + ConceptMapGroupField builds groups: retired and deleted mappings aren't read; a group's system is the from or to + source's canonical URL when the caller is shown that source, else the URL the mapping stores (an OCL URL as a + ConceptMap path, /orgs/{org}/ConceptMap/{id}/). OCL keeps no source or target value set on a ConceptMap, so + source-uri and target-uri are read as the systems. + + `sources` are the ids of the sources with that canonical the base shows the caller (viewable_rows), listed first + (sources_with), and `holding` the repositories and versions the mappings index lists as holding mappings from or + to them (mapping_index.holding). Only those versions, and their repositories' releases created within + mapping_index.LAG_WINDOW (which the index may not list yet), are candidates: the database decides each by its own + mappings, so a heavily mapped source costs no more than another. A mapping whose source isn't one the caller is + shown by a canonical URL is matched by the URL it stores: a source the caller isn't shown answers exactly as one + OCL doesn't hold. Alone, that is one EXISTS, which the planner runs as a semi-join; beside the candidates it's an + uncorrelated IN, which it runs once. + """ + field = f'mapping__{side}_source' + unnamed = ~Exists(viewable_rows(Source, namespace, user).filter(id=OuterRef(f'{field}_id'), canonical_url__gt='')) + stored = Q(**{f'{field}_url': value}) & unnamed + path = re.fullmatch(r'/(orgs|users)/([^/]+)/ConceptMap/([^/]+)/', value) + if path: + stored |= reduce(or_, (Q(**{f'{field}_url__startswith': f'/{path[1]}/{path[2]}/{kind}/{path[3]}/'}) + for kind in ('sources', 'collections', 'ConceptMap'))) & unnamed + links = Mapping.sources.through.objects.filter(mapping__retired=False, mapping__is_active=True) + if not holding: + return Exists(links.filter(stored, source_id=OuterRef('id'))) + recent = Q(created_at__gte=timezone.now() - mapping_index.LAG_WINDOW) + candidates = reduce(or_, (indexed_repository(*repository) & (Q(version__in=sorted(versions)) | recent) + for repository, versions in sorted(holding.items()))) + # each candidate is read through its own links, each link's mapping a scalar subquery the planner can't turn into + # a join: it can't start from all the source's mappings instead, which say nothing of where the version's are + live_side = Mapping.objects.filter(id=OuterRef('mapping_id'), retired=False, is_active=True).values( + f'{side}_source_id')[:1] + named = Exists(Mapping.sources.through.objects.filter(In(Subquery(live_side), sources), source_id=OuterRef('id'))) + return Q(id__in=links.filter(stored).values('source_id')) | (candidates & named) + + +class Search: + """The criteria of one search's parameters, on the versions of a resource type at a base, for a caller.""" + + def __init__(self, resource_type, kwargs, user=None): + self.resource_type = resource_type + self.model = REPO_TYPES[resource_type] + self.parameters = PARAMETERS[resource_type] + self.namespace, self.user = namespace_of(kwargs), user + self.access = owner_access(kwargs, user) + self.found = {} # (lookup, canonical URL): what look_up found for it + self.deadline = time.monotonic() + SEARCH_TIMEOUT_MS / 1000 # for what isn't a query: the mappings index + + def viewable_sources(self): + """ + The Source rows the base shows the caller, whoever owns them: what source-system, target-system and reference + match. At an owner base that's another owner's private source too, if the caller may view it themselves. + """ + return viewable_rows(Source, self.namespace, self.user) + + def criteria(self, query): + """ + One Q for a query string: every parameter holds (a repeated one each time), any of a value's comma-separated + alternatives does. An empty value isn't a criterion; the format and paging parameters aren't criteria. + """ + criteria = Q() + for name, modifier, repeats in self.parameters_of(query): + self.look_up(name, [item for alternatives in repeats for item in alternatives]) + for alternatives in repeats: + if alternatives: + criteria &= reduce(or_, [self.criterion(name, modifier, item) for item in alternatives]) + return criteria + + def parameters_of(self, query): + """ + (name, modifier, each repeat's alternatives) for every search parameter of a query string, each supported and + within MAX_ALTERNATIVES values (alternatives and repeats together) and the search within + MAX_SEARCH_ALTERNATIVES, else a 400 too-costly naming the parameter. Checked before any query runs. + """ + parameters, given, total = [], {}, 0 + for key, values in query.lists(): + if key in CONTROL_PARAMETERS or key in PAGING_PARAMETERS: + continue + name, _, modifier = key.partition(':') + self.check(key, name, modifier) + repeats = [[item for item in split(value, ',') if item] for value in values] + count = sum(map(len, repeats)) + given[name], total = given.get(name, 0) + count, total + count + if given[name] > MAX_ALTERNATIVES or total > MAX_SEARCH_ALTERNATIVES: + raise outcomes.TooCostlyError( + f'A search takes at most {MAX_ALTERNATIVES} values of a parameter, its alternatives and repeats ' + f'together, and {MAX_SEARCH_ALTERNATIVES} in all.', expression=key, status_code=400) + parameters.append((name, modifier, repeats)) + return parameters + + def check(self, key, name, modifier): + modifiers = STRING_LOOKUPS if name in STRING_FIELDS else ('',) if name in self.parameters else () + if modifier not in modifiers: + raise outcomes.NotSupportedError(f"The search parameter '{key}' is not supported.", expression=key) + + def look_up(self, name, alternatives): + """ + The canonical URLs a parameter's alternatives (still escaped) name, looked up together in one query, for its + criteria: the repositories with them (url, system, code's system), or among the sources the base shows the + caller, the repositories (reference) or the sources (source-system, source-uri, target-system, target-uri). + The same query lists the ConceptMap versions the base shows the caller (visible_versions). For the sources + found, one request to the mappings index lists the repositories and versions holding mappings from (source-) or + to (target-) them, among those versions; none is sent for a URL no source the caller is shown has, or where the + base shows no ConceptMap. + """ + lookup = LOOKUPS.get(name) + urls = {token(item)[0] if name == 'code' else unescape(item) for item in alternatives} - {None, ''} + if not lookup or not urls: + return + if lookup == 'repositories': + found = repositories_with(self.model.objects, urls) + elif lookup == 'referenced': + found = repositories_with(self.viewable_sources(), urls) + else: + found, shown = sources_with( + self.viewable_sources(), urls, visible_versions(self.resource_type, access=self.access)) + side = SIDES[self.parameters[name]] + holding = mapping_index.holding( + side, {url: {repository for _, repository in rows} for url, rows in found.items()}, shown, + self.deadline, search_too_costly) + self.found.update({(side, url): holding.get(url, {}) for url in found}) + found = {url: [source_id for source_id, _ in rows] for url, rows in found.items()} + self.found.update({(lookup, url): value for url, value in found.items()}) + + def criterion(self, name, modifier, value): + """The Q for one alternative (still escaped) of a parameter's value.""" + if name == 'title' and self.resource_type != 'ConceptMap': + return self.by_string(SHOWN_TITLE, modifier, unescape(value)) + if name in STRING_FIELDS: + return self.by_string(STRING_FIELDS[name], modifier, unescape(value)) + if name == 'identifier': + return self.by_identifier(value) + if name in TOKEN_SYSTEMS: + return self.by_token(name, *token(value)) + return getattr(self, self.parameters[name])(unescape(value)) + + def by_token(self, name, system, code): + """ + A token parameter: code (any system), system|code, system| (any code of that system) or |code (explicitly no + system), against the system its codes belong to (TOKEN_SYSTEMS). A parameter with no system takes |code as + code; any other system is no match. + """ + handler, implicit = getattr(self, self.parameters[name]), TOKEN_SYSTEMS[name] + if system is None or (system == '' and implicit is None): + return handler(code) + if not system or implicit is None: + return NOTHING + if implicit == CODE_SYSTEM: # system| is any code: a version of that code system with a concept + return self.by_url(system) & (handler(code) if code else defines_concept()) + if system != implicit: + return NOTHING + return handler(code) if code else ANY_CODE[name] + + @staticmethod + def by_string(field, modifier, value): + """ + FHIR string search: :exact is the whole value, case and accents included; no modifier (starts with) and + :contains ignore case and accents, unaccenting both sides in the database (LIKE's wildcards are escaped). + """ + lookup = STRING_LOOKUPS[modifier] + if lookup is None: + return Q(Exact(field, Value(value))) if isinstance(field, Case) else Q(**{field: value}) + return lookup(ImmutableUnaccent(field), ImmutableUnaccent(Value(value))) + + def by_id(self, value): + """The opaque id at the global base, the mnemonic at an owner base.""" + if self.namespace is None: + return Q(id=int(value)) if OPAQUE_ID.fullmatch(value) else NOTHING + return Q(mnemonic=value) + + @staticmethod + def by_last_updated(value): + prefix, start, end = date_range(value) + before_end = Q(updated_at__lt=end) if end else EVERYTHING + after_end = Q(updated_at__gte=end) if end else NOTHING + return { + 'eq': Q(updated_at__gte=start) & before_end, + 'ne': Q(updated_at__lt=start) | after_end, + 'gt': after_end, 'sa': after_end, + 'ge': Q(updated_at__gte=start), + 'lt': Q(updated_at__lt=start), 'eb': Q(updated_at__lt=start), + 'le': before_end, + }[prefix] + + def by_url(self, value): + return canonical_q(value, self.found[('repositories', value)]) + + @staticmethod + def by_version(value): + return Q(version=value) + + @staticmethod + def by_status(value): + return STATUSES.get(value, NOTHING) + + def by_identifier(self, value): + """ + A token, matched on one identifier as it's shown: value (any system), system|value, |value (no system) or + system| (any value). Besides the identifiers a repository holds, every resource without a stored accession + identifier shows a generated one: system API_BASE_URL, value /{orgs|users}/{owner}/{type}/{id}/. + """ + system, value = token(value) + value = value or None + if system == '' and value is None: + return NOTHING + criterion = held_identifier(system, value) + if system not in (None, settings.API_BASE_URL): + return criterion + generated = Q(identifier__isnull=True) | ~Q( + Exact(JsonbTypeof('identifier'), 'array') & JsonPathExists('identifier', STORED_ACCESSION, {})) + if value is None: + return criterion | generated + accession = re.fullmatch(r'/(orgs|users)/([^/]+)/([^/]+)/([^/]+)/?', value) + if accession and accession.group(3) == self.resource_type: + owner = 'organization__mnemonic' if accession.group(1) == 'orgs' else 'user__username' + criterion |= Q(Q(**{owner: accession.group(2), 'mnemonic': accession.group(4)}) & generated) + return criterion + + @staticmethod + def by_code(value): + """A CodeSystem that defines the code.""" + return defines_concept(concept__mnemonic=value) + + @staticmethod + def by_language(value): + return Q(default_locale=value) + + def by_reference(self, value): + """ + A ValueSet whose stored default expansion draws on a version of the code system (canonical URL) named, among + those the base shows the caller. An uncorrelated IN, which the planner hashes once, rather than a correlated + subquery. + """ + sources = self.viewable_sources().filter(canonical_q(value, self.found[('referenced', value)])).values('id') + return reduce(or_, ( + Q(expansion_uri__in=versions.through.objects.filter(source_id__in=sources).values('expansion__uri')) + for versions in (Expansion.explicit_source_versions, Expansion.evaluated_source_versions))) + + def by_source(self, value): + """A group's source is the system named.""" + return mapped('from', value, self.found[('sources', value)], self.found[('from', value)], self.namespace, + self.user) + + def by_target(self, value): + """A group's target is the system named.""" + return mapped('to', value, self.found[('sources', value)], self.found[('to', value)], self.namespace, + self.user) + + +def page_size(query): + value = query.get('_count') + if value is None: + return MAX_COUNT + if not re.fullmatch(r'[0-9]{1,9}', value.strip()): + raise outcomes.InvalidError(f"_count must be a whole number, got '{value}'.", expression='_count') + return min(int(value), MAX_COUNT) + + +def within_search_timeout(build, too_costly=None): + """ + build() within SEARCH_TIMEOUT_MS in all: each query it runs may take only the time the search has left, so a + search of several queries is bounded as a whole. A query the database cancels, or one with no time left, makes the + search too costly (422): too_costly() says so, the search's own wording by default (an operation passes its own). + Inside a transaction (tests) the timeout would outlive the search, so it isn't set there. + """ + too_costly = too_costly or search_too_costly + if connection.vendor != 'postgresql' or connection.in_atomic_block: + return build() + deadline = time.monotonic() + SEARCH_TIMEOUT_MS / 1000 + + def within_time_left(execute, sql, params, many, context): + left = math.ceil((deadline - time.monotonic()) * 1000) + if left <= 0: + raise too_costly() + with context['connection'].connection.cursor() as cursor: # the driver's own cursor, past this wrapper + cursor.execute('SET LOCAL statement_timeout = %s', [left]) + return execute(sql, params, many, context) + + try: + with transaction.atomic(), connection.execute_wrapper(within_time_left): + return build() + except OperationalError as ex: + if getattr(ex.__cause__, 'pgcode', None) != errorcodes.QUERY_CANCELED: + raise + raise too_costly() from ex + + +def search_too_costly(): + return outcomes.TooCostlyError( + 'The search would take too long to answer. Narrow it, for example with url, _id or version.') + + +def search_bundle(view): + """The searchset Bundle a FHIR search view answers: the base's matching versions, one page of them.""" + request, resource_type = view.request, view.resource_type + count = page_size(request.query_params) + search = Search(resource_type, view.kwargs, request.user) + ids = list(visible_versions(resource_type, access=search.access).filter( + search.criteria(request.query_params)).order_by('-id').values_list('id', flat=True)) + paginator, entries = None, [] + if count: + page = to_int(request.GET.get('page', '1'), 1) + rows = with_repository_canonical(REPO_TYPES[resource_type].objects.filter( + id__in=ids[(page - 1) * count:page * count])).order_by('-id') + if resource_type == 'CodeSystem': + rows = with_concepts_over_limit(rows) + paginator = CustomPaginator( + request=request, total_count=len(ids), queryset=list(rows), page_size=count, is_sliced=True) + resources = view.get_serializer(paginator.current_page_results, many=True).data + entries = FHIRBundleSerializer.convert_to_entry( + resources, f'{fhir_base_url(request, view.kwargs)}/{resource_type}') + bundle = {'meta': {'lastUpdated': timezone.now()}, 'type': 'searchset', 'total': len(ids)} + if entries: + bundle['entry'] = entries + return FHIRBundleSerializer(bundle, context={'paginator': paginator, 'request': request}).data + + +def read(view): + """ + The repository version a FHIR read view names: at the global base by its opaque id, at an owner base by its + mnemonic, the version ?version= names (an OCL extension) or else the latest release the requester is shown. It + must be one search lists: released, never HEAD, visible to the requester there. Raises NotFoundError. + """ + return read_version(view.resource_type, view.kwargs[view.lookup_field], view.request.query_params.get('version'), + view.kwargs, view.request.user) + + +def read_version(resource_type, resource_id, version, kwargs, user): + """ + read's rule for a resource `resource_id` names at the base `kwargs` names, for `user`: the id at the global base, + the mnemonic and `version` (else the latest release shown) at an owner base. For the instance-level operations too. + """ + namespace = namespace_of(kwargs) + versions = REPO_TYPES[resource_type].objects.filter(is_active=True, released=True).exclude(version=HEAD) + latest = False + if namespace is None: + versions = versions.filter(id=int(resource_id)) if OPAQUE_ID.fullmatch(resource_id) else versions.none() + else: + owner = {'user__username': kwargs['user']} if kwargs.get('user') else {'organization__mnemonic': kwargs['org']} + versions = versions.filter(mnemonic=resource_id, **owner) + latest = not version + versions = versions.filter(version=version) if version else versions.filter( + retired=False).order_by('-created_at') + + instance = with_repository_access(with_repository_canonical(versions)).first() + if latest and instance and not is_shown(instance, namespace, user) and is_public( + instance.repository_public_access or ''): + # the latest is private, in a public repository: the latest public one + instance = with_repository_access(with_repository_canonical( + versions.filter(public_access__in=PUBLIC_ACCESS))).first() + if not instance or not is_shown(instance, namespace, user): + raise outcomes.NotFoundError(f'{resource_type}/{resource_id} not found') + return instance + + +class FhirRepositoryViewMixin: + """The search and read views of CodeSystem, ValueSet and ConceptMap (after FhirViewMixin).""" + resource_type = None + + def get_serializer_context(self): + return {**super().get_serializer_context(), 'opaque_ids': namespace_of(self.kwargs) is None} + + +class FhirSearchMixin(FhirRepositoryViewMixin): + """GET on a resource type is its search.""" + + def get(self, request, *args, **kwargs): # pylint: disable=unused-argument + """The searchset Bundle, in at most SEARCH_TIMEOUT_MS of database time.""" + return Response(within_search_timeout(lambda: search_bundle(self))) + + head = get + + +class FhirReadMixin(FhirRepositoryViewMixin): + """ + GET on an id is its read, at the global base without the owner check the view's writes keep (verify_scope), and + without the source and collection views' recounts. + """ + + def verify_scope(self): + if self.request.method not in ('GET', 'HEAD'): + super().verify_scope() + + def get_object(self, queryset=None): + if self.request.method not in ('GET', 'HEAD'): + return super().get_object(queryset) + return read(self) + + +class CanonicalUrlField(CharField): + """A resource's canonical URL: the version's own, else its repository's (repository_canonical_url, on a read).""" + + def get_attribute(self, instance): + return super().get_attribute(instance) or getattr(instance, 'repository_canonical_url', None) + + +class TitleField(CharField): + """ + A repository's FHIR title: its full name, or its name when it has no full name (empty or only whitespace). Writes + still go to full_name. + """ + + def get_attribute(self, instance): + full_name = instance.full_name + return full_name if full_name and full_name.strip() else instance.name + + +class FhirRepositorySerializerMixin: + """ + For the CodeSystem, ValueSet and ConceptMap serializers: the opaque id at the global base (context opaque_ids), and + in a search (many=True, context has_many) the summary, marked SUBSETTED, without `content_field`. + """ + content_field = None + + @classmethod + def many_init(cls, *args, **kwargs): + kwargs['context'] = {**kwargs.get('context', {}), 'has_many': True} + return super().many_init(*args, **kwargs) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.context.get('has_many'): + self.fields.pop(self.content_field, None) + + def with_fhir_view(self, instance, rep): + """ + The id (opaque at the global base), and meta.versionId: the version's opaque id at both bases, so the + releases of one repository, which share their mnemonic id at an owner base, stay distinct in a Bundle (bdl-7). + A search entry or a read whose content field is subsetted is tagged SUBSETTED. meta keeps its existing values + and tags. + """ + if instance.id: + if self.context.get('opaque_ids'): + rep['id'] = str(instance.id) + rep['meta'] = {'versionId': str(instance.id), **(rep.get('meta') or {})} + if self.context.get('has_many') or getattr(self.fields.get(self.content_field), 'subsetted', False): + meta = rep.get('meta') or {} + tags = list(meta.get('tag') or []) + if SUBSETTED not in tags: + tags.append(SUBSETTED) + rep['meta'] = {**meta, 'tag': tags} + return rep diff --git a/core/fhir/swagger.py b/core/fhir/swagger.py new file mode 100644 index 000000000..282601a65 --- /dev/null +++ b/core/fhir/swagger.py @@ -0,0 +1,304 @@ +"""Curated Swagger 2.0 descriptions of the FHIR terminology reads and operations.""" +from copy import deepcopy +from urllib.parse import urlsplit, urlunsplit + +from django.conf import settings +from drf_yasg import openapi +from drf_yasg.generators import OpenAPISchemaGenerator + +from core.fhir.base import FHIR_JSON, FHIR_XML +from core.fhir.ops.expand import MAX_COUNT as EXPANSION_MAX_COUNT +from core.fhir.search import MAX_COUNT, PARAMETERS, STRING_FIELDS, STRING_LOOKUPS + +BASES = ('/fhir', '/orgs/{org}', '/users/{user}') +TAGS = ('CodeSystem', 'ValueSet', 'ConceptMap', 'Capabilities') +VERSION_SELECTION = ( + 'The default is the latest released version visible at this base; HEAD is used only when explicitly named. ' + 'An instance id at /fhir names a released version; at an owner base it is the repository mnemonic.') +SEARCH_DESCRIPTIONS = { + '_id': 'Resource id: an opaque release id at /fhir, or the repository mnemonic at an owner base.', + '_lastUpdated': 'FHIR date or dateTime, optionally prefixed by eq, ne, gt, lt, ge, le, sa or eb.', + 'url': 'Canonical URL of the resource.', + 'version': 'Business version token. Searches include released versions only, never HEAD.', + 'status': 'Publication status token: active or retired.', + 'identifier': 'Identifier token: value, system|value, |value or system|.', + 'system': 'Canonical URL of the code system (an alias of url).', + 'code': 'A code defined by the code system; code or system|code.', + 'language': 'Language token for the code system (urn:ietf:bcp:47).', + 'reference': 'Canonical code system URL referenced by the value set\'s stored default expansion.', + 'source-system': 'Canonical URL of a source system in a ConceptMap group.', + 'source-uri': 'Alias of source-system.', + 'target-system': 'Canonical URL of a target system in a ConceptMap group.', + 'target-uri': 'Alias of target-system.', +} + + +def on_fhir_host(request): + """Use the same host boundary for the schema and the FHIR middleware.""" + return bool(request and settings.FHIR_SUBDOMAIN and + request.get_host().startswith(settings.FHIR_SUBDOMAIN + '.')) + + +def fhir_swagger_url(request, api_url): + """The FHIR sibling of the API host, keeping the request's scheme and port.""" + url = urlsplit(request.build_absolute_uri('/') if request else api_url) + host = settings.FHIR_SUBDOMAIN + '.' + (url.netloc.partition('.')[2] or url.netloc) + return urlunsplit((url.scheme, host, '/swagger/', '', '')) + + +def query(name, description, type_=openapi.TYPE_STRING, **kwargs): + """A typed query parameter; complex FHIR values are described in the POST body instead.""" + return openapi.Parameter(name, openapi.IN_QUERY, type=type_, description=description, **kwargs) + + +NOT_WITH_CODING = ' Not read when coding or codeableConcept is given.' +FORMAT = query('_format', 'Response format; overrides Accept.', enum=['json', 'xml']) +CODE = query('code', 'Code to look up, validate or translate; alternatively supply coding.' + NOT_WITH_CODING) +SYSTEM = query('system', 'Canonical URL or OCL path of the code system the code is from.' + NOT_WITH_CODING) +VERSION = query('version', 'Business version of the code system; defaults to the latest released version.') +LOOKUP_SYSTEM = query('system', 'Canonical URL or OCL path of the code system the code is from. At an instance it ' + 'must name that code system. With coding at the type level, the coding\'s system ' + 'is used instead.') +URL = query('url', 'Canonical URL or OCL path of the target resource; may include |version.') +VALUE_SET_VERSION = query('valueSetVersion', 'Business version of the value set.') +LANGUAGE = query('displayLanguage', 'Preferred language for the returned display, when a name in it exists.') +DISPLAY = query('display', 'Display to validate against the concept\'s names, ignoring case.' + NOT_WITH_CODING) + + +def coding(replaces, applies=''): + """The coding input of one operation: a system|code token here, a full Coding in a POSTed body.""" + return query('coding', f'A system|code token. It replaces {replaces}.{applies} A GET token carries no version ' + 'or display: send those inside a POSTed valueCoding.') + + +OPERATION_PARAMETERS = { + ('CodeSystem', '$lookup'): [ + LOOKUP_SYSTEM, VERSION, CODE, + coding('code and system', ' version still selects the code system version; at an instance, a separate ' + 'system must still name it.'), + LANGUAGE], + ('CodeSystem', '$validate-code'): [ + URL, VERSION, CODE, + coding('code and display', ' url and version still name the code system validated against.'), + DISPLAY, LANGUAGE], + ('ValueSet', '$validate-code'): [ + URL, VALUE_SET_VERSION, SYSTEM, + query('systemVersion', 'Business version of the input code system.' + NOT_WITH_CODING), + CODE, coding('code, system, systemVersion and display'), DISPLAY, LANGUAGE], + ('ValueSet', '$expand'): [ + URL, VALUE_SET_VERSION, + query('filter', 'Each word must prefix a word in the code or one of its names, ignoring case.'), LANGUAGE, + query('includeDesignations', 'Include every name as a designation.', openapi.TYPE_BOOLEAN), + query('activeOnly', 'Exclude retired concepts.', openapi.TYPE_BOOLEAN), + query('offset', 'Number of matching members to skip.', openapi.TYPE_INTEGER, minimum=0, default=0), + query('count', 'Page size, capped at 1,000; zero returns only the total.', openapi.TYPE_INTEGER, + minimum=0, maximum=EXPANSION_MAX_COUNT, default=EXPANSION_MAX_COUNT), + query('includeDefinition', 'Only false is supported; definitions are not included.', openapi.TYPE_BOOLEAN, + enum=[False]), + query('excludePostCoordinated', 'Only false is supported; stored expansions do not record this distinction.', + openapi.TYPE_BOOLEAN, enum=[False]), + query('excludeNested', 'Either boolean is accepted; stored expansions are flat.', openapi.TYPE_BOOLEAN), + query('excludeNotForUI', 'Either boolean is accepted; stored expansions have no abstract codes.', + openapi.TYPE_BOOLEAN)], + ('ConceptMap', '$translate'): [ + URL, query('conceptMapVersion', 'Business version of the map; without url, selects the source\'s map version.'), + CODE, SYSTEM, + query('version', 'Input code system version; omitted means no mapping-side version filter. ' + 'Without url in forward translation, also selects the map version unless conceptMapVersion ' + 'is given.' + NOT_WITH_CODING), + coding('code, system and version'), + query('source', 'Input value set scope for forward translation; ignored when reverse=true.'), + query('target', 'Input value set scope for reverse translation; ignored in forward translation.'), + query('targetsystem', 'Restrict matches to this code system; in reverse, the system of the codes sought.'), + query('reverse', 'Translate from the target side of the mappings.', openapi.TYPE_BOOLEAN, default=False)], +} +OPERATION_DESCRIPTIONS = { + ('CodeSystem', '$lookup'): ( + 'Look up a code', 'Returns the code\'s display, names and inactive flag, including retired concepts. ' + 'A code and, at type level, its system are required (or supply coding).', (400, 404, 422)), + ('CodeSystem', '$validate-code'): ( + 'Validate a code in a code system', + 'Checks the code and optional display in url (or coding.system). A code and a target system are required. ' + 'An unknown code or system returns Parameters with result=false; retired codes are valid with a warning.', + (400,)), + ('ValueSet', '$validate-code'): ( + 'Validate a code in a value set', + 'Checks membership in the stored default expansion and an optional display. A code and its system are ' + 'required (or supply coding). Without url at type level, checks the code system alone. ' + 'A failed validation returns Parameters with result=false.', (400, 422)), + ('ValueSet', '$expand'): ( + 'Read a stored value set expansion', + 'Returns a page of the stored default expansion; never computes or saves an expansion. ' + 'At type level url is required. count is capped at 1,000. Missing, unfinished or incomplete stored ' + 'expansions return 422. Inline value sets, contexts, dates, selected designations and code system version ' + 'overrides are not supported.', (400, 422)), + ('ConceptMap', '$translate'): ( + 'Translate a code', + 'Returns live mappings from the named ConceptMap. Without url, uses the input system\'s own mappings ' + 'in forward translation, or targetsystem\'s mappings in reverse; without targetsystem, reverse has no match. ' + 'A code and system are required (or supply coding). Supports reverse translation and a target system filter. ' + 'No match returns Parameters with result=false. A POSTed codeableConcept uses its first coding.', + (400, 404, 422)), +} + + +def resource_schema(*resource_types): + """Identify the FHIR response type without duplicating the complete FHIR resource definitions.""" + return openapi.Schema(type=openapi.TYPE_OBJECT, required=['resourceType'], properties={ + 'resourceType': openapi.Schema(type=openapi.TYPE_STRING, enum=list(resource_types)), + }) + + +def parameters_schema(components): + """One shared Parameters body, including the structured alternatives to query-string coding.""" + definitions = components.with_scope(openapi.SCHEMA_DEFINITIONS) + if not definitions.has('FhirParameters'): + schema = resource_schema('Parameters') + schema.properties['parameter'] = openapi.Schema(type=openapi.TYPE_ARRAY, items=openapi.Schema( + type=openapi.TYPE_OBJECT, required=['name'], properties={ + 'name': openapi.Schema(type=openapi.TYPE_STRING), + 'valueString': openapi.Schema(type=openapi.TYPE_STRING), + 'valueCode': openapi.Schema(type=openapi.TYPE_STRING), + 'valueUri': openapi.Schema(type=openapi.TYPE_STRING), + 'valueBoolean': openapi.Schema(type=openapi.TYPE_BOOLEAN), + 'valueInteger': openapi.Schema(type=openapi.TYPE_INTEGER), + 'valueCoding': openapi.Schema(type=openapi.TYPE_OBJECT, properties={ + key: openapi.Schema(type=openapi.TYPE_STRING) for key in ('system', 'code', 'version', 'display')}), + 'valueCodeableConcept': openapi.Schema(type=openapi.TYPE_OBJECT, properties={ + 'coding': openapi.Schema(type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_OBJECT))}), + })) + definitions.set('FhirParameters', schema) + return openapi.SchemaRef(definitions, 'FhirParameters') + + +def operation(summary, description, parameters, response_type, errors=(400, 422)): + """A read with FHIR response media types and OperationOutcome errors; a token shows private content.""" + responses = {'200': openapi.Response(response_type, schema=resource_schema(response_type))} + errors_by_status = { + 400: 'Invalid or unsupported input.', 404: 'Resource or code not found or not visible at this base.', + 422: 'The request cannot be processed or exceeds the operation\'s processing bound.', + } + responses.update({str(code): openapi.Response(errors_by_status[code], schema=resource_schema('OperationOutcome')) + for code in errors}) + return openapi.Operation( + operation_id='', summary=summary, description=description, parameters=[*parameters, FORMAT], + responses=openapi.Responses(responses), produces=[FHIR_JSON, FHIR_XML]) + + +def search_operation(resource_type): + """Take the supported names and modifiers from the search parser itself.""" + parameters = [query(name, SEARCH_DESCRIPTIONS[name]) for name in PARAMETERS[resource_type]] + modifiers = {'': 'Starts with, ignoring case and accents.', 'exact': 'Exact, including case and accents.', + 'contains': 'Contains, ignoring case and accents.'} + parameters.extend(query(f'{name}:{modifier}' if modifier else name, f'{name}: {modifiers[modifier]}') + for name in STRING_FIELDS for modifier in STRING_LOOKUPS) + parameters.extend([ + query('_count', 'Page size, capped at 100; zero returns only the total.', openapi.TYPE_INTEGER, + minimum=0, maximum=MAX_COUNT, default=MAX_COUNT), + query('page', 'Page number, starting at 1.', openapi.TYPE_INTEGER, minimum=1, default=1), + ]) + return operation(f'Search {resource_type} resources', + 'Returns a searchset Bundle of visible released versions, never HEAD. Entries are summaries ' + 'tagged SUBSETTED. Commas separate alternatives; repeated parameters must all match. ' + 'Unsupported parameters or modifiers return 400.', parameters, 'Bundle') + + +def resource_operations(resource_type, base): + """The type's search, read and supported terminology operations, once per canonical suffix.""" + read_parameters = [] if base == '/fhir' else [query('version', 'Released business version; never HEAD.')] + read_description = ( + 'The opaque id selects one released version.' if base == '/fhir' else + 'The id is the repository mnemonic. Returns the latest visible release unless version is named.') + paths = { + f'/{resource_type}': {'get': search_operation(resource_type)}, + f'/{resource_type}/{{id}}': {'get': operation( + f'Read a {resource_type}', read_description + ' HEAD is never returned. A cut read is tagged SUBSETTED.', + read_parameters, resource_type, (400, 404, 422))}, + } + for (resource, name), (summary, description, errors) in OPERATION_DESCRIPTIONS.items(): + if resource != resource_type: + continue + for instance in (False, True) if resource != 'ConceptMap' else (False,): + suffix = f'/{resource}/' + ('{id}/' if instance else '') + name + operation_errors = tuple(sorted(set(errors) | ({404} if instance else set()))) + documented = operation(summary, description + ' ' + VERSION_SELECTION, + OPERATION_PARAMETERS[(resource, name)], + 'ValueSet' if name == '$expand' else 'Parameters', operation_errors) + paths[suffix] = {'get': documented, 'post': deepcopy(documented)} + return paths + + +def capability_operations(): + """The metadata variants share a route; $versions only supports GET.""" + metadata = operation('Read service capabilities', + 'Returns CapabilityStatement, or TerminologyCapabilities for mode=terminology.', + [query('mode', 'Use terminology to list the code systems and versions visible at this base.', + enum=['terminology'])], 'CapabilityStatement') + metadata.responses['200'] = openapi.Response( + 'CapabilityStatement; TerminologyCapabilities when mode=terminology.', + schema=resource_schema('CapabilityStatement', 'TerminologyCapabilities')) + return { + '/metadata': {'get': metadata}, + '/$versions': {'get': operation('List supported FHIR versions', + 'Returns the supported FHIR versions and the default (4.0 for R4).', + [], 'Parameters', (400,))}, + } + + +def path_parameters(path): + """Document the FHIR spelling of the route variables, independent of Django's lookup kwarg names.""" + descriptions = { + 'org': 'Organization identifier.', 'user': 'User identifier.', + 'id': 'Opaque release id at /fhir; repository mnemonic at an owner base.', + } + return [openapi.Parameter(name, openapi.IN_PATH, type=openapi.TYPE_STRING, required=True, description=description) + for name, description in descriptions.items() if '{' + name + '}' in path] + + +def fhir_paths(components): + """One source of paths for the dedicated FHIR schema and the combined API schema.""" + paths = {} + body = parameters_schema(components) + for tag in TAGS: + for base in BASES: + templates = capability_operations() if tag == 'Capabilities' else resource_operations(tag, base) + for suffix, methods in templates.items(): + path = base + suffix + for method, documented in methods.items(): + documented.operation_id = 'fhir_' + '_'.join( + path.strip('/').replace('{', '').replace('}', '').replace('$', '').replace('-', '_').split('/') + ) + '_' + method + documented.tags = [tag] + documented.parameters = deepcopy(documented.parameters) + path_parameters(path) + if method == 'post': + documented.consumes = [FHIR_JSON] + documented.parameters.append(openapi.Parameter( + 'parameters', openapi.IN_BODY, schema=body, + description='FHIR Parameters: use the query input names with one value[x] per parameter. ' + 'coding may use valueCoding; codeableConcept may use valueCodeableConcept ' + '(one coding for lookup/validation). Body values take precedence over query ' + 'values. _format stays in the query string.')) + paths[path] = openapi.PathItem(**methods) + return openapi.Paths(paths) + + +class FhirSwaggerSchemaGenerator(OpenAPISchemaGenerator): + """Generate the curated service schema without inspecting inherited resource write methods.""" + + def get_schema(self, request=None, public=False): # pylint: disable=unused-argument + components = self.reference_resolver_class(openapi.SCHEMA_DEFINITIONS, force_init=True) + paths = fhir_paths(components) + api_url = settings.API_BASE_URL.rstrip('/') + '/swagger/' + info = openapi.Info( + title='OCL FHIR terminology service (R4)', default_version='4.0.1', + description='FHIR R4 terminology in JSON and XML. Serves the SVCM transactions Query Value Set (ITI-95), ' + 'Query Code System (ITI-96), Expand Value Set (ITI-97), Lookup Code (ITI-98), ' + 'Validate Code (ITI-99), Query Concept Map (ITI-100) and Translate Code (ITI-101). ' + 'Bases: /fhir, /orgs/{org} and /users/{user}. The global base shows public resources; ' + 'owner bases show resources visible to the caller. ' + f'[OCL API Swagger]({api_url}) describes the rest of OCL.') + security_definitions = self.get_security_definitions() + return openapi.Swagger( + info=info, paths=paths, security_definitions=security_definitions, + security=self.get_security_requirements(security_definitions), + _url=request.build_absolute_uri('/') if request else self.url, _prefix='', **dict(components)) diff --git a/core/fhir/tests.py b/core/fhir/tests.py index ac2ae745d..ef6f90fd6 100644 --- a/core/fhir/tests.py +++ b/core/fhir/tests.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - from rest_framework.test import APIClient from core.common.tests import OCLTestCase @@ -21,10 +19,9 @@ def test_public_can_vew_terminology_statement(self): self.assertEqual(response.status_code, 200) self.assertEqual(response.data['resourceType'], "TerminologyCapabilities") - def test_fhir_subdomain_replaces_api_base_url(self): - with patch('core.fhir.views.settings.FHIR_SUBDOMAIN', 'fhir'): - with patch('core.fhir.views.settings.API_BASE_URL', 'https://api.openconceptlab.org'): - response = self.client.get('/fhir/metadata?mode=terminology') + def test_implementation_url_is_the_fhir_base_of_the_request_host(self): + response = self.client.get( + '/fhir/metadata?mode=terminology', HTTP_HOST='fhir.openconceptlab.org', HTTP_X_FORWARDED_PROTO='https') self.assertEqual(response.status_code, 200) self.assertEqual( diff --git a/core/fhir/tests_base.py b/core/fhir/tests_base.py new file mode 100644 index 000000000..6e49bb2e5 --- /dev/null +++ b/core/fhir/tests_base.py @@ -0,0 +1,750 @@ +import datetime +import json +import re +import uuid +from decimal import Decimal +from unittest.mock import patch + +from django.test import SimpleTestCase, override_settings +from django.http import HttpResponse, JsonResponse +from fhir.resources.R4B.bundle import Bundle +from fhir.resources.R4B.codesystem import CodeSystem, CodeSystemConcept +from fhir.resources.R4B.operationoutcome import OperationOutcome +from lxml import etree +from rest_framework.exceptions import NotAcceptable, ParseError, UnsupportedMediaType +from rest_framework.parsers import JSONParser +from rest_framework.renderers import JSONRenderer +from rest_framework.permissions import AllowAny +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.test import APIRequestFactory +from rest_framework.views import APIView + +from core.fhir import base, outcomes +from core.fhir.base import FhirContentNegotiation, FhirJsonParser, FhirJsonRenderer, FhirXmlParser, \ + FhirXmlRenderer, FhirParameters, FhirVersioning, FhirViewMixin + +FHIR = '{http://hl7.org/fhir}' +BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' + + +def child_names(element): + return [etree.QName(child).localname for child in element] + + +def assert_in_model_order(test, element, model_class): + """The element's children are elements of the R4B model, in the model's order (repeats kept together).""" + names, sequence = child_names(element), model_class.elements_sequence() + test.assertEqual([name for name in names if name not in sequence], [], 'elements the model does not define') + test.assertEqual(names, sorted(names, key=sequence.index)) + + +def released_code_system(**overrides): + """What CodeSystemDetailSerializer gives for a released source version, in its key order.""" + return { + 'resourceType': 'CodeSystem', 'url': 'http://example.org/cs', 'title': 'Codes', 'status': 'active', + 'id': 'codes', 'language': 'en', 'count': 1, 'content': 'complete', + 'property': [{'code': 'conceptclass', 'uri': 'http://testserver/orgs/OCL/sources/Classes/concepts', + 'description': 'Standard list of concept classes.', 'type': 'string'}], + 'meta': {'lastUpdated': '2026-10-14T10:00:00.123456Z'}, 'version': 'v1', + 'identifier': [{'system': 'http://testserver', 'value': '/orgs/X/CodeSystem/codes/', 'type': { + 'text': 'Accession ID', 'coding': [{'system': 'http://hl7.org/fhir/v2/0203', 'code': 'ACSN'}]}}], + 'name': 'codes', 'publisher': 'X', 'revisionDate': '2026-10-14T10:00:00Z', 'experimental': False, + 'caseSensitive': True, 'compositional': False, 'versionNeeded': False, + 'collectionReference': '/orgs/X/collections/c/', 'hierarchyMeaning': 'is-a', + 'concept': [{'code': 'c1', 'display': 'One', 'designation': [{'language': 'en', 'value': 'One'}], + 'property': [{'code': 'conceptclass', 'valueString': 'Diagnosis'}]}], + 'text': {'status': 'generated', 'div': '
Codes
'}, + 'filters': [{'code': 'x'}], + **overrides, + } + + +def renderers(): + return [klass() for klass in FhirViewMixin.renderer_classes] + + +def parsers(): + return [klass() for klass in FhirViewMixin.parser_classes] + + +class NegotiationTest(SimpleTestCase): + factory = APIRequestFactory() + + def select(self, path='/fhir/CodeSystem', **headers): + renderer, media_type = FhirContentNegotiation().select_renderer(Request(self.factory.get(path, **headers)), + renderers()) + return type(renderer), media_type + + def test_fhir_json_is_the_default(self): + self.assertEqual(self.select(), (FhirJsonRenderer, 'application/fhir+json')) + self.assertEqual(self.select(HTTP_ACCEPT='*/*'), (FhirJsonRenderer, 'application/fhir+json')) + + def test_accept_header(self): + self.assertEqual(self.select(HTTP_ACCEPT='application/fhir+xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='text/xml')[0], FhirXmlRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/json+fhir')[0], FhirJsonRenderer) + self.assertEqual(self.select(HTTP_ACCEPT='application/json')[0], JSONRenderer) + + def test_quality_values_are_honoured(self): + accept = 'application/fhir+json;q=0.5, application/xml+fhir;q=0.9' + + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], FhirXmlRenderer) + + def test_explicit_exclusions_override_wildcards(self): + """A broader range cannot make an explicitly excluded renderer acceptable.""" + cases = [ + ('application/fhir+json;q=0, */*;q=1', JSONRenderer), + ('application/json+fhir;q=0, application/json;q=0, application/*;q=1', FhirXmlRenderer), + ] + for accept, expected in cases: + with self.subTest(accept=accept): + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], expected) + + def test_most_specific_range_sets_each_renderers_quality(self): + """Specific ranges determine quality before renderers compete with one another.""" + cases = [ + 'application/fhir+json;q=0.2, application/fhir+xml;q=0.5, application/json;q=0, */*;q=1', + 'application/fhir+xml;q=0.5, application/*;q=0, */*;q=1', + '*/*;q=1, application/fhir+xml;q=1', + ] + for accept in cases: + with self.subTest(accept=accept): + self.assertEqual(self.select(HTTP_ACCEPT=accept)[0], FhirXmlRenderer) + + def test_type_wildcard_exclusion_overrides_universal_wildcard(self): + """Reject the request when all available application renderers are excluded.""" + with self.assertRaises(NotAcceptable): + self.select(HTTP_ACCEPT='application/*;q=0, */*;q=1') + + def test_a_malformed_quality_value_ranks_last(self): + self.assertEqual(self.select(HTTP_ACCEPT='application/fhir+xml;q=high, application/fhir+json')[0], + FhirJsonRenderer) + + def test_a_browser_gets_json_rather_than_the_xml_it_also_lists(self): + self.assertEqual(self.select(HTTP_ACCEPT=BROWSER_ACCEPT), (JSONRenderer, 'application/json')) + + def test_format_parameter_overrides_accept(self): + cases = [ + ('_format=xml', FhirXmlRenderer), ('_format=json', FhirJsonRenderer), + ('_format=application/fhir%2Bxml', FhirXmlRenderer), + ('_format=application/fhir+xml', FhirXmlRenderer), # an unencoded + arrives as a space + ('_format=application/xml', FhirXmlRenderer), ('_format=text/xml', FhirXmlRenderer), + ('_format=application/json', JSONRenderer), ('format=xml', FhirXmlRenderer), + ] + for query, renderer in cases: + self.assertEqual( + self.select(f'/fhir/CodeSystem?{query}', HTTP_ACCEPT='application/fhir+json')[0], renderer, query) + + def test_the_fhir_version_parameter_is_kept_on_the_accepted_media_type(self): + self.assertEqual( + self.select(HTTP_ACCEPT='application/fhir+json; fhirVersion=4.0'), + (FhirJsonRenderer, 'application/fhir+json; fhirVersion=4.0')) + self.assertEqual( + self.select('/fhir/CodeSystem?_format=application/fhir%2Bxml;fhirVersion=4.0'), + (FhirXmlRenderer, 'application/fhir+xml; fhirVersion=4.0')) + + def test_unsupported_formats_are_not_acceptable(self): + for path, accept in (('/fhir/CodeSystem?_format=turtle', '*/*'), ('/fhir/CodeSystem', 'text/csv'), + ('/fhir/CodeSystem', 'application/fhir+json;q=0')): + with self.assertRaises(NotAcceptable, msg=(path, accept)): + self.select(path, HTTP_ACCEPT=accept) + + def test_parser_selection_understands_fhir_media_types(self): + cases = [ + ('application/fhir+json', FhirJsonParser), ('application/fhir+json; charset=utf-8', FhirJsonParser), + ('application/json+fhir', FhirJsonParser), ('application/json', JSONParser), + ('application/fhir+xml', FhirXmlParser), ('application/xml', FhirXmlParser), + ('text/xml; charset=utf-8', FhirXmlParser), ('application/fhir+json; fhirVersion=4.0', FhirJsonParser), + ] + for content_type, parser in cases: + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', '{}', content_type=content_type)) + self.assertIsInstance(FhirContentNegotiation().select_parser(request, parsers()), parser, content_type) + + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', 'x', content_type='text/plain')) + self.assertIsNone(FhirContentNegotiation().select_parser(request, parsers())) + + +class VersioningTest(SimpleTestCase): + factory = APIRequestFactory() + + def version(self, accepted_media_type, content_type=None): + if content_type: + request = Request(self.factory.post('/fhir/CodeSystem/$lookup', '{}', content_type=content_type)) + else: + request = Request(self.factory.get('/fhir/CodeSystem')) + request.accepted_media_type = accepted_media_type + return FhirVersioning().determine_version(request) + + def test_r4_is_served(self): + self.assertEqual(self.version('application/fhir+json'), '4.0') + self.assertEqual(self.version('application/fhir+json; fhirVersion=4.0'), '4.0') + self.assertEqual(self.version('application/fhir+xml; fhirVersion=4.0.1'), '4.0') + self.assertEqual(self.version('application/fhir+json', 'application/fhir+json; fhirVersion=4.0'), '4.0') + + def test_another_version_is_refused(self): + with self.assertRaises(NotAcceptable): + self.version('application/fhir+json; fhirVersion=4.3') + with self.assertRaises(NotAcceptable): + self.version('application/fhir+json; fhirVersion=5.0') + with self.assertRaises(UnsupportedMediaType): + self.version('application/fhir+json', 'application/fhir+json; fhirVersion=5.0') + + +class XmlTest(SimpleTestCase): + def assert_in_model_order(self, element, model_class): + assert_in_model_order(self, element, model_class) + + def test_a_valid_resource_round_trips_through_its_model(self): + parameters = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'result', 'valueBoolean': True}, {'name': 'display', 'valueString': 'Malaria'}, + {'name': 'match', 'part': [ + {'name': 'concept', 'valueCoding': {'system': 'http://a', 'code': '1', 'display': 'One'}}]}, + {'name': 'issues', 'resource': {'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'code': 'invalid', 'details': {'text': 'Bad'}}]}}]} + + xml = base.to_xml(parameters) + + self.assertTrue(xml.startswith(b'

Codes

'}, + 'id': 'my_codes@v1', + } + + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(root.tag, f'{FHIR}CodeSystem') + self.assert_in_model_order(root, CodeSystem) + self.assertEqual(root.find(f'{FHIR}id').get('value'), 'my_codes@v1') + self.assertEqual(root.find(f'{FHIR}caseSensitive').get('value'), 'true') + self.assertEqual(root.find(f'{FHIR}count').get('value'), '2') + self.assertEqual(root.find(f'{FHIR}extension').get('url'), 'http://ext') + self.assertEqual(root.find(f'{FHIR}extension/{FHIR}valueString').get('value'), 'x') + self.assertEqual(root.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div/' + f'{{http://www.w3.org/1999/xhtml}}p').text, 'Codes') + concepts = root.findall(f'{FHIR}concept') + self.assertEqual(len(concepts), 2) + self.assert_in_model_order(concepts[0], CodeSystemConcept) + self.assertEqual(concepts[0].find(f'{FHIR}designation/{FHIR}language').get('value'), 'en') + + def test_contained_resources_and_narratives_that_are_not_xhtml(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'total': 1, 'unknownElement': 'x', + 'entry': [{'resource': {'resourceType': 'CodeSystem', 'id': 'a_b', 'status': 'active', + 'text': {'status': 'generated', 'div': 'plain Not a div

'}}}, + {'resource': {'resourceType': 'NoSuchResource', 'id': 'x'}}, + {'resource': 'not a resource'}]} + + root = etree.fromstring(base.to_xml(bundle)) + + self.assertEqual(child_names(root), ['type', 'total', 'entry', 'entry']) # the other two had nothing R4 + resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'a_b') + self.assertEqual(resource.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div').text, 'plain Not a div

') + + def test_data_that_is_not_a_resource_is_answered_with_an_outcome(self): + root = etree.fromstring(base.to_xml({'versions': ['4.0']})) + + self.assertEqual(root.tag, f'{FHIR}OperationOutcome') + self.assertEqual(base.to_xml(None), b'') + + def test_invalid_xml_bodies_are_parse_errors(self): + for body in (b'not xml', b'', b'', + b''): + with self.assertRaises(ParseError, msg=body): + base.from_xml(body) + + def test_entities_are_not_resolved(self): + body = (b']>' + b'' + b'') + try: + parsed = base.from_xml(body) + except ParseError: + return + self.assertNotIn('root:', json.dumps(parsed)) + + def test_the_xml_parser_and_renderer(self): + body = b'' \ + b'' + stream = type('Stream', (), {'read': lambda self: body})() + + parsed = FhirXmlParser().parse(stream) + + self.assertEqual(parsed, {'resourceType': 'Parameters', 'parameter': [{'name': 'code', 'valueCode': '1'}]}) + self.assertEqual(etree.fromstring(FhirXmlRenderer().render(parsed)).tag, f'{FHIR}Parameters') + + +class XmlElementOrderTest(SimpleTestCase): + """FHIR XML has a fixed element order: the R4B model's, whatever the serializer's key order.""" + + def assert_in_model_order(self, element, model_class): + assert_in_model_order(self, element, model_class) + + def test_a_released_code_system_comes_out_in_fhir_element_order(self): + xml = base.to_xml(released_code_system()) + + root = etree.fromstring(xml) + self.assert_in_model_order(root, CodeSystem) + self.assertEqual(child_names(root)[:4], ['id', 'meta', 'language', 'text']) + for name in ('revisionDate', 'collectionReference', 'filters'): + self.assertNotIn(name, child_names(root)) + concept = root.find(f'{FHIR}concept') + self.assert_in_model_order(concept, CodeSystemConcept) + self.assertEqual(base.from_xml(xml)['concept'][0]['display'], 'One') # valid R4B XML + + def test_a_search_bundle_holding_one_comes_out_in_fhir_element_order(self): + code_system = released_code_system(id='my_codes') # an owner-base id the model refuses + bundle = {'resourceType': 'Bundle', 'meta': {'lastUpdated': '2026-10-14T10:00:00Z'}, 'type': 'searchset', + 'total': 1, 'entry': [{'resource': code_system}], + 'link': [{'relation': 'self', 'url': 'http://testserver/fhir/CodeSystem/?page=1'}]} + + root = etree.fromstring(base.to_xml(bundle)) + + self.assertEqual(child_names(root), ['meta', 'type', 'total', 'link', 'entry']) + resource = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assert_in_model_order(resource, CodeSystem) + self.assertEqual(child_names(resource)[:4], ['id', 'meta', 'language', 'text']) + self.assertEqual(resource.find(f'{FHIR}id').get('value'), 'my_codes') + + def test_bundle_children_keep_the_model_order_with_an_id(self): + root = etree.fromstring(base.to_xml({ + 'resourceType': 'Bundle', 'entry': [{'resource': {'resourceType': 'CodeSystem', 'id': 'codes@1', + 'status': 'active', 'content': 'complete'}}], + 'link': [{'relation': 'self', 'url': 'http://x'}], 'total': 1, 'type': 'searchset', 'id': 'b1', + 'meta': {'lastUpdated': '2026-10-14T10:00:00Z'}})) + + self.assertEqual(child_names(root), ['id', 'meta', 'type', 'total', 'link', 'entry']) + + def test_keys_r4_does_not_define_are_dropped_at_every_depth(self): + code_system = {'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', + 'contact': [{'name': 'Jo', 'my key': 'x', 'telecom': [{'value': 'a@b.c', 'oops': 1}]}], + 'concept': [{'code': 'a', 'concept_class': 'Misc'}]} + + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(child_names(root.find(f'{FHIR}contact')), ['name', 'telecom']) + self.assertEqual(child_names(root.find(f'{FHIR}contact/{FHIR}telecom')), ['value']) + self.assertEqual(child_names(root.find(f'{FHIR}concept')), ['code']) + + def test_each_dropped_key_is_logged_once(self): + base.LOGGED_DROPPED_ELEMENTS.clear() + with self.assertLogs('oclapi', level='WARNING') as logs: + base.to_xml(released_code_system()) + base.to_xml(released_code_system()) + + dropped = [line for line in logs.output if 'does not define' in line] + self.assertEqual(len(dropped), 3, dropped) + self.assertTrue(any('CodeSystem.revisionDate' in line for line in dropped), dropped) + + def test_values_of_the_wrong_shape_are_dropped(self): + outcome = {'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'details': 'Failed to represent "/x" as CodeSystem', 'expression': 'code'}], + 'text': ['not', 'a', 'narrative'], 'id': {'not': 'an id'}} + + root = etree.fromstring(base.to_xml(outcome)) + + self.assertEqual(child_names(root), ['issue']) + self.assertEqual(child_names(root.find(f'{FHIR}issue')), ['severity', 'expression']) + + def test_a_resource_the_model_cannot_write_falls_back_to_the_element_writer(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'entry': [{'resource': { + 'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', 'experimental': False, + 'text': {'status': 'generated', 'div': '

A

'}, + 'extension': [{'url': 'http://ext', 'valueString': 'x'}], 'concept': [{'id': 'c1', 'code': 'x'}]}}]} + with patch.object(Bundle, 'model_dump_xml', side_effect=ValueError('boom')), \ + self.assertLogs('oclapi', level='ERROR'): + root = etree.fromstring(base.to_xml(bundle)) + + code_system = root.find(f'{FHIR}entry/{FHIR}resource/{FHIR}CodeSystem') + self.assertEqual(code_system.find(f'{FHIR}id').get('value'), 'a') + self.assertEqual(code_system.find(f'{FHIR}experimental').get('value'), 'false') + self.assertEqual(code_system.find(f'{FHIR}extension').get('url'), 'http://ext') + self.assertEqual(code_system.find(f'{FHIR}concept').get('id'), 'c1') + self.assertEqual(code_system.find(f'{FHIR}concept/{FHIR}code').get('value'), 'x') + self.assertEqual(code_system.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div/' + f'{{http://www.w3.org/1999/xhtml}}p').text, 'A') + + +# FHIR's instant: a full date and time with seconds and a zone +FHIR_INSTANT = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$') +UPDATED = datetime.datetime(2026, 10, 14, 12, 34, 56, 123456, tzinfo=datetime.timezone.utc) + + +def serialized_value_set(id_): + """What ValueSetDetailSerializer gives: meta.lastUpdated is the model's raw datetime.""" + return {'resourceType': 'ValueSet', 'id': id_, 'version': 'v1', 'url': f'http://example.org/vs/{id_}', + 'status': 'active', 'meta': {'lastUpdated': UPDATED}, 'date': UPDATED} + + +def json_of(data): + """What the JSON response carries for the same data.""" + return json.loads(FhirJsonRenderer().render(data)) + + +class XmlValuesTest(SimpleTestCase): + """XML carries the values the JSON response carries, on both the model and the unvalidated path.""" + + def last_updated(self, element): + return element.find(f'{FHIR}meta/{FHIR}lastUpdated').get('value') + + def test_a_refused_value_set_writes_its_datetimes_as_the_json_does(self): + data = serialized_value_set('my_vs') # an owner-base id the model refuses + + root = etree.fromstring(base.to_xml(data)) + + expected = json_of(data)['meta']['lastUpdated'] + self.assertEqual(self.last_updated(root), expected) + self.assertRegex(self.last_updated(root), FHIR_INSTANT) + self.assertEqual(root.find(f'{FHIR}date').get('value'), json_of(data)['date']) + + def test_every_entry_of_a_bundle_holding_a_refused_one(self): + bundle = {'resourceType': 'Bundle', 'type': 'searchset', 'total': 2, + 'entry': [{'resource': serialized_value_set('good')}, {'resource': serialized_value_set('bad_id')}]} + + root = etree.fromstring(base.to_xml(bundle)) + + expected = json_of(bundle)['entry'][0]['resource']['meta']['lastUpdated'] + for resource in root.iterfind(f'{FHIR}entry/{FHIR}resource/{FHIR}ValueSet'): + self.assertEqual(self.last_updated(resource), expected) + self.assertRegex(self.last_updated(resource), FHIR_INSTANT) + + def test_a_valid_resource_writes_the_same_instant(self): + data = serialized_value_set('good') + + written = self.last_updated(etree.fromstring(base.to_xml(data))) + + self.assertRegex(written, FHIR_INSTANT) + self.assertEqual(datetime.datetime.fromisoformat(written), + datetime.datetime.fromisoformat(json_of(data)['meta']['lastUpdated'])) + + def test_other_python_values_are_written_as_json_writes_them(self): + code = uuid.UUID('12345678-1234-5678-1234-567812345678') + parameters = {'resourceType': 'Parameters', 'id': 'p_1', 'parameter': [ + {'name': 'amount', 'valueDecimal': Decimal('1.50')}, {'name': 'when', 'valueDate': UPDATED.date()}, + {'name': 'code', 'valueString': code}]} + + values = [element.get('value') for element in etree.fromstring(base.to_xml(parameters)).iter() + if element.get('value') and etree.QName(element).localname.startswith('value')] + + self.assertEqual(values, ['1.5', '2026-10-14', str(code)]) + + +class XmlSingleElementTest(SimpleTestCase): + """A list for an element R4 allows once keeps its first item.""" + + def concept_map(self, identifiers): + return {'resourceType': 'ConceptMap', 'id': 'cm', 'status': 'active', 'identifier': identifiers} + + def test_a_concept_map_keeps_its_identifier(self): + first = {'system': 'http://testserver', 'value': '/orgs/X/ConceptMap/cm/'} + for identifiers in ([first], [first, {'value': 'another'}]): + root = etree.fromstring(base.to_xml(self.concept_map(identifiers))) + + self.assertEqual(len(root.findall(f'{FHIR}identifier')), 1, identifiers) + self.assertEqual(root.find(f'{FHIR}identifier/{FHIR}value').get('value'), '/orgs/X/ConceptMap/cm/') + + def test_the_items_left_out_are_logged_once(self): + base.LOGGED_DROPPED_ELEMENTS.clear() + identifiers = [{'value': 'a'}, {'value': 'b'}, {'value': 'c'}] + with self.assertLogs('oclapi', level='WARNING') as logs: + base.to_xml(self.concept_map(identifiers)) + base.to_xml(self.concept_map(identifiers)) + + self.assertEqual(len([line for line in logs.output if 'ConceptMap.identifier' in line]), 1, logs.output) + + +class XmlCharactersTest(SimpleTestCase): + """Characters XML 1.0 forbids never stop a response: each becomes U+FFFD, the replacement character.""" + + def test_a_control_character_in_a_value_becomes_the_replacement_character(self): + parameters = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'display', 'valueString': 'Mal\x0baria\x00'}, + {'name': 'name', 'valueString': 'Tab\tand\nnew line'}]} + + with self.assertLogs('oclapi', level='WARNING') as logs: + xml = base.to_xml(parameters) + + values = [element.get('value') for element in etree.fromstring(xml).iter(f'{FHIR}valueString')] + self.assertEqual(values, ['Mal\ufffdaria\ufffd', 'Tab\tand\nnew line']) + self.assertIn('2', logs.output[0]) + + def test_lone_surrogates_and_non_characters_in_text_and_narrative(self): + code_system = {'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', 'content': 'complete', + 'title': 'A\ud800B\uffffC', 'text': {'status': 'generated', 'div': 'x\x1fy'}} + + with self.assertLogs('oclapi', level='WARNING'): + root = etree.fromstring(base.to_xml(code_system)) + + self.assertEqual(root.find(f'{FHIR}title').get('value'), 'A\ufffdB\ufffdC') + self.assertEqual(root.find(f'{FHIR}text/{{http://www.w3.org/1999/xhtml}}div').text, 'x\ufffdy') + + def test_the_element_writer_is_safe_too(self): + with patch.object(CodeSystem, 'model_dump_xml', side_effect=ValueError('boom')), \ + self.assertLogs('oclapi', level='WARNING'): + root = etree.fromstring(base.to_xml({'resourceType': 'CodeSystem', 'id': 'a', 'status': 'active', + 'content': 'complete', 'title': 'bad\x0b'})) + + self.assertEqual(root.find(f'{FHIR}title').get('value'), 'bad\ufffd') + + +class FhirParametersTest(SimpleTestCase): + factory = APIRequestFactory() + + def read(self, django_request): + request = Request(django_request, parsers=parsers(), negotiator=FhirContentNegotiation()) + return FhirParameters.from_request(request) + + def test_query_parameters(self): + params = self.read(self.factory.get( + '/fhir/CodeSystem/$lookup?system=http://a&code=1&property=x&property=y&_format=xml&format=json&_pretty=1')) + + self.assertEqual(params.get('system'), 'http://a') + self.assertEqual(params.getlist('property'), ['x', 'y']) + self.assertNotIn('_format', params) + self.assertIsNone(params.get('version')) + self.assertEqual(params.get('version', 'v1'), 'v1') + + def test_a_posted_parameters_resource(self): + body = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'url', 'valueUri': 'http://a'}, {'name': 'activeOnly', 'valueBoolean': False}, + {'name': 'count', 'valueInteger': 10}, + {'name': 'coding', 'valueCoding': {'system': 'http://a', 'code': '1'}}, + {'name': 'valueSet', 'resource': {'resourceType': 'ValueSet', 'status': 'active'}}, + {'name': 'dependency', 'part': [{'name': 'element', 'valueUri': 'http://e'}]}, + {'name': 'empty'}]} + + params = self.read(self.factory.post( + '/fhir/ValueSet/$expand?_format=xml&offset=5', json.dumps(body), content_type='application/fhir+json')) + + self.assertEqual(params.get('url'), 'http://a') + self.assertIs(params.get('activeOnly'), False) + self.assertEqual(params.get('count'), 10) + self.assertEqual(params.get('coding'), {'system': 'http://a', 'code': '1'}) + self.assertEqual(params.get('valueSet')['resourceType'], 'ValueSet') + self.assertEqual(params.get('dependency').get('element'), 'http://e') + self.assertIsNone(params.get('empty')) + self.assertEqual(params.get('offset'), '5') + self.assertNotIn('_format', params) + + def test_a_posted_xml_parameters_resource(self): + body = '' \ + '' + + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup', body, content_type='application/fhir+xml')) + + self.assertEqual(params.get('code'), '1') + + def test_a_form_post(self): + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup', {'code': '1', 'system': 'http://a'})) + + self.assertEqual(params.get('code'), '1') + self.assertEqual(params.get('system'), 'http://a') + + def test_a_post_without_a_body_reads_the_query(self): + params = self.read(self.factory.post('/fhir/CodeSystem/$lookup?code=1')) + + self.assertEqual(params.get('code'), '1') + + def test_a_posted_resource_that_is_not_parameters_is_invalid(self): + with self.assertRaises(outcomes.InvalidError): + self.read(self.factory.post( + '/fhir/CodeSystem/$lookup', json.dumps({'resourceType': 'CodeSystem'}), + content_type='application/fhir+json')) + + def test_a_parameter_with_more_than_one_value_is_invalid_whatever_their_order(self): + """R4 gives a parameter one value[x], resource or part: two are refused, never read as whichever came first. A + primitive's _value[x] is its own value[x], and another's is a second one.""" + absent = {'extension': [{'url': 'http://hl7.org/fhir/StructureDefinition/data-absent-reason', + 'valueCode': 'unknown'}]} + body = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'code', 'valueCode': 'c1', '_valueCode': {'id': 'c'}}, {'name': 'system', '_valueUri': absent}]} + params = self.read(self.factory.post( + '/fhir/CodeSystem/$lookup', json.dumps(body), content_type='application/fhir+json')) + self.assertEqual((params.get('code'), params.get('system')), ('c1', None)) + for given in ({'valueString': 'c2', 'valueCode': 'c1'}, {'valueCode': 'c1', 'valueString': 'c2'}, + {'valueCode': 'c1', '_valueString': absent}, {'_valueString': absent, 'valueCode': 'c1'}, + {'valueUri': 'http://a', 'part': [{'name': 'x', 'valueCode': 'y'}]}, + {'valueString': 'c1', 'resource': {'resourceType': 'ValueSet'}}): + for nested in (False, True): + parameter = {'name': 'code', **given} + body = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'property', 'part': [parameter]} if nested else parameter]} + with self.subTest((given, nested)): + with self.assertRaises(outcomes.InvalidError) as raised: + self.read(self.factory.post( + '/fhir/CodeSystem/$lookup', json.dumps(body), content_type='application/fhir+json')) + self.assertEqual(raised.exception.expression, 'code') + + +class BaseUrlTest(SimpleTestCase): + factory = APIRequestFactory() + + def test_the_base_comes_from_the_request_host(self): + request = self.factory.get( + '/orgs/CIEL/CodeSystem/', HTTP_HOST='fhir.openconceptlab.org', HTTP_X_FORWARDED_PROTO='https') + + self.assertEqual(base.fhir_base_url(request), 'https://fhir.openconceptlab.org/fhir') + self.assertEqual(base.fhir_base_url(request, {'org': 'CIEL'}), 'https://fhir.openconceptlab.org/orgs/CIEL') + self.assertEqual(base.fhir_base_url(request, {'user': 'jo'}), 'https://fhir.openconceptlab.org/users/jo') + + def test_fhir_paths(self): + for path in ('/fhir', '/fhir/', '/fhir/metadata', '/fhir/CodeSystem/$lookup/', '/orgs/CIEL/CodeSystem', + '/orgs/CIEL/ValueSet/x/$expand/', '/users/jo/ConceptMap/$translate'): + self.assertTrue(base.is_fhir_path(path), path) + for path in ('/', '/orgs/CIEL/', '/orgs/CIEL/sources/', '/fhirx/', '/version/', '/user/sources/CodeSystem/', + '/sources/x/ConceptMap/'): + self.assertFalse(base.is_fhir_path(path), path) + + +class OutcomeResponseTest(SimpleTestCase): + factory = APIRequestFactory() + outcome = outcomes.operation_outcome(outcomes.issue('not-found', 'Nothing here')) + + def test_json_by_default(self): + response = base.outcome_response(self.factory.get('/fhir/Patient'), self.outcome, 404) + + self.assertEqual(response.status_code, 404) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + self.assertEqual(json.loads(response.content), self.outcome) + + def test_xml_when_asked_and_never_a_5xx(self): + response = base.outcome_response( + self.factory.get('/fhir/Patient?_format=xml'), self.outcome, 500, {'Retry-After': '1'}) + + self.assertEqual(response.status_code, 422) + self.assertEqual(response['Content-Type'], 'application/fhir+xml; charset=utf-8') + self.assertEqual(response['Retry-After'], '1') + self.assertEqual(etree.fromstring(response.content).tag, f'{FHIR}OperationOutcome') + + def test_an_unsupported_format_still_gets_json(self): + response = base.outcome_response(self.factory.get('/fhir/Patient?_format=turtle'), self.outcome, 404) + + self.assertEqual(response['Content-Type'], 'application/fhir+json') + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_unparseable_query_uses_a_400_json_outcome(self): + """Error negotiation must not repeat a failed query parse or retain the original error status.""" + query = '&'.join(['_format=xml'] + ['x='] * 1001) + response = base.outcome_response( + self.factory.get(f'/fhir/missing?{query}', HTTP_ACCEPT='application/fhir+xml'), + self.outcome, 404, {'Retry-After': '1'}) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + self.assertEqual(response['Retry-After'], '1') + outcome = OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome.issue[0].code, 'invalid') + self.assertEqual(outcome.issue[0].details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(outcome.issue[0].details.coding[0].code, 'invalid-data') + + def test_is_outcome_response(self): + self.assertTrue(base.is_outcome_response(base.outcome_response( + self.factory.get('/fhir/x'), self.outcome, 404))) + self.assertTrue(base.is_outcome_response(JsonResponse(self.outcome, status=404))) + self.assertFalse(base.is_outcome_response(JsonResponse({'detail': 'Not found.'}, status=404))) + self.assertFalse(base.is_outcome_response(HttpResponse(b'

Not Found

', status=404))) + self.assertFalse(base.is_outcome_response(HttpResponse(b'{oops', content_type='application/json'))) + + +class EchoView(FhirViewMixin, APIView): + """A FHIR view that answers with its parameters, or with the status it is asked for.""" + permission_classes = (AllowAny,) + authentication_classes = () + + def get(self, request): + status = int(request.query_params.get('status', 200)) + if status != 200: + return Response({'detail': 'As asked.'} if status == 409 else None, status=status) + return self.post(request) + + def post(self, _request): + params = self.get_fhir_parameters() + return Response({'resourceType': 'Parameters', 'parameter': [ + {'name': name, 'valueString': str(value)} for name, value in params.items] + [ + {'name': 'base', 'valueUri': self.get_fhir_base_url()}]}) + + +class ReturnedOutcomeView(FhirViewMixin, APIView): + """Return the legacy serializer error shape with a configurable HTTP status.""" + permission_classes = (AllowAny,) + authentication_classes = () + status_code = 200 + + def get(self, _request): + """Leave normalization to the production view mixin.""" + return Response({'resourceType': 'OperationOutcome', 'issue': [ + {'severity': 'error', 'details': 'Failed to serialize the resource.'}]}, status=self.status_code) + + +class FhirViewMixinTest(SimpleTestCase): + factory = APIRequestFactory() + + def test_returned_outcomes_are_normalized_at_every_status(self): + """Successful and error statuses both retain the diagnostic in valid JSON and XML outcomes.""" + for status, expected in ((200, 200), (201, 201), (400, 400), (503, 422)): + for media_type in ('application/fhir+json', 'application/fhir+xml'): + with self.subTest(status=status, media_type=media_type): + response = ReturnedOutcomeView.as_view(status_code=status)( + self.factory.get('/fhir/x', HTTP_ACCEPT=media_type)) + response.render() + + self.assertEqual(response.status_code, expected) + self.assertEqual(response['Content-Type'].split(';')[0], media_type) + outcome = (OperationOutcome.model_validate_xml(response.content) if media_type.endswith('xml') + else OperationOutcome.model_validate_json(response.content)) + issue = outcome.issue[0] + self.assertEqual(issue.code, 'processing') + self.assertEqual(issue.details.text, 'Failed to serialize the resource.') + self.assertEqual(issue.details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(issue.details.coding[0].code, 'invalid-data') + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_query_parsing_failure_in_a_view_is_a_400_json_outcome(self): + """A query rejected during initial negotiation is a client error, even when XML was requested.""" + query = '&'.join(['x='] * 1001) + response = EchoView.as_view()(self.factory.get(f'/fhir/x?{query}', HTTP_ACCEPT='application/fhir+xml')) + response.render() + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + outcome = OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome.issue[0].code, 'invalid') + self.assertEqual(outcome.issue[0].details.coding[0].system, outcomes.TX_ISSUE_TYPE_SYSTEM) + self.assertEqual(outcome.issue[0].details.coding[0].code, 'invalid-data') + + def test_a_view_reads_its_parameters_the_same_way_from_get_and_post(self): + get = EchoView.as_view()(self.factory.get('/fhir/x?code=1')) + post = EchoView.as_view()(self.factory.post( + '/fhir/x', json.dumps({'resourceType': 'Parameters', 'parameter': [{'name': 'code', 'valueCode': '1'}]}), + content_type='application/fhir+json')) + + self.assertEqual(get.data, post.data) + self.assertEqual(get.data['parameter'][0], {'name': 'code', 'valueString': '1'}) + self.assertEqual(get.data['parameter'][1], {'name': 'base', 'valueUri': 'http://testserver/fhir'}) + + def test_errors_a_view_returns_go_out_as_outcomes_and_never_as_a_5xx(self): + for status, expected, code in ((404, 404, 'not-found'), (409, 409, 'invalid'), (503, 422, 'exception')): + response = EchoView.as_view()(self.factory.get(f'/fhir/x?status={status}')) + + self.assertEqual(response.status_code, expected, status) + self.assertEqual(response.data['resourceType'], 'OperationOutcome', status) + self.assertEqual(response.data['issue'][0]['code'], code, status) diff --git a/core/fhir/tests_capabilities.py b/core/fhir/tests_capabilities.py new file mode 100644 index 000000000..d96eab630 --- /dev/null +++ b/core/fhir/tests_capabilities.py @@ -0,0 +1,772 @@ +""" +FHIR metadata at the global base /fhir/ and at owner bases: the CapabilityStatement built from the routes and views +that exist (fhirVersion 4.0.1, each resource's interactions, search parameters and operations, $versions, SVCM's +Terminology Repository, the software), $versions, and TerminologyCapabilities listing the code systems the base shows +the caller, with the versions it serves and the default one, in a fixed number of queries, with the URL registry +toggle off and on. OpenConceptLab/ocl_online#451. +""" +import json + +from django.db import connection, transaction +from django.test import SimpleTestCase, override_settings +from django.test.utils import CaptureQueriesContext +from django.urls import Resolver404, include, path, resolve as resolve_path +from fhir.resources.R4B.capabilitystatement import CapabilityStatement +from fhir.resources.R4B.parameters import Parameters +from fhir.resources.R4B.terminologycapabilities import TerminologyCapabilities +from mock import patch +from rest_framework.exceptions import ValidationError +from rest_framework.test import APIClient + +from core import __version__ +from core.common.constants import ACCESS_TYPE_NONE, HEAD +from core.common.serializers import IdentifierSerializer +from core.common.tests import OCLTestCase, OCLAPITransactionTestCase +from core.concepts.tests.factories import ConceptFactory +from core.fhir import base, search +from core.fhir.base import FhirOperationView +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.resolve import ( + ServedHead, UnresolvedError, in_namespace, namespace_of, ocl_path, resolve, viewable_rows, visible_versions) +from core.fhir.tests_ops_lookup import OpsCorpusMixin, CIEL, ICD10, REST, SHARED, NEWEST_PRIVATE, HIV +from core.fhir.tests_ops_lookup import name as concept_name +from core.fhir.tests_search import source +from core.orgs.tests.factories import OrganizationFactory +from core.sources.models import Source +from core.url_registry.factories import OrganizationURLRegistryFactory +from core.url_registry.tests_resolution import set_registry_toggle +from core.users.tests.factories import UserProfileFactory + +SVCM_REPOSITORY = 'https://profiles.ihe.net/ITI/SVCM/CapabilityStatement/IHE.SVCM.TerminologyRepository|1.5.1' +OPERATION_DEFINITION = 'http://hl7.org/fhir/OperationDefinition/' +CLOSED = 'http://moh.example/fhir/CodeSystem/closed' +DRAFT = 'http://example.org/fhir/CodeSystem/draft' +UNREGISTERED = 'http://example.org/fhir/CodeSystem/unregistered' +REDIRECTED = 'http://example.org/fhir/CodeSystem/redirected' +COLLIDE = 'http://example.org/fhir/CodeSystem/collide' +OLD, NEW = 'http://example.org/fhir/CodeSystem/renamed', 'http://example.org/fhir/CodeSystem/renamed-new' +RETIRED_HEAD = 'http://example.org/fhir/CodeSystem/retired-head' +NAMELESS = '/orgs/Acme/CodeSystem/nameless/' # a code system without a canonical URL, by its OCL path +# every R4 operation on the three resources and at the base, routed or not +R4_OPERATIONS = { + 'CodeSystem': ('lookup', 'validate-code', 'subsumes', 'find-matches'), + 'ValueSet': ('expand', 'validate-code'), + 'ConceptMap': ('translate', 'closure'), +} +R4_SYSTEM_OPERATIONS = ('versions', 'closure', 'convert', 'meta') +BASES = ('/fhir', '/orgs/CIEL', '/users/jdoe') + + +def is_operation_view(url): + """Whether Django routes a path to a FHIR operation view.""" + try: + view = resolve_path(url).func.view_class + except (Resolver404, AttributeError): + return False + return issubclass(view, FhirOperationView) + + +def routed_operations(fhir_base='/fhir'): + """(resource type, operation) for each R4 operation Django routes to an operation view, at the type or instance + level.""" + return {(resource_type, name) for resource_type, names in R4_OPERATIONS.items() for name in names + if is_operation_view(f'{fhir_base}/{resource_type}/${name}') or + is_operation_view(f'{fhir_base}/{resource_type}/some-id/${name}')} + + +def code_system_path(owner, mnemonic): + """An org's repository's OCL path as a CodeSystem.""" + return f'/orgs/{owner}/CodeSystem/{mnemonic}/' + + +def client_for(user=None): + client = APIClient() + if user: + client.force_authenticate(user) + return client + + +def listed_operations(statement): + return {(resource['type'], operation['name']) for resource in statement['rest'][0]['resource'] + for operation in resource.get('operation', [])} + + +class CapabilityStatementTest(OCLTestCase): + def setUp(self): + super().setUp() + OrganizationFactory(mnemonic='CIEL') + UserProfileFactory(username='jdoe') + + def statement(self, fhir_base='/fhir', **params): + response = APIClient().get(f'{fhir_base}/metadata', params) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['resourceType'], 'CapabilityStatement') + return response.data + + def test_fhir_version_is_r4s(self): + statement = self.statement() + self.assertEqual(statement['fhirVersion'], base.FHIR_RELEASE) + self.assertEqual(statement['fhirVersion'], '4.0.1') + self.assertEqual((statement['kind'], statement['status']), ('instance', 'active')) + + def test_the_operations_are_the_routed_ones(self): + for fhir_base in BASES: + statement = self.statement(fhir_base) + self.assertEqual(listed_operations(statement), routed_operations(fhir_base), fhir_base) + for resource in statement['rest'][0]['resource']: + for operation in resource.get('operation', []): + self.assertEqual( + operation['definition'], f"{OPERATION_DEFINITION}{resource['type']}-{operation['name']}") + self.assertEqual(statement['rest'][0]['operation'], [ + {'name': 'versions', 'definition': f'{OPERATION_DEFINITION}CapabilityStatement-versions'}]) + self.assertTrue({('CodeSystem', 'lookup'), ('CodeSystem', 'validate-code'), ('ValueSet', 'validate-code'), + ('ConceptMap', 'translate')} <= listed_operations(self.statement())) + self.assertNotIn(('CodeSystem', 'subsumes'), listed_operations(self.statement())) + + def test_system_operations_are_the_routed_ones(self): + routed = [name for name in R4_SYSTEM_OPERATIONS if is_operation_view(f'/fhir/${name}')] + self.assertEqual(routed, ['versions']) + self.assertEqual([operation['name'] for operation in self.statement()['rest'][0]['operation']], routed) + + def test_interactions_and_search_parameters_are_the_searchs(self): + resources = {resource['type']: resource for resource in self.statement()['rest'][0]['resource']} + self.assertEqual(list(resources), ['CodeSystem', 'ValueSet', 'ConceptMap']) + for resource_type, resource in resources.items(): + self.assertEqual([item['code'] for item in resource['interaction']], ['read', 'search-type']) + names = [item['name'] for item in resource['searchParam']] + self.assertEqual(names, [*search.PARAMETERS[resource_type], *search.STRING_FIELDS]) + for item in resource['searchParam']: + self.assertTrue(item['definition'].startswith('http://hl7.org/fhir/SearchParameter/'), item) + code_system = {item['name']: item for item in resources['CodeSystem']['searchParam']} + self.assertEqual(code_system['url'], { + 'name': 'url', 'definition': 'http://hl7.org/fhir/SearchParameter/conformance-url', 'type': 'uri'}) + self.assertEqual(code_system['_lastUpdated']['type'], 'date') + self.assertEqual(code_system['title']['type'], 'string') + self.assertEqual(code_system['code']['definition'], 'http://hl7.org/fhir/SearchParameter/CodeSystem-code') + concept_map = {item['name']: item['type'] for item in resources['ConceptMap']['searchParam']} + self.assertEqual((concept_map['source-system'], concept_map['source-uri']), ('uri', 'reference')) + + def test_source_uri_and_target_uri_are_documented_as_read_as_the_systems(self): + """OCL keeps no source or target value set on a ConceptMap, so source-uri and target-uri are read as the + groups' systems, and only they carry documentation saying so.""" + for fhir_base in BASES: + resources = self.statement(fhir_base)['rest'][0]['resource'] + documented = {(resource['type'], item['name']): item['documentation'] + for resource in resources for item in resource['searchParam'] if 'documentation' in item} + self.assertEqual(set(documented), {('ConceptMap', 'source-uri'), ('ConceptMap', 'target-uri')}, fhir_base) + for side in ('source', 'target'): + self.assertTrue(documented[('ConceptMap', f'{side}-uri')].startswith(f'Read as `{side}-system`')) + + def test_it_instantiates_svcms_terminology_repository_and_names_the_software(self): + statement = self.statement() + self.assertEqual(statement['instantiates'], [SVCM_REPOSITORY]) + self.assertEqual(statement['software'], {'name': 'OCL API', 'version': __version__}) + self.assertEqual(statement['format'], ['application/fhir+json', 'application/fhir+xml']) + self.assertIn('date', statement) + + def test_the_implementation_url_is_the_requests_base(self): + response = APIClient().get( + '/orgs/CIEL/metadata', HTTP_HOST='fhir.example.org', HTTP_X_FORWARDED_PROTO='https') + self.assertEqual(response.data['implementation']['url'], 'https://fhir.example.org/orgs/CIEL') + self.assertEqual(self.statement('/users/jdoe')['implementation']['url'], 'http://testserver/users/jdoe') + self.assertEqual(self.statement()['implementation']['url'], 'http://testserver/fhir') + + def test_every_base_with_and_without_a_trailing_slash(self): + expected = self.statement() + for fhir_base in ('/fhir', *BASES[1:], '/orgs/no-such-org', '/users/no-such-user'): + for suffix in ('metadata', 'metadata/'): + response = APIClient().get(f'{fhir_base}/{suffix}') + self.assertEqual(response.status_code, 200, (fhir_base, suffix)) + answer = response.data + self.assertEqual(answer['implementation']['url'], f'http://testserver{fhir_base}') + self.assertEqual({**answer, 'implementation': None}, {**expected, 'implementation': None}, fhir_base) + self.assertEqual(APIClient().get('/fhir/').data, expected) + + def test_it_parses_as_r4_json_and_xml(self): + for fhir_base in BASES: + for media_type in ('application/fhir+json', 'application/fhir+xml'): + response = APIClient().get(f'{fhir_base}/metadata', HTTP_ACCEPT=media_type) + self.assertEqual(response['Content-Type'].split(';')[0], media_type) + statement = (CapabilityStatement.model_validate_xml(response.content) if media_type.endswith('xml') + else CapabilityStatement.model_validate_json(response.content)) + self.assertEqual(statement.fhirVersion, '4.0.1') + self.assertEqual(len(statement.rest[0].resource), 3) + + def test_get_only(self): + for fhir_base in BASES: + response = APIClient().post(f'{fhir_base}/metadata', {}, format='json') + self.assertEqual(response.status_code, 405, fhir_base) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + + def test_another_fhir_version_is_refused(self): + for fhir_base in BASES: + for suffix in ('metadata', 'metadata?mode=terminology', '$versions'): + response = APIClient().get( + f'{fhir_base}/{suffix}', HTTP_ACCEPT='application/fhir+json; fhirVersion=3.0') + self.assertEqual(response.status_code, 406, (fhir_base, suffix)) + issue = response.data['issue'][0] + self.assertEqual(issue['code'], 'not-supported') + self.assertEqual(issue['details']['coding'][0]['system'], TX_ISSUE_TYPE_SYSTEM) + response = APIClient().get( + f'{fhir_base}/{suffix}', HTTP_ACCEPT='application/fhir+json; fhirVersion=4.0') + self.assertEqual(response.status_code, 200, (fhir_base, suffix)) + + def test_reads_nothing_from_the_database(self): + self.statement() + with self.assertNumQueries(0): + self.statement() + with self.assertNumQueries(0): + self.statement('/orgs/CIEL') + + +class RoutedOperationView(FhirOperationView): + """An operation the statement has not heard of, routed for the test below.""" + operation = staticmethod(lambda *args: {'resourceType': 'Parameters'}) + adapter = staticmethod(lambda answer: answer) + + +# A URL configuration with CodeSystem/$subsumes and ValueSet/$expand routed, and no ConceptMap at all +urlpatterns = [ + path('fhir/', include('core.fhir.urls')), + path('fhir/CodeSystem/$subsumes', RoutedOperationView.as_view()), + path('fhir/CodeSystem/', include('core.code_systems.urls')), + path('fhir/ValueSet/$expand', RoutedOperationView.as_view()), + path('fhir/ValueSet/', include('core.value_sets.urls')), +] + + +@override_settings(ROOT_URLCONF=__name__) +class TheRoutesMakeTheStatementTest(OCLTestCase): + """Routing an operation lists it, and a resource type with no routes isn't listed: nothing else changes.""" + + def test_an_added_route_is_listed_and_a_removed_one_is_not(self): + statement = APIClient().get('/fhir/metadata').data + resources = {resource['type']: resource for resource in statement['rest'][0]['resource']} + self.assertEqual(list(resources), ['CodeSystem', 'ValueSet']) + self.assertEqual([item['name'] for item in resources['CodeSystem']['operation']], + ['subsumes', 'lookup', 'validate-code']) + self.assertIn('expand', [item['name'] for item in resources['ValueSet']['operation']]) + self.assertEqual(listed_operations(statement), routed_operations()) + + capabilities = APIClient().get('/fhir/metadata', {'mode': 'terminology'}).data + self.assertEqual(capabilities['expansion'], {'paging': True}) + self.assertNotIn('translation', capabilities) + + +class VersionsTest(OCLTestCase): + def setUp(self): + super().setUp() + OrganizationFactory(mnemonic='CIEL') + UserProfileFactory(username='jdoe') + + def test_versions_at_every_base_with_and_without_a_trailing_slash(self): + expected = {'resourceType': 'Parameters', 'parameter': [ + {'name': 'version', 'valueCode': '4.0'}, {'name': 'default', 'valueCode': '4.0'}]} + for fhir_base in (*BASES, '/orgs/no-such-org'): + for suffix in ('$versions', '$versions/'): + response = APIClient().get(f'{fhir_base}/{suffix}') + self.assertEqual((response.status_code, response.data), (200, expected), (fhir_base, suffix)) + + def test_the_versions_are_the_negotiated_ones(self): + response = APIClient().get('/fhir/$versions') + self.assertEqual([item['valueCode'] for item in response.data['parameter']], + [base.FHIR_VERSION, base.FhirVersioning.default_version]) + + def test_it_parses_as_r4_json_and_xml(self): + for media_type in ('application/fhir+json', 'application/fhir+xml'): + response = APIClient().get('/orgs/CIEL/$versions', HTTP_ACCEPT=media_type) + parameters = (Parameters.model_validate_xml(response.content) if media_type.endswith('xml') + else Parameters.model_validate_json(response.content)) + self.assertEqual([(item.name, item.valueCode) for item in parameters.parameter], + [('version', '4.0'), ('default', '4.0')]) + + def test_get_only(self): + for fhir_base in BASES: + response = APIClient().post(f'{fhir_base}/$versions', {}, format='json') + self.assertEqual(response.status_code, 405, fhir_base) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + + +class OwnerBaseFhirPathTest(SimpleTestCase): + """An owner base's metadata and $versions are FHIR paths, so the FHIR host serves them and errors are outcomes.""" + + def test_owner_metadata_and_versions_are_fhir_paths(self): + for path_ in ('/orgs/CIEL/metadata', '/orgs/CIEL/metadata/', '/users/jo/$versions', '/users/jo/$versions/'): + self.assertTrue(base.is_fhir_path(path_), path_) + for path_ in ('/orgs/CIEL/', '/orgs/CIEL/metadata/x/', '/orgs/metadata/', '/sources/x/metadata', + '/users/jo/extras/'): + self.assertFalse(base.is_fhir_path(path_), path_) + + +class OclPathAsFhirTest(SimpleTestCase): + """IdentifierSerializer.convert_ocl_uri_to_fhir_url: an OCL URL as FHIR names it, the repository type's segment.""" + + def test_only_the_repository_type_segment_is_replaced(self): + convert = IdentifierSerializer.convert_ocl_uri_to_fhir_url + for uri, resource_type, expected in ( + ('/orgs/Acme/sources/collections/', 'CodeSystem', '/orgs/Acme/CodeSystem/collections/'), + ('/orgs/Acme/collections/sources/', 'ValueSet', '/orgs/Acme/ValueSet/sources/'), + ('/orgs/sources/sources/codes/', 'CodeSystem', '/orgs/sources/CodeSystem/codes/'), + ('/users/collections/collections/set/v1/', 'ValueSet', '/users/collections/ValueSet/set/'), + # as before for every other name + ('/orgs/CIEL/sources/CIEL/', 'CodeSystem', '/orgs/CIEL/CodeSystem/CIEL/'), + ('/orgs/CIEL/sources/CIEL/v1/concepts/1/', 'ConceptMap', '/orgs/CIEL/ConceptMap/CIEL/'), + ('/users/jo/collections/vs/', 'ValueSet', '/users/jo/ValueSet/vs/'), + ('/orgs/CIEL/CodeSystem/CIEL/', 'CodeSystem', '/orgs/CIEL/CodeSystem/CIEL/'), + ('http://example.org/fhir/CodeSystem/x', 'CodeSystem', 'http://example.org/fhir/CodeSystem/x'), + (None, 'CodeSystem', None)): + self.assertEqual(convert(uri, resource_type), expected, uri) + with self.assertRaisesMessage(ValidationError, 'given: orgs/CIEL/CodeSystem'): + convert('/orgs/CIEL/sources/', 'CodeSystem') + + +class ResolverIdentifiersTest(SimpleTestCase): + """The identifiers the resolver reads as written: an OCL path back to its repository, a registry namespace.""" + + def test_an_ocl_path_is_one_the_resolver_reads_back_to_the_repository(self): + self.assertEqual(ocl_path('/orgs/Acme/sources/nameless/'), '/orgs/Acme/CodeSystem/nameless/') + self.assertEqual(ocl_path('/users/jo/sources/codes/'), '/users/jo/CodeSystem/codes/') + # too short to name a repository; read as a canonical URL; read as a mapping's URL + for uri in ('/orgs/Acme/', '/orgs/Acme/sources/a:b/', '/orgs/Acme/sources/mappings/'): + self.assertIsNone(ocl_path(uri), uri) + + def test_a_registry_namespace_names_an_org_or_a_user(self): + head = ServedHead(1, 'codes', None, 7, 'urn:x', '/users/jo/sources/codes/', None, '/users/jo/') + self.assertTrue(in_namespace(head, '/users/jo/')) + for namespace in ('/orgs/jo/', '/users/someone/', '/', None, ''): + self.assertFalse(in_namespace(head, namespace), namespace) + + +class Rollback(Exception): + """Ends a savepoint, undoing what ran in it.""" + + +class TerminologyCapabilitiesCases(OpsCorpusMixin): # pylint: disable=too-many-instance-attributes + """ + Each repository a CodeSystem search at a base shows the caller, under the identifier the resolver serves it at + there (its canonical URL when the resolver picks it for that URL, else its OCL path), with the released versions it + serves by name, newest first, and the one it serves without a version marked isDefault; as the resolver serves them + were the repositories the caller isn't shown not there. On top of the operations' corpus: HL7's retired release + beside its live one, a member of HL7, MOH's public release of a repository MOH made private, sehatapp's copy of + ICD-10 (a canonical two public repositories share), an org with no sources, and these, mostly Acme's: a code system + without a canonical URL; one with nothing released; one the registry doesn't hold; one whose newest release has a + canonical URL of its own; one retired (its HEAD); a canonical Acme holds and HL7 holds too, which Acme's registry + sends to HL7; two of HL7's sharing a canonical, the older private; releases named 2.0+build and r%41; and a source + named collections. + """ + + def setUp(self): # pylint: disable=too-many-statements + super().setUp() + self.rest_retired = source(self.hl7, 'restful-interaction', '3.0.1', released=True, retired=True, + canonical_url=REST) + self.hl7_member = UserProfileFactory(username='hl7-member') + self.hl7.members.add(self.hl7_member) + self.closed = source(self.moh, 'closed', canonical_url=CLOSED, public_access=ACCESS_TYPE_NONE) + self.closed_v1 = source(self.moh, 'closed', 'v1', released=True, canonical_url=CLOSED) + self.sehatapp = OrganizationFactory(mnemonic='sehatapp') + source(self.sehatapp, 'ICD-10', canonical_url=ICD10) + source(self.sehatapp, 'ICD-10', 'v1.0', released=True, canonical_url=ICD10) + source(self.acme, 'nameless') + source(self.acme, 'nameless', 'v1', released=True) + source(self.acme, 'draft', canonical_url=DRAFT) + source(self.acme, 'draft', 'v1', canonical_url=DRAFT) + source(self.acme, 'unregistered', canonical_url=UNREGISTERED) + source(self.acme, 'unregistered', 'v1', released=True, canonical_url=UNREGISTERED) + source(self.acme, 'renamed', canonical_url=OLD) + source(self.acme, 'renamed', 'v1', released=True, canonical_url=None) + source(self.acme, 'renamed', 'v2', released=True, canonical_url=NEW) + source(self.acme, 'retired-head', canonical_url=RETIRED_HEAD, retired=True) + source(self.acme, 'retired-head', 'v1', released=True, canonical_url=RETIRED_HEAD) + source(self.acme, 'local', canonical_url=REDIRECTED) + source(self.acme, 'local', 'v1', released=True, canonical_url=REDIRECTED) + source(self.hl7, 'redirected', canonical_url=REDIRECTED) + source(self.hl7, 'redirected', 'r1', released=True, canonical_url=REDIRECTED) + OrganizationURLRegistryFactory(organization=self.acme, url=REDIRECTED, namespace=self.hl7.uri) + source(self.hl7, 'collide-private', canonical_url=COLLIDE, public_access=ACCESS_TYPE_NONE) + source(self.hl7, 'collide-private', 'p1', released=True, canonical_url=COLLIDE, + public_access=ACCESS_TYPE_NONE) + source(self.hl7, 'collide-public', canonical_url=COLLIDE) + source(self.hl7, 'collide-public', 'v1', released=True, canonical_url=COLLIDE) + plus = source(self.acme, 'plus') + plus_releases = [source(self.acme, 'plus', '1.0', released=True), + source(self.acme, 'plus', '2.0+build', released=True)] + ConceptFactory(parent=plus, mnemonic='P1', sources=plus_releases, + names=[concept_name('Plus one', preferred=True)]) + encoded = source(self.acme, 'encoded') + ConceptFactory(parent=encoded, mnemonic='E1', sources=[source(self.acme, 'encoded', 'r%41', released=True)], + names=[concept_name('Encoded one', preferred=True)]) + source(self.acme, 'collections') + source(self.acme, 'collections', 'v1', released=True) + OrganizationFactory(mnemonic='Empty') + self.users = (None, self.outsider, self.moh_member, self.hl7_member, self.jdoe, self.admin) + + def capabilities(self, fhir_base='/fhir', user=None, **params): + response = client_for(user).get(f'{fhir_base}/metadata', {'mode': 'terminology', **params}) + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['resourceType'], 'TerminologyCapabilities') + return response + + def code_systems(self, fhir_base='/fhir', user=None): + """{uri: [version, with a * on the default]} of a base's TerminologyCapabilities for a caller.""" + return {item['uri']: [version['code'] + ('*' if version.get('isDefault') else '') + for version in item.get('version', [])] + for item in self.capabilities(fhir_base, user).data.get('codeSystem', [])} + + @staticmethod + def named(identifier, version, kwargs, user): + """The version, if resolve() serves it named as it is stored (a business version, as R4 lists it).""" + try: + instance = resolve('CodeSystem', identifier, version, kwargs, user) + except UnresolvedError: + return None + return version if instance.version == version and instance.released and not instance.is_head else None + + def served(self, kwargs, user): # pylint: disable=too-many-locals + """ + {identifier: (released versions served by name, sorted; the one served without a version)} by search and + resolve() themselves: each repository a CodeSystem search at the base shows the caller, under its canonical URL + if resolve() serves that repository for it, else its OCL path if it serves it there; each of its released + versions resolve() serves named as stored, and the one it serves without a version. + """ + found = {} + searched = {(row.mnemonic, row.organization_id, row.user_id) for row in visible_versions( + 'CodeSystem', kwargs, user)} + for mnemonic, organization_id, user_id in searched: + rows = Source.objects.filter(mnemonic=mnemonic, organization_id=organization_id, user_id=user_id) + head = rows.get(version=HEAD) + owner_type, owner_id, _, repository_id = head.uri.strip('/').split('/') + for identifier in (head.canonical_url, f'/{owner_type}/{owner_id}/CodeSystem/{repository_id}/'): + try: + default = resolve('CodeSystem', identifier, None, kwargs, user) if identifier else None + except UnresolvedError: + continue + if not default or (default.mnemonic, default.organization_id, default.user_id) != ( + mnemonic, organization_id, user_id): + continue + # the releases served by name there: the repository's own; under a canonical URL, every release of a + # repository with it that resolve() serves, as a version named resolves to its holder (decision 9) + holders = Source.objects.filter( + version=HEAD, canonical_url=identifier) if identifier == head.canonical_url else [head] + releases = Source.objects.filter(released=True, id__in=[ + row.id for holder in holders for row in Source.objects.filter( + mnemonic=holder.mnemonic, organization_id=holder.organization_id, user_id=holder.user_id)]) + versions = [self.named(identifier, version, kwargs, user) for version in set( + releases.exclude(version=HEAD).values_list('version', flat=True))] + found[identifier] = (sorted(name for name in versions if name), + self.named(identifier, default.version, kwargs, user)) + break + return found + + def served_were_hidden_missing(self, kwargs, user): + """served(), with the repositories the base doesn't show the caller (their HEAD) deleted, then restored.""" + hidden = Source.objects.filter(version=HEAD).exclude( + id__in=viewable_rows(Source, namespace_of(kwargs), user).filter(version=HEAD).values('id')) + found = None + try: + with transaction.atomic(): + for mnemonic, organization_id, user_id in hidden.values_list('mnemonic', 'organization_id', 'user_id'): + Source.objects.filter(mnemonic=mnemonic, organization_id=organization_id, user_id=user_id).delete() + found = self.served(kwargs, user) + raise Rollback + except Rollback: + pass + return found + + def test_it_lists_what_the_resolver_serves_were_hidden_repositories_missing(self): + """ + At every base, to every caller: each repository a CodeSystem search shows, under the identifier the operations + serve it at, with the released versions they serve by name and the one they serve without a version, as they + serve them were the repositories the caller isn't shown missing. With them there, every version listed is + served, but where a hidden repository of the base's owner holds the canonical too: the resolver tries it and + stops there (open decision A.3, Codex F2.3). + """ + cases = [({}, '/fhir'), ({'user': 'jdoe'}, '/users/jdoe')] + [ + ({'org': org}, f'/orgs/{org}') for org in ('HL7', 'MOH', 'CIEL', 'WHO', 'sehatapp', 'Acme', 'Empty')] + for kwargs, fhir_base in cases: + for user in self.users: + listed = self.code_systems(fhir_base, user) + self.assertEqual( + {uri: (sorted(code.rstrip('*') for code in codes), + next((code.rstrip('*') for code in codes if code.endswith('*')), None)) + for uri, codes in listed.items()}, + self.served_were_hidden_missing(kwargs, user), (fhir_base, user)) + unserved = set() + for uri, codes in listed.items(): + for code in [None, *(code.rstrip('*') for code in codes)]: + try: + resolve('CodeSystem', uri, code, kwargs, user) + except UnresolvedError: + unserved.add(uri) + self.assertEqual(unserved, self.hidden_collisions(fhir_base, user), (fhir_base, user)) + + def hidden_collisions(self, fhir_base, user): + """The canonicals a hidden repository of the base's owner holds that the listing serves from another one.""" + if fhir_base == '/orgs/HL7' and user not in (self.hl7_member, self.admin): + return {COLLIDE} + return set() + + def test_the_global_base(self): + """Every repository search shows, under its canonical URL where the resolver picks it for it, else its path.""" + by_url = {CIEL: ['v2*', 'v1'], ICD10: ['ICD102019*'], NEWEST_PRIVATE: ['v1*'], REST: ['3.0.1', '4.0.1*'], + SHARED: ['v1*']} + # a canonical another public repository holds too lists that one's releases after its own (decision 9) + unregistered = {UNREGISTERED: ['v1*'], REDIRECTED: ['v1*', 'r1'], COLLIDE: ['v1*'], OLD: ['v2*', 'v1']} + if not self.registry: + by_url[ICD10] = ['ICD102019*', 'v1.0'] + if self.registry: # what the registry doesn't hold is served by its path + unregistered = { + code_system_path('Acme', 'unregistered'): ['v1*'], code_system_path('Acme', 'local'): ['v1*'], + code_system_path('HL7', 'collide-public'): ['v1*'], code_system_path('Acme', 'renamed'): ['v2*', 'v1']} + expected = { + **by_url, **unregistered, NAMELESS: ['v1*'], code_system_path('sehatapp', 'ICD-10'): ['v1.0*'], + code_system_path('HL7', 'redirected'): ['r1*'], code_system_path('Acme', 'plus'): ['2.0+build*', '1.0'], + code_system_path('Acme', 'encoded'): ['r%41*'], '/orgs/Acme/CodeSystem/collections/': ['v1*']} + for user in self.users: + self.assertEqual(self.code_systems(user=user), expected, user) + self.assertEqual(list(self.code_systems()), sorted(expected)) + + def test_with_the_registry_on_only_what_the_registry_resolves(self): + """ + F1.1, F2.1: at /fhir/ a canonical the global registry doesn't hold isn't served, so its repository is listed by + its path. F2.2: an owner base lists the owner's repositories only; one whose canonical the owner's registry + sends elsewhere is listed by its path, and the repository it's sent to isn't the owner's. + """ + self.assertEqual(UNREGISTERED in self.code_systems(), not self.registry) + self.assertEqual(code_system_path('Acme', 'unregistered') in self.code_systems(), self.registry) + acme = self.code_systems('/orgs/Acme') + self.assertEqual(acme[UNREGISTERED], ['v1*']) + self.assertEqual((acme.get(REDIRECTED), acme.get(code_system_path('Acme', 'local'))), + (None, ['v1*']) if self.registry else (['v1*'], None)) + self.assertEqual(resolve('CodeSystem', REDIRECTED, None, {'org': 'Acme'}).version, + 'r1' if self.registry else 'v1') + self.assertEqual(resolve('CodeSystem', code_system_path('Acme', 'local'), None, {'org': 'Acme'}).version, 'v1') + self.assertNotIn(CIEL, acme) # the global registry's: resolution falls through to it, but it isn't Acme's + self.assertEqual(self.code_systems('/orgs/Empty'), {}) + + def test_a_shared_canonical_is_the_repository_the_resolver_picks(self): + """ + F1.2: WHO's ICD-10 (the older), not sehatapp's, with either toggle; F2.1: sehatapp's by its path. Without the + registry, sehatapp's release is also served named under the canonical, after WHO's (decision 9); with it, the + global entry decides. + """ + listed = self.code_systems() + self.assertEqual((listed[ICD10], listed[code_system_path('sehatapp', 'ICD-10')]), ( + ['ICD102019*'] if self.registry else ['ICD102019*', 'v1.0'], ['v1.0*'])) + if self.registry: + with self.assertRaises(UnresolvedError): + resolve('CodeSystem', ICD10, 'v1.0') + else: + self.assertEqual(resolve('CodeSystem', ICD10, 'v1.0').organization, self.sehatapp) + self.assertEqual(resolve('CodeSystem', code_system_path('sehatapp', 'ICD-10'), 'v1.0').version, 'v1.0') + self.assertEqual(self.code_systems('/orgs/sehatapp'), {ICD10: ['v1.0*']}) + + def test_a_version_is_listed_as_stored_and_served_by_it(self): + """ + F3.1: TerminologyCapabilities lists a release's business version as it is stored (2.0+build, r%41), the newest + the default; the operations serve it named so: GET with it encoded in the query, as HTTP does, and POST as a + Parameters string or a Coding's version. + """ + listed = self.code_systems('/orgs/Acme') + self.assertEqual(listed[code_system_path('Acme', 'plus')], ['2.0+build*', '1.0']) + self.assertEqual(listed[code_system_path('Acme', 'encoded')], ['r%41*']) + for fhir_base in ('/fhir', '/orgs/Acme'): + for mnemonic, code, version in (('plus', 'P1', '2.0+build'), ('encoded', 'E1', 'r%41')): + system = code_system_path('Acme', mnemonic) + for operation, valid in (('$lookup', False), ('$validate-code', True)): + named = {'url' if valid else 'system': system, 'code': code} + as_coding = [{'name': 'url', 'valueUri': system}] if valid else [] + answers = [ + client_for().get(f'{fhir_base}/CodeSystem/{operation}', {**named, 'version': version}), + client_for().get(f'{fhir_base}/CodeSystem/{operation}', named), # the default + self.post_parameters(fhir_base, operation, [ + {'name': key, 'valueCode' if key == 'code' else 'valueUri': value} + for key, value in named.items()] + [{'name': 'version', 'valueString': version}]), + self.post_parameters(fhir_base, operation, as_coding + [{'name': 'coding', 'valueCoding': { + 'system': system, 'code': code, 'version': version}}])] + for response in answers: + self.assertEqual(response.status_code, 200, (fhir_base, operation, response.data)) + parameters = {item['name']: item for item in response.data['parameter']} + self.assertEqual(parameters['version']['valueString'], version, (fhir_base, operation)) + if valid: + self.assertIs(parameters['result']['valueBoolean'], True, (fhir_base, operation)) + + @staticmethod + def post_parameters(fhir_base, operation, parameters): + return client_for().post( + f'{fhir_base}/CodeSystem/{operation}', json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), + content_type='application/fhir+json') + + def test_a_repository_named_like_a_repository_type_is_listed_by_its_path(self): + """F3.2: a source named collections, by /orgs/Acme/CodeSystem/collections/, which the operations resolve.""" + for fhir_base, kwargs in (('/fhir', {}), ('/orgs/Acme', {'org': 'Acme'})): + self.assertEqual(self.code_systems(fhir_base)['/orgs/Acme/CodeSystem/collections/'], ['v1*'], fhir_base) + self.assertEqual(resolve('CodeSystem', '/orgs/Acme/CodeSystem/collections/', None, kwargs).version, 'v1') + + def test_a_code_system_without_a_canonical_url_is_listed_by_its_ocl_path(self): + """F1.3: as the operations resolve it, and as $expand names its members' system.""" + for fhir_base, kwargs in (('/fhir', {}), ('/orgs/Acme', {'org': 'Acme'})): + self.assertEqual(self.code_systems(fhir_base)[NAMELESS], ['v1*'], fhir_base) + self.assertEqual(resolve('CodeSystem', NAMELESS, None, kwargs).version, 'v1') + self.assertNotIn(NAMELESS, self.code_systems('/orgs/HL7')) # another owner's + + def test_a_hidden_repository_of_the_owner_is_listed_as_missing(self): + """ + F1.4: HL7's private collide-private is older than collide-public. To those who may view it, the resolver serves + it, and so does the listing. To others, the listing answers byte for byte, in as many queries, as if it + weren't there: collide-public's, which the resolver at HL7's base doesn't serve them (it tries the hidden one + and stops: open decision A.3). + """ + for user in (self.hl7_member, self.admin): # and collide-public's release, served named (decision 9) + self.assertEqual(self.code_systems('/orgs/HL7', user)[COLLIDE], ['p1*', 'v1'], user) + others = (None, self.outsider, self.moh_member, self.jdoe) + before = {} + for user in others: + with CaptureQueriesContext(connection) as queries: + before[user] = (self.capabilities('/orgs/HL7', user).content, len(queries)) + self.assertEqual(self.code_systems('/orgs/HL7', user)[COLLIDE], ['v1*'], user) + with self.assertRaises(UnresolvedError): + resolve('CodeSystem', COLLIDE, None, {'org': 'HL7'}, user) + Source.objects.filter(mnemonic='collide-private', organization=self.hl7).delete() + for user in others: + with CaptureQueriesContext(connection) as queries: + content = self.capabilities('/orgs/HL7', user).content + self.assertEqual((content, len(queries)), before[user], user) + self.assertEqual(resolve('CodeSystem', COLLIDE, None, {'org': 'HL7'}, user).version, 'v1') + + def test_a_release_is_listed_under_the_url_the_resolver_serves_it_at(self): + """F1.5: the repository's canonical URL (its HEAD's), not a release's own (open decision DI19).""" + acme = self.code_systems('/orgs/Acme') + self.assertEqual(acme[OLD], ['v2*', 'v1']) + self.assertNotIn(NEW, acme) + self.assertEqual(resolve('CodeSystem', OLD, 'v2', {'org': 'Acme'}).version, 'v2') + with self.assertRaises(UnresolvedError): + resolve('CodeSystem', NEW, None, {'org': 'Acme'}) + + def test_a_retired_repository_is_not_listed(self): + """F1.6: the resolver serves no repository whose HEAD is retired, by canonical URL or path.""" + for fhir_base, kwargs in (('/fhir', {}), ('/orgs/Acme', {'org': 'Acme'})): + listed = self.code_systems(fhir_base) + self.assertNotIn(RETIRED_HEAD, listed) + self.assertNotIn('/orgs/Acme/CodeSystem/retired-head/', listed) + for url in (RETIRED_HEAD, '/orgs/Acme/CodeSystem/retired-head/'): + with self.assertRaises(UnresolvedError): + resolve('CodeSystem', url, None, kwargs) + + def test_an_owner_base_shows_the_owners_private_code_systems_to_those_who_may_view_them(self): + for user in self.users: + hl7 = self.code_systems('/orgs/HL7', user) + self.assertEqual(hl7[NEWEST_PRIVATE], ['v2*', 'v1'] if user in (self.hl7_member, self.admin) else ['v1*']) + moh = self.code_systems('/orgs/MOH', user) + self.assertEqual({url: moh.get(url) for url in (HIV, CLOSED)}, {HIV: ['v1*'], CLOSED: ['v1*']} if user in ( + self.moh_member, self.admin) else {HIV: None, CLOSED: None}, user) + self.assertEqual(self.code_systems('/orgs/no-such-org', user), {}, user) + self.assertEqual(self.code_systems('/users/jdoe', self.jdoe)[SHARED], ['v1*']) + self.assertEqual(self.code_systems('/orgs/CIEL')[CIEL], ['v2*', 'v1']) + self.assertEqual(self.code_systems('/orgs/WHO')[ICD10], ['ICD102019*']) + + def test_a_hidden_repository_answers_as_one_that_does_not_exist(self): + """MOH's private repositories, to those who may not view them, and at /fhir/ to everyone: the same bytes and + the same number of queries as when they don't exist.""" + hidden = [('/orgs/MOH', None), ('/orgs/MOH', self.outsider), ('/orgs/MOH', self.hl7_member), + *[('/fhir', user) for user in self.users]] + + def answers(): + found = {} + for fhir_base, user in hidden: + with CaptureQueriesContext(connection) as queries: + content = self.capabilities(fhir_base, user).content + found[(fhir_base, user)] = (content, len(queries)) + return found + + with_them = answers() + self.assertNotIn(HIV.encode(), b''.join(content for content, _ in with_them.values())) + self.assertNotIn(CLOSED.encode(), b''.join(content for content, _ in with_them.values())) + Source.objects.filter(mnemonic__in=['hiv', 'closed'], organization=self.moh).delete() + self.assertEqual(answers(), with_them) + + def test_a_fixed_number_of_queries(self): + """ + The toggle, the versions search shows, the HEADs (with the registry on, its entries too); at an owner base, the + owner and the caller's rights over it first: whatever the number of code systems. + """ + registry = 1 if self.registry else 0 + + def count(expected, fhir_base='/fhir', user=None): + with self.assertNumQueries(expected + registry): + return len(self.capabilities(fhir_base, user).data.get('codeSystem', [])) + + found = [count(3), count(4, '/orgs/HL7'), count(5, '/orgs/HL7', self.hl7_member)] + for number in range(30): + url = f'http://example.org/fhir/CodeSystem/many-{number}' + source(self.hl7, f'many-{number}', canonical_url=url) + for version in ('v1', 'v2'): + source(self.hl7, f'many-{number}', version, released=True, canonical_url=url) + self.assertEqual([count(3), count(4, '/orgs/HL7'), count(5, '/orgs/HL7', self.hl7_member)], [ + found[0] + 30, found[1] + 30, found[2] + 30]) + + def test_the_blocks_are_the_routed_operations(self): + capabilities = self.capabilities().data + routed = routed_operations() + self.assertEqual(capabilities.get('translation'), + {'needsMap': False} if ('ConceptMap', 'translate') in routed else None) + self.assertEqual(capabilities.get('expansion'), + {'paging': True} if ('ValueSet', 'expand') in routed else None) + self.assertNotIn('validateCode', capabilities) + + def test_it_describes_the_service_as_the_statement_does(self): + capabilities = self.capabilities('/orgs/HL7').data + statement = client_for().get('/orgs/HL7/metadata').data + for key in ('status', 'date', 'kind', 'software', 'implementation'): + self.assertEqual(capabilities[key], statement[key], key) + + def test_it_parses_as_r4_json_and_xml(self): + for fhir_base, user in (('/fhir', None), ('/orgs/HL7', self.hl7_member), ('/orgs/no-such-org', None)): + for media_type in ('application/fhir+json', 'application/fhir+xml'): + response = client_for(user).get( + f'{fhir_base}/metadata', {'mode': 'terminology'}, HTTP_ACCEPT=media_type) + self.assertEqual(response['Content-Type'].split(';')[0], media_type) + capabilities = ( + TerminologyCapabilities.model_validate_xml(response.content) if media_type.endswith('xml') + else TerminologyCapabilities.model_validate_json(response.content)) + self.assertEqual(capabilities.kind, 'instance') + listed = TerminologyCapabilities.model_validate_xml(client_for().get( + '/fhir/metadata', {'mode': 'terminology'}, HTTP_ACCEPT='application/fhir+xml').content) + self.assertEqual([(item.uri, [(version.code, version.isDefault) for version in item.version]) + for item in listed.codeSystem if item.uri in (CIEL, NAMELESS)], + [(NAMELESS, [('v1', True)]), (CIEL, [('v2', True), ('v1', None)])]) + + +class TerminologyCapabilitiesWithoutRegistryTest(TerminologyCapabilitiesCases, OCLTestCase): + registry = False + + +class TerminologyCapabilitiesWithRegistryTest(TerminologyCapabilitiesCases, OCLTestCase): + registry = True + + def tearDown(self): + set_registry_toggle(False) + super().tearDown() + + +class TerminologyCapabilitiesTimeoutTest(OCLAPITransactionTestCase): + """The list's query is bounded as a search's is: one the database cancels is too costly (422), never a 5xx.""" + + def test_a_cancelled_statement_is_too_costly(self): + set_registry_toggle(False) + org = OrganizationFactory(mnemonic='Acme') + source(org, 'codes', canonical_url=DRAFT) + source(org, 'codes', 'v1', released=True, canonical_url=DRAFT) + + def slow(*args, **kwargs): # pylint: disable=unused-argument + with connection.cursor() as cursor: + cursor.execute('SELECT pg_sleep(1)') + return [] + + with patch('core.fhir.search.SEARCH_TIMEOUT_MS', 50), patch('core.fhir.capabilities.served_repositories', slow): + response = APIClient().get('/fhir/metadata', {'mode': 'terminology'}) + self.assertEqual(response.status_code, 422, response.data) + issue = response.data['issue'][0] + self.assertEqual(issue['code'], 'too-costly') + self.assertIn({'system': TX_ISSUE_TYPE_SYSTEM, 'code': 'too-costly'}, issue['details']['coding']) + self.assertEqual([item['uri'] for item in APIClient().get( + '/fhir/metadata', {'mode': 'terminology'}).data['codeSystem']], [DRAFT]) diff --git a/core/fhir/tests_middleware.py b/core/fhir/tests_middleware.py new file mode 100644 index 000000000..7fdf58307 --- /dev/null +++ b/core/fhir/tests_middleware.py @@ -0,0 +1,272 @@ +"""The middleware's answers on FHIR paths are OperationOutcomes (OpenConceptLab/ocl_online#451).""" +import gzip +import json +from unittest.mock import patch + +import requests +from django.conf import settings +from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse, JsonResponse, HttpResponseNotFound +from django.test import RequestFactory, SimpleTestCase, TestCase, override_settings +from fhir.resources.R4B.operationoutcome import OperationOutcome +from lxml import etree +from rest_framework.test import APIClient + +from core.common.exceptions import DeactivatedAccountLoginRefused +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.middlewares.middlewares import FhirErrorMiddleware, FhirMiddleware, RequireAuthenticationMiddleware +from core.wsgi import application + +REQUIRE_AUTHENTICATION = 'core.middlewares.middlewares.RequireAuthenticationMiddleware' + + +def outcome_issue(response): + """The first issue of an OperationOutcome response, as severity|code|tx-issue-type.""" + outcome = json.loads(response.content) + issue = outcome['issue'][0] + tx_issue_type = next(c['code'] for c in issue['details']['coding'] if c['system'] == TX_ISSUE_TYPE_SYSTEM) + return f"{issue['severity']}|{issue['code']}|{tx_issue_type}" + + +class FhirMiddlewareTest(SimpleTestCase): + factory = RequestFactory() + + def answer(self, path, response=None, host='api.example.org', **headers): + middleware = FhirMiddleware(lambda request: response or HttpResponse('ok')) + return middleware(self.factory.get(path, HTTP_HOST=host, **headers)) + + @override_settings(FHIR_SUBDOMAIN='fhir') + def test_fhir_paths_off_the_fhir_host_are_a_404_outcome(self): + for path in ('/fhir/CodeSystem/', '/orgs/CIEL/CodeSystem/$lookup', '/users/jo/ValueSet/x/$expand/'): + response = self.answer(path) + + self.assertEqual(response.status_code, 404, path) + self.assertEqual(response['Content-Type'], 'application/fhir+json', path) + self.assertEqual(outcome_issue(response), 'error|not-found|not-found', path) + + for path in ('/orgs/CIEL/sources/', '/user/sources/CodeSystem/'): + self.assertEqual(self.answer(path).content, b'ok', path) + + @override_settings(FHIR_SUBDOMAIN='fhir') + def test_other_paths_on_the_fhir_host_are_a_404_outcome(self): + response = self.answer('/orgs/CIEL/sources/?q=x', host='fhir.example.org', HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 404) + self.assertEqual(etree.fromstring(response.content).tag, '{http://hl7.org/fhir}OperationOutcome') + + for path in ('/fhir/CodeSystem/?url=http://x', '/orgs/CIEL/CodeSystem?url=http://x', '/version/'): + self.assertEqual(self.answer(path, host='fhir.example.org').content, b'ok', path) + + + +class FhirErrorMiddlewareTest(SimpleTestCase): + """The outermost middleware: an error anywhere in the stack on a FHIR path goes out as an OperationOutcome.""" + factory = RequestFactory() + + def answer(self, path, response, **headers): + return FhirErrorMiddleware(lambda request: response)(self.factory.get(path, **headers)) + + def test_is_the_outermost_middleware(self): + self.assertEqual(settings.MIDDLEWARE[0], 'core.middlewares.middlewares.FhirErrorMiddleware') + + def test_an_error_no_fhir_view_answered_becomes_an_outcome(self): + response = self.answer('/fhir/Patient/', HttpResponseNotFound(b'

Not Found

')) + + self.assertEqual(response.status_code, 404) + self.assertEqual(outcome_issue(response), 'error|not-found|not-found') + self.assertEqual( + json.loads(response.content)['issue'][0]['details']['text'], 'Unknown FHIR path: GET /fhir/Patient/') + + server_error = JsonResponse({'error': 'Server Error (500)'}, status=500) + server_error['Access-Control-Allow-Origin'] = '*' + response = self.answer('/fhir/CodeSystem/', server_error, HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 422) + self.assertEqual(response['Access-Control-Allow-Origin'], '*') # the stack's headers are kept + issue = etree.fromstring(response.content).find('{http://hl7.org/fhir}issue/{http://hl7.org/fhir}code') + self.assertEqual(issue.get('value'), 'exception') + + def test_a_json_error_keeps_its_message(self): + response = self.answer('/orgs/CIEL/CodeSystem/', JsonResponse({'detail': 'Slow down.'}, status=429)) + + self.assertEqual(response.status_code, 429) + self.assertEqual(json.loads(response.content)['issue'][0]['details']['text'], 'Slow down.') + + def test_answers_left_alone(self): + outcome = JsonResponse({'resourceType': 'OperationOutcome', 'issue': []}, status=404) + self.assertIs(self.answer('/fhir/CodeSystem/x', outcome), outcome) + + compressed = HttpResponse(gzip.compress(json.dumps({'resourceType': 'OperationOutcome', 'issue': []}).encode()), + status=404, content_type='application/json') + compressed['Content-Encoding'] = 'gzip' + self.assertIs(self.answer('/fhir/CodeSystem/x', compressed), compressed) + + not_found = HttpResponseNotFound(b'

Not Found

') + self.assertIs(self.answer('/orgs/CIEL/sources/x/', not_found), not_found) + ok = HttpResponse('ok') + self.assertIs(self.answer('/fhir/CodeSystem/', ok), ok) + + +@override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000, FHIR_SUBDOMAIN=None, REQUIRE_AUTHENTICATION=False, + DEBUG=False, DEBUG_PROPAGATE_EXCEPTIONS=False) +class QueryParsingFailureTest(TestCase): + """Malformed query strings remain FHIR errors through the complete middleware stack.""" + + def test_too_many_query_fields_return_a_400_json_outcome(self): + """Both routed and unknown paths use safe JSON when the query cannot be parsed.""" + query = '&'.join(['_format=xml'] + ['x='] * 1001) + client = APIClient(raise_request_exception=False) + for path in ('/fhir/missing', '/fhir/metadata'): + with self.subTest(path=path): + response = client.get(f'{path}?{query}', HTTP_ACCEPT='application/fhir+xml') + + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome_issue(response), 'error|invalid|invalid-data') + + +@override_settings(FHIR_SUBDOMAIN=None, REQUIRE_AUTHENTICATION=False, + DEBUG=False, DEBUG_PROPAGATE_EXCEPTIONS=False) +class MediaTypeParsingFailureTest(TestCase): + """Malformed MIME parameters remain client errors through the complete middleware stack.""" + + def assert_bad_request_outcome(self, response): + """Malformed media types always use safe FHIR JSON, including when XML was requested.""" + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/fhir+json') + OperationOutcome.model_validate_json(response.content) + self.assertEqual(outcome_issue(response), 'error|invalid|invalid-data') + + def test_unknown_accept_encoding_on_unknown_path_is_a_400_json_outcome(self): + """An unrouted path must not fail again while negotiating its error response.""" + response = APIClient(raise_request_exception=False).get( + '/fhir/missing', HTTP_ACCEPT="application/fhir+xml; foo*=bad''%41") + + self.assert_bad_request_outcome(response) + + def test_unknown_accept_encoding_on_routed_path_is_a_400_json_outcome(self): + """Initial view negotiation rejects the same malformed Accept as middleware negotiation.""" + client = APIClient(raise_request_exception=False) + for path in ('/fhir/metadata', '/fhir/metadata?_format=xml'): + for accept in ("application/fhir+xml; foo*=bad''%41", "text/html; foo*=bad''%41, application/xml"): + with self.subTest(path=path, accept=accept): + response = client.get(path, HTTP_ACCEPT=accept) + + self.assert_bad_request_outcome(response) + + def test_unknown_content_type_encoding_on_post_is_a_400_json_outcome(self): + """Use the production request class so Content-Type parsing reaches the middleware stack.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + for path in ('/fhir/missing', '/fhir/CodeSystem/$lookup'): + with self.subTest(path=path): + response = client.post( + path, b'{"resourceType":"Parameters"}', + content_type="application/fhir+json; foo*=bad''%41", HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1000) + def test_too_many_query_fields_with_charset_are_a_400_json_outcome(self): + """Query parsing in the production request's encoding setter remains a client error.""" + query = '&'.join(['x='] * 1001) + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get( + f'/fhir/metadata?{query}', content_type='application/fhir+json; charset=utf-8', + HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + + def test_unknown_charset_name_is_ignored_as_django_does(self): + """An unknown charset name was never an error: Django ignores it and decodes with the default.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get('/fhir/metadata', content_type='application/fhir+json; charset=not-a-charset') + + self.assertEqual(response.status_code, 200) + + + def test_non_text_charset_with_query_is_a_400_json_outcome(self): + """A failed charset is cleared and cannot break error rendering on an unknown path.""" + client = APIClient(raise_request_exception=False) + with patch('django.test.client.WSGIRequest', application.request_class): + response = client.get( + '/fhir/missing?x=1', content_type='application/fhir+json; charset=base64', + HTTP_ACCEPT='application/fhir+xml') + + self.assert_bad_request_outcome(response) + self.assertIsNone(response.wsgi_request.encoding) + + +@override_settings(REQUIRE_AUTHENTICATION=True, APPROVED_ANONYMOUS_CLIENTS=[], APPROVED_ANONYMOUS_API_KEYS=[], + APPROVED_ANONYMOUS_IPS=[], MIDDLEWARE=[ + *settings.MIDDLEWARE[:settings.MIDDLEWARE.index( + 'django.contrib.auth.middleware.AuthenticationMiddleware') + 1], + REQUIRE_AUTHENTICATION, + *[name for name in settings.MIDDLEWARE[settings.MIDDLEWARE.index( + 'django.contrib.auth.middleware.AuthenticationMiddleware') + 1:] + if name != REQUIRE_AUTHENTICATION]]) +class OuterMiddlewareFailureTest(TestCase): + """An outer middleware failing on a FHIR path (the sign-in provider down) is not a 5xx.""" + + @patch('core.middlewares.middlewares.OCLAuthentication.authenticate', + side_effect=requests.exceptions.ConnectionError('sign-in provider unreachable')) + def test_an_outer_middleware_error_on_a_fhir_path_is_a_4xx_outcome(self, _authenticate): + client = APIClient(raise_request_exception=False) + with self.assertLogs('django.request', level='ERROR'): + response = client.get('/orgs/CIEL/CodeSystem/$lookup?system=http://x&code=1', HTTP_AUTHORIZATION='Bearer x') + + self.assertEqual(response.status_code, 422) + self.assertEqual(outcome_issue(response), 'error|exception|invalid-data') + + with self.assertLogs('django.request', level='ERROR'): + response = client.get('/orgs/CIEL/sources/', HTTP_AUTHORIZATION='Bearer x') + + self.assertEqual(response.status_code, 500) # other paths keep Django's answer + self.assertEqual(json.loads(response.content), {'error': 'Server Error (500)'}) + + +@override_settings(REQUIRE_AUTHENTICATION=True, APPROVED_ANONYMOUS_CLIENTS=[], APPROVED_ANONYMOUS_API_KEYS=[], + APPROVED_ANONYMOUS_IPS=[]) +class RequireAuthenticationOnFhirPathsTest(SimpleTestCase): + factory = RequestFactory() + + def answer(self, path, **headers): + request = self.factory.get(path, **headers) + request.user = AnonymousUser() + return RequireAuthenticationMiddleware(lambda request: HttpResponse('ok'))(request) + + def test_anonymous_access_to_an_owner_base_is_a_403_outcome(self): + response = self.answer('/orgs/CIEL/CodeSystem/$lookup?system=x&code=1') + + self.assertEqual(response.status_code, 403) + self.assertEqual(outcome_issue(response), 'error|login|invalid-data') + issue = json.loads(response.content)['issue'][0] + self.assertEqual(issue['details']['text'], RequireAuthenticationMiddleware.forbidden_response['detail']) + self.assertNotIn('diagnostics', issue) + + def test_other_paths_keep_their_json_403(self): + response = self.answer('/orgs/CIEL/sources/') + + self.assertEqual(response.status_code, 403) + self.assertEqual(json.loads(response.content), RequireAuthenticationMiddleware.forbidden_response) + + def test_the_global_fhir_base_stays_open(self): + self.assertEqual(self.answer('/fhir/CodeSystem/$lookup?system=x&code=1').content, b'ok') + + def test_a_refused_sign_in_on_a_fhir_path_is_an_outcome(self): + middleware = RequireAuthenticationMiddleware(lambda request: HttpResponse('ok')) + middleware.is_request_allowed = lambda request: (_ for _ in ()).throw(DeactivatedAccountLoginRefused()) + + request = self.factory.get('/orgs/CIEL/CodeSystem/', HTTP_AUTHORIZATION='Bearer x') + response = middleware(request) + + self.assertEqual(response.status_code, 401) + self.assertEqual(response['WWW-Authenticate'], 'Bearer realm="api"') + self.assertEqual(outcome_issue(response), 'error|login|invalid-data') + + response = middleware(self.factory.get('/orgs/CIEL/sources/', HTTP_AUTHORIZATION='Bearer x')) + self.assertEqual(response.status_code, 401) + self.assertIn('detail', json.loads(response.content)) diff --git a/core/code_systems/tests_operation_bounds.py b/core/fhir/tests_operation_bounds.py similarity index 61% rename from core/code_systems/tests_operation_bounds.py rename to core/fhir/tests_operation_bounds.py index f000b9ffe..ccb81a586 100644 --- a/core/code_systems/tests_operation_bounds.py +++ b/core/fhir/tests_operation_bounds.py @@ -2,8 +2,9 @@ FHIR operations read only the concepts and mappings their input names (OpenConceptLab/ocl_online#451). A $validate-code or $translate without a code, or without the system the code is in, is a 400 OperationOutcome, -never a read of every concept or mapping. A POSTed Parameters resource is read with the value types R4 gives each -input, so a code sent as valueString is still the code. +never a read of every concept or mapping. A $expand reads the stored expansion its value set names, one page of it at a +time, and without a value set it is a 400 that reads none. A POSTed Parameters resource is read with the value types R4 +gives each input, so a code sent as valueString is still the code. """ import json import re @@ -16,25 +17,28 @@ from rest_framework.response import Response from rest_framework.test import APIClient -from core.code_systems.serializers import ValidateCodeParametersSerializer from core.collections.models import CollectionReference from core.collections.tests.factories import OrganizationCollectionFactory, ExpansionFactory from core.common.mixins import ListWithHeadersMixin from core.common.tests import OCLTestCase from core.common.views import BaseAPIView +from core.concepts.models import Concept from core.concepts.tests.factories import ConceptFactory +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM from core.mappings.models import Mapping from core.mappings.tests.factories import MappingFactory from core.orgs.tests.factories import OrganizationFactory from core.sources.models import Source from core.sources.tests.factories import OrganizationSourceFactory +from core.url_registry.tests_resolution import set_registry_toggle SYSTEM = 'http://example.org/fhir/CodeSystem/bounds' VALUE_SET = 'http://example.org/fhir/ValueSet/bounds' DISPLAY = 'Malaria' WRONG_DISPLAY = 'Not the display of c1' VALUE_TYPES = {'url': 'valueUri', 'system': 'valueUri', 'code': 'valueCode', 'display': 'valueString', - 'systemVersion': 'valueString'} + 'systemVersion': 'valueString', 'count': 'valueInteger', 'offset': 'valueInteger', + 'activeOnly': 'valueBoolean', 'filter': 'valueString'} # A statement that reads concept or mapping rows: that table is its own, not a joined one ROW_READ = re.compile(r'^SELECT .+? FROM "(concepts|mappings)"(?: |$)') # A literal equality, or a literal IN list, on a column that names rows: a code, an id, or the concept whose @@ -65,10 +69,31 @@ def unbounded_reads(queries): return [sql for sql in row_reads(queries) if not NAMED_ROWS.search(sql)] +def unbounded_expansion_reads(queries, expansion_id): + """ + The statements a $expand runs that read concept rows or their names other than those of the expansion named, or + more than 1,000 of them at once: each must be limited to that expansion (in the statement or in the subquery that + chooses its page), or name its rows by id. + """ + unbounded = [] + for query in queries: + sql = query['sql'] + if not re.match(r'^SELECT .+? FROM "(concepts|concept_names)"(?: |$)', sql): + continue + alias = re.search(r'"collection_expansions_concepts" ([A-Z]\d+) ON', sql) + table = f'(?:"collection_expansions_concepts"|{alias[1]})' if alias else '"collection_expansions_concepts"' + expansion = re.search(rf'{table}\."expansion_id" = {expansion_id}\b', sql) + named = NAMED_ROWS.search(sql) or re.search(r'"concept_id" IN \(\d', sql) + if not (expansion or named) or any(int(limit) > 1000 for limit in re.findall(r'LIMIT (\d+)', sql)): + unbounded.append(sql) + return unbounded + + class FhirOperationBoundsTest(OCLTestCase): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) def setUp(self): + set_registry_toggle(False) # the operations resolve through core.fhir.resolve: no registry, as in production super().setUp() self.org = OrganizationFactory(mnemonic='BoundsOrg') self.source = OrganizationSourceFactory(organization=self.org, mnemonic='bounds', canonical_url=SYSTEM) @@ -90,6 +115,8 @@ def setUp(self): self.collection.add_references([CollectionReference( expression=self.concept.uri, collection=self.collection, code='c1', system=self.source.uri, version='v1')]) self.collection_v1.seed_references() + self.collection_v1.expansion.concepts.add( # as the release's expansion holds it: the code system release's c1 + *Concept.objects.filter(sources=self.source_v1, mnemonic='c1')) self.client = APIClient() def request(self, path, method, headers=None, **inputs): @@ -103,7 +130,7 @@ def request(self, path, method, headers=None, **inputs): parameters = [{'name': name, ('valueString' if 'valueString' in method else VALUE_TYPES[name]): value} for name, value in inputs.items()] response = self.client.post(path, json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), - content_type='application/json') + content_type='application/fhir+json') return response, context.captured_queries def assert_required(self, response, name, msg): @@ -112,6 +139,7 @@ def assert_required(self, response, name, msg): self.assertEqual(outcome['resourceType'], 'OperationOutcome', msg) self.assertEqual(outcome['issue'][0]['code'], 'invalid', msg) self.assertEqual(outcome['issue'][0]['expression'], [name], msg) + self.assertEqual(outcome['issue'][0]['details']['coding'][0]['system'], TX_ISSUE_TYPE_SYSTEM, msg) def assert_result(self, response, expected, msg): self.assertEqual(response.status_code, 200, msg) @@ -195,6 +223,39 @@ def test_lookup_reads_only_the_named_code(self): self.assertEqual(unbounded_reads(queries), [], (system, code)) self.assertEqual(response.status_code, 200 if system and code else 400, (system, code)) + def test_codings_and_instance_routes_read_only_the_named_code(self): + """The Coding inputs and the instance-level routes read only the code they name too; without one, a 400.""" + operations = [ + ('/fhir/CodeSystem/$lookup', None), ('/fhir/CodeSystem/$validate-code', None), + ('/fhir/ValueSet/$validate-code', VALUE_SET), + (f'/fhir/CodeSystem/{self.source_v1.id}/$lookup', None), + (f'/fhir/CodeSystem/{self.source_v1.id}/$validate-code', None), + (f'/orgs/{self.org.mnemonic}/ValueSet/boundsvs/$validate-code', None), + ] + inputs = [('coding', 'valueCoding', lambda coding: coding), + ('codeableConcept', 'valueCodeableConcept', lambda coding: {'coding': [coding]})] + for (path, url), (name, value_type, wrap), code in product(operations, inputs, (None, 'c1')): + msg = (path, name, code) + with self.subTest(msg): + coding = {'system': SYSTEM, **({'code': code} if code else {})} + parameters = [{'name': name, value_type: wrap(coding)}] + if url: + parameters.append({'name': 'url', 'valueUri': url}) + with CaptureQueriesContext(connection) as context: + response = self.client.post( + path, json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), + content_type='application/fhir+json') + + self.assertEqual(unbounded_reads(context.captured_queries), [], msg) + if code: + self.assertEqual(response.status_code, 200, msg) + else: + self.assert_required(response, name, msg) + for path in ('/fhir/CodeSystem/$lookup', '/fhir/CodeSystem/$validate-code'): + response, queries = self.request(path, 'GET', coding=f'{SYSTEM}|c1') # a GET's Coding: system|code + self.assertEqual(unbounded_reads(queries), [], path) + self.assertEqual(response.status_code, 200, path) + def test_operations_take_no_list_shortcut(self): """A search, sort, facets, search stats or CSV export never stands in for an operation's own checks.""" operations = [ @@ -235,19 +296,63 @@ def test_operations_take_no_list_shortcut(self): self.assertEqual(response.status_code, 400, msg) self.assertEqual({name: mock.call_count for name, mock in mocks.items()}, dict.fromkeys(mocks, 0)) - def test_parameters_are_read_with_the_value_types_r4_allows(self): - serializer = ValidateCodeParametersSerializer(data={'resourceType': 'Parameters', 'parameter': [ - {'name': 'url', 'valueString': SYSTEM}, - {'name': 'code', 'valueString': 'c1'}, - {'name': 'display', 'valueString': DISPLAY}, - {'name': 'version', 'valueCode': 'v1'}, - {'name': 'system', 'valueBoolean': True}, - ]}) - - self.assertTrue(serializer.is_valid()) - self.assertEqual(serializer.validated_data['parameters'], {'url': SYSTEM, 'code': 'c1', 'display': DISPLAY}) - - serializer = ValidateCodeParametersSerializer(data={'resourceType': 'Parameters', 'parameter': [ - {'name': 'code', 'valueCode': 'c1', 'valueString': 'c2'}]}) - self.assertTrue(serializer.is_valid()) - self.assertEqual(serializer.validated_data['parameters'], {'code': 'c1'}) + def test_expand_reads_only_the_expansion_named(self): + """$expand reads the concepts of the stored expansion its value set names, a page of them at a time; without + a value set it is a 400 that reads no concept, GET or POST, whatever value types the inputs come in.""" + expansion_id = self.collection_v1.expansion.id + typed = {'count': 1, 'offset': 0, 'activeOnly': True, 'filter': 'mal'} + for method, path, url, names in product( + ('GET', 'POST', 'POST valueString'), + ('/fhir/ValueSet/$expand', f'/orgs/{self.org.mnemonic}/ValueSet/boundsvs/$expand'), (None, VALUE_SET), + ((), ('count', 'offset'), ('filter', 'activeOnly'))): + values = {name: typed[name] if method == 'POST' else str(typed[name]).lower() for name in names} + msg = (method, path, url, names) + with self.subTest(msg): + response, queries = self.request(path, method, url=url, **values) + + self.assertEqual(unbounded_expansion_reads(queries, expansion_id), [], msg) + if not url and '/fhir/' in path: + self.assert_required(response, 'url', msg) + self.assertEqual(row_reads(queries), [], msg) + else: + self.assertEqual(response.status_code, 200, msg) + self.assertEqual( + [member['code'] for member in response.data['expansion'].get('contains', [])], ['c1'], msg) + self.assertTrue(row_reads(queries), msg) # the concepts it read were checked + + def test_expand_inputs_it_refuses_read_nothing(self): + """An $expand input that would need an expansion computed for the request is a 400 before anything is read.""" + for name in ('date', 'exclude-system', 'system-version', 'check-system-version', 'force-system-version', + 'designation', 'contextDirection', 'context', 'includeDefinition', 'excludePostCoordinated'): + for method in ('GET', 'POST valueString'): + response, queries = self.request( + '/fhir/ValueSet/$expand', method, url=VALUE_SET, + **{name: 'true' if name in ('includeDefinition', 'excludePostCoordinated') else SYSTEM}) + self.assertEqual(response.status_code, 400, (name, method)) + self.assertEqual(response.data['issue'][0]['expression'], [name], (name, method)) + self.assertEqual(row_reads(queries), [], (name, method)) + + def test_expand_takes_no_list_shortcut(self): + """A search, sort, facets, search stats or CSV export never stands in for $expand's own reading.""" + shortcuts = { + (BaseAPIView, 'get_search_results_qs'): (Concept.objects.none(), {}, None, {}), + (BaseAPIView, 'get_fuzzy_search_results_qs'): (Concept.objects.none(), {}, None, {}), + (BaseAPIView, 'get_facets'): {}, + (BaseAPIView, 'get_search_stats'): {}, + (ListWithHeadersMixin, 'get_csv'): Response({'url': 'https://example.org/export.csv'}), + } + mocks = {name: Mock(return_value=value) for (_, name), value in shortcuts.items()} + for (klass, name), mock in zip(shortcuts, mocks.values()): + patcher = patch.object(klass, name, mock) + patcher.start() + self.addCleanup(patcher.stop) + expected, _ = self.request('/fhir/ValueSet/$expand', 'GET', url=VALUE_SET) + + for params, headers in LIST_SHORTCUTS: + with self.subTest((params, headers)): + response, queries = self.request( + '/fhir/ValueSet/$expand', 'GET', headers=headers, url=VALUE_SET, **params) + + self.assertEqual(unbounded_expansion_reads(queries, self.collection_v1.expansion.id), []) + self.assertEqual(json.loads(response.content), json.loads(expected.content), (params, headers)) + self.assertEqual({name: mock.call_count for name, mock in mocks.items()}, dict.fromkeys(mocks, 0)) diff --git a/core/fhir/tests_ops_concept_versions.py b/core/fhir/tests_ops_concept_versions.py new file mode 100644 index 000000000..2a5f88999 --- /dev/null +++ b/core/fhir/tests_ops_concept_versions.py @@ -0,0 +1,241 @@ +""" +core.fhir.ops: the release a value set member is reported in, when the stored expansion holds a concept's +versioned-object row, which only HEAD holds. It carries the concept's newest content, so the newest live release the +caller is shown that holds another, not deleted, version answers, whatever the expansion pins. A member a release +holds itself follows the explicit pin, then the evaluated pin, then the newest holding release. $expand +(contains.version, used-codesystem) and ValueSet $validate-code (version, and systemVersion naming a holding release) +agree, with the URL registry off and on. +""" +import re + +from django.db import connection +from django.test.utils import CaptureQueriesContext + +from core.collections.tests.factories import ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE +from core.common.tests import OCLTestCase +from core.concepts.models import Concept +from core.concepts.tests.factories import ConceptFactory +from core.fhir.tests_operation_bounds import unbounded_expansion_reads +from core.fhir.tests_ops_lookup import OpsCorpusMixin, name +from core.fhir.tests_search import collection, source +from core.orgs.tests.factories import OrganizationFactory +from core.sources.models import Source +from core.url_registry.factories import GlobalURLRegistryFactory + +DEMO = 'http://example.org/fhir/CodeSystem/demo' +DEMO_SET = 'http://example.org/fhir/ValueSet/demo-set' + + +class ConceptVersionCases(OpsCorpusMixin): + """ + Acme's code system demo has releases 1 and 2 and the private release 3. Codes A and B each have a versioned-object + row, which HEAD holds, and a version row in each release. The value set demo-set 1's stored expansion holds the + versioned-object rows. + """ + def setUp(self): + super().setUp() + self.demo = source(self.acme, 'demo', canonical_url=DEMO) + self.demo_v1 = source(self.acme, 'demo', '1', released=True, canonical_url=DEMO) + self.demo_v2 = source(self.acme, 'demo', '2', released=True, canonical_url=DEMO) + self.demo_v3 = source(self.acme, 'demo', '3', released=True, canonical_url=DEMO, + public_access=ACCESS_TYPE_NONE) + self.roots = {} + for code in ('A', 'B'): + root = ConceptFactory(parent=self.demo, mnemonic=code, names=[name(f'Code {code}', preferred=True)]) + for release in (self.demo_v1, self.demo_v2, self.demo_v3): + ConceptFactory(parent=self.demo, mnemonic=code, version=f'{code}-{release.version}', + versioned_object=root, sources=[release], + names=[name(f'Code {code} in {release.version}', preferred=True)]) + self.roots[code] = root + self.value_set = collection(self.acme, 'demo-set', '1', released=True, canonical_url=DEMO_SET) + collection(self.acme, 'demo-set', canonical_url=DEMO_SET) + self.stored = ExpansionFactory(mnemonic='autoexpand-1', collection_version=self.value_set) + self.stored.concepts.add(*self.roots.values()) + self.stored.explicit_source_versions.add(self.demo_v1) + self.value_set.expansion_uri = self.stored.uri + self.value_set.save() + if self.registry: + for url in (DEMO, DEMO_SET): + GlobalURLRegistryFactory(url=url, namespace=self.acme.uri) + + def expanded(self, base='/fhir', user=None, **params): + """The expansion answered at this base, with the given paging and filtering inputs.""" + response = self.get('$expand', base=base, user=user, resource='ValueSet', url=DEMO_SET, **params) + self.assertEqual(response.status_code, 200, response.data) + return response.data['expansion'] + + def members(self, base='/fhir', user=None): + return [(member['code'], member.get('version')) for member in self.expanded(base, user)['contains']] + + def used(self, base='/fhir', user=None): + return [item['valueUri'] for item in self.expanded(base, user)['parameter'] + if item['name'] == 'used-codesystem'] + + def validated(self, system=DEMO, **params): + response = self.get('$validate-code', resource='ValueSet', url=DEMO_SET, system=system, **params) + self.assertEqual(response.status_code, 200, response.data) + return {item['name']: next(value for key, value in item.items() if key != 'name') + for item in response.data['parameter'] if item['name'] in ('result', 'version', 'display')} + + def test_a_versioned_object_member_is_reported_in_the_newest_holding_release_the_caller_is_shown(self): + self.assertEqual(self.members(), [('A', '2'), ('B', '2')]) + self.assertEqual(self.used(), [f'{DEMO}|2']) + + def test_pins_do_not_change_the_release_of_a_versioned_object_member(self): + for pinned in ('explicit', 'evaluated', 'none'): + with self.subTest(pinned=pinned): + self.stored.explicit_source_versions.clear() + self.stored.evaluated_source_versions.clear() + if pinned == 'explicit': + self.stored.explicit_source_versions.add(self.demo_v1) + elif pinned == 'evaluated': + self.stored.evaluated_source_versions.add(self.demo_v1) + self.assertEqual(self.members(), [('A', '2'), ('B', '2')]) + self.assertEqual(self.used(), [f'{DEMO}|2']) + self.assertEqual(self.validated(code='A'), {'result': True, 'version': '2', 'display': 'Code A'}) + + def test_a_release_the_caller_is_not_shown_is_never_reported(self): + self.stored.explicit_source_versions.clear() + self.stored.explicit_source_versions.add(self.demo_v3) + for user in (None, self.outsider): + self.assertEqual(self.members(user=user), [('A', '2'), ('B', '2')]) + self.assertEqual(self.members(base='/orgs/Acme', user=self.admin), [('A', '3'), ('B', '3')]) + + def test_validate_code_answers_the_release_and_finds_the_code_in_it(self): + """The version is the release's; the concept, and so the display, is the member's, as $expand shows it.""" + self.assertEqual(self.validated(code='A'), {'result': True, 'version': '2', 'display': 'Code A'}) + for version in ('1', '2'): + self.assertEqual(self.validated(code='A', systemVersion=version)['result'], True) + self.assertEqual(self.validated(code='A', systemVersion='3')['result'], False) # not shown + + def test_a_coding_expand_returns_validates(self): + for member in self.expanded()['contains']: + self.assertEqual(self.validated(system=member['system'], code=member['code'], + systemVersion=member['version'], display=member['display']), + {'result': True, 'version': member['version'], 'display': member['display']}) + + def test_a_deleted_version_of_the_concept_is_never_the_release(self): + """A release whose version of the concept is deleted doesn't hold it: the next in order answers.""" + Concept.objects.filter(versioned_object=self.roots['A'], sources=self.demo_v2).update(is_active=False) + self.assertEqual(self.members(), [('A', '1'), ('B', '2')]) + self.assertEqual(self.validated(code='A', systemVersion='2')['result'], False) + self.assertEqual(self.validated(code='A'), {'result': True, 'version': '1', 'display': 'Code A'}) + + def test_a_member_a_release_holds_itself_answers_as_before(self): + """A version row stays in the release that holds it, though the expansion pins another one.""" + held = ConceptFactory(parent=self.demo, mnemonic='C', sources=[self.demo_v2], + names=[name('Code C', preferred=True)]) + ConceptFactory(parent=self.demo, mnemonic='C', version='C-1', versioned_object=held, sources=[self.demo_v1], + names=[name('Code C in 1', preferred=True)]) + self.stored.concepts.add(held) + self.assertEqual(self.members()[-1], ('C', '2')) + self.assertEqual(self.validated(code='C'), {'result': True, 'version': '2', 'display': 'Code C'}) + self.assertEqual(self.validated(code='C', systemVersion='1')['result'], False) # 1 holds another version + self.assertEqual(self.validated(code='C', systemVersion='2')['result'], True) + + def test_a_member_its_release_holds_outranks_another_concepts_pinned_version(self): + """Two members share the canonical, code and release name: the one a release holds itself answers first.""" + other = OrganizationFactory(mnemonic='Other') + source(other, 'demo', canonical_url=DEMO) + other_v1 = source(other, 'demo', '1', released=True, canonical_url=DEMO) + held = ConceptFactory(parent=Source.objects.get(organization=other, mnemonic='demo', version='HEAD'), + mnemonic='A', sources=[other_v1], names=[name('Other A', preferred=True)]) + self.stored.concepts.add(held) + self.assertEqual(self.validated(code='A', systemVersion='1', display='Other A'), + {'result': True, 'version': '1', 'display': 'Other A'}) + + def test_a_member_a_release_holds_itself_still_follows_the_pins(self): + """For a member held by both releases, an explicit pin wins over an evaluated pin.""" + root = ConceptFactory(parent=self.demo, mnemonic='C') + held = ConceptFactory(parent=self.demo, mnemonic='C', version='C-1', versioned_object=root, + sources=[self.demo_v1, self.demo_v2], names=[name('Code C', preferred=True)]) + self.stored.concepts.set([held]) + for explicit, evaluated, version in ((True, False, '1'), (False, True, '2'), (True, True, '1')): + with self.subTest(explicit=explicit, evaluated=evaluated): + self.stored.explicit_source_versions.set([self.demo_v1] if explicit else []) + self.stored.evaluated_source_versions.set([self.demo_v2] if evaluated else []) + self.assertEqual(self.members(), [('C', version)]) + self.assertEqual(self.used(), [f'{DEMO}|{version}']) + self.assertEqual(self.validated(code='C'), {'result': True, 'version': version, 'display': 'Code C'}) + + def test_the_statement_count_does_not_grow_with_either_kind_of_member(self): + """Adding members costs no more statements, whether releases hold their rows or other versions.""" + def statements(roots): + self.stored.concepts.set(roots if roots else self.demo_v1.concepts.all()) + with CaptureQueriesContext(connection) as context: + expansion = self.expanded() + self.assertEqual(len(expansion['contains']), len(self.roots)) + self.assertEqual(unbounded_expansion_reads(context.captured_queries, self.stored.id), []) + return len(context.captured_queries) + + before = statements(None), statements(self.roots.values()) + for code in ('C', 'D', 'E'): + root = ConceptFactory(parent=self.demo, mnemonic=code) + for release in (self.demo_v1, self.demo_v2, self.demo_v3): + ConceptFactory(parent=self.demo, mnemonic=code, version=f'{code}-{release.version}', + versioned_object=root, sources=[release]) + self.roots[code] = root + self.assertEqual((statements(None), statements(self.roots.values())), before) + + def test_releases_holding_every_concept_need_no_correlated_version_lookup(self): + """The newest releases are read as sets, whatever the pins, even when only the total is requested.""" + correlated = re.compile(r'"versioned_object_id" = \("concepts"\."versioned_object_id"\)') + for pinned in ('explicit', 'evaluated', 'none'): + self.stored.explicit_source_versions.clear() + self.stored.evaluated_source_versions.clear() + if pinned == 'explicit': + self.stored.explicit_source_versions.add(self.demo_v1) + elif pinned == 'evaluated': + self.stored.explicit_source_versions.add(self.demo) + self.stored.evaluated_source_versions.add(self.demo_v1) + for count in (0, 1, 1000): + with self.subTest(pinned=pinned, count=count): + with CaptureQueriesContext(connection) as context: + expansion = self.expanded(count=count) + self.assertEqual(expansion['total'], 2) + self.assertEqual([item.get('version') for item in expansion.get('contains', [])], + ['2'] * min(count, 2)) + self.assertEqual([item['valueUri'] for item in expansion['parameter'] + if item['name'] == 'used-codesystem'], [f'{DEMO}|2']) + self.assertEqual([query['sql'] for query in context.captured_queries + if correlated.search(query['sql'])], []) + self.assertEqual(unbounded_expansion_reads(context.captured_queries, self.stored.id), []) + + def test_the_newest_visible_release_of_each_repository_answers(self): + """A private newest release answers only at a base where the caller may view it.""" + root = ConceptFactory(parent=self.ciel, mnemonic='another') + ConceptFactory(parent=self.ciel, mnemonic='another', version='another-1', versioned_object=root, + sources=[self.ciel_v1, self.ciel_v2]) + self.stored.concepts.add(root) + for base, user, version in (('/fhir', None, '2'), ('/orgs/Acme', self.outsider, '2'), + ('/orgs/Acme', self.admin, '3')): + with self.subTest(base=base, version=version): + self.assertEqual(self.members(base=base, user=user), + [('A', version), ('B', version), ('another', 'v2')]) + self.assertEqual(self.used(base=base, user=user), + sorted([f'{DEMO}|{version}', f'{self.ciel.canonical_url}|v2'])) + + def test_an_older_release_answers_when_the_newest_no_longer_holds_the_concept(self): + """Missing newest links fall back to an older holding release, or no version when none holds it.""" + self.stored.explicit_source_versions.clear() + Concept.sources.through.objects.filter( + source=self.demo_v2, concept__versioned_object=self.roots['A']).delete() + Concept.sources.through.objects.filter( + source__in=[self.demo_v1, self.demo_v2, self.demo_v3], + concept__versioned_object=self.roots['B']).delete() + with CaptureQueriesContext(connection) as context: + expansion = self.expanded() + self.assertEqual([(item['code'], item.get('version')) for item in expansion['contains']], + [('A', '1'), ('B', None)]) + self.assertEqual([item['valueUri'] for item in expansion['parameter'] if item['name'] == 'used-codesystem'], + [DEMO, f'{DEMO}|1']) + self.assertEqual(unbounded_expansion_reads(context.captured_queries, self.stored.id), []) + + +class ConceptVersionWithoutRegistryTest(ConceptVersionCases, OCLTestCase): + registry = False + + +class ConceptVersionWithRegistryTest(ConceptVersionCases, OCLTestCase): + registry = True diff --git a/core/fhir/tests_ops_expand.py b/core/fhir/tests_ops_expand.py new file mode 100644 index 000000000..0dbc18dcd --- /dev/null +++ b/core/fhir/tests_ops_expand.py @@ -0,0 +1,616 @@ +""" +ValueSet/$expand (SVCM ITI-97, FHIR R4 $expand) at the global base /fhir/ and at owner bases, GET and POST, at the type +and instance levels, with the URL registry toggle off and on: the value set version resolved as core.fhir.resolve does; +a page of its default stored expansion, never computed or saved; contains with each member's system, version, code, +display and inactive flag; total, offset, the inputs echoed and used-codesystem; count, offset, activeOnly, filter, +displayLanguage and includeDesignations; FHIRsmith's errors; members of code systems the caller isn't shown leave no +trace; a fixed number of queries. OpenConceptLab/ocl_online#451. +""" +import json +import re +from urllib.parse import quote + +from django.db import connection +from django.test.utils import CaptureQueriesContext +from lxml import etree +from mock.mock import patch +from rest_framework.test import APIClient + +from core.collections.models import Collection, Expansion +from core.collections.tests.factories import ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE +from core.common.tests import OCLTestCase, OCLAPITransactionTestCase +from core.concepts.models import Concept +from core.concepts.tests.factories import ConceptFactory +from core.fhir.base import from_xml +from core.fhir.outcomes import MESSAGE_ID_EXTENSION, TX_ISSUE_TYPE_SYSTEM +from core.fhir.tests_ops_lookup import OpsCorpusMixin, name, CIEL, REST, HIV, FEVERS, NEWEST_PRIVATE +from core.fhir.tests_search import source, collection +from core.orgs.tests.factories import OrganizationFactory +from core.url_registry.factories import GlobalURLRegistryFactory +from core.url_registry.tests_resolution import set_registry_toggle + +UNKNOWN_VALUE_SET = 'http://example.org/ocl-fhir-conformance/unknown-valueset' +PINNED = 'https://example.org/fhir/ValueSet/pinned' +DRAFTS = 'https://example.org/fhir/ValueSet/drafts' +DRAFT_ONLY = 'http://example.org/fhir/CodeSystem/head-only' +INFECTIOUS = {'system': CIEL, 'version': 'v2', 'code': '117053', 'display': 'Infectious enteritis'} +RETIRED = {'system': CIEL, 'version': 'v2', 'code': '103', 'display': 'HIV antibody', 'inactive': True} +BATCH = {'system': REST, 'version': '4.0.1', 'code': 'batch'} +PRIVATE = {'system': HIV, 'version': 'v1', 'code': 'P1', 'display': 'Private code'} +USED = [f'{REST}|4.0.1', f'{CIEL}|v2'] +OLD_URL = 'http://example.org/fhir/CodeSystem/renamed-old' +NEW_URL = 'http://example.org/fhir/CodeSystem/renamed' + + +def used(*systems): + return [{'name': 'used-codesystem', 'valueUri': system} for system in systems] + + +def issues_of(response): + """An OperationOutcome's issues as severity|code|tx-issue-type|expression|message id.""" + return [ + '|'.join([item['severity'], item['code'], item['details']['coding'][0]['code'], + ','.join(item.get('expression', [])), + ','.join(e['valueString'] for e in item.get('extension', []) if e['url'] == MESSAGE_ID_EXTENSION)]) + for item in response.data['issue']] + + +def name_matches(queries): + """The statements that match a filter against concept names.""" + return [query['sql'] for query in queries if re.search(r'\."name"(::text)? ~\*', query['sql'])] + + +class ExpandCases(OpsCorpusMixin): + def expand(self, base='/fhir', user=None, **params): + return self.get('$expand', base=base, user=user, resource='ValueSet', **params) + + def post_expand(self, parameters, base='/fhir', user=None): + return self.post('$expand', parameters, base=base, user=user, resource='ValueSet') + + def expansion(self, response): + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['resourceType'], 'ValueSet') + return response.data['expansion'] + + def register(self, url, owner): + if self.registry: + GlobalURLRegistryFactory(url=url, namespace=owner.uri) + + # the route, the envelope and the members + + def test_a_value_set_version_named_with_a_path_takes_its_repository_canonical(self): + draft = collection(self.acme, 'fevers', 'v2') + expansion = ExpansionFactory(mnemonic='autoexpand-v2', collection_version=draft) + draft.expansion_uri = expansion.uri + draft.save() + + for params in ({'url': '/orgs/Acme/ValueSet/fevers/', 'valueSetVersion': 'v2'}, + {'url': '/orgs/Acme/ValueSet/fevers/v2/'}): + response = self.expand(**params) + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual((response.data['url'], response.data['version']), (FEVERS, 'v2')) + + def test_a_type_level_get_answers_the_value_set_with_its_stored_expansion(self): + response = self.expand(url=FEVERS) + + expansion = self.expansion(response) + self.assertEqual((response.data['url'], response.data['version'], response.data['id']), + (FEVERS, 'v1', str(self.fevers_v1.id))) + self.assertEqual(response.data['status'], 'active') + self.assertNotIn('compose', response.data) + self.assertEqual(expansion['identifier'], self.fevers_v1.expansion_uri) + self.assertIn('timestamp', expansion) + self.assertEqual((expansion['total'], expansion['offset']), (3, 0)) + self.assertEqual(expansion['parameter'], used(*USED)) + self.assertEqual(expansion['contains'], [INFECTIOUS, RETIRED, BATCH]) + self.assertEqual(self.expand(url=FEVERS, valueSetVersion='v1').content, response.content) + self.assertEqual(self.expand(url=f'{FEVERS}|v1').content, response.content) + self.assertEqual(self.client_for().get('/fhir/ValueSet/$expand/', {'url': FEVERS}).content, response.content) + + def test_a_post_answers_as_a_get_does(self): + response = self.expand(url=FEVERS, count='2', activeOnly='true') + for parameters in ( + [{'name': 'url', 'valueUri': FEVERS}, {'name': 'count', 'valueInteger': 2}, + {'name': 'activeOnly', 'valueBoolean': True}], + [{'name': 'url', 'valueString': FEVERS}, {'name': 'count', 'valueString': '2'}, + {'name': 'activeOnly', 'valueString': 'true'}]): + posted = self.post_expand(parameters) + self.assertEqual((posted.status_code, posted.content), (response.status_code, response.content)) + + def test_an_owner_base_shows_the_callers_private_members(self): + expansion = self.expansion(self.expand(base='/orgs/Acme', user=self.moh_member, url=FEVERS)) + self.assertEqual(expansion['total'], 4) + self.assertEqual(expansion['contains'], [INFECTIOUS, RETIRED, BATCH, PRIVATE]) + self.assertEqual(expansion['parameter'], used(f'{REST}|4.0.1', f'{HIV}|v1', f'{CIEL}|v2')) + for base, user in (('/fhir', self.moh_member), ('/orgs/Acme', None), ('/orgs/Acme', self.outsider)): + self.assertEqual(self.expansion(self.expand(base=base, user=user, url=FEVERS))['total'], 3, (base, user)) + + # paging, activeOnly, filter and the echo + + def test_count_and_offset_page_the_members(self): + first = self.expansion(self.expand(url=FEVERS, count='2', offset='0')) + second = self.expansion(self.expand(url=FEVERS, count='2', offset='2')) + self.assertEqual((first['total'], first['offset'], first['contains']), (3, 0, [INFECTIOUS, RETIRED])) + self.assertEqual((second['total'], second['offset'], second['contains']), (3, 2, [BATCH])) + self.assertEqual(first['parameter'], [ + {'name': 'offset', 'valueInteger': 0}, {'name': 'count', 'valueInteger': 2}, *used(*USED)]) + past = self.expansion(self.expand(url=FEVERS, offset='10')) + self.assertEqual((past['total'], past['offset'], 'contains' in past), (3, 10, False)) + + def test_count_0_answers_the_total_alone(self): + expansion = self.expansion(self.expand(url=FEVERS, count='0')) + self.assertEqual(expansion['total'], 3) + self.assertNotIn('contains', expansion) + self.assertEqual(expansion['parameter'], [{'name': 'count', 'valueInteger': 0}, *used(*USED)]) + + def test_count_is_at_most_1000(self): + with CaptureQueriesContext(connection) as context: + expansion = self.expansion(self.expand(url=FEVERS, count='5000')) + self.assertEqual(expansion['parameter'][0], {'name': 'count', 'valueInteger': 1000}) + self.assertEqual(len(expansion['contains']), 3) + pages = [query['sql'] for query in context.captured_queries if re.search(r'\bLIMIT 1000\b', query['sql'])] + self.assertEqual(len(pages), 1) + self.assertTrue(pages[0].startswith('SELECT') and 'FROM "concepts"' in pages[0]) + with CaptureQueriesContext(connection) as context: + self.expand(url=FEVERS) # without count: a page of 1000 too, not echoed + self.assertEqual(len([query for query in context.captured_queries if 'LIMIT 1000' in query['sql']]), 1) + + def test_active_only_leaves_retired_members_out(self): + expansion = self.expansion(self.expand(url=FEVERS, activeOnly='true')) + self.assertEqual((expansion['total'], expansion['contains']), (2, [INFECTIOUS, BATCH])) + self.assertEqual(expansion['parameter'][0], {'name': 'activeOnly', 'valueBoolean': True}) + self.assertEqual(expansion['parameter'][1:], used(*USED)) # used-codesystem: the whole value set's + self.assertEqual(self.expansion(self.expand(url=FEVERS, activeOnly='false'))['total'], 3) + + def test_a_filter_keeps_members_with_a_word_starting_with_each_of_its_words(self): + for text, members in (('infect', [INFECTIOUS]), ('INFECT', [INFECTIOUS]), ('entérite', [INFECTIOUS]), + ('origin enteritis', [INFECTIOUS]), ('bat', [BATCH]), ('10', [RETIRED]), + ('hiv', [RETIRED]), ('fect', []), ('antibody enteritis', []), ('', [INFECTIOUS, RETIRED, + BATCH])): + expansion = self.expansion(self.expand(url=FEVERS, filter=text)) + self.assertEqual((expansion['total'], expansion.get('contains', [])), (len(members), members), text) + expansion = self.expansion(self.expand(url=FEVERS, filter='infect', activeOnly='true', count='1')) + self.assertEqual(expansion['parameter'], [ + {'name': 'filter', 'valueString': 'infect'}, {'name': 'activeOnly', 'valueBoolean': True}, + {'name': 'count', 'valueInteger': 1}, *used(*USED)]) + + def test_display_language_and_designations(self): + expansion = self.expansion(self.expand(url=FEVERS, displayLanguage='fr', includeDesignations='true', count='1')) + member = expansion['contains'][0] + self.assertEqual(member['display'], 'Entérite infectieuse') + self.assertEqual(sorted((item['language'], item['value']) for item in member['designation']), [ + ('en', 'Enteritis of infectious origin'), ('en', 'Infectious enteritis'), ('es', 'enteritis infecciosa'), + ('fr', 'Entérite infectieuse')]) + self.assertEqual(expansion['parameter'][:2], [ + {'name': 'displayLanguage', 'valueCode': 'fr'}, {'name': 'includeDesignations', 'valueBoolean': True}]) + self.assertNotIn('designation', self.expansion(self.expand(url=FEVERS))['contains'][0]) + + # versions: only releases the caller is shown + + def test_a_members_version_is_the_release_the_expansion_pins_then_evaluated_then_newest(self): + pinned = collection(self.acme, 'pinned', 'v1', released=True, canonical_url=PINNED) + collection(self.acme, 'pinned', canonical_url=PINNED) + expansion = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=pinned) + expansion.concepts.add(self.infectious, self.older) # 117053 in CIEL v1 and v2, 999 in v1 only + expansion.explicit_source_versions.add(self.ciel_v1) + expansion.evaluated_source_versions.add(self.ciel_v2) + pinned.expansion_uri = expansion.uri + pinned.save() + self.register(PINNED, self.acme) + + answer = self.expansion(self.expand(url=PINNED)) + self.assertEqual([(member['code'], member['version']) for member in answer['contains']], + [('117053', 'v1'), ('999', 'v1')]) + self.assertEqual(answer['parameter'], used(f'{CIEL}|v1')) + expansion.explicit_source_versions.clear() + answer = self.expansion(self.expand(url=PINNED)) + self.assertEqual([(member['code'], member['version']) for member in answer['contains']], + [('117053', 'v2'), ('999', 'v1')]) + self.assertEqual(answer['parameter'], used(f'{CIEL}|v1', f'{CIEL}|v2')) + expansion.evaluated_source_versions.clear() + self.assertEqual([member['version'] for member in self.expansion(self.expand(url=PINNED))['contains']], + ['v2', 'v1']) # the newest release holding it + + def test_a_version_is_only_a_release_the_caller_is_shown_never_head(self): + drafts = OrganizationFactory(mnemonic='Drafts') + head_only = source(drafts, 'head-only', canonical_url=DRAFT_ONLY) + value_set = collection(self.acme, 'drafts', 'v1', released=True, canonical_url=DRAFTS) + collection(self.acme, 'drafts', canonical_url=DRAFTS) + expansion = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=value_set) + newest = Concept.objects.filter(parent=self.newest_private, mnemonic='N1').first() # in public v1, private v2 + expansion.concepts.add( + ConceptFactory(parent=head_only, mnemonic='D1', names=[name('D one', preferred=True)]), newest) + expansion.explicit_source_versions.add(head_only, self.newest_private_v2) + value_set.expansion_uri = expansion.uri + value_set.save() + self.register(DRAFTS, self.acme) + + answer = self.expansion(self.expand(url=DRAFTS)) + self.assertEqual(answer['contains'], [ # N1 is the older concept + {'system': NEWEST_PRIVATE, 'version': 'v1', 'code': 'N1', 'display': 'N one'}, + {'system': DRAFT_ONLY, 'code': 'D1', 'display': 'D one'}]) + self.assertEqual(answer['parameter'], used(DRAFT_ONLY, f'{NEWEST_PRIVATE}|v1')) + staff = self.expansion(self.expand(base='/orgs/Acme', user=self.admin, url=DRAFTS)) + self.assertIn(f'{NEWEST_PRIVATE}|v2', [item['valueUri'] for item in staff['parameter']]) + + def test_head_is_expanded_only_when_named(self): + """HEAD's own stored expansion answers when valueSetVersion names HEAD; the default stays the release.""" + head_expansion = ExpansionFactory(mnemonic='autoexpand-HEAD', collection_version=self.fevers) + head_expansion.concepts.add(self.infectious) + self.fevers.expansion_uri = head_expansion.uri + self.fevers.save() + response = self.expand(url=FEVERS, valueSetVersion='HEAD') + self.assertEqual((response.data['version'], self.expansion(response)['contains']), ('HEAD', [INFECTIOUS])) + self.assertEqual(self.expansion(self.expand(url=FEVERS))['total'], 3) + + def test_a_member_of_a_code_system_without_a_canonical_url_is_named_by_its_path(self): + local = source(self.acme, 'local', canonical_url=None) + local_v1 = source(self.acme, 'local', 'v1', released=True, canonical_url=None) + self.fevers_v1.expansion.concepts.add(ConceptFactory( + parent=local, mnemonic='L1', sources=[local_v1], names=[name('Local one', preferred=True)])) + answer = self.expansion(self.expand(url=FEVERS, filter='local')) + self.assertEqual(answer['contains'], [ + {'system': '/orgs/Acme/CodeSystem/local/', 'version': 'v1', 'code': 'L1', 'display': 'Local one'}]) + self.assertIn('/orgs/Acme/CodeSystem/local/|v1', [item.get('valueUri') for item in answer['parameter']]) + + def test_deleted_concepts_and_releases_are_gone(self): + deleted = ConceptFactory(parent=self.ciel, mnemonic='555', sources=[self.ciel_v2], is_active=False, + names=[name('Deleted', preferred=True)]) + self.fevers_v1.expansion.concepts.add(deleted) + self.assertEqual(self.expansion(self.expand(url=FEVERS))['total'], 3) + self.rest_v1.is_active = False + self.rest_v1.save() + answer = self.expansion(self.expand(url=FEVERS)) + self.assertEqual(answer['contains'][2], {'system': REST, 'code': 'batch'}) + self.assertEqual(answer['parameter'], used(REST, f'{CIEL}|v2')) + + def test_a_filter_matches_the_words_of_a_code_as_fhir_spells_it(self): + """filter matches a stored code by its decoded words, ignoring case, non-ASCII letters too: never by the + characters of an escape, and a word starts after an encoded character only when that character, decoded, isn't + a word character (Codex F1.2, F2.1).""" + codes = {'ND53.Y%26XK70': 'Encoded', quote('café'): 'Coffee shop', quote('caféine'): 'Stimulant', + quote('été'): 'Summer', quote('QX–ZZ9'): 'Range', '50%off': 'Discount', '%FFyy': 'Replacement', + quote('cm.²'): 'Area'} + for code, display in codes.items(): + self.fevers_v1.expansion.concepts.add(ConceptFactory( + parent=self.ciel, mnemonic=code, sources=[self.ciel_v2], names=[name(display, preferred=True)])) + for text, decoded in ( + ('26', []), ('c3', []), ('A9', []), ('80', []), # the characters of an escape + ('CAFÉ', ['café', 'caféine']), ('Café', ['café', 'caféine']), ('ÉTÉ', ['été']), # case, non-ASCII too + ('CAFÉINE', ['caféine']), ('ine', []), ('té', []), # é is a word character: no word starts after it + ('zz9', ['QX–ZZ9']), ('XK70', ['ND53.Y&XK70']), # – and & are not + ('ND53.Y&XK70', ['ND53.Y&XK70']), ('xk', ['ND53.Y&XK70']), ('Y&XK', ['ND53.Y&XK70']), # decoded, whole + ('encoded', ['ND53.Y&XK70']), ('K70', []), # by a name; a word's middle + ('CAF', ['café', 'caféine']), + ('off', ['50%off']), ('yy', ['\ufffdyy']), # a % that is no escape; a byte that is no character + ('²', []), ('cm', ['cm.²'])): # \m, as for names: ² is no word character to Postgres + with self.subTest(text): + expansion = self.expansion(self.expand(url=FEVERS, filter=text)) + self.assertEqual([member['code'] for member in expansion.get('contains', [])], decoded) + self.assertEqual(expansion['total'], len(decoded)) + + def test_a_members_system_is_its_releases_own_canonical_url(self): + """A release with a canonical URL of its own names its members' system, else its repository's does, in contains + and in used-codesystem (Codex F1.3; search and read follow the same rule).""" + renamed = source(self.acme, 'renamed', canonical_url=NEW_URL) + old = source(self.acme, 'renamed', 'v1', released=True, canonical_url=OLD_URL) + plain = source(self.acme, 'renamed', 'v2', released=True, canonical_url=None) + for code, sources in (('R1', [old]), ('R2', [plain]), ('R3', [])): + self.fevers_v1.expansion.concepts.add(ConceptFactory( + parent=renamed, mnemonic=code, sources=sources, names=[name(f'Renamed {code}', preferred=True)])) + answer = self.expansion(self.expand(url=FEVERS, filter='renamed')) + self.assertEqual([(member['system'], member.get('version'), member['code']) for member in answer['contains']], [ + (OLD_URL, 'v1', 'R1'), (NEW_URL, 'v2', 'R2'), (NEW_URL, None, 'R3')]) + self.assertEqual([item['valueUri'] for item in answer['parameter'] if item['name'] == 'used-codesystem'], + sorted([*USED, f'{OLD_URL}|v1', f'{NEW_URL}|v2', NEW_URL])) + + # what the caller isn't shown leaves no trace + + def sent(self, requests): + """{(method, path, params, caller): (status, bytes, statements run, ids and numbers normalised)}, for each of + requests' (method, path, params, callers).""" + answers = {} + for method, path, params, callers in requests: + for who, user in callers: + with CaptureQueriesContext(connection) as context: + if method == 'GET': + response = self.client_for(user).get(path, params) + else: + response = self.client_for(user).post( + path, json.dumps({'resourceType': 'Parameters', 'parameter': params}), + content_type='application/fhir+json') + answers[(method, path, json.dumps(params), who)] = ( + response.status_code, response.content, + [re.sub(r'\b\d+\b', 'N', query['sql']) for query in context.captured_queries]) + return answers + + def assert_same_when_missing(self, hidden, missing, bytes_only=lambda key: False): + """The two worlds' answers are alike in status and bytes, and in the statements run, except for the requests + bytes_only(key) names (statements that resolve() itself runs differently, outside this operation).""" + self.assertEqual(hidden.keys(), missing.keys()) + for key, (status, content, statements) in hidden.items(): + self.assertEqual((status, content), missing[key][:2], key) + if not bytes_only(key): + self.assertEqual(statements, missing[key][2], key) + + def test_a_member_the_caller_is_not_shown_leaves_no_trace(self): + """An expansion with a hidden member answers byte for byte, and with the same statements, as one without it: + in contains, total, used-codesystem and the versions reported, for any input that shapes the answer.""" + late = ConceptFactory(parent=self.ciel, mnemonic='Z9', sources=[self.ciel_v2], names=[ + name('Zed nine code', preferred=True), name('Zède', 'fr', True)]) + expansion = self.fevers_v1.expansion + expansion.concepts.add(late) # after the hidden member in id order: a page can run across it + expansion.explicit_source_versions.add(self.hiv_v1) # its release, pinned + expansion.evaluated_source_versions.add(self.hiv_v1) + callers = [('anonymous', None), ('outsider', self.outsider), ('moh-member', self.moh_member), + ('staff', self.admin)] + inputs = ({}, {'count': '1', 'offset': '1'}, {'count': '1', 'offset': '2'}, {'count': '2', 'offset': '2'}, + {'offset': '4'}, {'offset': '10'}, {'activeOnly': 'true'}, {'filter': 'private'}, {'filter': 'p1'}, + {'filter': 'private', 'count': '0'}, {'count': '0'}, + {'displayLanguage': 'fr', 'includeDesignations': 'true'}, {'includeDesignations': 'true'}) + requests = [('GET', '/fhir/ValueSet/$expand', {'url': FEVERS, **params}, callers) for params in inputs] + [ + ('GET', '/orgs/Acme/ValueSet/$expand', {'url': FEVERS, **params}, callers[:2]) for params in inputs] + [ + ('POST', '/fhir/ValueSet/$expand', [{'name': 'url', 'valueUri': FEVERS}, {'name': 'count', + 'valueInteger': 1}], callers), + ('GET', f'/fhir/ValueSet/{self.fevers_v1.id}/$expand', {'offset': '2'}, callers), + ('GET', '/orgs/Acme/ValueSet/fevers/$expand', {'filter': 'p'}, callers[:2])] + hidden = self.sent(requests) + self.assertEqual(hidden[('GET', '/fhir/ValueSet/$expand', json.dumps({'url': FEVERS}), 'moh-member')][0], 200) + expansion.concepts.remove(self.private_code) + expansion.explicit_source_versions.remove(self.hiv_v1) + expansion.evaluated_source_versions.remove(self.hiv_v1) + self.assert_same_when_missing(hidden, self.sent(requests)) + + def test_a_value_set_the_caller_is_not_shown_answers_as_a_missing_one(self): + url = 'https://example.org/hidden' + private = collection(self.moh, 'hidden', 'v1', released=True, canonical_url=url, public_access=ACCESS_TYPE_NONE) + collection(self.moh, 'hidden', canonical_url=url, public_access=ACCESS_TYPE_NONE) + private.expansion_uri = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=private).uri + private.save() + self.register(url, self.moh) + self.assertEqual(self.expand(base='/orgs/MOH', user=self.moh_member, url=url).status_code, 200) + everyone = [('anonymous', None), ('outsider', self.outsider), ('moh-member', self.moh_member), + ('staff', self.admin)] + outside = everyone[:2] + by_path = {'url': '/orgs/MOH/ValueSet/hidden/'} + path_v1 = {'url': '/orgs/MOH/ValueSet/hidden/v1/'} # resolve() reads the version's HEAD first, hidden or not + requests = [(method, f'{base}/ValueSet/$expand', params, callers) for base, callers in ( + ('/fhir', everyone), ('/orgs/MOH', outside)) for method, params in ( + ('GET', {'url': url}), ('GET', {'url': f'{url}|v1'}), ('GET', {'url': url, 'valueSetVersion': 'v1'}), + ('GET', {'url': url, 'valueSetVersion': 'HEAD'}), ('GET', {'url': url, 'valueSetVersion': 'v9'}), + ('GET', by_path), ('GET', path_v1), ('GET', {'url': url, 'filter': 'inf', 'count': '0'}), + ('POST', [{'name': 'url', 'valueUri': url}]), + ('POST', [{'name': 'url', 'valueString': url}, {'name': 'valueSetVersion', 'valueString': 'v1'}]))] + requests += [('GET', f'/fhir/ValueSet/{private.id}/$expand', params, everyone) for params in ( + {}, {'valueSetVersion': 'HEAD'})] + [ + ('POST', f'/fhir/ValueSet/{private.id}/$expand', [], everyone), + ('GET', '/orgs/MOH/ValueSet/hidden/$expand', {}, outside), + ('GET', '/orgs/MOH/ValueSet/hidden/$expand', {'valueSetVersion': 'HEAD'}, outside)] + hidden = self.sent(requests) + Collection.objects.filter(mnemonic='hidden').delete() + missing = self.sent(requests) + self.assertEqual(issues_of(self.expand(url=url)), ['error|not-found|not-found||Unable_to_resolve_value_Set_']) + for key, (status, _, _) in hidden.items(): + self.assertIn(status, (404, 422), key) + # the same statements everywhere but where resolve() itself differs: for a caller who is signed in, at an owner + # base (a membership check, only when a private row is found), with the registry on at an owner base (its + # lookup is skipped for a private canonical), and for an OCL path naming a version (its HEAD is read first) + self.assert_same_when_missing( + hidden, missing, lambda key: key[1].startswith('/orgs/MOH') or key[2] == json.dumps(path_v1)) + + def test_a_private_version_of_a_value_set_answers_as_a_missing_one(self): + """The newest release is private: named, or by its id, it answers as a version that doesn't exist.""" + v2 = collection(self.acme, 'fevers', 'v2', released=True, canonical_url=FEVERS, public_access=ACCESS_TYPE_NONE) + v2.expansion_uri = ExpansionFactory(mnemonic='autoexpand-v2', collection_version=v2).uri + v2.save() + everyone = [('anonymous', None), ('outsider', self.outsider), ('moh-member', self.moh_member), + ('staff', self.admin)] + requests = [('GET', '/fhir/ValueSet/$expand', params, everyone) for params in ( + {'url': FEVERS, 'valueSetVersion': 'v2'}, {'url': f'{FEVERS}|v2'}, + {'url': '/orgs/Acme/ValueSet/fevers/v2/'})] + [ + ('POST', '/fhir/ValueSet/$expand', [{'name': 'url', 'valueUri': FEVERS}, + {'name': 'valueSetVersion', 'valueString': 'v2'}], everyone), + ('GET', f'/fhir/ValueSet/{v2.id}/$expand', {}, everyone), + ('GET', '/orgs/Acme/ValueSet/$expand', {'url': FEVERS, 'valueSetVersion': 'v2'}, everyone[:3])] + hidden = self.sent(requests) + Collection.objects.filter(id=v2.id).delete() + # an OCL path naming a version reads its HEAD first, and an owner base checks a signed-in caller's membership + # once a private row is found: statements resolve() itself runs differently, not this operation + self.assert_same_when_missing(hidden, self.sent(requests), lambda key: key[1].startswith('/orgs/Acme') or + '/fevers/v2/' in key[2]) + + # errors + + def test_errors(self): + response = self.expand(url=UNKNOWN_VALUE_SET) + self.assert_outcome(response, 422, 'not-found', 'not-found') + self.assertEqual(self.expand(url=FEVERS, valueSetVersion='ocl-fhir-conformance-no-such-version').status_code, + 422) + for params, expression in (({}, 'url'), ({'count': '-1'}, 'count'), ({'count': 'all'}, 'count'), + ({'offset': '1.5'}, 'offset'), ({'activeOnly': 'maybe'}, 'activeOnly'), + ({'includeDesignations': '1'}, 'includeDesignations')): + response = self.expand(**({'url': FEVERS} if expression != 'url' else {}), **params) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression], params) + for parameter in ({'name': 'valueSet', 'resource': {'resourceType': 'ValueSet', 'status': 'active'}}, + {'name': 'context', 'valueUri': 'http://example.org/context'}): + response = self.post_expand([{'name': 'url', 'valueUri': FEVERS}, parameter]) + self.assert_outcome(response, 400, 'not-supported', 'not-supported') + self.assertEqual(response.data['issue'][0]['expression'], [parameter['name']]) + response = self.post_expand([{'name': 'url', 'valueUri': FEVERS}, {'name': 'count', 'valueBoolean': True}]) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + + def test_inputs_it_does_not_honour_are_refused(self): + """Every R4 $expand input either shapes the answer, is one a stored expansion always satisfies, or is a 400 + not-supported naming it, GET and POST: none is ignored (Codex F1.1). Names that aren't $expand inputs are.""" + refused = { + 'date': ('valueDateTime', '2020-01-01'), 'exclude-system': ('valueCanonical', CIEL), + 'system-version': ('valueCanonical', f'{CIEL}|v1'), + 'check-system-version': ('valueCanonical', f'{CIEL}|v1'), + 'force-system-version': ('valueCanonical', f'{CIEL}|v1'), 'designation': ('valueString', 'fr'), + 'contextDirection': ('valueCode', 'incoming'), 'includeDefinition': ('valueBoolean', True), + 'excludePostCoordinated': ('valueBoolean', True)} # Codex F2.2: nothing leaves such codes out + for name_, (value_type, value) in refused.items(): + responses = (self.expand(url=FEVERS, **{name_: 'true' if value is True else value}), + self.post_expand([{'name': 'url', 'valueUri': FEVERS}, {'name': name_, value_type: value}])) + for response in responses: + self.assert_outcome(response, 400, 'not-supported', 'not-supported') + self.assertEqual(response.data['issue'][0]['expression'], [name_], name_) + expected = self.expand(url=FEVERS).content + for params in ({'excludeNested': 'true'}, {'excludeNested': 'false'}, {'excludeNotForUI': 'true'}, + {'excludePostCoordinated': 'false'}, {'includeDefinition': 'false'}, {'q': 'anything'}): + self.assertEqual(self.expand(url=FEVERS, **params).content, expected, params) + for name_ in ('excludePostCoordinated', 'includeDefinition'): + posted = self.post_expand([{'name': 'url', 'valueUri': FEVERS}, {'name': name_, 'valueBoolean': False}]) + self.assertEqual((posted.status_code, posted.content), (200, expected), name_) + self.assert_outcome(self.expand(url=FEVERS, excludeNested='perhaps'), 400, 'invalid', 'invalid-data') + + def test_an_expansion_that_cannot_be_served_is_a_422(self): + expansion = self.fevers_v1.expansion + expansion.unresolved_repo_versions = [ + {'url': 'http://snomed.info/sct', 'type': 'reference.system', 'version': '', 'namespace': None}] + expansion.save() + response = self.expand(url=FEVERS) + self.assertEqual(issues_of(response), ['error|not-found|not-found||UNKNOWN_CODESYSTEM_EXP']) + self.assertEqual(response.status_code, 422) + self.assertIn("CodeSystem 'http://snomed.info/sct' could not be found", response.data['issue'][0]['details'][ + 'text']) + expansion.unresolved_repo_versions = [{'url': 'http://example.org/vs', 'type': 'reference.valueset', + 'version': '2'}] + expansion.save() + self.assertEqual(issues_of(self.expand(url=FEVERS)), + ['error|not-found|not-found||VS_EXP_IMPORT_UNK_PINNED']) + expansion.unresolved_repo_versions = [] + expansion.is_processing = True + expansion.save() + self.assert_outcome(self.expand(url=FEVERS), 422, 'processing', 'invalid-data') + self.fevers_v1.expansion_uri = None + self.fevers_v1.save() + self.assert_outcome(self.expand(url=FEVERS), 422, 'invalid', 'vs-invalid') + + def test_a_post_never_persists(self): + expansions = list(Expansion.objects.values_list('id', 'parameters')) + members = self.fevers_v1.expansion.concepts.count() + for parameters in ([{'name': 'url', 'valueUri': FEVERS}, {'name': 'filter', 'valueString': 'infect'}], + [{'name': 'url', 'valueUri': FEVERS}, {'name': 'activeOnly', 'valueBoolean': True}]): + self.assertLess(self.expansion(self.post_expand(parameters))['total'], 3) + self.assertEqual(list(Expansion.objects.values_list('id', 'parameters')), expansions) + self.assertEqual(self.fevers_v1.expansion.concepts.count(), members) + + # the instance level, XML + + def test_the_instance_level(self): + expected = self.expansion(self.expand(url=FEVERS)) + for path in (f'/fhir/ValueSet/{self.fevers_v1.id}/$expand', '/orgs/Acme/ValueSet/fevers/$expand', + '/orgs/Acme/ValueSet/fevers/$expand/'): + response = self.client_for().get(path) + self.assertEqual(self.expansion(response), expected, path) + response = self.client_for().get('/orgs/Acme/ValueSet/fevers/$expand', {'valueSetVersion': 'v1'}) + self.assertEqual(self.expansion(response), expected) + for params, expression in (({'url': CIEL}, 'url'), ({'valueSetVersion': 'v9'}, 'valueSetVersion')): + response = self.client_for().get('/orgs/Acme/ValueSet/fevers/$expand', params) + self.assertEqual(response.status_code, 400 if expression == 'url' else 404, params) + self.assertEqual(self.client_for().get(f'/fhir/ValueSet/{self.fevers_v1.id}/$expand', {'url': CIEL}) + .status_code, 400) + self.assertEqual(self.client_for().get('/orgs/Acme/ValueSet/nothing/$expand').status_code, 404) + + def test_xml(self): + response = self.client_for().get('/fhir/ValueSet/$expand', {'url': FEVERS, '_format': 'xml'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/fhir+xml; charset=utf-8') + root = etree.fromstring(response.content) + self.assertEqual(root.tag, '{http://hl7.org/fhir}ValueSet') + value_set = from_xml(response.content) + self.assertEqual(value_set['expansion']['total'], 3) + self.assertEqual([member['code'] for member in value_set['expansion']['contains']], ['117053', '103', 'batch']) + self.assertTrue(value_set['expansion']['contains'][1]['inactive']) + + # cost + + def test_a_filtered_page_matches_names_once(self): + """A page larger than the matching members reads the name match once, for both total and contains.""" + with CaptureQueriesContext(connection) as context: + expansion = self.expansion(self.expand(url=FEVERS, filter='infect', count='1000')) + self.assertEqual((expansion['total'], expansion['offset'], expansion['contains']), (1, 0, [INFECTIOUS])) + self.assertEqual(len(name_matches(context.captured_queries)), 1) + + def test_a_filtered_page_with_an_offset_matches_names_once(self): + """The total counts every match, and the offset selects a page in id order without matching names again.""" + for number in range(30): + self.fevers_v1.expansion.concepts.add(ConceptFactory( + parent=self.ciel, mnemonic=f'x{number}', sources=[self.ciel_v1, self.ciel_v2], + names=[name(f'Extra {number}', preferred=True)])) + with CaptureQueriesContext(connection) as context: + expansion = self.expansion(self.expand(url=FEVERS, filter='extra', count='2', offset='1')) + self.assertEqual((expansion['total'], expansion['offset']), (30, 1)) + self.assertEqual(expansion['contains'], [ + {'system': CIEL, 'version': 'v2', 'code': 'x1', 'display': 'Extra 1'}, + {'system': CIEL, 'version': 'v2', 'code': 'x2', 'display': 'Extra 2'}]) + self.assertEqual(len(name_matches(context.captured_queries)), 1) + + def test_an_empty_filtered_page_reads_no_page_or_names(self): + """A zero count, an offset past the matches and no matches read no page or names.""" + resolution = 5 if self.registry else 3 + for text, count, offset, total in (('infect', '0', '0', 1), ('infect', '2', '1', 1), + ('missing', '1000', '0', 0)): + with self.assertNumQueries(resolution + 3): + expansion = self.expansion(self.expand(url=FEVERS, filter=text, count=count, offset=offset)) + self.assertEqual((expansion['total'], expansion['offset']), (total, int(offset))) + self.assertNotIn('contains', expansion) + + def test_a_fixed_number_of_queries(self): + """Resolution, the stored expansion, the total, used-codesystem, one page and its names: whatever the size of + the expansion, and with count=0 no page.""" + resolution = 5 if self.registry else 3 # the toggle, the repository, its release (+ the registry's two) + with self.assertNumQueries(resolution + 5): + self.expand(url=FEVERS) + for number in range(30): + self.fevers_v1.expansion.concepts.add(ConceptFactory( + parent=self.ciel, mnemonic=f'x{number}', sources=[self.ciel_v1, self.ciel_v2], + names=[name(f'Extra {number}', preferred=True)])) + self.fevers_v1.expansion.concepts.add(ConceptFactory( # a word after an encoded character: same statements + parent=self.ciel, mnemonic=quote('QX–ZZ9'), sources=[self.ciel_v2], names=[name('Range', preferred=True)])) + for params in ({}, {'count': '10', 'offset': '20', 'filter': 'extra', 'activeOnly': 'true'}, + {'displayLanguage': 'fr', 'includeDesignations': 'true'}, {'filter': 'zz9 range'}): + with self.assertNumQueries(resolution + 5): + self.assertEqual(self.expand(url=FEVERS, **params).status_code, 200) + with self.assertNumQueries(resolution + 3): + self.expand(url=FEVERS, count='0') + with self.assertNumQueries(resolution + 4): # an empty page: no names to read + self.expand(url=FEVERS, offset='100') + + +class ExpandWithoutRegistryTest(ExpandCases, OCLTestCase): + registry = False + + +class ExpandWithRegistryTest(ExpandCases, OCLTestCase): + registry = True + + +class ExpandTimeoutTest(OCLAPITransactionTestCase): + """$expand's statements are bounded as a search's are: one the database cancels is too costly (422).""" + + def test_a_cancelled_statement_is_too_costly(self): + set_registry_toggle(False) + org = OrganizationFactory(mnemonic='Acme') + value_set = collection(org, 'fevers', 'v1', released=True, canonical_url=FEVERS) + collection(org, 'fevers', canonical_url=FEVERS) + value_set.expansion_uri = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=value_set).uri + value_set.save() + + def slow(*_): + with connection.cursor() as cursor: + cursor.execute('SELECT pg_sleep(1)') + + with patch('core.fhir.search.SEARCH_TIMEOUT_MS', 50), patch('core.fhir.ops.expand.Expander.used', slow): + response = APIClient().get('/fhir/ValueSet/$expand', {'url': FEVERS}) + self.assertEqual(response.status_code, 422, response.data) + issue = response.data['issue'][0] + self.assertEqual(issue['code'], 'too-costly') + self.assertIn({'system': TX_ISSUE_TYPE_SYSTEM, 'code': 'too-costly'}, issue['details']['coding']) + self.assertIn('expansion', issue['details']['text']) + self.assertEqual(json.loads(APIClient().get('/fhir/ValueSet/$expand', {'url': FEVERS}).content)['expansion'][ + 'total'], 0) diff --git a/core/fhir/tests_ops_lookup.py b/core/fhir/tests_ops_lookup.py new file mode 100644 index 000000000..72015fdf3 --- /dev/null +++ b/core/fhir/tests_ops_lookup.py @@ -0,0 +1,529 @@ +""" +CodeSystem/$lookup (SVCM ITI-98, FHIR R4 $lookup) at the global base /fhir/ and at owner bases, GET and POST, at the +type and instance levels: the code system resolved as core.fhir.resolve does (released versions only, the newest the +caller is shown unless one is named, public only at /fhir/), with the URL registry toggle off and on; name, version +(the one resolved), display, code, system, abstract, designations and the inactive property; retired codes found and +flagged; displayLanguage; FHIRsmith's errors; a code system the caller isn't shown answers as a missing one; a fixed +number of queries. OpenConceptLab/ocl_online#451. +""" +import json + +from django.db import connection +from django.test.utils import CaptureQueriesContext +from rest_framework.test import APIClient + +from core.collections.tests.factories import ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory +from core.fhir.outcomes import TX_ISSUE_TYPE_SYSTEM +from core.fhir.tests_search import source, collection +from core.orgs.tests.factories import OrganizationFactory +from core.url_registry.factories import GlobalURLRegistryFactory +from core.url_registry.tests_resolution import set_registry_toggle +from core.users.models import UserProfile +from core.users.tests.factories import UserProfileFactory + +CIEL = 'https://CIELterminology.org' +ICD10 = 'http://hl7.org/fhir/sid/icd-10' +HIV = 'http://moh.example/fhir/CodeSystem/hiv' +REST = 'http://hl7.org/fhir/restful-interaction' +SHARED = 'http://example.org/fhir/CodeSystem/shared' +NEWEST_PRIVATE = 'http://example.org/fhir/CodeSystem/newest-private' +UNKNOWN = 'http://example.org/ocl-fhir-conformance/unknown-system' +FEVERS = 'https://example.org/fhir/ValueSet/fevers' +INACTIVE = {'name': 'property', 'part': [ + {'name': 'code', 'valueCode': 'inactive'}, {'name': 'value', 'valueBoolean': False}]} + + +def name(text, locale='en', preferred=False, name_type='Fully-Specified'): + return ConceptNameFactory.build(name=text, locale=locale, locale_preferred=preferred, type=name_type) + + +class OpsCorpusMixin: + """ + CIEL with two releases: 117053 in both (names in three languages), retired 103 in the newer, 999 in the older only. + WHO's ICD-10, whose release has no canonical of its own; HL7's restful-interaction, whose 'batch' has no names. + MOH's private code system (members see it at MOH's base). A canonical held privately (lower id) and publicly; a + code system whose newest release is private. Acme's public value set 'fevers', whose release's default expansion + holds CIEL 117053 and 103, MOH's P1 and HL7's batch. `registry` is the URL registry toggle: off as in production, + or on with the curated registry's global entries (none for private repositories). + """ + registry = False + + def setUp(self): # pylint: disable=too-many-statements + set_registry_toggle(self.registry) + super().setUp() + self.admin = UserProfile.objects.get(username='ocladmin') + self.ciel_org = OrganizationFactory(mnemonic='CIEL') + self.who = OrganizationFactory(mnemonic='WHO') + self.hl7 = OrganizationFactory(mnemonic='HL7') + self.moh = OrganizationFactory(mnemonic='MOH') + self.acme = OrganizationFactory(mnemonic='Acme') + self.moh_member = UserProfileFactory(username='moh-member') + self.moh.members.add(self.moh_member) + self.outsider = UserProfileFactory(username='outsider') + self.jdoe = UserProfileFactory(username='jdoe') + + self.ciel = source(self.ciel_org, 'CIEL', canonical_url=CIEL, default_locale='en', supported_locales=['en']) + self.ciel_v1 = source(self.ciel_org, 'CIEL', 'v1', released=True, canonical_url=CIEL) + self.ciel_v2 = source(self.ciel_org, 'CIEL', 'v2', released=True, canonical_url=CIEL) + self.infectious = ConceptFactory( + parent=self.ciel, mnemonic='117053', sources=[self.ciel_v1, self.ciel_v2], names=[ + name('Infectious enteritis', preferred=True), name('Entérite infectieuse', 'fr', True), + name('enteritis infecciosa', 'es'), name('Enteritis of infectious origin', name_type='Synonym')]) + self.retired = ConceptFactory(parent=self.ciel, mnemonic='103', retired=True, sources=[self.ciel_v2], + names=[name('HIV antibody', preferred=True)]) + self.older = ConceptFactory(parent=self.ciel, mnemonic='999', sources=[self.ciel_v1], + names=[name('Older code', preferred=True)]) + + self.icd = source(self.who, 'ICD-10-WHO', canonical_url=ICD10) + self.icd_2019 = source(self.who, 'ICD-10-WHO', 'ICD102019', released=True, canonical_url=None) + ConceptFactory(parent=self.icd, mnemonic='S20.4', sources=[self.icd_2019], + names=[name('Other superficial injuries of back wall of thorax', preferred=True)]) + self.rest = source(self.hl7, 'restful-interaction', canonical_url=REST) + self.rest_v1 = source(self.hl7, 'restful-interaction', '4.0.1', released=True, canonical_url=REST) + self.batch = ConceptFactory(parent=self.rest, mnemonic='batch', sources=[self.rest_v1]) + + self.hiv = source(self.moh, 'hiv', canonical_url=HIV, public_access=ACCESS_TYPE_NONE) + self.hiv_v1 = source(self.moh, 'hiv', 'v1', released=True, canonical_url=HIV, public_access=ACCESS_TYPE_NONE) + self.private_code = ConceptFactory(parent=self.hiv, mnemonic='P1', sources=[self.hiv_v1], + names=[name('Private code', preferred=True)]) + + private_shared = source(self.jdoe, 'shared', canonical_url=SHARED, public_access=ACCESS_TYPE_NONE) + private_shared_v1 = source(self.jdoe, 'shared', 'v1', released=True, canonical_url=SHARED, + public_access=ACCESS_TYPE_NONE) + ConceptFactory(parent=private_shared, mnemonic='S1', sources=[private_shared_v1], + names=[name('Private copy', preferred=True)]) + self.public_shared = source(self.hl7, 'shared', canonical_url=SHARED) + self.public_shared_v1 = source(self.hl7, 'shared', 'v1', released=True, canonical_url=SHARED) + ConceptFactory(parent=self.public_shared, mnemonic='S1', sources=[self.public_shared_v1], + names=[name('Public copy', preferred=True)]) + + self.newest_private = source(self.hl7, 'newest', canonical_url=NEWEST_PRIVATE) + self.newest_public_v1 = source(self.hl7, 'newest', 'v1', released=True, canonical_url=NEWEST_PRIVATE) + self.newest_private_v2 = source(self.hl7, 'newest', 'v2', released=True, canonical_url=NEWEST_PRIVATE, + public_access=ACCESS_TYPE_NONE) + ConceptFactory(parent=self.newest_private, mnemonic='N1', + sources=[self.newest_public_v1, self.newest_private_v2], names=[name('N one', preferred=True)]) + + self.fevers = collection(self.acme, 'fevers', canonical_url=FEVERS) + self.fevers_v1 = collection(self.acme, 'fevers', 'v1', released=True, canonical_url=FEVERS) + expansion = ExpansionFactory(mnemonic='autoexpand-v1', collection_version=self.fevers_v1) + expansion.concepts.add(self.infectious, self.retired, self.private_code, self.batch) + expansion.evaluated_source_versions.add(self.ciel_v2) + self.fevers_v1.expansion_uri = expansion.uri + self.fevers_v1.save() + + if self.registry: + for url, owner in ((CIEL, self.ciel_org), (ICD10, self.who), (REST, self.hl7), (SHARED, self.hl7), + (NEWEST_PRIVATE, self.hl7), (FEVERS, self.acme)): + GlobalURLRegistryFactory(url=url, namespace=owner.uri) + + @staticmethod + def client_for(user=None): + client = APIClient() + if user: + client.force_authenticate(user) + return client + + def get(self, operation, base='/fhir', user=None, resource='CodeSystem', **params): + return self.client_for(user).get(f'{base}/{resource}/{operation}', params) + + def post( # pylint: disable=too-many-arguments + self, operation, parameters, base='/fhir', user=None, resource='CodeSystem'): + return self.client_for(user).post( + f'{base}/{resource}/{operation}', json.dumps({'resourceType': 'Parameters', 'parameter': parameters}), + content_type='application/fhir+json') + + @staticmethod + def by_name(response): + """A Parameters response's parameters: {name: parameter}, each designation in `designations`.""" + parameters = {} + for parameter in response.data['parameter']: + if parameter['name'] == 'designation': + parameters.setdefault('designations', []).append(parameter) + else: + parameters[parameter['name']] = parameter + return parameters + + def assert_outcome(self, response, status_code, issue, tx_issue_type): + self.assertEqual(response.status_code, status_code, response.data) + self.assertEqual(response.data['resourceType'], 'OperationOutcome') + self.assertEqual(response.data['issue'][0]['code'], issue) + self.assertEqual(response.data['issue'][0]['details']['coding'], + [{'system': TX_ISSUE_TYPE_SYSTEM, 'code': tx_issue_type}]) + + +class LookupCases(OpsCorpusMixin): + def lookup(self, **params): + return self.get('$lookup', **params) + + def test_answers_name_version_display_code_system_abstract_designations_and_inactive(self): + response = self.lookup(system=CIEL, code='117053') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['parameter'][:6], [ + {'name': 'name', 'valueString': 'CIEL'}, + {'name': 'version', 'valueString': 'v2'}, + {'name': 'display', 'valueString': 'Infectious enteritis'}, + {'name': 'code', 'valueCode': '117053'}, + {'name': 'system', 'valueUri': CIEL}, + {'name': 'abstract', 'valueBoolean': False}, + ]) + self.assertEqual(response.data['parameter'][-1], INACTIVE) + designations = sorted( + [{part['name']: next(value for key, value in part.items() if key.startswith('value')) + for part in designation['part']} for designation in self.by_name(response)['designations']], + key=lambda designation: designation['value']) + self.assertEqual(designations, [ + {'language': 'en', 'use': {'code': 'Synonym'}, 'value': 'Enteritis of infectious origin'}, + {'language': 'fr', 'use': {'code': 'Fully-Specified'}, 'value': 'Entérite infectieuse'}, + {'language': 'en', 'use': {'code': 'Fully-Specified'}, 'value': 'Infectious enteritis'}, + {'language': 'es', 'use': {'code': 'Fully-Specified'}, 'value': 'enteritis infecciosa'}, + ]) + + def test_answers_the_version_it_resolved(self): + """The version named, else the newest release; never the repository's latest when another was named.""" + self.assertEqual(self.by_name(self.lookup(system=CIEL, code='117053', version='v1'))['version']['valueString'], + 'v1') + older = self.lookup(system=CIEL, code='999', version='v1') + self.assertEqual(older.status_code, 200, older.data) + self.assertEqual((self.by_name(older)['version']['valueString'], self.by_name(older)['display']['valueString']), + ('v1', 'Older code')) + self.assert_outcome(self.lookup(system=CIEL, code='999'), 404, 'not-found', 'invalid-code') + self.assertEqual(self.by_name(self.lookup(system=f'{CIEL}|v1', code='999'))['version']['valueString'], 'v1') + + def test_a_release_without_a_canonical_of_its_own_resolves_by_its_repository(self): + response = self.lookup(system=ICD10, code='S20.4', version='ICD102019') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'ICD102019') + self.assertEqual(self.by_name(response)['system']['valueUri'], ICD10) + + def test_a_retired_code_is_found_and_flagged_inactive(self): + response = self.lookup(system=CIEL, code='103') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['property']['part'][1], {'name': 'value', 'valueBoolean': True}) + self.assertEqual(self.by_name(response)['display']['valueString'], 'HIV antibody') + + def test_display_language(self): + response = self.lookup(system=CIEL, code='117053', displayLanguage='fr') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Entérite infectieuse') + # a language the concept has no name in: its own display + response = self.lookup(system=CIEL, code='117053', displayLanguage='ru') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Infectious enteritis') + + def test_a_concept_without_names_is_displayed_by_its_code(self): + response = self.lookup(system=REST, code='batch') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['display'], {'name': 'display', 'valueString': 'batch'}) + self.assertNotIn('designations', self.by_name(response)) + + def test_errors(self): + self.assert_outcome(self.lookup(system=CIEL, code='nope'), 404, 'not-found', 'invalid-code') + self.assertNotIn('expression', self.lookup(system=CIEL, code='nope').data['issue'][0]) + for params in ({'system': UNKNOWN, 'code': '1'}, {'system': CIEL, 'code': '117053', 'version': 'nope'}): + response = self.lookup(**params) + self.assert_outcome(response, 422, 'not-found', 'not-found') + self.assertNotIn('expression', response.data['issue'][0]) + for params, missing in (({'system': CIEL}, 'code'), ({'code': '117053'}, 'system'), ({}, 'code')): + response = self.lookup(**params) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [missing]) + + def test_post(self): + for value_types in (('valueUri', 'valueCode', 'valueString'), ('valueString', 'valueString', 'valueString')): + response = self.post('$lookup', [ + {'name': 'system', value_types[0]: CIEL}, {'name': 'code', value_types[1]: '117053'}, + {'name': 'version', value_types[2]: 'v1'}]) + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + self.assertEqual(self.by_name(response)['code']['valueCode'], '117053') + response = self.client_for().post( + '/fhir/CodeSystem/$lookup', + '', + content_type='application/fhir+xml', HTTP_ACCEPT='application/fhir+xml') + self.assertEqual(response.status_code, 200) + self.assertIn(b'', response.content) + + def test_a_coding(self): + response = self.post('$lookup', [ + {'name': 'coding', 'valueCoding': {'system': CIEL, 'code': '999', 'version': 'v1'}}]) + self.assertEqual(self.by_name(response)['display']['valueString'], 'Older code') + response = self.post('$lookup', [{'name': 'coding', 'valueCoding': {'system': CIEL}}]) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['coding']) + + def test_the_instance_level(self): + response = self.get(f'{self.ciel_v1.id}/$lookup', code='999') + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.get(f'{self.icd_2019.id}/$lookup', code='S20.4') # a release with no canonical of its own + self.assertEqual(self.by_name(response)['system']['valueUri'], ICD10) + + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='117053') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v2') + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='999', version='v1') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Older code') + response = self.post('CIEL/$lookup/', [{'name': 'code', 'valueCode': '103'}], base='/orgs/CIEL') + self.assertEqual(self.by_name(response)['display']['valueString'], 'HIV antibody') + + self.assert_outcome(self.get('CIEL/$lookup', code='117053'), 404, 'not-found', 'not-found') + self.assert_outcome(self.get(f'{self.hiv_v1.id}/$lookup', code='P1', user=self.moh_member), + 404, 'not-found', 'not-found') + response = self.get('hiv/$lookup', base='/orgs/MOH', code='P1', user=self.moh_member) + self.assertEqual(self.by_name(response)['display']['valueString'], 'Private code') + + def test_a_code_system_the_caller_is_not_shown_answers_as_a_missing_one(self): + member = self.get('$lookup', base='/orgs/MOH', user=self.moh_member, system=HIV, code='P1') + self.assertEqual(member.status_code, 200, member.data) + self.assertEqual(self.by_name(member)['display']['valueString'], 'Private code') + + hidden = [self.get('$lookup', base=base, user=user, system=HIV, code=code) + for base, user in (('/fhir', None), ('/fhir', self.outsider), ('/fhir', self.moh_member), + ('/orgs/MOH', None), ('/orgs/MOH', self.outsider)) + for code in ('P1', 'nope')] + self.hiv.is_active = False # now it doesn't exist + self.hiv.save() + missing = self.get('$lookup', system=HIV, code='P1') + self.assertEqual(missing.status_code, 422) + for response in hidden: + self.assertEqual((response.status_code, response.content), (missing.status_code, missing.content)) + + def test_a_private_copy_of_a_canonical_does_not_hide_the_public_one(self): + response = self.lookup(system=SHARED, code='S1') + self.assertEqual(self.by_name(response)['display']['valueString'], 'Public copy') + + def test_the_default_version_is_the_newest_release_the_caller_is_shown(self): + response = self.lookup(system=NEWEST_PRIVATE, code='N1') + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.get('$lookup', base='/orgs/HL7', user=self.admin, system=NEWEST_PRIVATE, code='N1') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v2') + + def test_a_hidden_version_in_a_path_answers_as_a_missing_one(self): + """A private version named in an OCL path is not found, exactly as one that doesn't exist (Codex F1.1).""" + path = '/orgs/HL7/CodeSystem/newest/v2/' + hidden = self.lookup(system=path, code='N1') + self.assertEqual(hidden.status_code, 422, hidden.data) + self.assertEqual(self.by_name(self.lookup(system='/orgs/HL7/CodeSystem/newest/v1/', code='N1'))['version'], + {'name': 'version', 'valueString': 'v1'}) + self.newest_private_v2.is_active = False + self.newest_private_v2.save() + self.assertEqual(self.lookup(system=path, code='N1').content, hidden.content) + + def test_a_path_naming_a_version_and_another_version_named(self): + """ + An OCL path naming one version with another named (version, url|version, HEAD, a coding's) names no version + OCL holds: a private version in the path answers byte for byte as a missing one, at /fhir/ and at an owner base + for a non-member (adversarial A.1). + """ + path = '/orgs/HL7/CodeSystem/newest/v2/' + requests = [ + ('get', {'system': path, 'version': 'v1', 'code': 'N1'}), ('get', {'system': f'{path}|v1', 'code': 'N1'}), + ('get', {'system': path, 'version': 'HEAD', 'code': 'N1'}), + ('post', [{'name': 'coding', 'valueCoding': {'system': path, 'version': 'v1', 'code': 'N1'}}])] + def ask(base, user, method, inputs): + if method == 'get': + return self.get('$lookup', base=base, user=user, **inputs) + return self.post('$lookup', inputs, base=base, user=user) + bases = [('/fhir', None), ('/orgs/HL7', None), ('/orgs/HL7', self.outsider)] + hidden = [(base, user, method, inputs, ask(base, user, method, inputs)) + for base, user in bases for method, inputs in requests] + self.newest_private_v2.is_active = False + self.newest_private_v2.save() + for base, user, method, inputs, response in hidden: + missing = ask(base, user, method, inputs) + self.assertEqual((response.status_code, response.content), (missing.status_code, missing.content), + (base, method, inputs)) + self.assertEqual(response.status_code, 422, (base, method, inputs)) + + def test_an_owner_base_costs_the_same_for_a_hidden_and_an_unknown_code_system(self): + """Anonymous, at an owner base: a private code system costs the queries an unknown one does (adversarial A.3). + For a signed-in non-member the membership check remains, and with the registry an owner-base match stops the + fall-through to the global entry: both are decision A.3's.""" + if self.registry: + self.skipTest('with the registry, an owner-base match stops the fall-through (decision A.3)') + counts = {} + for name_, operation, params, status_code in ( + ('hidden', '$lookup', {'system': HIV}, 422), + ('unknown', '$lookup', {'system': 'http://moh.example/fhir/CodeSystem/nope'}, 422), + ('hidden instance', 'hiv/$lookup', {}, 404), ('unknown instance', 'nope/$lookup', {}, 404)): + with CaptureQueriesContext(connection) as context: + response = self.get(operation, base='/orgs/MOH', code='P1', **params) + self.assertEqual(response.status_code, status_code, name_) + counts[name_] = len(context.captured_queries) + self.assertEqual(counts['hidden'], counts['unknown']) + self.assertEqual(counts['hidden instance'], counts['unknown instance']) + + def test_an_instance_and_a_coding_of_another_system(self): + """At the instance level, a coding names the code system it is from: another one's code isn't looked up.""" + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': {'system': ICD10, 'code': '117053'}}], + base='/orgs/CIEL') + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['Coding.system']) + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': {'system': CIEL, 'code': '117053'}}], + base='/orgs/CIEL') + self.assertEqual(response.status_code, 200, response.data) + + def test_an_instance_and_another_version_or_system_named(self): + """ + At the instance level, the id names the code system version: a version (the version parameter, or a coding's) + or a system parameter naming another one is a 400, never answered from the instance (Codex F2.1, F2.2). + """ + global_ciel_v1 = f'{self.ciel_v1.id}/$lookup' + for base, operation, params, expression in ( + ('/fhir', global_ciel_v1, {'code': '999', 'version': 'v2'}, 'version'), + ('/fhir', global_ciel_v1, {'code': '999', 'system': ICD10}, 'system'), + ('/orgs/CIEL', 'CIEL/$lookup', {'code': '117053', 'system': ICD10}, 'system')): + response = self.get(operation, base=base, **params) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression], (base, params)) + for base, operation, extra, expression in ( + ('/fhir', global_ciel_v1, [], 'Coding.version'), + ('/orgs/CIEL', 'CIEL/$lookup', [{'name': 'version', 'valueString': 'v1'}], 'Coding.version'), + ('/orgs/CIEL', 'CIEL/$lookup', [{'name': 'system', 'valueUri': ICD10}], 'system')): + coding = {'system': CIEL, 'code': '999', 'version': 'v2' if expression.endswith('version') else 'v1'} + response = self.post(operation, [{'name': 'coding', 'valueCoding': coding}, *extra], base=base) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression], (base, extra)) + # the instance's own version, named any of these ways, is answered + response = self.get(global_ciel_v1, code='999', version='v1', system=f'{CIEL}|v1') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + response = self.post('CIEL/$lookup', [{'name': 'coding', 'valueCoding': { + 'system': CIEL, 'code': '999', 'version': 'v1'}}], base='/orgs/CIEL') + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + + def test_an_owner_instance_serves_head_when_named(self): + """ + At an owner base, CodeSystem/{id}/$lookup serves HEAD when version=HEAD names it, as the type level does + (design D6), with HEAD's visibility rule; a global id names a release, so HEAD there is a conflict (Codex F3.1). + """ + ConceptFactory(parent=self.ciel, mnemonic='H1', names=[name('In HEAD only', preferred=True)]) + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='H1', version='HEAD') + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual((answer['version']['valueString'], answer['display']['valueString']), ('HEAD', 'In HEAD only')) + self.assert_outcome(self.get('CIEL/$lookup', base='/orgs/CIEL', code='H1'), 404, 'not-found', 'invalid-code') + response = self.get('hiv/$lookup', base='/orgs/MOH', user=self.moh_member, code='P1', version='HEAD') + self.assertEqual(self.by_name(response)['version']['valueString'], 'HEAD') + hidden = self.get('hiv/$lookup', base='/orgs/MOH', user=self.outsider, code='P1', version='HEAD') + self.assertEqual(hidden.status_code, 404) + self.hiv.is_active = False + self.hiv.save() + self.assertEqual(self.get('hiv/$lookup', base='/orgs/MOH', code='P1', version='HEAD').content, hidden.content) + response = self.get(f'{self.ciel_v1.id}/$lookup', code='999', version='HEAD') + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], ['version']) + + def test_a_deleted_concept_version_is_not_found(self): + """A concept version deleted (is_active false) is gone, a retired one stays (Codex F1.4).""" + ConceptFactory(parent=self.ciel, mnemonic='555', sources=[self.ciel_v2], is_active=False, + names=[name('Deleted', preferred=True)]) + self.assert_outcome(self.lookup(system=CIEL, code='555'), 404, 'not-found', 'invalid-code') + + def test_an_owner_instance_serves_a_named_unreleased_version(self): + draft = source(self.ciel_org, 'CIEL', 'v3', canonical_url=CIEL) + ConceptFactory(parent=self.ciel, mnemonic='D1', sources=[draft], + names=[name('In v3 only', preferred=True)]) + + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='D1', version='v3') + + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual((answer['version']['valueString'], answer['display']['valueString']), ('v3', 'In v3 only')) + self.assert_outcome(self.get('CIEL/$lookup', base='/orgs/CIEL', code='D1'), + 404, 'not-found', 'invalid-code') + self.assert_outcome(self.get(f'{draft.id}/$lookup', code='D1', version='v3'), + 404, 'not-found', 'not-found') + response = self.client_for().get('/orgs/CIEL/CodeSystem/CIEL/', {'version': 'v3'}) + self.assert_outcome(response, 404, 'not-found', 'not-found') + + def test_a_private_unreleased_instance_version_is_visible_to_its_owner(self): + draft = source(self.ciel_org, 'CIEL', 'v3', canonical_url=CIEL, public_access=ACCESS_TYPE_NONE) + ConceptFactory(parent=self.ciel, mnemonic='D1', sources=[draft], + names=[name('In v3 only', preferred=True)]) + self.ciel_org.members.add(self.moh_member) + for user in (None, self.outsider): + hidden = self.get('CIEL/$lookup', base='/orgs/CIEL', user=user, code='D1', version='v3') + missing = self.get('CIEL/$lookup', base='/orgs/CIEL', user=user, code='D1', version='missing') + self.assert_outcome(hidden, 404, 'not-found', 'not-found') + self.assert_outcome(missing, 404, 'not-found', 'not-found') + self.assertEqual(hidden.content, missing.content) + + response = self.get('CIEL/$lookup', base='/orgs/CIEL', user=self.moh_member, code='D1', version='v3') + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual((answer['version']['valueString'], answer['display']['valueString']), ('v3', 'In v3 only')) + self.assert_outcome(self.get('CIEL/$lookup', base='/orgs/CIEL', user=self.moh_member, + code='D1', version='missing'), 404, 'not-found', 'not-found') + + def test_a_named_unreleased_instance_version_without_a_canonical_takes_its_repository_canonical(self): + draft = source(self.ciel_org, 'CIEL', 'v3') + ConceptFactory(parent=self.ciel, mnemonic='D1', sources=[draft], + names=[name('In v3 only', preferred=True)]) + + response = self.get('CIEL/$lookup', base='/orgs/CIEL', system=CIEL, code='D1', version='v3') + + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual( + (answer['system']['valueUri'], answer['version']['valueString'], answer['display']['valueString']), + (CIEL, 'v3', 'In v3 only')) + + def test_a_named_unreleased_instance_version_with_its_own_canonical_takes_its_repository_canonical_too(self): + draft = source(self.ciel_org, 'CIEL', 'v3', canonical_url='https://example.org/old-ciel') + ConceptFactory(parent=self.ciel, mnemonic='D1', sources=[draft], + names=[name('In v3 only', preferred=True)]) + + for system in (CIEL, 'https://example.org/old-ciel'): + response = self.get('CIEL/$lookup', base='/orgs/CIEL', system=system, code='D1', version='v3') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['display']['valueString'], 'In v3 only') + + def test_a_version_named_with_a_path_takes_its_repository_canonical(self): + draft = source(self.ciel_org, 'CIEL', 'v3') + ConceptFactory(parent=self.ciel, mnemonic='D1', sources=[draft], + names=[name('In v3 only', preferred=True)]) + + for params in ({'system': '/orgs/CIEL/CodeSystem/CIEL/', 'version': 'v3'}, + {'system': '/orgs/CIEL/CodeSystem/CIEL/v3/'}): + response = self.lookup(code='D1', **params) + + self.assertEqual(response.status_code, 200, response.data) + answer = self.by_name(response) + self.assertEqual((answer['system']['valueUri'], answer['version']['valueString']), (CIEL, 'v3')) + + def test_a_released_owner_instance_version_uses_three_queries(self): + with self.assertNumQueries(3): + response = self.get('CIEL/$lookup', base='/orgs/CIEL', code='999', version='v1') + + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(self.by_name(response)['version']['valueString'], 'v1') + + def test_a_fixed_number_of_queries(self): + """Resolution (the toggle, the repository, its release; the registry adds its entry and namespace owner), the + concept and its names, however many concepts the code system holds.""" + count = 7 if self.registry else 5 + with self.assertNumQueries(count): + self.lookup(system=CIEL, code='117053') + for number in range(30): + ConceptFactory(parent=self.ciel, mnemonic=f'x{number}', sources=[self.ciel_v2], names=[name(f'x {number}')]) + with self.assertNumQueries(count): + self.lookup(system=CIEL, code='117053') + with self.assertNumQueries(count - 1): # no names to read + self.lookup(system=CIEL, code='nope') + + +class LookupWithoutRegistryTest(LookupCases, OCLTestCase): + registry = False + + +class LookupWithRegistryTest(LookupCases, OCLTestCase): + registry = True diff --git a/core/fhir/tests_ops_validate_code.py b/core/fhir/tests_ops_validate_code.py new file mode 100644 index 000000000..00234b0da --- /dev/null +++ b/core/fhir/tests_ops_validate_code.py @@ -0,0 +1,571 @@ +""" +CodeSystem/$validate-code and ValueSet/$validate-code (SVCM ITI-99, FHIR R4 $validate-code) at the global base /fhir/ +and at owner bases, GET and POST, at the type and instance levels, with the URL registry toggle off and on: the +answer's code, system, version used and display; FHIRsmith's issues and message ids (unknown system or version, unknown +code, wrong display, inactive, not in the value set); value set membership from the resolved release's default +expansion; a value set that doesn't resolve is a 422 OperationOutcome; whatever the caller isn't shown answers as +missing; a fixed number of queries. OpenConceptLab/ocl_online#451. +""" +from django.db import connection +from django.test.utils import CaptureQueriesContext + +from core.collections.models import Collection +from core.collections.tests.factories import ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE +from core.common.tests import OCLTestCase +from core.concepts.tests.factories import ConceptFactory +from core.fhir.outcomes import MESSAGE_ID_EXTENSION +from core.fhir.tests_ops_lookup import ( + OpsCorpusMixin, name, CIEL, ICD10, HIV, REST, UNKNOWN, FEVERS, SHARED) +from core.fhir.tests_search import source, collection +from core.orgs.tests.factories import OrganizationFactory +from core.url_registry.factories import GlobalURLRegistryFactory + +DRAFT_ONLY = 'http://example.org/fhir/CodeSystem/head-only' +GONE = 'http://example.org/fhir/CodeSystem/gone' +DRAFT_UNRELEASED = 'http://example.org/fhir/CodeSystem/unreleased' +RENAMED = 'http://example.org/fhir/CodeSystem/renamed' +RENAMED_OLD = 'http://example.org/fhir/CodeSystem/renamed-old' + + +def issues_of(parameters): + """The `issues` OperationOutcome's issues as severity|code|tx-issue-type|expression|message id.""" + outcome = parameters.get('issues', {}).get('resource', {'issue': []}) + return sorted( + '|'.join([item['severity'], item['code'], item['details']['coding'][0]['code'], + ','.join(item.get('expression', [])), + ','.join(e['valueString'] for e in item.get('extension', []) if e['url'] == MESSAGE_ID_EXTENSION)]) + for item in outcome['issue']) + + +class ValidateCodeCases(OpsCorpusMixin): + def validate(self, resource='CodeSystem', base='/fhir', user=None, **params): + return self.get('$validate-code', base=base, user=user, resource=resource, **params) + + def answer(self, response): + """{name: value} of a Parameters answer, the issues as issues_of has them.""" + self.assertEqual(response.status_code, 200, response.data) + self.assertEqual(response.data['parameter'][0]['name'], 'result') + parameters = self.by_name(response) + values = {key: next(value for name_, value in parameter.items() if name_ not in ('name',)) + for key, parameter in parameters.items() if key != 'issues'} + if 'issues' in parameters: + values['issues'] = issues_of(parameters) + self.assertIn('message', values) + return values + + # CodeSystem + + def test_a_valid_code_answers_its_code_system_version_and_display(self): + self.assertEqual(self.answer(self.validate(url=CIEL, code='117053')), { + 'result': True, 'system': CIEL, 'code': '117053', 'version': 'v2', 'display': 'Infectious enteritis'}) + self.assertEqual(self.answer(self.validate(url=CIEL, code='999', version='v1'))['version'], 'v1') + self.assertEqual(self.answer(self.validate(url=ICD10, code='S20.4', version='ICD102019'))['result'], True) + + def test_a_display_is_one_of_the_concepts_names_in_any_case(self): + for display in ('Infectious enteritis', 'INFECTIOUS ENTERITIS', 'Entérite infectieuse', + 'Enteritis of infectious origin'): + self.assertTrue(self.answer(self.validate(url=CIEL, code='117053', display=display))['result'], display) + self.assertEqual(self.answer(self.validate(url=CIEL, code='117053', display='Not it')), { + 'result': False, 'system': CIEL, 'code': '117053', 'version': 'v2', 'display': 'Infectious enteritis', + 'message': self.answer(self.validate(url=CIEL, code='117053', display='Not it'))['message'], + 'issues': ['error|invalid|invalid-display|display|Display_Name_for__should_be_one_of__instead_of']}) + # a concept without names has none to check a display against + self.assertEqual(self.answer(self.validate(url=REST, code='batch', display='Anything')), { + 'result': True, 'system': REST, 'code': 'batch', 'version': '4.0.1'}) + + def test_a_retired_code_is_valid_and_inactive_with_a_warning(self): + answer = self.answer(self.validate(url=CIEL, code='103')) + self.assertEqual((answer['result'], answer['inactive'], answer['issues']), + (True, True, ['warning|business-rule|code-comment|code|INACTIVE_CONCEPT_FOUND'])) + + def test_an_unknown_code(self): + answer = self.answer(self.validate(url=CIEL, code='nope')) + self.assertEqual((answer['result'], answer['version'], answer['issues']), + (False, 'v2', ['error|code-invalid|invalid-code|code|Unknown_Code_in_Version'])) + + def test_an_unknown_system_or_version_with_or_without_a_code(self): + self.assertEqual(self.answer(self.validate(url=UNKNOWN, code='1')), { + 'result': False, 'system': UNKNOWN, 'code': '1', 'x-caused-by-unknown-system': UNKNOWN, + 'message': f"A definition for CodeSystem '{UNKNOWN}' could not be found, so the code cannot be validated", + 'issues': ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM']}) + answer = self.answer(self.validate(url=CIEL, code='117053', version='nope')) + self.assertEqual((answer['version'], answer['x-caused-by-unknown-system'], answer['issues']), + ('nope', f'{CIEL}|nope', ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM_VERSION'])) + self.assertNotIn('code', self.answer(self.validate(url=UNKNOWN))) + answer = self.answer(self.validate(url=f'{CIEL}|nope', code='117053')) + self.assertEqual((answer['x-caused-by-unknown-system'], answer['issues']), + (f'{CIEL}|nope', ['error|not-found|not-found|system|UNKNOWN_CODESYSTEM_VERSION'])) + + def test_a_code_system_the_caller_is_not_shown_answers_as_a_missing_one(self): + member = self.validate(base='/orgs/MOH', user=self.moh_member, url=HIV, code='P1') + self.assertTrue(self.answer(member)['result']) + hidden = [self.validate(base=base, user=user, url=HIV, **code) + for base, user in (('/fhir', None), ('/fhir', self.moh_member), ('/orgs/MOH', None), + ('/orgs/MOH', self.outsider)) + for code in ({'code': 'P1'}, {'code': 'nope'}, {})] + self.hiv.is_active = False + self.hiv.save() + missing = {code: self.validate(url=HIV, **({'code': code} if code else {})).content + for code in ('P1', 'nope', None)} + for response, code in zip(hidden, ['P1', 'nope', None] * 4): + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, missing[code]) + + def test_a_private_copy_of_a_canonical_does_not_hide_the_public_one(self): + self.assertEqual(self.answer(self.validate(url=SHARED, code='S1'))['display'], 'Public copy') + + def test_post_and_a_coding(self): + response = self.post('$validate-code', [ + {'name': 'url', 'valueUri': CIEL}, {'name': 'code', 'valueString': '117053'}, + {'name': 'display', 'valueString': 'Infectious enteritis'}]) + self.assertTrue(self.answer(response)['result']) + response = self.post('$validate-code', [{'name': 'coding', 'valueCoding': { + 'system': CIEL, 'code': '117053', 'version': 'v1', 'display': 'Wrong'}}]) + answer = self.answer(response) + self.assertEqual((answer['version'], answer['issues']), ('v1', [ + 'error|invalid|invalid-display|Coding.display|Display_Name_for__should_be_one_of__instead_of'])) + response = self.validate(coding=f'{CIEL}|103') + self.assertTrue(self.answer(response)['inactive']) + + def test_a_codeable_concept_and_inputs_that_are_not_text(self): + response = self.post('$validate-code', [{'name': 'codeableConcept', 'valueCodeableConcept': { + 'coding': [{'system': CIEL, 'code': '117053', 'display': 'Infectious enteritis'}]}}]) + self.assertTrue(self.answer(response)['result']) + response = self.post('$validate-code', [{'name': 'codeableConcept', 'valueCodeableConcept': { + 'coding': [{'system': CIEL, 'code': '117053'}, {'system': ICD10, 'code': 'S20.4'}]}}]) + self.assert_outcome(response, 400, 'not-supported', 'not-supported') + self.assertEqual(response.data['issue'][0]['expression'], ['codeableConcept']) + for parameter, expression in (({'name': 'url', 'valueBoolean': True}, 'url'), + ({'name': 'coding', 'valueCoding': {'system': CIEL, 'code': 7}}, 'coding')): + response = self.post('$validate-code', [parameter, {'name': 'code', 'valueCode': '117053'}]) + self.assert_outcome(response, 400, 'invalid', 'invalid-data') + self.assertEqual(response.data['issue'][0]['expression'], [expression]) + + def test_a_code_stored_percent_encoded(self): + """OCL stores a code with reserved characters percent-encoded (ICD-11's ND53.Y&XK70); FHIR names it as it is.""" + ConceptFactory(parent=self.ciel, mnemonic='ND53.Y%26XK70', sources=[self.ciel_v2], + names=[name('Encoded', preferred=True)]) + self.assertEqual(self.answer(self.validate(url=CIEL, code='ND53.Y&XK70')), { + 'result': True, 'system': CIEL, 'code': 'ND53.Y&XK70', 'version': 'v2', 'display': 'Encoded'}) + + def test_the_instance_level(self): + response = self.get(f'{self.ciel_v1.id}/$validate-code', code='999') + self.assertEqual(self.answer(response)['version'], 'v1') + response = self.get('CIEL/$validate-code', base='/orgs/CIEL', code='117053', display='Nope') + self.assertFalse(self.answer(response)['result']) + + def test_xml(self): + response = self.client_for().get( + f'/fhir/CodeSystem/$validate-code?url={UNKNOWN}&code=1&_format=xml') + self.assertEqual(response.status_code, 200) + self.assertIn(b'', response.content) + self.assertIn(b'