From 11e9c1278d3d167fd138d7a33144ded9eb36f714 Mon Sep 17 00:00:00 2001 From: Filipe Lopes Date: Thu, 22 Oct 2026 17:33:36 -0300 Subject: [PATCH] feat(resolvers): implement post-coordinated name resolution * Introduce a dedicated mechanism to resolve display names for concepts and mappings, including complex compositions * This allows the system to display human-readable names derived from multiple components (e.g., a pattern and a placeholder) in a single code * The resolution process is cacheable and is designed to fall back to the stored/original code name if a part of the composition is missing --- core/concepts/serializers.py | 10 +- core/integration_tests/test_utils.py | 5 +- core/integration_tests/tests_mappings.py | 64 ++++++ core/integration_tests/tests_sources.py | 43 +++- core/mappings/models.py | 15 ++ core/mappings/name_resolution.py | 110 ++++++++++ core/mappings/serializers.py | 32 ++- core/resolvers/__init__.py | 0 core/resolvers/core.py | 153 ++++++++++++++ core/resolvers/names.py | 194 ++++++++++++++++++ core/resolvers/tests.py | 93 +++++++++ .../migrations/0047_source_resolvers.py | 18 ++ core/sources/models.py | 4 + core/sources/serializers.py | 17 +- core/sources/views.py | 1 + 15 files changed, 739 insertions(+), 20 deletions(-) create mode 100644 core/mappings/name_resolution.py create mode 100644 core/resolvers/__init__.py create mode 100644 core/resolvers/core.py create mode 100644 core/resolvers/names.py create mode 100644 core/resolvers/tests.py create mode 100644 core/sources/migrations/0047_source_resolvers.py diff --git a/core/concepts/serializers.py b/core/concepts/serializers.py index 9a9fc8493..f289e37bd 100644 --- a/core/concepts/serializers.py +++ b/core/concepts/serializers.py @@ -252,7 +252,7 @@ def get_mappings(self, obj): ) if is_collection else obj.get_bidirectional_mappings() if not include_retired: mappings = mappings.filter(retired=False) - return serializer_class(mappings, many=True).data + return serializer_class(mappings, many=True, context=context).data if self.include_direct_mappings: mappings = obj.get_unidirectional_mappings_for_collection( parent_uri, collection_version) if is_collection else obj.get_unidirectional_mappings() @@ -262,7 +262,7 @@ def get_mappings(self, obj): mappings = mappings.filter(to_source_url__in=target_repo_urls) if not include_retired: mappings = mappings.filter(retired=False) - return serializer_class(mappings, many=True).data + return serializer_class(mappings, many=True, context=context).data return [] @@ -458,9 +458,11 @@ def get_entries(self, obj): from core.mappings.serializers import MappingMinimalSerializer, MappingReverseMinimalSerializer if get(self, 'context.request.query_params.reverse') in TRUTHY: - result += MappingReverseMinimalSerializer(obj.cascaded_entries['mappings'], many=True).data + result += MappingReverseMinimalSerializer( + obj.cascaded_entries['mappings'], many=True, context=self.context).data else: - result += MappingMinimalSerializer(obj.cascaded_entries['mappings'], many=True).data + result += MappingMinimalSerializer( + obj.cascaded_entries['mappings'], many=True, context=self.context).data return result diff --git a/core/integration_tests/test_utils.py b/core/integration_tests/test_utils.py index f5b23c724..2e0cac918 100644 --- a/core/integration_tests/test_utils.py +++ b/core/integration_tests/test_utils.py @@ -53,6 +53,7 @@ class SourceChildListQueryCountMixin: relation = None list_serializer = None detail_serializer = None + small_page_size = 2 # the smaller page must hold the same kinds of rows as the page of 4 def get_page(self, url, limit, params=''): with CaptureQueriesContext(connection) as context: @@ -62,8 +63,8 @@ def get_page(self, url, limit, params=''): return response, len(context.captured_queries) def assert_page(self, url, serializer_class, params=''): - """Same query count for a page of 2 and of 4; same output as serializing each row on its own.""" - _, small_page_queries = self.get_page(url, 2, params) + """Same query count for a small page and a page of 4; same output as serializing each row on its own.""" + _, small_page_queries = self.get_page(url, self.small_page_size, params) response, page_queries = self.get_page(url, 4, params) self.assertEqual(page_queries, small_page_queries) diff --git a/core/integration_tests/tests_mappings.py b/core/integration_tests/tests_mappings.py index db7e9a022..32e98b212 100644 --- a/core/integration_tests/tests_mappings.py +++ b/core/integration_tests/tests_mappings.py @@ -3,6 +3,7 @@ from rest_framework.exceptions import ErrorDetail from core.collections.tests.factories import OrganizationCollectionFactory, ExpansionFactory +from core.common.constants import ACCESS_TYPE_NONE from core.common.tests import OCLAPITestCase from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory from core.integration_tests.test_utils import SourceChildListQueryCountMixin @@ -12,6 +13,7 @@ from core.mappings.serializers import MappingListSerializer, MappingDetailSerializer from core.mappings.tests.factories import MappingFactory from core.orgs.tests.factories import OrganizationFactory +from core.resolvers.tests import CONCEPTS, RESOLVERS, make_concept, make_source from core.sources.tests.factories import UserSourceFactory, OrganizationSourceFactory from core.users.models import UserProfile from core.users.tests.factories import UserProfileFactory @@ -1086,6 +1088,9 @@ class MappingListQueryCountTest(SourceChildListQueryCountMixin, OCLAPITestCase): relation = 'mappings' list_serializer = MappingListSerializer detail_serializer = MappingDetailSerializer + # A source version lists a mapping's head and latest version, which are the same kind of row. Unlinked (external) + # mappings cost one batched name lookup per page, so the smaller page needs 3 rows to hold both kinds as well. + small_page_size = 3 def setUp(self): super().setUp() @@ -1106,3 +1111,62 @@ def setUp(self): for mapping in mappings: mapping.set_checksums() # a GET calculates and saves missing checksums, one row at a time self.set_released_versions(mappings) + + +class MappingResolvedConceptNameTest(OCLAPITestCase): + """A mapping to a post-coordinated code shows the name its destination Source resolves.""" + CODE = 'ND13.2%26XJ44E%2FND13.3%26XJ44E' # as stored: percent-encoded once + NAME = 'Fracture of unspecified tarsal bone [Closed fracture] / Fracture of metatarsal bone [Closed fracture]' + + def setUp(self): + super().setUp() + self.origin = OrganizationSourceFactory() + self.origin_concept = make_concept(self.origin, 'O1', 'Origin one') + self.destination = make_source(RESOLVERS, CONCEPTS) + + def make_mapping(self, code, destination=None): + """A mapping whose destination code is not linked to a concept, as post-coordinated codes are.""" + destination = destination or self.destination + return MappingFactory( + parent=self.origin, from_concept=self.origin_concept, from_source=self.origin, + from_source_url=self.origin.uri, to_concept=None, to_source=destination, + to_source_url=destination.uri, to_concept_code=code, to_concept_name='Stored name') + + def get(self, url, user=None, **params): + headers = {'HTTP_AUTHORIZATION': 'Token ' + user.get_token()} if user else {} + response = self.client.get(url, params, **headers) + self.assertEqual(response.status_code, 200, response.content) + return response.data + + def test_every_representation_shows_the_resolved_name(self): + mapping = self.make_mapping(self.CODE) + + for data in (self.get(self.origin.uri + 'mappings/')[0], self.get(mapping.uri), + self.get(mapping.uri + 'versions/')[0]): + self.assertEqual(data['to_concept_name_resolved'], self.NAME) + self.assertEqual(data['to_concept_name'], 'Stored name') # the stored value is untouched + self.assertIsNone(data['to_concept_url']) # no concept, no url + brief = self.get(self.origin.uri + 'mappings/', brief='true') + self.assertEqual(brief[0]['cascade_target_concept_name'], self.NAME) + cascade = self.get(self.origin_concept.uri + '$cascade/', method='sourceMappings', cascadeLevels='1') + entries = [entry for entry in cascade['entry'] if entry.get('cascade_target_concept_code')] + self.assertEqual([entry['cascade_target_concept_name'] for entry in entries], [self.NAME]) + + def test_the_stored_name_is_kept_when_a_part_is_missing(self): + mapping = self.make_mapping('ND13.2%26NOPE') + + # as before the resolver: no linked concept, so the field is omitted + self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri)) + brief = self.get(self.origin.uri + 'mappings/', brief='true') + self.assertEqual(brief[0]['cascade_target_concept_name'], 'Stored name') + + def test_a_private_destination_is_resolved_only_for_users_who_can_see_it(self): + organization = OrganizationFactory() + private = make_source(RESOLVERS, CONCEPTS, organization=organization, public_access=ACCESS_TYPE_NONE) + mapping = self.make_mapping(self.CODE, private) + member = UserProfileFactory() + organization.members.add(member) + + self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri)) # omitted, as without a resolver + self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri, UserProfileFactory())) + self.assertEqual(self.get(mapping.uri, member)['to_concept_name_resolved'], self.NAME) diff --git a/core/integration_tests/tests_sources.py b/core/integration_tests/tests_sources.py index d4361f395..67d65d428 100644 --- a/core/integration_tests/tests_sources.py +++ b/core/integration_tests/tests_sources.py @@ -28,6 +28,7 @@ from core.mappings.serializers import MappingVersionExportSerializer from core.mappings.tests.factories import MappingFactory from core.orgs.models import Organization +from core.resolvers.tests import RESOLVERS from core.sources.models import Source, CloneLimitExceeded from core.sources.serializers import SourceDetailSerializer, SourceVersionExportSerializer from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory @@ -196,7 +197,7 @@ def test_post_201(self): 'autoid_concept_name_external_id', 'autoid_concept_description_external_id', 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from', 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'checksums', - 'properties', 'filters', 'match_algorithms' + 'properties', 'filters', 'match_algorithms', 'resolvers' ]) ) source = Source.objects.last() @@ -318,7 +319,7 @@ def test_put_200(self): 'autoid_concept_name_external_id', 'autoid_concept_description_external_id', 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from', 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'checksums', - 'properties', 'filters', 'match_algorithms' + 'properties', 'filters', 'match_algorithms', 'resolvers' ]) ) source = Source.objects.last() @@ -332,6 +333,28 @@ def test_put_200(self): self.assertEqual(source.default_locale, 'en') self.assertEqual(source.supported_locales, ['fr']) + + def test_put_resolvers(self): + source = OrganizationSourceFactory(organization=self.organization) + + response = self.client.put( + source.uri, {'resolvers': RESOLVERS}, HTTP_AUTHORIZATION='Token ' + self.token, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['resolvers'], RESOLVERS) + + response = self.client.put( + source.uri, {'resolvers': {'names': {'replace': []}}}, HTTP_AUTHORIZATION='Token ' + self.token, + format='json') + self.assertEqual(response.status_code, 400) + self.assertIn('resolvers', response.data) + source.refresh_from_db() + self.assertEqual(source.resolvers, RESOLVERS) + + response = self.client.put( + source.uri, {'resolvers': None}, HTTP_AUTHORIZATION='Token ' + self.token, format='json') + self.assertEqual(response.status_code, 200) + self.assertIsNone(response.data['resolvers']) + def test_put_hierarchy_root(self): source = OrganizationSourceFactory(organization=self.organization) self.assertTrue(source.is_head) @@ -477,9 +500,10 @@ def test_post_201(self): @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) - def test_post_201_copies_properties_and_filters_from_head(self): + def test_post_201_copies_properties_filters_and_resolvers_from_head(self): self.source.properties = [{'code': 'p1'}] self.source.filters = [{'code': 'f1'}] + self.source.resolvers = RESOLVERS self.source.save() response = self.client.post( @@ -495,10 +519,12 @@ def test_post_201_copies_properties_and_filters_from_head(self): self.assertEqual(response.status_code, 201) self.assertEqual(response.data['properties'], [{'code': 'p1'}]) self.assertEqual(response.data['filters'], [{'code': 'f1'}]) + self.assertEqual(response.data['resolvers'], RESOLVERS) new_version = self.source.versions.get(version='v1') self.assertEqual(new_version.properties, [{'code': 'p1'}]) self.assertEqual(new_version.filters, [{'code': 'f1'}]) + self.assertEqual(new_version.resolvers, RESOLVERS) @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) @@ -598,6 +624,17 @@ def test_post_400_released_invalid_string(self): response.data, {'released': [ErrorDetail(string='Must be a valid boolean.', code='invalid')]}) self.assertEqual(self.source.versions.count(), 1) + @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts')) + @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings')) + def test_post_400_invalid_resolvers(self): + response = self.client.post( + self.source.uri + 'versions/', {'id': 'v1', 'resolvers': {'names': {'replace': []}}}, + HTTP_AUTHORIZATION='Token ' + self.token, format='json') + + self.assertEqual(response.status_code, 400) + self.assertIn('resolvers', response.data) + self.assertFalse(self.source.versions.filter(version='v1').exists()) + @patch('core.sources.views.export_source') def test_post_400(self, export_source_mock): response = self.client.post( diff --git a/core/mappings/models.py b/core/mappings/models.py index e183d8370..d496021ee 100644 --- a/core/mappings/models.py +++ b/core/mappings/models.py @@ -16,6 +16,7 @@ from core.mappings.constants import MAPPING_TYPE, MAPPING_IS_ALREADY_RETIRED, MAPPING_WAS_RETIRED, \ MAPPING_IS_ALREADY_NOT_RETIRED, MAPPING_WAS_UNRETIRED, ALREADY_EXISTS from core.mappings.mixins import MappingValidationMixin +from core.mappings.name_resolution import load_resolved_names, SIDES from core.services.storages.postgres import PostgresQL @@ -25,6 +26,12 @@ class Mapping(MappingValidationMixin, SourceChildMixin, VersionedModel): 'to_source__organization', 'to_source__user', 'created_by', 'updated_by', 'from_concept__names', 'to_concept__names' ) + # serializer fields showing a resolved concept name, and the mapping sides (from/to) each one reads + NAME_RESOLUTION_FIELDS = { + 'from_concept_name_resolved': ('from', ), + 'to_concept_name_resolved': ('to', ), + 'cascade_target_concept_name': SIDES, # the cascade direction is a serializer choice; the unused side is cheap + } class Meta: db_table = 'mappings' @@ -323,6 +330,14 @@ def create_initial_version(cls, mapping, **kwargs): initial_version.save() return initial_version + @classmethod + def load_list_fields(cls, instances, fields, collection=None, user=None): + """Besides the generic page lookups, resolves the concept names the serialized fields will show.""" + super().load_list_fields(instances, fields, collection, user) + sides = {side for field in fields for side in cls.NAME_RESOLUTION_FIELDS.get(field, ())} + if sides: + load_resolved_names(instances, [side for side in SIDES if side in sides], user) + def populate_fields_from_relations( # pylint: disable=too-many-locals,too-many-statements self, data, cache=None, user=None): """Concepts and repos the user can't view are not linked; they are kept as external codes/URLs.""" diff --git a/core/mappings/name_resolution.py b/core/mappings/name_resolution.py new file mode 100644 index 000000000..e63dafec9 --- /dev/null +++ b/core/mappings/name_resolution.py @@ -0,0 +1,110 @@ +""" +Adapter between Mappings and the ``names`` resolver (``core.resolvers.names``). + +Picks, for each side (`from` / `to`) of a mapping, the Source (version) its code must be resolved against and projects +the resolver result onto the mapping. The grammar and the lookups live in the resolver; nothing here is persisted. +""" +from pydash import get + +from core.common.constants import HEAD +from core.common.utils import decode_string +from core.resolvers.core import user_key +from core.resolvers.names import NamesResolver + +SIDES = ('from', 'to') +RESOLVED_NAMES_ATTR = '_resolved_concept_names' + + +def context_user(context): + """The user of a serializer context. Without one the caller is anonymous: access is never elevated.""" + return get(context, 'request.user') or get(context, 'user') + + +def _find_source(mapping, side, cache): + """ + The Source (version) the code of `side` belongs to: the declared version when there is one, otherwise HEAD. + The stored FK / URL identify the repo; a version declared in the mapping always wins over the HEAD FK. + Returns None when the repo cannot be identified locally. + """ + from core.sources.models import Source + version = getattr(mapping, f'{side}_source_version') or HEAD + source_id = getattr(mapping, f'{side}_source_id') + if source_id: + head = getattr(mapping, f'{side}_source') + if version == HEAD: + return head + key = ('version', head.id, version) + if key not in cache: + cache[key] = head.active_versions.filter(version=version).first() + return cache[key] + + url = getattr(mapping, f'{side}_source_url') + if not url: + return None + key = ('url', url, version) + if key not in cache: + source, _ = Source.resolve_reference_expression(url, None, version) + cache[key] = source if source.id else None + return cache[key] + + +def _target(mapping, side, cache): + """ + `(source, logical_code)` the side must be resolved against, or None when the stored link already gives the name: + a concept linked to the mapping without a declared source version is the very concept the resolver would find. + """ + code = getattr(mapping, f'{side}_concept_code') + if not code: + return None + declared_version = getattr(mapping, f'{side}_source_version') + if getattr(mapping, f'{side}_concept_id') and (not declared_version or declared_version == HEAD): + return None + source = _find_source(mapping, side, cache) + # stored codes are percent-encoded once; the resolver works on logical codes + return (source, decode_string(code, plus=False)) if source else None + + +def _resolve_targets(targets, user): + """Resolved names keyed by `(source id, code)`: one resolver call per distinct target Source (version).""" + wanted = {} # source id -> (source, {code}) + for target in targets: + if target: + source, code = target + wanted.setdefault(source.id, (source, set()))[1].add(code) + + resolved = {} + resolver_cache = {} + for source_id, (source, codes) in wanted.items(): + ordered_codes = sorted(codes) + for code, name in zip(ordered_codes, NamesResolver(source, user, resolver_cache).resolve(ordered_codes)): + resolved[(source_id, code)] = name + return resolved + + +def load_resolved_names(mappings, sides, user): + """ + Resolves, for a whole page, the names of the given sides and keeps them on each mapping. + Not saved anywhere; scoped to the user it was computed for. + """ + cache = {} + targets = [(mapping, side, _target(mapping, side, cache)) for mapping in mappings for side in sides] + resolved = _resolve_targets([target for _, _, target in targets], user) + + owner = user_key(user) + for mapping, side, target in targets: + names = mapping.__dict__.get(RESOLVED_NAMES_ATTR) + if not names or names['user'] != owner: + names = mapping.__dict__[RESOLVED_NAMES_ATTR] = {'user': owner} + names[side] = resolved.get((target[0].id, target[1])) if target else None + + +def get_resolved_name(mapping, side, user): + """ + The name of the `side` concept computed by the resolver (exact match or composition), or None when the serializer + must keep its stored/legacy value. Loads the single mapping on demand when no page loader prepared it. + """ + names = mapping.__dict__.get(RESOLVED_NAMES_ATTR) + if not names or names['user'] != user_key(user) or side not in names: + load_resolved_names([mapping], (side,), user) + names = mapping.__dict__[RESOLVED_NAMES_ATTR] + return names[side] diff --git a/core/mappings/serializers.py b/core/mappings/serializers.py index 35dd4767f..d7b40a04e 100644 --- a/core/mappings/serializers.py +++ b/core/mappings/serializers.py @@ -12,12 +12,26 @@ from core.common.utils import get_truthy_values from core.concepts.serializers import ConceptDetailSerializer from core.mappings.models import Mapping +from core.mappings.name_resolution import get_resolved_name, context_user from core.sources.serializers import SourceListSerializer, SourceDetailSerializer TRUTHY = get_truthy_values() +class ResolvedConceptNameField(CharField): + """ + Name of a mapping side's concept: the name the Source name resolver computes for the stored code (exact concept or + post-coordinated composition) and, when it has none, the linked concept's display name as before. + """ + def __init__(self, side, **kwargs): + self.side = side + super().__init__(source=f'{side}_concept.display_name', read_only=True, **kwargs) + + def get_attribute(self, instance): + return get_resolved_name(instance, self.side, context_user(self.context)) or super().get_attribute(instance) + + class AbstractMappingSerializer(SourceVersionFieldMixin, AbstractResourceSerializer): from_concept = SerializerMethodField() to_concept = SerializerMethodField() @@ -113,8 +127,8 @@ class MappingListSerializer(AbstractMappingSerializer): url = CharField(required=False, source='versioned_object_url') version = CharField(read_only=True) version_created_on = DateTimeField(source='created_at', read_only=True) - from_concept_name_resolved = CharField(source='from_concept.display_name', read_only=True) - to_concept_name_resolved = CharField(source='to_concept.display_name', read_only=True) + from_concept_name_resolved = ResolvedConceptNameField('from') + to_concept_name_resolved = ResolvedConceptNameField('to') to_concept_code = EncodedDecodedCharField(required=False) from_concept_code = EncodedDecodedCharField(required=False) sort_weight = FloatField(required=False, allow_null=True) @@ -191,6 +205,7 @@ class MappingMinimalSerializer(AbstractMappingSerializer): class Meta: model = Mapping + list_serializer_class = SourceChildPageSerializer fields = AbstractMappingSerializer.Meta.fields + ( 'id', 'type', 'map_type', 'url', 'version_url', 'to_concept_code', 'to_concept_url', 'cascade_target_concept_code', 'cascade_target_concept_url', 'cascade_target_source_owner', @@ -198,9 +213,9 @@ class Meta: 'from_concept_code', 'from_concept_url' ) - @staticmethod - def get_cascade_target_concept_name(obj): - return obj.to_concept_name or get(obj, 'to_concept.display_name') + def get_cascade_target_concept_name(self, obj): + return get_resolved_name(obj, 'to', context_user(self.context)) or obj.to_concept_name or get( + obj, 'to_concept.display_name') class MappingReverseMinimalSerializer(ModelSerializer): @@ -217,6 +232,7 @@ class MappingReverseMinimalSerializer(ModelSerializer): class Meta: model = Mapping + list_serializer_class = SourceChildPageSerializer fields = ( 'id', 'type', 'map_type', 'url', 'version_url', 'from_concept_code', 'from_concept_url', 'cascade_target_concept_code', 'cascade_target_concept_url', 'cascade_target_source_owner', @@ -224,9 +240,9 @@ class Meta: 'checksums' ) - @staticmethod - def get_cascade_target_concept_name(obj): - return obj.from_concept_name or get(obj, 'from_concept.display_name') + def get_cascade_target_concept_name(self, obj): + return get_resolved_name(obj, 'from', context_user(self.context)) or obj.from_concept_name or get( + obj, 'from_concept.display_name') @staticmethod def get_checksums(obj): diff --git a/core/resolvers/__init__.py b/core/resolvers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/resolvers/core.py b/core/resolvers/core.py new file mode 100644 index 000000000..9f7a6f1de --- /dev/null +++ b/core/resolvers/core.py @@ -0,0 +1,153 @@ +""" +Base of the read-time resolvers a Source declares in ``Source.resolvers`` (one key per resolver family). + +A resolver answers on read, from the content of ONE concrete Source / Source version, and never writes anything. +``names`` is the first family; ``concepts`` and ``mappings`` (see DOCS_RESOLVERS.md) will subclass ``Resolver`` and be +added to ``get_resolver_classes`` when they are built. +""" +import unicodedata + +from django.core.exceptions import ValidationError +from pydash import get + +from core.common.utils import encode_string + +# Server-side safety limit of one batched concept lookup. The Source configuration can never raise it. +MAX_LOOKUP_CHUNK = 500 + + +def get_resolver_classes(): + """The resolver families a Source can declare, by their key in ``Source.resolvers``.""" + from core.resolvers.names import NamesResolver + return {NamesResolver.key: NamesResolver} + + +def validate_resolvers(resolvers): + """ + Validates a ``Source.resolvers`` value: ``None`` (no resolver) or an object whose keys are resolver families, each + holding a valid configuration. Raises ``ValidationError`` keyed by ``resolvers`` naming the first problem found. + """ + if resolvers is None: + return + resolver_classes = get_resolver_classes() + try: + check_keys(resolvers, 'resolvers', resolver_classes) + for key, config in resolvers.items(): + resolver_classes[key].validate(config, f'resolvers.{key}') + except ValueError as ex: + raise ValidationError({'resolvers': [str(ex)]}) from ex + + +def check_keys(value, path, allowed, required=()): + """``value`` must be an object with only ``allowed`` keys and every ``required`` one.""" + if not isinstance(value, dict): + raise ValueError(f'{path} must be an object') + unknown = set(value) - set(allowed) + if unknown: + raise ValueError(f'{path} has unsupported keys: {sorted(unknown)}') + missing = set(required) - set(value) + if missing: + raise ValueError(f'{path} is missing keys: {sorted(missing)}') + + +def check_list(value, path, keys, min_items, max_items): + """``value`` must be a list of ``min_items`` to ``max_items`` objects holding exactly ``keys``.""" + if not isinstance(value, list) or not min_items <= len(value) <= max_items: + raise ValueError(f'{path} must be a list of {min_items} to {max_items} items') + for index, item in enumerate(value): + check_keys(item, f'{path}[{index}]', keys, keys) + + +def check_string(value, path, min_length, max_length, allow_whitespace=True): + """Strict string: no coercion, bounded length, no control characters and, if asked, no whitespace.""" + if not isinstance(value, str): + raise ValueError(f'{path} must be a string') + if not min_length <= len(value) <= max_length: + raise ValueError(f'{path} must have between {min_length} and {max_length} characters') + if any(unicodedata.category(char).startswith('C') for char in value): + raise ValueError(f'{path} must not contain control characters') + if not allow_whitespace and any(char.isspace() for char in value): + raise ValueError(f'{path} must not contain whitespace') + + +def user_key(user): + """Cache key of a user: its id when authenticated, None for an anonymous caller.""" + return get(user, 'id') if get(user, 'is_authenticated') else None + + +class Resolver: + """ + A resolver family bound to one concrete Source (version) and one user. + + ``cache`` is an optional dict owned by the caller (one request, one user). Resolvers sharing it check access and + look up each concept once. + """ + key = None # key of the family in ``Source.resolvers`` + + def __init__(self, source, user, cache=None): + self.source = source + self.user = user + self.cache = {} if cache is None else cache + self.config = self.get_config(source) + + @classmethod + def validate(cls, config, path): + """Raises ``ValueError`` naming the first problem of ``config``; ``path`` prefixes the message.""" + raise NotImplementedError + + @classmethod + def get_config(cls, source): + """The stored config of the family, or None when absent or not valid: a stored invalid value never raises.""" + config = get(source.resolvers, cls.key) if isinstance(source.resolvers, dict) else None + if config is None: + return None + try: + cls.validate(config, cls.key) + except ValueError: + return None + return config + + def has_view_access(self): + """Whether the user can see the Source (version); checked once per cache.""" + key = (self.source.id, user_key(self.user)) + access = self.cache.setdefault('access', {}) + if key not in access: + access[key] = bool(self.source.has_view_access(self.user)) + return access[key] + + @staticmethod + def stored_mnemonics(code): + """ + Concept mnemonics hold reserved characters percent-encoded (``1B1Y%26XK5A``), so a logical code is looked up + as typed and in its stored form. The code is encoded once, never decoded repeatedly. + """ + encoded = encode_string(code, safe=' ') + return (code, ) if encoded == code else (code, encoded) + + def load_concepts(self, codes): + """ + Fills ``cache['concepts'][(source id, mnemonic)]`` for every stored form of the codes, with one batched query + per chunk: None when the Source (version) holds no such active concept, otherwise its name and retired flag. + """ + mnemonics = {mnemonic for code in codes for mnemonic in self.stored_mnemonics(code)} + concepts = self.cache.setdefault('concepts', {}) + pending = sorted({mnemonic for mnemonic in mnemonics if (self.source.id, mnemonic) not in concepts}) + for start in range(0, len(pending), MAX_LOOKUP_CHUNK): + chunk = pending[start:start + MAX_LOOKUP_CHUNK] + queryset = self.source.get_concepts_queryset().filter( + mnemonic__in=chunk, is_active=True + ).select_related('parent').prefetch_related('names') + for concept in queryset: + concepts[(self.source.id, concept.mnemonic)] = { + 'name': concept.display_name, 'retired': concept.retired} + for mnemonic in chunk: + concepts.setdefault((self.source.id, mnemonic), None) + + def find_concept(self, code): + """The loaded concept of a logical code, trying each stored form; None when absent.""" + concepts = self.cache.get('concepts', {}) + for mnemonic in self.stored_mnemonics(code): + concept = concepts.get((self.source.id, mnemonic)) + if concept: + return concept + return None diff --git a/core/resolvers/names.py b/core/resolvers/names.py new file mode 100644 index 000000000..1a051400a --- /dev/null +++ b/core/resolvers/names.py @@ -0,0 +1,194 @@ +""" +``names`` resolver: the display name of a code of a Source (version), including post-coordinated codes. + +1. Exact first: the whole code is looked up as a concept of the Source (version) and its own name is used. +2. Composition, only when the code is not a concept and the Source declares ``resolvers.names``: the code is split by + the configured ``replace`` patterns and ``literals``. Each code becomes the name of its concept, each pattern its + ``substitution`` (where ``{{name}}`` stands for the name of the code or literal right after the pattern) and each + literal its own text without the delimiters. All or nothing: a missing concept or name gives no name, never a + partial one. + +The resolver knows no terminology; each use case is configuration only, e.g.: +- ICD-11 ``ND13.2&XJ44E/ND13.3&XJ44E``: ``&`` -> `` [{{name}}]``, ``/`` -> `` / ``; +- SNOMED CT ``71388002:{260686004=129304002}``: ``:`` -> `` with ``, ``{`` and ``}`` -> ````, ``=`` -> ``: ``; +- LOINC ``104765-3:{Specimen}=LP7681-2``: literal ``{`` ... ``}``, ``:`` -> ``, ``, ``=`` -> ``: ``. + +The result is a computed presentation value: nothing is created or saved. See DOCS_RESOLVERS.md. +""" +from functools import cached_property + +from core.resolvers.core import Resolver, check_keys, check_list, check_string + +NAME_PLACEHOLDER = '{{name}}' + +# Server-side safety limits. The Source configuration can never raise them. +MAX_EXPRESSION_LENGTH = 2048 +MAX_EXPRESSION_COMPONENTS = 64 + +# Limits of the configuration schema. +MAX_REPLACE_RULES = 16 +MAX_LITERALS = 4 +MAX_PATTERN_LENGTH = 8 +MAX_SUBSTITUTION_LENGTH = 32 + +TOKEN_CODE = 'code' +TOKEN_LITERAL = 'literal' +TOKEN_PATTERN = 'pattern' + + +def is_lookup_code(code): + """Only non-blank strings within the expression limit are looked up.""" + return isinstance(code, str) and bool(code.strip()) and len(code) <= MAX_EXPRESSION_LENGTH + + +class NamesResolver(Resolver): + """Display names of codes, composed from the names of their parts when the Source configures it.""" + key = 'names' + + @classmethod + def validate(cls, config, path): + check_keys(config, path, {'replace', 'literals'}, {'replace'}) + check_list(config['replace'], f'{path}.replace', {'pattern', 'substitution'}, 1, MAX_REPLACE_RULES) + patterns = set() + for index, rule in enumerate(config['replace']): + rule_path = f'{path}.replace[{index}]' + check_string(rule['pattern'], f'{rule_path}.pattern', 1, MAX_PATTERN_LENGTH, allow_whitespace=False) + check_string(rule['substitution'], f'{rule_path}.substitution', 0, MAX_SUBSTITUTION_LENGTH) + if rule['substitution'].count(NAME_PLACEHOLDER) > 1: + raise ValueError(f'{rule_path}.substitution may use {NAME_PLACEHOLDER} only once') + if rule['pattern'] in patterns: + raise ValueError(f'{rule_path}.pattern is repeated') + patterns.add(rule['pattern']) + literals = config.get('literals', []) + check_list(literals, f'{path}.literals', {'start', 'end'}, 0, MAX_LITERALS) + for index, literal in enumerate(literals): + for key in ('start', 'end'): + key_path = f'{path}.literals[{index}].{key}' + check_string(literal[key], key_path, 1, MAX_PATTERN_LENGTH, allow_whitespace=False) + if literal[key] in patterns: + raise ValueError(f'{key_path} is also a replace pattern') + + @cached_property + def substitutions(self): + """``{pattern: substitution}`` of the configuration.""" + return {rule['pattern']: rule['substitution'] for rule in self.config['replace']} + + def tokenize(self, expression): + """ + ``[(TOKEN_CODE | TOKEN_LITERAL | TOKEN_PATTERN, text), ...]``, or None when a literal is not closed. At each + position a literal start wins, then the longest pattern; a code is the text in between. Codes and literals + lose their surrounding whitespace; nothing is reordered or case-changed. + """ + patterns = sorted(self.substitutions, key=len, reverse=True) + literals = self.config.get('literals', []) + tokens, buffer, position = [], [], 0 + + def flush(): + code = ''.join(buffer).strip() + if code: + tokens.append((TOKEN_CODE, code)) + buffer.clear() + + while position < len(expression): + literal = next((item for item in literals if expression.startswith(item['start'], position)), None) + pattern = None if literal else next( + (item for item in patterns if expression.startswith(item, position)), None) + if literal: + flush() + end = expression.find(literal['end'], position + len(literal['start'])) + if end < 0: + return None + tokens.append((TOKEN_LITERAL, expression[position + len(literal['start']):end].strip())) + position = end + len(literal['end']) + elif pattern: + flush() + tokens.append((TOKEN_PATTERN, pattern)) + position += len(pattern) + else: + buffer.append(expression[position]) + position += 1 + flush() + return tokens + + def parse(self, expression): + """ + The tokens of a code that can be composed, or None. It must be within the limits, hold a pattern or a literal, + and not start or end with a pattern whose substitution has text. Structural only: no clinical validation. + """ + if len(expression) > MAX_EXPRESSION_LENGTH: + return None + tokens = self.tokenize(expression) + if not tokens or all(kind == TOKEN_CODE for kind, _ in tokens): + return None + if sum(kind != TOKEN_PATTERN for kind, _ in tokens) > MAX_EXPRESSION_COMPONENTS: + return None + for kind, text in (tokens[0], tokens[-1]): + if kind == TOKEN_PATTERN and self.substitutions[text]: + return None + return tokens + + def render(self, tokens, names): + """ + Joins the name: a code becomes ``names[code]``, a literal its text, a pattern its substitution, and + ``{{name}}`` takes the name of the token right after the pattern. None (all or nothing) when a code or literal + has no name or ``{{name}}`` is not followed by a code or literal. + """ + def value(token): + kind, text = token + return text if kind == TOKEN_LITERAL else names.get(text) if kind == TOKEN_CODE else None + + parts, position = [], 0 + while position < len(tokens): + kind, text = tokens[position] + if kind == TOKEN_PATTERN: + substitution = self.substitutions[text] + if NAME_PLACEHOLDER in substitution: + name = value(tokens[position + 1]) if position + 1 < len(tokens) else None + if not name: + return None + parts.append(substitution.replace(NAME_PLACEHOLDER, name)) + position += 2 + continue + parts.append(substitution) + else: + name = value(tokens[position]) + if not name: + return None + parts.append(name) + position += 1 + return ''.join(parts) + + def part_names(self, tokens): + """Names of the codes of the tokens whose concept can be a part: active, not retired and named.""" + names = {} + for kind, text in tokens: + if kind == TOKEN_CODE: + concept = self.find_concept(text) + if concept and not concept['retired'] and concept['name']: + names[text] = concept['name'] + return names + + def resolve(self, codes): + """ + One name (or None) per logical code (never percent-encoded), in the same order and multiplicity. A user + without view access to the Source (version) gets no name and no concept is queried. + """ + if not self.has_view_access(): + return [None] * len(codes) + + distinct = {code for code in codes if is_lookup_code(code)} # a repeated code is processed once + self.load_concepts(distinct) + names, to_compose = {}, {} + for code in distinct: + concept = self.find_concept(code) + if concept: # exact first; a concept without a name is not recomposed + names[code] = concept['name'] or None + elif self.config: + tokens = self.parse(code) + if tokens: + to_compose[code] = tokens + + self.load_concepts({text for tokens in to_compose.values() for kind, text in tokens if kind == TOKEN_CODE}) + for code, tokens in to_compose.items(): + names[code] = self.render(tokens, self.part_names(tokens)) + return [names.get(code) if is_lookup_code(code) else None for code in codes] diff --git a/core/resolvers/tests.py b/core/resolvers/tests.py new file mode 100644 index 000000000..2035e2267 --- /dev/null +++ b/core/resolvers/tests.py @@ -0,0 +1,93 @@ +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import ValidationError + +from core.common.tests import OCLAPITestCase +from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory +from core.resolvers.core import validate_resolvers +from core.resolvers.names import NamesResolver +from core.sources.tests.factories import OrganizationSourceFactory + +# ICD-11 use case: '&' attaches an extension code in brackets and '/' joins clusters. Also used by the API tests. +RESOLVERS = {'names': {'replace': [ + {'pattern': '&', 'substitution': ' [{{name}}]'}, {'pattern': '/', 'substitution': ' / '}]}} +CONCEPTS = { + 'ND13.2': 'Fracture of unspecified tarsal bone', 'ND13.3': 'Fracture of metatarsal bone', + 'XJ44E': 'Closed fracture', +} + +# (resolvers, concepts of the Source, code, expected name): one resolver, one configuration per use case +COMPOSITION_CASES = ( + (RESOLVERS, CONCEPTS, 'ND13.2&XJ44E/ND13.3&XJ44E', + 'Fracture of unspecified tarsal bone [Closed fracture] / Fracture of metatarsal bone [Closed fracture]'), + ( # SNOMED CT: the attribute is a code too; '{' and '}' only group attributes + {'names': {'replace': [ + {'pattern': ':', 'substitution': ' with '}, {'pattern': '{', 'substitution': ''}, + {'pattern': '}', 'substitution': ''}, {'pattern': ',', 'substitution': ' and '}, + {'pattern': '=', 'substitution': ': '}]}}, + {'71388002': 'Procedure', '260686004': 'Method', '129304002': 'Excision - action', + '405813007': 'Procedure site - Direct', '15497006': 'Ovarian structure'}, + '71388002:{260686004=129304002,405813007=15497006}', + 'Procedure with Method: Excision - action and Procedure site - Direct: Ovarian structure'), + ( # LOINC: the axis in braces is a label, copied without the braces and never looked up + {'names': {'literals': [{'start': '{', 'end': '}'}], 'replace': [ + {'pattern': ':', 'substitution': ', '}, {'pattern': '=', 'substitution': ': '}, + {'pattern': ',', 'substitution': ', '}]}}, + {'104765-3': 'Neisseria gonorrhoeae DNA [Presence] in Specimen by Molecular genetics method', + 'LP7681-2': 'Specimen part', 'LP6464-4': 'Probe.amp.tar'}, + '104765-3:{Specimen}=LP7681-2,{Molgen}=LP6464-4', + 'Neisseria gonorrhoeae DNA [Presence] in Specimen by Molecular genetics method, Specimen: Specimen part, ' + 'Molgen: Probe.amp.tar'), +) + + +def make_concept(source, code, name): + """Creates a concept of the source with a single preferred English name.""" + concept = ConceptFactory(mnemonic=code, parent=source) + ConceptNameFactory(concept=concept, name=name, locale='en', locale_preferred=True) + return concept + + +def make_source(resolvers, concepts, **kwargs): + """A Source with the given resolvers holding the given concepts ({code: name}).""" + source = OrganizationSourceFactory(resolvers=resolvers, **kwargs) + for code, name in concepts.items(): + make_concept(source, code, name) + return source + + +class NamesResolverTest(OCLAPITestCase): + @staticmethod + def resolve(source, *codes): + return NamesResolver(source, AnonymousUser()).resolve(list(codes)) + + def test_code_is_resolved_with_the_resolvers_of_the_source(self): + for resolvers, concepts, code, name in COMPOSITION_CASES: + with self.subTest(code=code): + source = make_source(resolvers, concepts) + simple_code = next(iter(concepts)) + self.assertEqual(self.resolve(source, code, simple_code), [name, concepts[simple_code]]) + + def test_a_missing_part_gives_no_name(self): + source = make_source(RESOLVERS, CONCEPTS) + self.assertEqual(self.resolve(source, 'ND13.2&NOPE', 'ND13.2/', 'ND13.2&&XJ44E'), [None, None, None]) + + def test_an_existing_concept_wins_and_without_resolvers_nothing_is_split(self): + source = make_source(RESOLVERS, {**CONCEPTS, 'ND13.2%26XJ44E': 'Stored name'}) + self.assertEqual(self.resolve(source, 'ND13.2&XJ44E'), ['Stored name']) + + source = make_source(None, CONCEPTS) + self.assertEqual( + self.resolve(source, 'ND13.2&XJ44E', 'ND13.2'), [None, 'Fracture of unspecified tarsal bone']) + + def test_validate_resolvers(self): + for resolvers, *_ in COMPOSITION_CASES: + validate_resolvers(resolvers) + for invalid in ( + 'text', {'concepts': {}}, {'names': {}}, {'names': {'replace': []}}, + {'names': {'replace': [{'pattern': '&', 'substitution': '{{name}}{{name}}'}]}}, + {'names': {'replace': [{'pattern': '{', 'substitution': ''}], + 'literals': [{'start': '{', 'end': '}'}]}}): + with self.subTest(invalid=invalid): + with self.assertRaises(ValidationError) as context: + validate_resolvers(invalid) + self.assertIn('resolvers', context.exception.message_dict) diff --git a/core/sources/migrations/0047_source_resolvers.py b/core/sources/migrations/0047_source_resolvers.py new file mode 100644 index 000000000..d6f9ce9e5 --- /dev/null +++ b/core/sources/migrations/0047_source_resolvers.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-10-21 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sources', '0046_retire_public_edit_access'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='resolvers', + field=models.JSONField(blank=True, null=True), + ), + ] diff --git a/core/sources/models.py b/core/sources/models.py index 079a473ba..8da9f0415 100644 --- a/core/sources/models.py +++ b/core/sources/models.py @@ -19,6 +19,7 @@ from core.common.utils import to_camel_case from core.common.validators import validate_non_negative from core.concepts.models import ConceptName, Concept +from core.resolvers.core import validate_resolvers from core.services.storages.postgres import PostgresQL from core.sources.constants import SOURCE_TYPE, SOURCE_VERSION_TYPE, HIERARCHY_ROOT_MUST_BELONG_TO_SAME_SOURCE, \ HIERARCHY_MEANINGS, AUTO_ID_CHOICES, AUTO_ID_SEQUENTIAL, AUTO_ID_UUID, LOCALE_EXTERNAL_AUTO_ID_CHOICES @@ -128,6 +129,8 @@ class Meta: properties = ArrayField(models.JSONField(), default=list, null=True, blank=True) filters = ArrayField(models.JSONField(), default=list, null=True, blank=True) + # Read-time resolvers by family, e.g. {"names": {"replace": [...], "literals": [...]}}. See DOCS_RESOLVERS.md. + resolvers = models.JSONField(null=True, blank=True) match_algorithms = ArrayField( models.CharField(max_length=50), default=list, null=True, blank=True) @@ -267,6 +270,7 @@ def is_hierarchy_root_belonging_to_self(self): def clean(self): self.hierarchy_meaning = self.hierarchy_meaning or None + validate_resolvers(self.resolvers) self.clean_properties() self.clean_filters() self.clean_match_algorithms() diff --git a/core/sources/serializers.py b/core/sources/serializers.py index 6da20e68f..3a24e3cfd 100644 --- a/core/sources/serializers.py +++ b/core/sources/serializers.py @@ -1,5 +1,6 @@ import json +from django.core.exceptions import ValidationError as DjangoValidationError from django.core.validators import RegexValidator from pydash import get, compact from rest_framework.fields import CharField, IntegerField, DateTimeField, JSONField, ListField, \ @@ -14,6 +15,7 @@ from core.common.serializers import AbstractRepoResourcesSerializer, AbstractResourceSerializer, PublicAccessField from core.common.utils import get_truthy_values from core.orgs.models import Organization +from core.resolvers.core import validate_resolvers as validate_resolvers_config from core.settings import DEFAULT_LOCALE from core.sources.models import Source from core.users.models import UserProfile @@ -181,7 +183,7 @@ def prepare_object(self, validated_data, instance=None): 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from', 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'autoid_concept_name_external_id', 'autoid_concept_description_external_id', 'properties', 'filters', - 'match_algorithms' + 'match_algorithms', 'resolvers' ]: setattr(source, attr, validated_data.get(attr, get(source, attr))) for attr in ['jurisdiction', 'identifier', 'contact', 'meta']: @@ -202,6 +204,15 @@ def prepare_object(self, validated_data, instance=None): uri=hierarchy_root_url).first() if hierarchy_root_url else None return source + @staticmethod + def validate_resolvers(value): + """New versions are persisted without full_clean, so the resolvers are validated here for every write.""" + try: + validate_resolvers_config(value) + except DjangoValidationError as ex: + raise ValidationError(ex.messages) from ex + return value + def update(self, instance, validated_data): original_schema = instance.custom_validation_schema source = self.prepare_object(validated_data, instance) @@ -428,7 +439,7 @@ class Meta: 'autoid_mapping_mnemonic', 'autoid_mapping_external_id', 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from', 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'checksums', - 'properties', 'filters', 'match_algorithms' + 'properties', 'filters', 'match_algorithms', 'resolvers' ) + AbstractRepoResourcesSerializer.Meta.fields def __init__(self, *args, **kwargs): @@ -518,7 +529,7 @@ class Meta: 'content_type', 'revision_date', 'summary', 'text', 'meta', 'experimental', 'case_sensitive', 'collection_reference', 'hierarchy_meaning', 'compositional', 'version_needed', 'hierarchy_root_url', 'checksums', 'states', 'tasks', 'properties', 'filters', - 'match_algorithms', 'external_exports' + 'match_algorithms', 'external_exports', 'resolvers' ) + AbstractRepoResourcesSerializer.Meta.fields def __init__(self, *args, **kwargs): diff --git a/core/sources/views.py b/core/sources/views.py index eb476310e..d3a88344f 100644 --- a/core/sources/views.py +++ b/core/sources/views.py @@ -272,6 +272,7 @@ def create(self, request, *args, **kwargs): "meta": request.data.get('meta', head_object.meta), "properties": request.data.get('properties', head_object.properties), "filters": request.data.get('filters', head_object.filters), + "resolvers": request.data.get('resolvers', head_object.resolvers), "match_algorithms": request.data.get( 'match_algorithms', head_object.get_match_algorithms_for_new_version()) }