feat(resolvers): implement post-coordinated name resolution

core/concepts/serializers.py

@@ -252,7 +252,7 @@ def get_mappings(self, obj):
             ) if is_collection else obj.get_bidirectional_mappings()
             if not include_retired:
                 mappings = mappings.filter(retired=False)
-            return serializer_class(mappings, many=True).data
+            return serializer_class(mappings, many=True, context=context).data
         if self.include_direct_mappings:
             mappings = obj.get_unidirectional_mappings_for_collection(
                 parent_uri, collection_version) if is_collection else obj.get_unidirectional_mappings()
@@ -262,7 +262,7 @@ def get_mappings(self, obj):
                 mappings = mappings.filter(to_source_url__in=target_repo_urls)
             if not include_retired:
                 mappings = mappings.filter(retired=False)
-            return serializer_class(mappings, many=True).data
+            return serializer_class(mappings, many=True, context=context).data
 
         return []
 
@@ -458,9 +458,11 @@ def get_entries(self, obj):
 
             from core.mappings.serializers import MappingMinimalSerializer, MappingReverseMinimalSerializer
             if get(self, 'context.request.query_params.reverse') in TRUTHY:
-                result += MappingReverseMinimalSerializer(obj.cascaded_entries['mappings'], many=True).data
+                result += MappingReverseMinimalSerializer(
+                    obj.cascaded_entries['mappings'], many=True, context=self.context).data
             else:
-                result += MappingMinimalSerializer(obj.cascaded_entries['mappings'], many=True).data
+                result += MappingMinimalSerializer(
+                    obj.cascaded_entries['mappings'], many=True, context=self.context).data
 
         return result
 

core/integration_tests/test_utils.py

@@ -53,6 +53,7 @@ class SourceChildListQueryCountMixin:
     relation = None
     list_serializer = None
     detail_serializer = None
+    small_page_size = 2  # the smaller page must hold the same kinds of rows as the page of 4
 
     def get_page(self, url, limit, params=''):
         with CaptureQueriesContext(connection) as context:
@@ -62,8 +63,8 @@ def get_page(self, url, limit, params=''):
         return response, len(context.captured_queries)
 
     def assert_page(self, url, serializer_class, params=''):
-        """Same query count for a page of 2 and of 4; same output as serializing each row on its own."""
-        _, small_page_queries = self.get_page(url, 2, params)
+        """Same query count for a small page and a page of 4; same output as serializing each row on its own."""
+        _, small_page_queries = self.get_page(url, self.small_page_size, params)
         response, page_queries = self.get_page(url, 4, params)
         self.assertEqual(page_queries, small_page_queries)
 

core/integration_tests/tests_mappings.py

@@ -3,6 +3,7 @@
 from rest_framework.exceptions import ErrorDetail
 
 from core.collections.tests.factories import OrganizationCollectionFactory, ExpansionFactory
+from core.common.constants import ACCESS_TYPE_NONE
 from core.common.tests import OCLAPITestCase
 from core.concepts.tests.factories import ConceptFactory, ConceptNameFactory
 from core.integration_tests.test_utils import SourceChildListQueryCountMixin
@@ -12,6 +13,7 @@
 from core.mappings.serializers import MappingListSerializer, MappingDetailSerializer
 from core.mappings.tests.factories import MappingFactory
 from core.orgs.tests.factories import OrganizationFactory
+from core.resolvers.tests import CONCEPTS, RESOLVERS, make_concept, make_source
 from core.sources.tests.factories import UserSourceFactory, OrganizationSourceFactory
 from core.users.models import UserProfile
 from core.users.tests.factories import UserProfileFactory
@@ -1086,6 +1088,9 @@ class MappingListQueryCountTest(SourceChildListQueryCountMixin, OCLAPITestCase):
     relation = 'mappings'
     list_serializer = MappingListSerializer
     detail_serializer = MappingDetailSerializer
+    # A source version lists a mapping's head and latest version, which are the same kind of row. Unlinked (external)
+    # mappings cost one batched name lookup per page, so the smaller page needs 3 rows to hold both kinds as well.
+    small_page_size = 3
 
     def setUp(self):
         super().setUp()
@@ -1106,3 +1111,62 @@ def setUp(self):
         for mapping in mappings:
             mapping.set_checksums()  # a GET calculates and saves missing checksums, one row at a time
         self.set_released_versions(mappings)
+
+
+class MappingResolvedConceptNameTest(OCLAPITestCase):
+    """A mapping to a post-coordinated code shows the name its destination Source resolves."""
+    CODE = 'ND13.2%26XJ44E%2FND13.3%26XJ44E'  # as stored: percent-encoded once
+    NAME = 'Fracture of unspecified tarsal bone [Closed fracture] / Fracture of metatarsal bone [Closed fracture]'
+
+    def setUp(self):
+        super().setUp()
+        self.origin = OrganizationSourceFactory()
+        self.origin_concept = make_concept(self.origin, 'O1', 'Origin one')
+        self.destination = make_source(RESOLVERS, CONCEPTS)
+
+    def make_mapping(self, code, destination=None):
+        """A mapping whose destination code is not linked to a concept, as post-coordinated codes are."""
+        destination = destination or self.destination
+        return MappingFactory(
+            parent=self.origin, from_concept=self.origin_concept, from_source=self.origin,
+            from_source_url=self.origin.uri, to_concept=None, to_source=destination,
+            to_source_url=destination.uri, to_concept_code=code, to_concept_name='Stored name')
+
+    def get(self, url, user=None, **params):
+        headers = {'HTTP_AUTHORIZATION': 'Token ' + user.get_token()} if user else {}
+        response = self.client.get(url, params, **headers)
+        self.assertEqual(response.status_code, 200, response.content)
+        return response.data
+
+    def test_every_representation_shows_the_resolved_name(self):
+        mapping = self.make_mapping(self.CODE)
+
+        for data in (self.get(self.origin.uri + 'mappings/')[0], self.get(mapping.uri),
+                     self.get(mapping.uri + 'versions/')[0]):
+            self.assertEqual(data['to_concept_name_resolved'], self.NAME)
+            self.assertEqual(data['to_concept_name'], 'Stored name')  # the stored value is untouched
+            self.assertIsNone(data['to_concept_url'])  # no concept, no url
+        brief = self.get(self.origin.uri + 'mappings/', brief='true')
+        self.assertEqual(brief[0]['cascade_target_concept_name'], self.NAME)
+        cascade = self.get(self.origin_concept.uri + '$cascade/', method='sourceMappings', cascadeLevels='1')
+        entries = [entry for entry in cascade['entry'] if entry.get('cascade_target_concept_code')]
+        self.assertEqual([entry['cascade_target_concept_name'] for entry in entries], [self.NAME])
+
+    def test_the_stored_name_is_kept_when_a_part_is_missing(self):
+        mapping = self.make_mapping('ND13.2%26NOPE')
+
+        # as before the resolver: no linked concept, so the field is omitted
+        self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri))
+        brief = self.get(self.origin.uri + 'mappings/', brief='true')
+        self.assertEqual(brief[0]['cascade_target_concept_name'], 'Stored name')
+
+    def test_a_private_destination_is_resolved_only_for_users_who_can_see_it(self):
+        organization = OrganizationFactory()
+        private = make_source(RESOLVERS, CONCEPTS, organization=organization, public_access=ACCESS_TYPE_NONE)
+        mapping = self.make_mapping(self.CODE, private)
+        member = UserProfileFactory()
+        organization.members.add(member)
+
+        self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri))  # omitted, as without a resolver
+        self.assertNotIn('to_concept_name_resolved', self.get(mapping.uri, UserProfileFactory()))
+        self.assertEqual(self.get(mapping.uri, member)['to_concept_name_resolved'], self.NAME)

core/integration_tests/tests_sources.py

@@ -28,6 +28,7 @@
 from core.mappings.serializers import MappingVersionExportSerializer
 from core.mappings.tests.factories import MappingFactory
 from core.orgs.models import Organization
+from core.resolvers.tests import RESOLVERS
 from core.sources.models import Source, CloneLimitExceeded
 from core.sources.serializers import SourceDetailSerializer, SourceVersionExportSerializer
 from core.sources.tests.factories import OrganizationSourceFactory, UserSourceFactory
@@ -196,7 +197,7 @@ def test_post_201(self):
                 'autoid_concept_name_external_id', 'autoid_concept_description_external_id',
                 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from',
                 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'checksums',
-                'properties', 'filters', 'match_algorithms'
+                'properties', 'filters', 'match_algorithms', 'resolvers'
             ])
         )
         source = Source.objects.last()
@@ -318,7 +319,7 @@ def test_put_200(self):
                 'autoid_concept_name_external_id', 'autoid_concept_description_external_id',
                 'autoid_concept_mnemonic_start_from', 'autoid_concept_external_id_start_from',
                 'autoid_mapping_mnemonic_start_from', 'autoid_mapping_external_id_start_from', 'checksums',
-                'properties', 'filters', 'match_algorithms'
+                'properties', 'filters', 'match_algorithms', 'resolvers'
             ])
         )
         source = Source.objects.last()
@@ -332,6 +333,28 @@ def test_put_200(self):
         self.assertEqual(source.default_locale, 'en')
         self.assertEqual(source.supported_locales, ['fr'])
 
+
+    def test_put_resolvers(self):
+        source = OrganizationSourceFactory(organization=self.organization)
+
+        response = self.client.put(
+            source.uri, {'resolvers': RESOLVERS}, HTTP_AUTHORIZATION='Token ' + self.token, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['resolvers'], RESOLVERS)
+
+        response = self.client.put(
+            source.uri, {'resolvers': {'names': {'replace': []}}}, HTTP_AUTHORIZATION='Token ' + self.token,
+            format='json')
+        self.assertEqual(response.status_code, 400)
+        self.assertIn('resolvers', response.data)
+        source.refresh_from_db()
+        self.assertEqual(source.resolvers, RESOLVERS)
+
+        response = self.client.put(
+            source.uri, {'resolvers': None}, HTTP_AUTHORIZATION='Token ' + self.token, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertIsNone(response.data['resolvers'])
+
     def test_put_hierarchy_root(self):
         source = OrganizationSourceFactory(organization=self.organization)
         self.assertTrue(source.is_head)
@@ -477,9 +500,10 @@ def test_post_201(self):
 
     @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts'))
     @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings'))
-    def test_post_201_copies_properties_and_filters_from_head(self):
+    def test_post_201_copies_properties_filters_and_resolvers_from_head(self):
         self.source.properties = [{'code': 'p1'}]
         self.source.filters = [{'code': 'f1'}]
+        self.source.resolvers = RESOLVERS
         self.source.save()
 
         response = self.client.post(
@@ -495,10 +519,12 @@ def test_post_201_copies_properties_and_filters_from_head(self):
         self.assertEqual(response.status_code, 201)
         self.assertEqual(response.data['properties'], [{'code': 'p1'}])
         self.assertEqual(response.data['filters'], [{'code': 'f1'}])
+        self.assertEqual(response.data['resolvers'], RESOLVERS)
 
         new_version = self.source.versions.get(version='v1')
         self.assertEqual(new_version.properties, [{'code': 'p1'}])
         self.assertEqual(new_version.filters, [{'code': 'f1'}])
+        self.assertEqual(new_version.resolvers, RESOLVERS)
 
     @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts'))
     @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings'))
@@ -598,6 +624,17 @@ def test_post_400_released_invalid_string(self):
             response.data, {'released': [ErrorDetail(string='Must be a valid boolean.', code='invalid')]})
         self.assertEqual(self.source.versions.count(), 1)
 
+    @patch('core.sources.models.index_source_concepts', Mock(__name__='index_source_concepts'))
+    @patch('core.sources.models.index_source_mappings', Mock(__name__='index_source_mappings'))
+    def test_post_400_invalid_resolvers(self):
+        response = self.client.post(
+            self.source.uri + 'versions/', {'id': 'v1', 'resolvers': {'names': {'replace': []}}},
+            HTTP_AUTHORIZATION='Token ' + self.token, format='json')
+
+        self.assertEqual(response.status_code, 400)
+        self.assertIn('resolvers', response.data)
+        self.assertFalse(self.source.versions.filter(version='v1').exists())
+
     @patch('core.sources.views.export_source')
     def test_post_400(self, export_source_mock):
         response = self.client.post(

core/mappings/models.py

@@ -16,6 +16,7 @@
 from core.mappings.constants import MAPPING_TYPE, MAPPING_IS_ALREADY_RETIRED, MAPPING_WAS_RETIRED, \
     MAPPING_IS_ALREADY_NOT_RETIRED, MAPPING_WAS_UNRETIRED, ALREADY_EXISTS
 from core.mappings.mixins import MappingValidationMixin
+from core.mappings.name_resolution import load_resolved_names, SIDES
 from core.services.storages.postgres import PostgresQL
 
 
@@ -25,6 +26,12 @@ class Mapping(MappingValidationMixin, SourceChildMixin, VersionedModel):
         'to_source__organization', 'to_source__user', 'created_by', 'updated_by', 'from_concept__names',
         'to_concept__names'
     )
+    # serializer fields showing a resolved concept name, and the mapping sides (from/to) each one reads
+    NAME_RESOLUTION_FIELDS = {
+        'from_concept_name_resolved': ('from', ),
+        'to_concept_name_resolved': ('to', ),
+        'cascade_target_concept_name': SIDES,  # the cascade direction is a serializer choice; the unused side is cheap
+    }
 
     class Meta:
         db_table = 'mappings'
@@ -323,6 +330,14 @@ def create_initial_version(cls, mapping, **kwargs):
         initial_version.save()
         return initial_version
 
+    @classmethod
+    def load_list_fields(cls, instances, fields, collection=None, user=None):
+        """Besides the generic page lookups, resolves the concept names the serialized fields will show."""
+        super().load_list_fields(instances, fields, collection, user)
+        sides = {side for field in fields for side in cls.NAME_RESOLUTION_FIELDS.get(field, ())}
+        if sides:
+            load_resolved_names(instances, [side for side in SIDES if side in sides], user)
+
     def populate_fields_from_relations(  # pylint: disable=too-many-locals,too-many-statements
             self, data, cache=None, user=None):
         """Concepts and repos the user can't view are not linked; they are kept as external codes/URLs."""

core/mappings/name_resolution.py

@@ -0,0 +1,110 @@
+"""
+Adapter between Mappings and the ``names`` resolver (``core.resolvers.names``).
+
+Picks, for each side (`from` / `to`) of a mapping, the Source (version) its code must be resolved against and projects
+the resolver result onto the mapping. The grammar and the lookups live in the resolver; nothing here is persisted.
+"""
+from pydash import get
+
+from core.common.constants import HEAD
+from core.common.utils import decode_string
+from core.resolvers.core import user_key
+from core.resolvers.names import NamesResolver
+
+SIDES = ('from', 'to')
+RESOLVED_NAMES_ATTR = '_resolved_concept_names'
+
+
+def context_user(context):
+    """The user of a serializer context. Without one the caller is anonymous: access is never elevated."""
+    return get(context, 'request.user') or get(context, 'user')
+
+
+def _find_source(mapping, side, cache):
+    """
+    The Source (version) the code of `side` belongs to: the declared version when there is one, otherwise HEAD.
+    The stored FK / URL identify the repo; a version declared in the mapping always wins over the HEAD FK.
+    Returns None when the repo cannot be identified locally.
+    """
+    from core.sources.models import Source
+    version = getattr(mapping, f'{side}_source_version') or HEAD
+    source_id = getattr(mapping, f'{side}_source_id')
+    if source_id:
+        head = getattr(mapping, f'{side}_source')
+        if version == HEAD:
+            return head
+        key = ('version', head.id, version)
+        if key not in cache:
+            cache[key] = head.active_versions.filter(version=version).first()
+        return cache[key]
+
+    url = getattr(mapping, f'{side}_source_url')
+    if not url:
+        return None
+    key = ('url', url, version)
+    if key not in cache:
+        source, _ = Source.resolve_reference_expression(url, None, version)
+        cache[key] = source if source.id else None
+    return cache[key]
+
+
+def _target(mapping, side, cache):
+    """
+    `(source, logical_code)` the side must be resolved against, or None when the stored link already gives the name:
+    a concept linked to the mapping without a declared source version is the very concept the resolver would find.
+    """
+    code = getattr(mapping, f'{side}_concept_code')
+    if not code:
+        return None
+    declared_version = getattr(mapping, f'{side}_source_version')
+    if getattr(mapping, f'{side}_concept_id') and (not declared_version or declared_version == HEAD):
+        return None
+    source = _find_source(mapping, side, cache)
+    # stored codes are percent-encoded once; the resolver works on logical codes
+    return (source, decode_string(code, plus=False)) if source else None
+
+
+def _resolve_targets(targets, user):
+    """Resolved names keyed by `(source id, code)`: one resolver call per distinct target Source (version)."""
+    wanted = {}  # source id -> (source, {code})
+    for target in targets:
+        if target:
+            source, code = target
+            wanted.setdefault(source.id, (source, set()))[1].add(code)
+
+    resolved = {}
+    resolver_cache = {}
+    for source_id, (source, codes) in wanted.items():
+        ordered_codes = sorted(codes)
+        for code, name in zip(ordered_codes, NamesResolver(source, user, resolver_cache).resolve(ordered_codes)):
+            resolved[(source_id, code)] = name
+    return resolved
+
+
+def load_resolved_names(mappings, sides, user):
+    """
+    Resolves, for a whole page, the names of the given sides and keeps them on each mapping.
+    Not saved anywhere; scoped to the user it was computed for.
+    """
+    cache = {}
+    targets = [(mapping, side, _target(mapping, side, cache)) for mapping in mappings for side in sides]
+    resolved = _resolve_targets([target for _, _, target in targets], user)
+
+    owner = user_key(user)
+    for mapping, side, target in targets:
+        names = mapping.__dict__.get(RESOLVED_NAMES_ATTR)
+        if not names or names['user'] != owner:
+            names = mapping.__dict__[RESOLVED_NAMES_ATTR] = {'user': owner}
+        names[side] = resolved.get((target[0].id, target[1])) if target else None
+
+
+def get_resolved_name(mapping, side, user):
+    """
+    The name of the `side` concept computed by the resolver (exact match or composition), or None when the serializer
+    must keep its stored/legacy value. Loads the single mapping on demand when no page loader prepared it.
+    """
+    names = mapping.__dict__.get(RESOLVED_NAMES_ATTR)
+    if not names or names['user'] != user_key(user) or side not in names:
+        load_resolved_names([mapping], (side,), user)
+        names = mapping.__dict__[RESOLVED_NAMES_ATTR]
+    return names[side]