diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index e66e3319b57..2cad91d7018 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -190,6 +190,7 @@ import org.opensearch.sql.calcite.utils.WildcardUtils; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.common.patterns.PatternUtils; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.type.ExprCoreType; @@ -4456,7 +4457,24 @@ public RelNode visitExpand(Expand expand, CalcitePlanContext context) { // 2. Get the field to expand and an optional alias. Field arrayField = expand.getField(); - RexInputRef arrayFieldRex = (RexInputRef) rexVisitor.analyze(arrayField, context); + RexNode resolved = rexVisitor.analyze(arrayField, context); + if (!(resolved instanceof RexInputRef arrayFieldRex)) { + // Anything that is not a column of the input -- a leaf of an object or a flat_object field, + // an expression -- has no column to correlate the expansion with. Raised the way the + // mvexpand check below does, rather than as the ClassCastException the cast used to throw. + String name = StringUtils.unquoteIdentifier(arrayField.getField().toString()); + throw ErrorReport.wrap( + new IllegalArgumentException(StringUtils.format("Cannot expand [%s]", name))) + .code(ErrorCode.UNSUPPORTED_OPERATION) + .stage(QueryProcessingStage.ANALYZING) + .location("while resolving the field for expand") + .context("command", "expand") + .context("field", name) + .suggestion( + "expand takes a field of the index, not a value computed from one. Expand the field" + + " itself, or project the value first.") + .build(); + } String alias = expand.getAlias(); buildExpandRelNode(arrayFieldRex, arrayField.getField().toString(), alias, null, context); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 99fef793f5d..81ce881f0ce 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -206,6 +206,15 @@ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boole return TYPE_FACTORY.createSqlType(SqlTypeName.GEOMETRY, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("text")) { return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); + } else if (fieldType.legacyTypeName().equalsIgnoreCase("flat_object")) { + // Keys are the dotted leaf paths, values are text: a flat_object gives its leaves no type + // of their own, and the index holds each as a keyword term. This is what spath already + // does for a JSON string -- every value reads as text, and a path holding several values + // reads as their JSON. + return TYPE_FACTORY.createMapType( + TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), + TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true), + nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("ip")) { return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable); } else if (fieldType.getOriginalPath().isPresent()) { diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index adefd59397b..fcd052133ef 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -103,6 +103,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | nested | array | STRUCT | +-----------------+---------------------+-----------+ +| flat_object | struct | STRUCT | ++-----------------+---------------------+-----------+ Notes: * Not all the OpenSearch SQL Type has correspond OpenSearch Type. e.g. data and time. To use function which required such data type, user should explicitly convert the data type. diff --git a/docs/user/ppl/general/datatypes.md b/docs/user/ppl/general/datatypes.md index b2f38418ea8..28cf7be5656 100644 --- a/docs/user/ppl/general/datatypes.md +++ b/docs/user/ppl/general/datatypes.md @@ -113,6 +113,16 @@ A string is a sequence of characters enclosed in either single or double quotes. ## Query Struct Data Types In PPL, the Struct Data Types corresponding to the [Object field type in OpenSearch](https://opensearch.org/docs/latest/field-types/supported-field-types/object/). The "." is used as the path selector when access the inner attribute of the struct data. + +A [flat_object field](https://docs.opensearch.org/latest/mappings/supported-field-types/flat-object/) is also presented as a struct, keyed by the dotted path of each leaf. A flat_object declares no sub-fields, so a leaf has no type of its own: every value reads as text, and a path holding more than one value reads as their JSON. This is the rule `spath` already applies to a JSON string, and `typeof` on a leaf reports `string`. + +Both a nested object and a literal dotted key reach the same leaf: `{"a": {"b": 1}}` and `{"a.b": 1}` each give the entry `a.b` with the value `"1"`. A path receives more than one value when it comes from an array, from several elements of an array of objects, or from both spellings of the same path appearing in one document; it then reads as a JSON array of those texts, in the order the document wrote them. + +A leaf is text, so a query does with it what it does with any text column, the same as a field `spath` extracted from a JSON string: read it, cast it (`eval ms = cast(attributes.duration_ms as double)`), filter by it, sort by it, group by it. + +A path holding more than one value reads as JSON, so it does not compare equal to any single one of those values. The index still matches each of them, so a filter answers differently depending on whether it reached the index — that applies to any field holding several values, not only to a flat_object leaf. + +`expand` takes a column of the index rather than a leaf; expand the field itself, or project the leaf first. ### Example: People There are three fields in test index `people`: 1) deep nested object field `city`; 2) object field of array value `account`; 3) nested field `projects` diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml new file mode 100644 index 00000000000..a961e3d5a99 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml @@ -0,0 +1,163 @@ +# Issue: https://github.com/opensearch-project/sql/issues/1604 +# https://github.com/opensearch-project/sql/issues/3067 +# A flat_object field used to be dropped by OpenSearchDataType.parseMapping ("unknown type, skip +# it"), so it was absent from the schema and any reference failed as "field not found". +# +# The field is now a struct keyed by the dotted path of each leaf. A leaf has no type of its own in +# the index, so every value reads as text and a path holding several values reads as their JSON -- +# the same rule spath applies to a JSON string. +setup: + - do: + indices.create: + index: otel + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + service: + type: keyword + attributes: + type: flat_object + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "otel", "_id": "1"}}' + - '{"service": "checkout", "attributes": {"http.response.status_code": 500, "duration_ms": 12.5, "ok": true}}' + - '{"index": {"_index": "otel", "_id": "2"}}' + - '{"service": "payments", "attributes": {"http": {"response": {"status_code": 503}}, "duration_ms": "4"}}' + - '{"index": {"_index": "otel", "_id": "3"}}' + - '{"service": "tags", "attributes": {"tags": ["a", "b"], "mixed": [1, "x"], "nested": [{"k": 1}, {"k": 2}]}}' + - '{"index": {"_index": "otel", "_id": "4"}}' + - '{"service": "collide", "attributes": {"a": {"b": 1}, "a.b": 2}}' + + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + plugins.calcite.fallback.allowed: false + +--- +teardown: + - do: + indices.delete: + index: otel + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: null + plugins.calcite.fallback.allowed: null + +--- +"describe lists the field as flat_object": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'describe otel | fields COLUMN_NAME, TYPE_NAME' + - match: {"datarows": [["attributes", "flat_object"], ["service", "string"]]} + +--- +"the whole field reads as a map of dotted leaf path to text": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where service = ''checkout'' | fields attributes' + # every value is text, and the dotted path is the key -- both spellings of a path reach the same one + - match: {"datarows": [[{"http.response.status_code": "500", "duration_ms": "12.5", "ok": "true"}]]} + +--- +"a leaf reads as text, under either spelling of its path": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | fields service, attributes.http.response.status_code, attributes.duration_ms | sort service' + - match: {"datarows": [["checkout", "500", "12.5"], ["collide", null, null], ["payments", "503", "4"], ["tags", null, null]]} + +--- +"a path holding several values reads as their JSON": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where service = ''tags'' | fields attributes.tags, attributes.mixed, attributes.nested.k' + - match: {"datarows": [["[\"a\",\"b\"]", "[\"1\",\"x\"]", "[\"1\",\"2\"]"]]} + # both spellings of a path in one document: neither value is dropped + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where service = ''collide'' | fields attributes.a.b' + - match: {"datarows": [["[\"1\",\"2\"]"]]} + +--- +"a leaf can be cast, and typeof reports text": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where service = ''payments'' | eval n = cast(attributes.duration_ms as double), t = typeof(attributes.duration_ms) | fields n, t' + - match: {"datarows": [[4.0, "STRING"]]} + +--- +# A leaf is text, so PPL does with it what it does with any text column -- the same as a field +# spath extracted from a JSON string. +"a leaf can be filtered, sorted and aggregated like any text column": + - skip: + features: [headers] + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where attributes.duration_ms = ''4'' | fields service' + - match: {"datarows": [["payments"]]} + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where attributes.duration_ms > 5 | fields service' + - match: {"datarows": [["checkout"]]} + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | stats avg(attributes.duration_ms)' + - match: {"datarows": [[8.25]]} + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | where isnotnull(attributes.duration_ms) | sort attributes.duration_ms | fields attributes.duration_ms' + - match: {"datarows": [["12.5"], ["4"]]} + +--- +# expand needs a column of the input to correlate the expansion with. A leaf is not one, so it is +# reported the way the mvexpand check reports it rather than as a ClassCastException. +"expand does not take a leaf": + - skip: + features: [headers] + - do: + catch: bad_request + headers: {Content-Type: 'application/json'} + ppl: + body: + query: 'source=otel | expand attributes.tags | fields service' + - match: {"$body": "/Cannot\\s+expand\\s+.attributes.tags.*/"} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 98bb69b7fdf..f3fbbe8d2c6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -39,6 +39,7 @@ public enum MappingType { DateNanos("date_nanos", ExprCoreType.TIMESTAMP), Object("object", ExprCoreType.STRUCT), Nested("nested", ExprCoreType.ARRAY), + FlatObject("flat_object", ExprCoreType.UNKNOWN), Byte("byte", ExprCoreType.BYTE), Short("short", ExprCoreType.SHORT), Integer("integer", ExprCoreType.INTEGER), @@ -170,6 +171,8 @@ public static OpenSearchDataType of(MappingType mappingType, Map return OpenSearchGeoPointType.of(); case Binary: return OpenSearchBinaryType.of(); + case FlatObject: + return OpenSearchFlatObjectType.of(); case Date: case DateNanos: // Default date formatter is used when "" is passed as the second parameter diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectType.java new file mode 100644 index 00000000000..ad718fd79cd --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectType.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.type; + +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import lombok.EqualsAndHashCode; + +/** + * The type of a flat_object field. See doc + * + *

A flat_object declares no sub-fields in the index mapping: its keys exist only inside the + * documents, and every leaf is indexed as a single keyword term with no numeric type. The engine + * therefore reads its values from {@code _source} and presents the field as a map keyed by the + * dotted leaf path, so that a nested object and a literal dotted key resolve to the same entry. + * + *

Like the other mapping types with no direct core-type counterpart (text, geo_point, binary), + * it carries {@code UNKNOWN} as its core type so that {@link #getExprType()} returns this instance + * and the Calcite type factory can map it by name. + */ +@EqualsAndHashCode(callSuper = false) +public class OpenSearchFlatObjectType extends OpenSearchDataType { + + private static final OpenSearchFlatObjectType instance = new OpenSearchFlatObjectType(); + + private OpenSearchFlatObjectType() { + super(MappingType.FlatObject); + exprCoreType = UNKNOWN; + } + + public static OpenSearchFlatObjectType of() { + return OpenSearchFlatObjectType.instance; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return instance; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java index d2fa68dac06..fe4ea935454 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java @@ -53,6 +53,9 @@ public interface Content { /** Is array value. */ boolean isArray(); + /** Is object value: a set of named members, as opposed to a scalar or an array. */ + boolean isObject(); + /** Get integer value. */ Integer intValue(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java index 6818e75ec07..69dd3cd7075 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java @@ -144,6 +144,11 @@ public boolean isArray() { return value instanceof ArrayNode || value instanceof List; } + @Override + public boolean isObject() { + return value instanceof Map; + } + @Override public boolean isString() { return value instanceof String; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java index 587e133a10f..092b6ba7a69 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java @@ -140,6 +140,11 @@ public boolean isArray() { return value().isArray(); } + @Override + public boolean isObject() { + return value().isObject(); + } + @Override public Object objectValue() { return value(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/FlatObjectValues.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/FlatObjectValues.java new file mode 100644 index 00000000000..b7ea7b52e2c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/FlatObjectValues.java @@ -0,0 +1,164 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.value; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.data.utils.Content; + +/** + * Flattens a flat_object value into the shape the engine presents it as: a single-level map from + * dotted leaf path to the text of the value written there. + * + *

A flat_object declares no sub-fields, so its shape is known only from the document, and the + * index files one term per leaf with the path folded in. This mirrors that: every way of writing a + * path reaches the same entry ({@code {"a": {"b": 1}}} and {@code {"a.b": 1}} both give {@code + * a.b}), a path written more than once holds every value, and each value is the text the index + * holds for it -- which is also the only form the field can be looked up by. + */ +@UtilityClass +public class FlatObjectValues { + + /** + * Depth guard. A flat_object exists so that documents can escape the mapping depth limit, so + * nothing upstream bounds this recursion. Levels beyond the cap are kept as one leaf rather than + * dropped. + */ + static final int MAX_DEPTH = 20; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Flatten a parsed flat_object value. + * + * @param content the field's value as parsed from {@code _source} + * @return a tuple keyed by dotted leaf path, or null if the value is neither an object nor an + * array (a scalar where the mapping promised an object, e.g. a wildcard over conflicting + * mappings) + */ + public static ExprValue flatten(Content content) { + if (content == null || content.isNull()) { + return ExprNullValue.of(); + } + LinkedHashMap leaves = new LinkedHashMap<>(); + if (content.isObject()) { + flattenInto(content, "", leaves, 0); + } else if (content.isArray()) { + // OpenSearch accepts an array at the top level and folds every element's paths into the same + // terms, so the elements merge into one map here too. + content + .array() + .forEachRemaining( + element -> { + if (element.isObject()) { + flattenInto(element, "", leaves, 0); + } + }); + } else { + return ExprNullValue.of(); + } + return ExprTupleValue.fromExprValueMap(leaves); + } + + private static void flattenInto( + Content content, String prefix, LinkedHashMap out, int depth) { + content + .map() + .forEachRemaining( + entry -> { + String key = prefix.isEmpty() ? entry.getKey() : prefix + "." + entry.getKey(); + fold(entry.getValue(), key, out, depth); + }); + } + + /** + * Fold one value into the map under {@code key}, the way the index folds it into terms: an object + * contributes its own leaves under the dotted path, and an array contributes each element under + * the same path, whatever the element is. So {@code {"a": [1, 2]}}, {@code [{"a": 1}, {"a": 2}]} + * and {@code {"a": {"b": 1}, "a.b": 2}} all reach the same shape the index holds. + */ + private static void fold( + Content value, String key, LinkedHashMap out, int depth) { + if (value.isObject() && depth < MAX_DEPTH) { + flattenInto(value, key, out, depth + 1); + return; + } + if (value.isArray()) { + value.array().forEachRemaining(element -> fold(element, key, out, depth)); + return; + } + put(out, key, text(value)); + } + + /** + * A leaf as the text the index holds for it. The index has no type for a leaf -- every value is a + * keyword term -- so the engine presents the same text, and a query reads and compares exactly + * what a lookup on the field would match. + */ + private static ExprValue text(Content value) { + ExprValue parsed = OpenSearchExprValueFactory.parseContent(value); + return parsed.isNull() + ? ExprNullValue.of() + : new ExprStringValue(String.valueOf(parsed.value())); + } + + /** + * A path written once holds its value. A path written more than once -- both spellings of a + * dotted path in one document, or one path across the elements of an array -- holds all of them, + * since the index files a term for each and answers a lookup for any of them, and they read as + * their JSON, the way {@code spath} reads an array out of a JSON string. + */ + private static void put(LinkedHashMap out, String key, ExprValue value) { + ExprValue existing = out.get(key); + if (existing == null) { + out.put(key, value); + return; + } + List values = new ArrayList<>(); + collect(existing, values); + collect(value, values); + out.put(key, new ExprStringValue(asJsonArray(values))); + } + + /** The values a key already holds, unwrapped from the JSON an earlier merge wrote. */ + private static void collect(ExprValue value, List into) { + if (value.isNull()) { + return; + } + String text = value.stringValue(); + if (text.startsWith("[") && text.endsWith("]")) { + into.addAll(fromJsonArray(text)); + } else { + into.add(text); + } + } + + private static String asJsonArray(List values) { + try { + return OBJECT_MAPPER.writeValueAsString(values); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Cannot render a flat_object leaf as JSON", e); + } + } + + private static List fromJsonArray(String json) { + try { + return OBJECT_MAPPER.readValue(json, new TypeReference>() {}); + } catch (JsonProcessingException e) { + // not JSON this class wrote: a single value that happens to look like an array + return List.of(json); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index aeda2da0ce6..be75bc5caa9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -61,6 +61,7 @@ import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; +import org.opensearch.sql.opensearch.data.type.OpenSearchFlatObjectType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; @@ -217,6 +218,11 @@ private ExprValue parse( } catch (Exception e) { return ExprNullValue.of(); } + } else if (type.equals(OpenSearchFlatObjectType.of())) { + // Before the isArray() branch below: OpenSearch accepts an array at the top level of a + // flat_object and folds its elements into the same terms, so flatten handles it rather than + // parseArray, which would leave the elements unflattened. + return FlatObjectValues.flatten(content); } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) || content.isArray()) { return parseArray(content, field, type, supportArrays); @@ -260,7 +266,7 @@ private static String stringOf(Content content) { } } - private ExprValue parseContent(Content content) { + static ExprValue parseContent(Content content) { if (content.isNumber()) { if (content.isInt()) { return new ExprIntegerValue(content.intValue()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java index a62661ee601..919736e05f0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java @@ -40,6 +40,7 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; +import org.opensearch.sql.opensearch.data.type.OpenSearchFlatObjectType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.CalciteScriptEngine.Source; @@ -110,8 +111,10 @@ public RexNode visitInputRef(RexInputRef inputRef, ScriptParameterHelper helper) String docFieldName = exprType == ExprCoreType.STRUCT || exprType == ExprCoreType.ARRAY - // A binary field has no doc values, so it has to be read from _source too. + // A binary field has no doc values, so it has to be read from _source too, and + // neither does a flat_object: each leaf is one folded path=value term. || exprType instanceof OpenSearchBinaryType + || exprType instanceof OpenSearchFlatObjectType ? null : OpenSearchTextType.toKeywordSubField(field.getName(), exprType); int newIndex = helper.sources.size(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 247b7a754e0..1f69e86be93 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -568,4 +568,16 @@ public void test_parseMapping_on_AliasType() { new OpenSearchAliasType("col1", OpenSearchDataType.of(MappingType.Invalid))), OpenSearchDataType.parseMapping(indexMapping2)); } + + @Test + // A flat_object declares no `properties`, so it must survive parseMapping as a leaf of its own + // type -- an unknown mapping type used to be dropped, which is why the field was invisible. + public void of_flat_object() { + Map parsed = + OpenSearchDataType.parseMapping(Map.of("attributes", Map.of("type", "flat_object"))); + assertAll( + () -> assertEquals(1, parsed.size()), + () -> assertEquals(OpenSearchFlatObjectType.of(), parsed.get("attributes")), + () -> assertEquals(1, OpenSearchDataType.traverseAndFlatten(parsed).size())); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectTypeTest.java new file mode 100644 index 00000000000..edbd137a62f --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectTypeTest.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.type; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import java.util.Map; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; + +class OpenSearchFlatObjectTypeTest { + + @Test + void of_isASingletonThatSurvivesCloning() { + OpenSearchFlatObjectType type = OpenSearchFlatObjectType.of(); + assertAll( + () -> assertSame(type, OpenSearchFlatObjectType.of()), + () -> assertSame(type, OpenSearchDataType.of(OpenSearchDataType.MappingType.FlatObject)), + () -> assertSame(type, type.cloneEmpty()), + () -> assertSame(type, type.cloneDeep())); + } + + // Like text, geo_point and binary: no core-type counterpart, so the type stands for itself in + // the type environment and is mapped to Calcite by name. + @Test + void hasNoCoreTypeAndResolvesToItself() { + OpenSearchFlatObjectType type = OpenSearchFlatObjectType.of(); + assertAll( + () -> assertEquals(UNKNOWN, type.getExprCoreType()), + () -> assertSame(type, type.getExprType()), + () -> assertEquals("FLAT_OBJECT", type.legacyTypeName()), + () -> assertEquals(OpenSearchDataType.MappingType.FlatObject, type.getMappingType()), + () -> assertTrue(type.getProperties().isEmpty())); + } + + // The field is presented to the planner as a map from dotted leaf path to text: a leaf has no + // type of its own, and the index holds each as a keyword term. A map of text to text is also what + // tells this field apart from an ordinary object, which is a map of text to ANY. + @Test + void mapsToCalciteMapOfVarcharToVarchar() { + RelDataType relType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(OpenSearchFlatObjectType.of()); + assertAll( + () -> assertEquals(SqlTypeName.MAP, relType.getSqlTypeName()), + () -> assertEquals(SqlTypeName.VARCHAR, relType.getKeyType().getSqlTypeName()), + () -> assertEquals(SqlTypeName.VARCHAR, relType.getValueType().getSqlTypeName()), + () -> assertTrue(relType.getValueType().isNullable())); + } + + // The parsed mapping keeps the singleton, so a schema built from it uses the map type above. + @Test + void parsedMappingUsesTheSingleton() { + Map parsed = + OpenSearchDataType.parseMapping(Map.of("attributes", Map.of("type", "flat_object"))); + assertSame(OpenSearchFlatObjectType.of(), parsed.get("attributes")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/FlatObjectValuesTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/FlatObjectValuesTest.java new file mode 100644 index 00000000000..565e2f99617 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/FlatObjectValuesTest.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.value; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.data.utils.ObjectContent; + +class FlatObjectValuesTest { + + private static ExprValue flatten(Object source) { + return FlatObjectValues.flatten(new ObjectContent(source)); + } + + // A leaf has no type of its own in the index, so every value reads as its text -- the same rule + // spath applies to a JSON string. + @Test + void everyLeafReadsAsText() { + Map source = new LinkedHashMap<>(); + source.put("n", 12.5); + source.put("i", 503); + source.put("s", "4"); + source.put("b", true); + source.put("z", null); + ExprValue value = flatten(source); + assertAll( + () -> assertEquals(stringValue("12.5"), value.tupleValue().get("n")), + () -> assertEquals(stringValue("503"), value.tupleValue().get("i")), + () -> assertEquals(stringValue("4"), value.tupleValue().get("s")), + () -> assertEquals(stringValue("true"), value.tupleValue().get("b")), + () -> assertEquals(nullValue(), value.tupleValue().get("z"))); + } + + // Both spellings of a path reach the same leaf, which is what the index holds: one term with the + // path folded in, whichever way the document wrote it. + @Test + void bothSpellingsOfAPathReachTheSameLeaf() { + assertAll( + () -> + assertEquals( + stringValue("1"), flatten(Map.of("a", Map.of("b", 1))).tupleValue().get("a.b")), + () -> assertEquals(stringValue("1"), flatten(Map.of("a.b", 1)).tupleValue().get("a.b"))); + } + + // The index files a term per value, so a path that receives several holds them all, as the JSON + // of those values -- again what spath returns for an array inside a JSON string. + @Test + void aPathWithSeveralValuesReadsAsJson() { + LinkedHashMap collide = new LinkedHashMap<>(); + collide.put("a", Map.of("b", 1)); + collide.put("a.b", 2); + LinkedHashMap collideOtherOrder = new LinkedHashMap<>(); + collideOtherOrder.put("a.b", 2); + collideOtherOrder.put("a", Map.of("b", 1)); + assertAll( + // an array leaf + () -> + assertEquals( + stringValue("[\"a\",\"b\"]"), + flatten(Map.of("tags", List.of("a", "b"))).tupleValue().get("tags")), + // elements of different types still read as text + () -> + assertEquals( + stringValue("[\"1\",\"x\"]"), + flatten(Map.of("mixed", List.of(1, "x"))).tupleValue().get("mixed")), + // an object inside an array folds into the dotted path, one term per element + () -> + assertEquals( + stringValue("[\"1\",\"2\"]"), + flatten(Map.of("nested", List.of(Map.of("k", 1), Map.of("k", 2)))) + .tupleValue() + .get("nested.k")), + // an array at the top level, which OpenSearch also accepts + () -> + assertEquals( + stringValue("[\"1\",\"2\"]"), + flatten(List.of(Map.of("p", 1), Map.of("p", 2))).tupleValue().get("p")), + // both spellings present in one document: neither value may be dropped. The order follows + // the document, as _source order does, so the two spellings read in the order written + () -> assertEquals(stringValue("[\"1\",\"2\"]"), flatten(collide).tupleValue().get("a.b")), + () -> + assertEquals( + stringValue("[\"2\",\"1\"]"), flatten(collideOtherOrder).tupleValue().get("a.b"))); + } + + @Test + void aScalarWhereAnObjectWasPromisedIsNull() { + assertEquals(nullValue(), flatten(5)); + } + + // A flat_object exists so documents can escape the mapping depth limit, so nothing upstream + // bounds the recursion: levels past the cap are kept as one leaf rather than dropped. + @Test + void depthIsCapped() { + Map deepest = Map.of("leaf", "v"); + Map source = deepest; + for (int i = 0; i < 25; i++) { + source = Map.of("l" + i, source); + } + ExprValue value = flatten(source); + assertEquals(1, value.tupleValue().size()); + String key = value.tupleValue().keySet().iterator().next(); + // MAX_DEPTH levels are descended, so the key is that many prefixes plus the leaf's own name + assertEquals(FlatObjectValues.MAX_DEPTH + 1, key.split("\\.").length); + } +}