Read a flat_object field in PPL

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -190,6 +190,7 @@
 import org.opensearch.sql.calcite.utils.WildcardUtils;
 import org.opensearch.sql.common.error.ErrorCode;
 import org.opensearch.sql.common.error.ErrorReport;
+import org.opensearch.sql.common.error.QueryProcessingStage;
 import org.opensearch.sql.common.patterns.PatternUtils;
 import org.opensearch.sql.common.utils.StringUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -4456,7 +4457,24 @@ public RelNode visitExpand(Expand expand, CalcitePlanContext context) {
 
     // 2. Get the field to expand and an optional alias.
     Field arrayField = expand.getField();
-    RexInputRef arrayFieldRex = (RexInputRef) rexVisitor.analyze(arrayField, context);
+    RexNode resolved = rexVisitor.analyze(arrayField, context);
+    if (!(resolved instanceof RexInputRef arrayFieldRex)) {
+      // Anything that is not a column of the input -- a leaf of an object or a flat_object field,
+      // an expression -- has no column to correlate the expansion with. Raised the way the
+      // mvexpand check below does, rather than as the ClassCastException the cast used to throw.
+      String name = StringUtils.unquoteIdentifier(arrayField.getField().toString());
+      throw ErrorReport.wrap(
+              new IllegalArgumentException(StringUtils.format("Cannot expand [%s]", name)))
+          .code(ErrorCode.UNSUPPORTED_OPERATION)
+          .stage(QueryProcessingStage.ANALYZING)
+          .location("while resolving the field for expand")
+          .context("command", "expand")
+          .context("field", name)
+          .suggestion(
+              "expand takes a field of the index, not a value computed from one. Expand the field"
+                  + " itself, or project the value first.")
+          .build();
+    }
     String alias = expand.getAlias();
 
     buildExpandRelNode(arrayFieldRex, arrayField.getField().toString(), alias, null, context);

core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java

@@ -206,6 +206,15 @@ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boole
         return TYPE_FACTORY.createSqlType(SqlTypeName.GEOMETRY, nullable);
       } else if (fieldType.legacyTypeName().equalsIgnoreCase("text")) {
         return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable);
+      } else if (fieldType.legacyTypeName().equalsIgnoreCase("flat_object")) {
+        // Keys are the dotted leaf paths, values are text: a flat_object gives its leaves no type
+        // of their own, and the index holds each as a keyword term. This is what spath already
+        // does for a JSON string -- every value reads as text, and a path holding several values
+        // reads as their JSON.
+        return TYPE_FACTORY.createMapType(
+            TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
+            TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true),
+            nullable);
       } else if (fieldType.legacyTypeName().equalsIgnoreCase("ip")) {
         return TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP, nullable);
       } else if (fieldType.getOriginalPath().isPresent()) {

docs/user/general/datatypes.rst

@@ -103,6 +103,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da
 +-----------------+---------------------+-----------+
 | nested          | array               | STRUCT    |
 +-----------------+---------------------+-----------+
+| flat_object     | struct              | STRUCT    |
++-----------------+---------------------+-----------+
 
 Notes:
 * Not all the OpenSearch SQL Type has correspond OpenSearch Type. e.g. data and time. To use function which required such data type, user should explicitly convert the data type.

docs/user/ppl/general/datatypes.md

@@ -113,6 +113,16 @@ A string is a sequence of characters enclosed in either single or double quotes.
 ## Query Struct Data Types  
 
 In PPL, the Struct Data Types corresponding to the [Object field type in OpenSearch](https://opensearch.org/docs/latest/field-types/supported-field-types/object/). The "." is used as the path selector when access the inner attribute of the struct data.
+
+A [flat_object field](https://docs.opensearch.org/latest/mappings/supported-field-types/flat-object/) is also presented as a struct, keyed by the dotted path of each leaf. A flat_object declares no sub-fields, so a leaf has no type of its own: every value reads as text, and a path holding more than one value reads as their JSON. This is the rule `spath` already applies to a JSON string, and `typeof` on a leaf reports `string`.
+
+Both a nested object and a literal dotted key reach the same leaf: `{"a": {"b": 1}}` and `{"a.b": 1}` each give the entry `a.b` with the value `"1"`. A path receives more than one value when it comes from an array, from several elements of an array of objects, or from both spellings of the same path appearing in one document; it then reads as a JSON array of those texts, in the order the document wrote them.
+
+A leaf is text, so a query does with it what it does with any text column, the same as a field `spath` extracted from a JSON string: read it, cast it (`eval ms = cast(attributes.duration_ms as double)`), filter by it, sort by it, group by it.
+
+A path holding more than one value reads as JSON, so it does not compare equal to any single one of those values. The index still matches each of them, so a filter answers differently depending on whether it reached the index — that applies to any field holding several values, not only to a flat_object leaf.
+
+`expand` takes a column of the index rather than a leaf; expand the field itself, or project the leaf first.
 ### Example: People  
 
 There are three fields in test index `people`: 1) deep nested object field `city`; 2) object field of array value `account`; 3) nested field `projects`

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/1604.yml

@@ -0,0 +1,163 @@
+# Issue: https://github.com/opensearch-project/sql/issues/1604
+#        https://github.com/opensearch-project/sql/issues/3067
+# A flat_object field used to be dropped by OpenSearchDataType.parseMapping ("unknown type, skip
+# it"), so it was absent from the schema and any reference failed as "field not found".
+#
+# The field is now a struct keyed by the dotted path of each leaf. A leaf has no type of its own in
+# the index, so every value reads as text and a path holding several values reads as their JSON --
+# the same rule spath applies to a JSON string.
+setup:
+  - do:
+      indices.create:
+        index: otel
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              service:
+                type: keyword
+              attributes:
+                type: flat_object
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "otel", "_id": "1"}}'
+          - '{"service": "checkout", "attributes": {"http.response.status_code": 500, "duration_ms": 12.5, "ok": true}}'
+          - '{"index": {"_index": "otel", "_id": "2"}}'
+          - '{"service": "payments", "attributes": {"http": {"response": {"status_code": 503}}, "duration_ms": "4"}}'
+          - '{"index": {"_index": "otel", "_id": "3"}}'
+          - '{"service": "tags", "attributes": {"tags": ["a", "b"], "mixed": [1, "x"], "nested": [{"k": 1}, {"k": 2}]}}'
+          - '{"index": {"_index": "otel", "_id": "4"}}'
+          - '{"service": "collide", "attributes": {"a": {"b": 1}, "a.b": 2}}'
+
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: true
+            plugins.calcite.fallback.allowed: false
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: otel
+        ignore_unavailable: true
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: null
+            plugins.calcite.fallback.allowed: null
+
+---
+"describe lists the field as flat_object":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'describe otel | fields COLUMN_NAME, TYPE_NAME'
+  - match: {"datarows": [["attributes", "flat_object"], ["service", "string"]]}
+
+---
+"the whole field reads as a map of dotted leaf path to text":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where service = ''checkout'' | fields attributes'
+  # every value is text, and the dotted path is the key -- both spellings of a path reach the same one
+  - match: {"datarows": [[{"http.response.status_code": "500", "duration_ms": "12.5", "ok": "true"}]]}
+
+---
+"a leaf reads as text, under either spelling of its path":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | fields service, attributes.http.response.status_code, attributes.duration_ms | sort service'
+  - match: {"datarows": [["checkout", "500", "12.5"], ["collide", null, null], ["payments", "503", "4"], ["tags", null, null]]}
+
+---
+"a path holding several values reads as their JSON":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where service = ''tags'' | fields attributes.tags, attributes.mixed, attributes.nested.k'
+  - match: {"datarows": [["[\"a\",\"b\"]", "[\"1\",\"x\"]", "[\"1\",\"2\"]"]]}
+  # both spellings of a path in one document: neither value is dropped
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where service = ''collide'' | fields attributes.a.b'
+  - match: {"datarows": [["[\"1\",\"2\"]"]]}
+
+---
+"a leaf can be cast, and typeof reports text":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where service = ''payments'' | eval n = cast(attributes.duration_ms as double), t = typeof(attributes.duration_ms) | fields n, t'
+  - match: {"datarows": [[4.0, "STRING"]]}
+
+---
+# A leaf is text, so PPL does with it what it does with any text column -- the same as a field
+# spath extracted from a JSON string.
+"a leaf can be filtered, sorted and aggregated like any text column":
+  - skip:
+      features: [headers]
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where attributes.duration_ms = ''4'' | fields service'
+  - match: {"datarows": [["payments"]]}
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where attributes.duration_ms > 5 | fields service'
+  - match: {"datarows": [["checkout"]]}
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | stats avg(attributes.duration_ms)'
+  - match: {"datarows": [[8.25]]}
+  - do:
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | where isnotnull(attributes.duration_ms) | sort attributes.duration_ms | fields attributes.duration_ms'
+  - match: {"datarows": [["12.5"], ["4"]]}
+
+---
+# expand needs a column of the input to correlate the expansion with. A leaf is not one, so it is
+# reported the way the mvexpand check reports it rather than as a ClassCastException.
+"expand does not take a leaf":
+  - skip:
+      features: [headers]
+  - do:
+      catch: bad_request
+      headers: {Content-Type: 'application/json'}
+      ppl:
+        body:
+          query: 'source=otel | expand attributes.tags | fields service'
+  - match: {"$body": "/Cannot\\s+expand\\s+.attributes.tags.*/"}

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java

@@ -39,6 +39,7 @@ public enum MappingType {
     DateNanos("date_nanos", ExprCoreType.TIMESTAMP),
     Object("object", ExprCoreType.STRUCT),
     Nested("nested", ExprCoreType.ARRAY),
+    FlatObject("flat_object", ExprCoreType.UNKNOWN),
     Byte("byte", ExprCoreType.BYTE),
     Short("short", ExprCoreType.SHORT),
     Integer("integer", ExprCoreType.INTEGER),
@@ -170,6 +171,8 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object>
         return OpenSearchGeoPointType.of();
       case Binary:
         return OpenSearchBinaryType.of();
+      case FlatObject:
+        return OpenSearchFlatObjectType.of();
       case Date:
       case DateNanos:
         // Default date formatter is used when "" is passed as the second parameter

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchFlatObjectType.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.opensearch.data.type;
+
+import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;
+
+import lombok.EqualsAndHashCode;
+
+/**
+ * The type of a flat_object field. See <a
+ * href="https://docs.opensearch.org/latest/mappings/supported-field-types/flat-object/">doc</a>
+ *
+ * <p>A flat_object declares no sub-fields in the index mapping: its keys exist only inside the
+ * documents, and every leaf is indexed as a single keyword term with no numeric type. The engine
+ * therefore reads its values from {@code _source} and presents the field as a map keyed by the
+ * dotted leaf path, so that a nested object and a literal dotted key resolve to the same entry.
+ *
+ * <p>Like the other mapping types with no direct core-type counterpart (text, geo_point, binary),
+ * it carries {@code UNKNOWN} as its core type so that {@link #getExprType()} returns this instance
+ * and the Calcite type factory can map it by name.
+ */
+@EqualsAndHashCode(callSuper = false)
+public class OpenSearchFlatObjectType extends OpenSearchDataType {
+
+  private static final OpenSearchFlatObjectType instance = new OpenSearchFlatObjectType();
+
+  private OpenSearchFlatObjectType() {
+    super(MappingType.FlatObject);
+    exprCoreType = UNKNOWN;
+  }
+
+  public static OpenSearchFlatObjectType of() {
+    return OpenSearchFlatObjectType.instance;
+  }
+
+  @Override
+  protected OpenSearchDataType cloneEmpty() {
+    return instance;
+  }
+}

opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java

@@ -53,6 +53,9 @@ public interface Content {
   /** Is array value. */
   boolean isArray();
 
+  /** Is object value: a set of named members, as opposed to a scalar or an array. */
+  boolean isObject();
+
   /** Get integer value. */
   Integer intValue();
 

opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java

@@ -144,6 +144,11 @@ public boolean isArray() {
     return value instanceof ArrayNode || value instanceof List;
   }
 
+  @Override
+  public boolean isObject() {
+    return value instanceof Map;
+  }
+
   @Override
   public boolean isString() {
     return value instanceof String;

opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java

@@ -140,6 +140,11 @@ public boolean isArray() {
     return value().isArray();
   }
 
+  @Override
+  public boolean isObject() {
+    return value().isObject();
+  }
+
   @Override
   public Object objectValue() {
     return value();