diff --git a/core/src/main/java/io/substrait/dialect/ExpandFieldType.java b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java index d6e76b34f..fea2f6edc 100644 --- a/core/src/main/java/io/substrait/dialect/ExpandFieldType.java +++ b/core/src/main/java/io/substrait/dialect/ExpandFieldType.java @@ -4,6 +4,6 @@ public enum ExpandFieldType { /** A field whose value switches between expressions per expansion. */ SWITCHING_FIELD, - /** A field with a constant value across expansions. */ - CONSTANT_FIELD + /** A field with the same value across expansions. */ + CONSISTENT_FIELD } diff --git a/core/src/main/java/io/substrait/dialect/ExpressionKind.java b/core/src/main/java/io/substrait/dialect/ExpressionKind.java index bb42360e1..eda9f0841 100644 --- a/core/src/main/java/io/substrait/dialect/ExpressionKind.java +++ b/core/src/main/java/io/substrait/dialect/ExpressionKind.java @@ -26,6 +26,10 @@ public enum ExpressionKind { NESTED, /** A dynamic parameter expression. */ DYNAMIC_PARAMETER, + /** A lambda expression. */ + LAMBDA, + /** A lambda invocation expression. */ + LAMBDA_INVOCATION, /** An execution-context variable expression. */ EXECUTION_CONTEXT_VARIABLE } diff --git a/core/src/main/java/io/substrait/dialect/RelationKind.java b/core/src/main/java/io/substrait/dialect/RelationKind.java index 03f409bfb..72814eb8c 100644 --- a/core/src/main/java/io/substrait/dialect/RelationKind.java +++ b/core/src/main/java/io/substrait/dialect/RelationKind.java @@ -42,6 +42,8 @@ public enum RelationKind { EXCHANGE, /** The expand relation. */ EXPAND, + /** The top-N relation. */ + TOP_N, /** The single-input extension relation. */ EXTENSION_SINGLE, /** The multi-input extension relation. */ diff --git a/core/src/main/java/io/substrait/dialect/SupportedRelation.java b/core/src/main/java/io/substrait/dialect/SupportedRelation.java index 8f58bf2c5..1b7907d0e 100644 --- a/core/src/main/java/io/substrait/dialect/SupportedRelation.java +++ b/core/src/main/java/io/substrait/dialect/SupportedRelation.java @@ -79,7 +79,8 @@ public abstract class SupportedRelation { public abstract List fieldTypes(); /** - * Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}. + * Supported message type URIs for {@code EXTENSION_SINGLE}/{@code MULTI}/{@code LEAF}, which must + * name at least one. * * @return the supported message type URIs */ @@ -100,21 +101,37 @@ protected void checkWriteTypes() { } /** - * Whether this entry can be written as a bare enum string. Extension relations are never bare: - * they are absent from the schema's bare-enum list. - * - * @return {@code true} if the entry carries no configuration + * The schema requires an extension relation entry to name at least one message type, the same way + * a {@code USER_DEFINED} type entry names its extension. */ - public boolean isBare() { - switch (relation()) { + @Value.Check + protected void checkMessageTypes() { + if (isExtension(relation()) && messageTypes().isEmpty()) { + throw new IllegalArgumentException( + relation() + " relations must declare at least one supported message type."); + } + } + + private static boolean isExtension(RelationKind relation) { + switch (relation) { case EXTENSION_SINGLE: case EXTENSION_MULTI: case EXTENSION_LEAF: - return false; + return true; default: - break; + return false; } - return !metadata().isPresent() + } + + /** + * Whether this entry can be written as a bare enum string. Extension relations are never bare: + * they are absent from the schema's bare-enum list. + * + * @return {@code true} if the entry carries no configuration + */ + public boolean isBare() { + return !isExtension(relation()) + && !metadata().isPresent() && joinTypes().isEmpty() && readTypes().isEmpty() && operations().isEmpty() diff --git a/core/src/main/java/io/substrait/dialect/TypeKind.java b/core/src/main/java/io/substrait/dialect/TypeKind.java index 30f274ba9..f62dd6ade 100644 --- a/core/src/main/java/io/substrait/dialect/TypeKind.java +++ b/core/src/main/java/io/substrait/dialect/TypeKind.java @@ -54,6 +54,8 @@ public enum TypeKind { LIST, /** The map type. */ MAP, + /** The function type, the type of a lambda expression. */ + FUNC, /** A user-defined extension type. */ USER_DEFINED } diff --git a/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java index 46806bb29..badd280d7 100644 --- a/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java +++ b/core/src/test/java/io/substrait/dialect/DialectBareStringCollapseTest.java @@ -48,7 +48,10 @@ void configuredRelationSerializesAsObject() { @Test void extensionRelationIsNeverBare() { SupportedRelation extension = - SupportedRelation.builder().relation(RelationKind.EXTENSION_LEAF).build(); + SupportedRelation.builder() + .relation(RelationKind.EXTENSION_LEAF) + .addMessageTypes("type.googleapis.com/google.profile.Person") + .build(); assertFalse(extension.isBare()); String yaml = Dialect.toYaml(Dialect.builder().addSupportedRelations(extension).build()); diff --git a/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java index 58d17c253..645322728 100644 --- a/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java +++ b/core/src/test/java/io/substrait/dialect/DialectRoundTripTest.java @@ -20,6 +20,7 @@ private static Dialect sampleDialect() { .putDependencies("spark", "extension:substrait:spark") // Types: bare, precision (max_precision), system_metadata, and user-defined. .addSupportedTypes(SupportedType.of(TypeKind.BOOL)) + .addSupportedTypes(SupportedType.of(TypeKind.FUNC)) .addSupportedTypes( SupportedType.builder() .type(TypeKind.PRECISION_TIMESTAMP) @@ -38,6 +39,7 @@ private static Dialect sampleDialect() { .build()) // Relations: bare, join, read, set, write, ddl, exchange, expand, extension. .addSupportedRelations(SupportedRelation.of(RelationKind.FILTER)) + .addSupportedRelations(SupportedRelation.of(RelationKind.TOP_N)) .addSupportedRelations( SupportedRelation.builder() .relation(RelationKind.JOIN) @@ -71,15 +73,17 @@ private static Dialect sampleDialect() { .addSupportedRelations( SupportedRelation.builder() .relation(RelationKind.EXPAND) - .addFieldTypes(ExpandFieldType.SWITCHING_FIELD) + .addFieldTypes(ExpandFieldType.SWITCHING_FIELD, ExpandFieldType.CONSISTENT_FIELD) .build()) .addSupportedRelations( SupportedRelation.builder() .relation(RelationKind.EXTENSION_SINGLE) .addMessageTypes("type.googleapis.com/google.profile.Person") .build()) - // Expressions: bare, cast, subquery, nested, execution-context-variable. + // Expressions: bare, lambda, cast, subquery, nested, execution-context-variable. .addSupportedExpressions(SupportedExpression.of(ExpressionKind.LITERAL)) + .addSupportedExpressions(SupportedExpression.of(ExpressionKind.LAMBDA)) + .addSupportedExpressions(SupportedExpression.of(ExpressionKind.LAMBDA_INVOCATION)) .addSupportedExpressions( SupportedExpression.builder() .expression(ExpressionKind.CAST) diff --git a/core/src/test/java/io/substrait/dialect/DialectValidationTest.java b/core/src/test/java/io/substrait/dialect/DialectValidationTest.java index 2ce573732..198e30811 100644 --- a/core/src/test/java/io/substrait/dialect/DialectValidationTest.java +++ b/core/src/test/java/io/substrait/dialect/DialectValidationTest.java @@ -30,4 +30,17 @@ void writeAndDdlWriteTypesAreMutuallyExclusive() { .addDdlWriteTypes(DdlWriteType.NAMED_OBJECT) .build()); } + + @Test + void extensionRelationsRequireMessageTypes() { + for (RelationKind kind : + new RelationKind[] { + RelationKind.EXTENSION_SINGLE, RelationKind.EXTENSION_MULTI, RelationKind.EXTENSION_LEAF + }) { + assertThrows( + IllegalArgumentException.class, + () -> SupportedRelation.builder().relation(kind).build(), + kind.name()); + } + } } diff --git a/core/src/test/java/io/substrait/type/ParameterizedReturnTypeTest.java b/core/src/test/java/io/substrait/type/ParameterizedReturnTypeTest.java index c870f38c5..2ba51d016 100644 --- a/core/src/test/java/io/substrait/type/ParameterizedReturnTypeTest.java +++ b/core/src/test/java/io/substrait/type/ParameterizedReturnTypeTest.java @@ -106,14 +106,11 @@ void aConcreteReturnStillChecksSharedParameters() { assertTrue( comparison.getMessage().contains("Inconsistent binding for type parameter 'any1'"), comparison.getMessage()); + } - UnsupportedOperationException strpos = - assertThrows( - UnsupportedOperationException.class, - () -> resolve("strpos:vchar_vchar", R.varChar(20), R.varChar(3))); - assertTrue( - strpos.getMessage().contains("Inconsistent binding for type parameter 'L1'"), - strpos.getMessage()); + @Test + void independentParametersBindSeparately() { + assertEquals(R.I64, resolve("strpos:vchar_vchar", R.varChar(20), R.varChar(3))); } @Test diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index eddea1f64..f9838575d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,7 +31,7 @@ spark-3-5 = "3.5.4" spark-4-0 = "4.0.2" spotless = "8.10.2" # substrait-packaging artifacts, versioned by the Substrait spec release they are generated from. -substrait-packaging = "0.102.0" +substrait-packaging = "0.103.0" testcontainers = "2.0.5" validator = "3.0.7"