From fd8782a14183bf5915803ecdf133517b5395f43d Mon Sep 17 00:00:00 2001 From: Ccompailer Date: Mon, 5 Oct 2026 02:06:43 +0300 Subject: [PATCH 1/2] in progress: Lift composite component readers into LiftableConstantExpression --- .../ClickHouseQueryCompilationContext.cs | 9 +- ...lickHouseQueryCompilationContextFactory.cs | 7 +- .../Mapping/ClickHouseComponentConversion.cs | 89 ++++++++++++++++--- .../Mapping/ClickHouseTupleTypeMapping.cs | 10 +-- 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContext.cs b/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContext.cs index 2df970f..31c35b1 100644 --- a/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContext.cs +++ b/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContext.cs @@ -1,14 +1,19 @@ +#pragma warning disable EF9100 + using Microsoft.EntityFrameworkCore.Query; namespace ClickHouse.EntityFrameworkCore.Query.Internal; public class ClickHouseQueryCompilationContext : RelationalQueryCompilationContext { + public override bool SupportsPrecompiledQuery => true; + public ClickHouseQueryCompilationContext( QueryCompilationContextDependencies dependencies, RelationalQueryCompilationContextDependencies relationalDependencies, - bool async) - : base(dependencies, relationalDependencies, async) + bool async, + bool precompiling) + : base(dependencies, relationalDependencies, async, precompiling: precompiling) { } } diff --git a/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContextFactory.cs b/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContextFactory.cs index 933da97..5f40857 100644 --- a/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContextFactory.cs +++ b/src/EFCore.ClickHouse/Query/Internal/ClickHouseQueryCompilationContextFactory.cs @@ -1,3 +1,5 @@ +#pragma warning disable EF9100 + using Microsoft.EntityFrameworkCore.Query; namespace ClickHouse.EntityFrameworkCore.Query.Internal; @@ -16,5 +18,8 @@ public ClickHouseQueryCompilationContextFactory( } public QueryCompilationContext Create(bool async) - => new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async); + => new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async, precompiling: false); + + public QueryCompilationContext CreatePrecompiled(bool async) + => new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async, precompiling: true); } diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs index 68d5dc7..5a56a7a 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs @@ -1,5 +1,8 @@ +#pragma warning disable EF9100 + using System.Collections.Concurrent; using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore.Query; using Microsoft.EntityFrameworkCore.Storage; namespace ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping; @@ -38,8 +41,9 @@ internal static class ClickHouseComponentConversion { // Keyed by (mapping, target type). The compiled converter is embedded in the materializer as a // constant, so it is built once per mapping rather than once per row. - private static readonly ConcurrentDictionary<(RelationalTypeMapping Mapping, Type Target), Delegate> ConverterCache = new(); + private sealed record ComponentReader(LambdaExpression Quotable, Delegate Compiled); + private static readonly ConcurrentDictionary<(RelationalTypeMapping Mapping, Type Target), ComponentReader> ReaderCache = new(); private static readonly ConcurrentDictionary NeedsConversionCache = new(); /// @@ -119,21 +123,84 @@ public static bool CanPassThrough(RelationalTypeMapping mapping) /// convert are not usable from precompiled queries. /// public static Expression CreateConverter(RelationalTypeMapping mapping, Type componentType) - => Expression.Constant( - GetConverter(mapping, componentType), - typeof(Func<,>).MakeGenericType(typeof(object), componentType)); + { + var reader = GetReader(mapping, componentType); + return Lift(reader.Compiled, reader.Quotable, "componentReader"); + } /// - /// Returns the compiled reader for one component as a Func<object, object?>, for a - /// caller that holds the readers in an array rather than embedding each one separately. + /// Returns an expression of type Func<object, object?>[] holding one reader per mapping. /// - public static Func CreateComponentReader(RelationalTypeMapping mapping) - => (Func)GetConverter(mapping, typeof(object)); + public static Expression CreateComponentReaders(IReadOnlyList mappings) + { + var readers = mappings.Select(m => GetReader(m, typeof(object))).ToArray(); + + var compiled = readers.Select(r => (Func)r.Compiled).ToArray(); + var resolver = Expression.NewArrayInit( + typeof(Func), + readers.Select(Expression (r) => r.Quotable)); + + return Lift(compiled, resolver, "componentReaders"); + } + + + private static LiftableConstantExpression Lift(object runtimeValue, Expression resolverBody, string variableName) + { + var context = Expression.Parameter(typeof(MaterializerLiftableConstantContext), "context"); + + return new LiftableConstantExpression( + runtimeValue, + Expression.Lambda(resolverBody, context), + variableName, + resolverBody.Type); + } + + private static ComponentReader BuildReader(RelationalTypeMapping mapping, Type componentType) + { + var parameter = Expression.Parameter(typeof(object), "component"); + var body = mapping.CustomizeDataReaderExpression(parameter); + + if (mapping.Converter is { } valueConverter) + { + body = Coerce(body, valueConverter.ProviderClrType, mapping, componentType); + body = Expression.Invoke(valueConverter.ConvertFromProviderExpression, body); + } + + body = Coerce(body, componentType, mapping, componentType); + + var delegateType = typeof(Func<,>).MakeGenericType(typeof(object), componentType); + + // A nested composite's own CustomizeDataReaderExpression already returned liftable constants. + // Neither lambda can keep them: the compiled one would not compile, and the quotable one + // must not hold a constant. So each gets the matching replacement. + var runtime = Expression.Lambda(delegateType, new LiftedConstantInliner(useResolvers: false).Visit(body), parameter); + var quotable = Expression.Lambda(delegateType, new LiftedConstantInliner(useResolvers: true).Visit(body), parameter); + + return new ComponentReader(quotable, runtime.Compile()); + } + + private sealed class LiftedConstantInliner(bool useResolvers) : ExpressionVisitor + { + protected override Expression VisitExtension(Expression node) + { + if (node is not LiftableConstantExpression lifted) + return base.VisitExtension(node); + + // Runtime path: back to the prebuilt delegate, so a nested reader costs nothing per row. + if (!useResolvers) + return Expression.Constant(lifted.OriginalExpression, lifted.Type); + + // Quotable path: the resolver body. It runs once, when the outer reader is created. + // Resolvers built by Lift() never reference their context parameter, so the body can be + // used directly. + return Visit(lifted.ResolverExpression.Body); + } + } - private static Delegate GetConverter(RelationalTypeMapping mapping, Type componentType) - => ConverterCache.GetOrAdd( + private static ComponentReader GetReader(RelationalTypeMapping mapping, Type componentType) + => ReaderCache.GetOrAdd( (mapping, componentType), - static key => Compile(key.Mapping, key.Target)); + static key => BuildReader(key.Mapping, key.Target)); private static Delegate Compile(RelationalTypeMapping mapping, Type componentType) { diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseTupleTypeMapping.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseTupleTypeMapping.cs index 595a4d9..bdd5e12 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseTupleTypeMapping.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseTupleTypeMapping.cs @@ -59,12 +59,10 @@ public override Expression CustomizeDataReaderExpression(Expression expression) if (!ClrType.IsValueType && !needsComponentConversion) return Expression.Convert(expression, ClrType); - // Build the delegate array once and embed it as a constant. Expression.NewArrayInit would - // instead make the allocation part of the materializer, so every row read would allocate a - // fresh array of the same delegates — about 40 bytes per row on a two-component tuple. - var componentConverters = Expression.Constant( - ElementMappings.Select(ClickHouseComponentConversion.CreateComponentReader).ToArray(), - typeof(Func[])); + // The readers are lifted: normal compilation resolves the array once, precompiled-query + // generation emits a local initialized from NewArrayInit. Either way the materializer + // itself does not allocate the array per row. + var componentConverters = ClickHouseComponentConversion.CreateComponentReaders(ElementMappings); return Expression.Call( ConvertMethod.MakeGenericMethod(ClrType), From 666394ba46a912d340749ef1ffafdcf7bc14d11f Mon Sep 17 00:00:00 2001 From: Ccompailer Date: Thu, 29 Oct 2026 00:17:31 +0300 Subject: [PATCH 2/2] Expose ConvertArray as a public helper and update composite tests Code generated for precompiled queries lives in the user's assembly and calls ConvertArray directly, so it cannot stay private (CS0117 when compiling the generated file). Move it to ClickHouseCompositeMaterialization, a public static class marked EditorBrowsable(Never) and documented as provider internals. Update CompositeElementConversionTests for liftable component readers: - Read inlines liftable constants before compiling the reader expression, as EF does when compiling a materializer. - Component_converters_are_embedded_as_liftable_constants asserts that converters are LiftableConstantExpression (the bool flag stays a plain constant), still guarding against an inline lambda that would allocate a delegate per row. --- .../Mapping/ClickHouseArrayTypeMapping.cs | 29 +------------ .../Mapping/ClickHouseComponentConversion.cs | 25 +++++++---- .../ClickHouseCompositeMaterialization.cs | 42 +++++++++++++++++++ .../CompositeElementConversionTests.cs | 38 +++++++++++++---- 4 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseCompositeMaterialization.cs diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseArrayTypeMapping.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseArrayTypeMapping.cs index 3b56ef3..ce26f98 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseArrayTypeMapping.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseArrayTypeMapping.cs @@ -15,7 +15,7 @@ public class ClickHouseArrayTypeMapping : RelationalTypeMapping typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetValue), [typeof(int)])!; private static readonly MethodInfo ConvertArrayMethod = - typeof(ClickHouseArrayTypeMapping).GetMethod(nameof(ConvertArray), BindingFlags.Static | BindingFlags.NonPublic)!; + typeof(ClickHouseCompositeMaterialization).GetMethod(nameof(ClickHouseCompositeMaterialization.ConvertArray))!; public RelationalTypeMapping ElementMapping { get; } @@ -102,33 +102,6 @@ public override Expression CustomizeDataReaderExpression(Expression expression) return converted.Type == targetType ? converted : Expression.Convert(converted, targetType); } - /// - /// Rebuilds the driver's array as TElement[], converting each element. Nested composites - /// compose through this: an Array(Array(DateTime64)) element mapping is itself a - /// , so its own conversion runs per element. - /// - private static TElement[] ConvertArray( - object value, - Func convertElement, - bool canPassThrough) - { - // The driver often already produces the target type, for example Array(Int32) -> int[]. - // See ClickHouseComponentConversion.CanPassThrough for when that proves there is no work - // left to do. - if (canPassThrough && value is TElement[] alreadyTyped) - return alreadyTyped; - - var source = (Array)value; - var result = new TElement[source.Length]; - for (var i = 0; i < source.Length; i++) - { - var element = source.GetValue(i); - result[i] = element is null or DBNull ? default! : convertElement(element); - } - - return result; - } - protected override string GenerateNonNullSqlLiteral(object value) { var sb = new StringBuilder("["); diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs index 5a56a7a..cddf195 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseComponentConversion.cs @@ -144,13 +144,20 @@ public static Expression CreateComponentReaders(IReadOnlyList, so the body is boxed to object. + // The liftable constant keeps the real type. + var resolver = Expression.Lambda>( + Expression.Convert(resolverBody, typeof(object)), + context); + return new LiftableConstantExpression( runtimeValue, - Expression.Lambda(resolverBody, context), + resolver, variableName, resolverBody.Type); } @@ -178,7 +185,7 @@ private static ComponentReader BuildReader(RelationalTypeMapping mapping, Type c return new ComponentReader(quotable, runtime.Compile()); } - + private sealed class LiftedConstantInliner(bool useResolvers) : ExpressionVisitor { protected override Expression VisitExtension(Expression node) @@ -188,12 +195,14 @@ protected override Expression VisitExtension(Expression node) // Runtime path: back to the prebuilt delegate, so a nested reader costs nothing per row. if (!useResolvers) - return Expression.Constant(lifted.OriginalExpression, lifted.Type); + return lifted.OriginalExpression; + + // Quotable path: the resolver body, without the object box added by Lift(). + var body = lifted.ResolverExpression.Body; + if (body is UnaryExpression { NodeType: ExpressionType.Convert } box && body.Type == typeof(object)) + body = box.Operand; - // Quotable path: the resolver body. It runs once, when the outer reader is created. - // Resolvers built by Lift() never reference their context parameter, so the body can be - // used directly. - return Visit(lifted.ResolverExpression.Body); + return Visit(body); } } diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseCompositeMaterialization.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseCompositeMaterialization.cs new file mode 100644 index 0000000..46c65e3 --- /dev/null +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseCompositeMaterialization.cs @@ -0,0 +1,42 @@ +using System.ComponentModel; + +namespace ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping; + +/// +/// Helpers that composite type mappings (Array, Map, Tuple) call from materializer expressions. +/// +/// +/// This is a provider-internal API. It is public only because code generated for precompiled +/// queries calls it directly and lives in the user's assembly. It is not intended to be used from +/// application code and may change or be removed in any release without notice. +/// +[EditorBrowsable(EditorBrowsableState.Never)] +public static class ClickHouseCompositeMaterialization +{ + /// + /// Rebuilds the driver's array as TElement[], converting each element. Nested composites + /// compose through this: an Array(Array(DateTime64)) element mapping is itself a + /// , so its own conversion runs per element. + /// + public static TElement[] ConvertArray( + object value, + Func convertElement, + bool canPassThrough) + { + // The driver often already produces the target type, for example Array(Int32) -> int[]. + // See ClickHouseComponentConversion.CanPassThrough for when that proves there is no work + // left to do. + if (canPassThrough && value is TElement[] alreadyTyped) + return alreadyTyped; + + var source = (Array)value; + var result = new TElement[source.Length]; + for (var i = 0; i < source.Length; i++) + { + var element = source.GetValue(i); + result[i] = element is null or DBNull ? default! : convertElement(element); + } + + return result; + } +} \ No newline at end of file diff --git a/test/EFCore.ClickHouse.Tests/CompositeElementConversionTests.cs b/test/EFCore.ClickHouse.Tests/CompositeElementConversionTests.cs index 2901c12..c46ef61 100644 --- a/test/EFCore.ClickHouse.Tests/CompositeElementConversionTests.cs +++ b/test/EFCore.ClickHouse.Tests/CompositeElementConversionTests.cs @@ -1,7 +1,10 @@ +#pragma warning disable EF9100 + using System.Linq.Expressions; using ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Query; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using Xunit; @@ -400,16 +403,17 @@ public void Converting_components_take_the_rebuild_path(string propertyName) } /// - /// The per-component converter must be embedded as a constant, not as an inline lambda. An inline - /// lambda is rebuilt on every materialization, which allocates a delegate per row for every - /// composite column that converts — including ones where the runtime fast path then returns the - /// driver's array untouched. + /// The per-component converter must be embedded as a liftable constant, not as an inline lambda. + /// An inline lambda is rebuilt on every materialization, which allocates a delegate per row for + /// every composite column that converts — including ones where the runtime fast path then returns + /// the driver's array untouched. A liftable constant keeps the compiled delegate as the runtime + /// value and still lets EF render the converter into precompiled query source. /// [Theory] [InlineData(nameof(CompositeElementEntity.Offsets))] [InlineData(nameof(CompositeElementEntity.Ints))] [InlineData(nameof(CompositeElementEntity.OffsetsByName))] - public void Component_converters_are_embedded_as_constants(string propertyName) + public void Component_converters_are_embedded_as_liftable_constants(string propertyName) { using var ctx = new CompositeElementDbContext(_fixture.ConnectionString); var mapping = ctx.Model @@ -420,10 +424,17 @@ public void Component_converters_are_embedded_as_constants(string propertyName) var read = (MethodCallExpression)mapping.CustomizeDataReaderExpression( Expression.Parameter(typeof(object), "v")); - // Argument 0 is the raw value; every argument after it is a component converter. + // Argument 0 is the raw value; every argument after it is either the plain bool flag + // or a component converter, which must be lifted. Assert.All( read.Arguments.Skip(1), - argument => Assert.Equal(ExpressionType.Constant, argument.NodeType)); + argument => + { + if (argument.Type == typeof(bool)) + Assert.Equal(ExpressionType.Constant, argument.NodeType); + else + Assert.IsType(argument); + }); } // --- components carrying a ValueConverter ------------------------------- @@ -655,9 +666,20 @@ public async Task An_element_converter_set_on_the_model_is_applied_on_read() private static T Read(RelationalTypeMapping mapping, object driverValue) { var parameter = Expression.Parameter(typeof(object), "value"); - var body = mapping.CustomizeDataReaderExpression(parameter); + + // EF inlines liftable constants before compiling a materializer; do the same here, since a + // LiftableConstantExpression cannot be compiled on its own. + var body = new InlineLiftables().Visit(mapping.CustomizeDataReaderExpression(parameter)); return Expression.Lambda>(Expression.Convert(body, typeof(T)), parameter) .Compile()(driverValue); } + + private sealed class InlineLiftables : ExpressionVisitor + { + protected override Expression VisitExtension(Expression node) + => node is LiftableConstantExpression lifted + ? lifted.OriginalExpression + : base.VisitExtension(node); + } }