Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
#pragma warning disable EF9100

using Microsoft.EntityFrameworkCore.Query;

namespace ClickHouse.EntityFrameworkCore.Query.Internal;

public class ClickHouseQueryCompilationContext : RelationalQueryCompilationContext
{
public override bool SupportsPrecompiledQuery => true;

public ClickHouseQueryCompilationContext(
QueryCompilationContextDependencies dependencies,
RelationalQueryCompilationContextDependencies relationalDependencies,
bool async)
: base(dependencies, relationalDependencies, async)
bool async,
bool precompiling)
: base(dependencies, relationalDependencies, async, precompiling: precompiling)
{
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#pragma warning disable EF9100

using Microsoft.EntityFrameworkCore.Query;

namespace ClickHouse.EntityFrameworkCore.Query.Internal;
Expand All @@ -16,5 +18,8 @@ public ClickHouseQueryCompilationContextFactory(
}

public QueryCompilationContext Create(bool async)
=> new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async);
=> new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async, precompiling: false);

public QueryCompilationContext CreatePrecompiled(bool async)
=> new ClickHouseQueryCompilationContext(_dependencies, _relationalDependencies, async, precompiling: true);
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public class ClickHouseArrayTypeMapping : RelationalTypeMapping
typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetValue), [typeof(int)])!;

private static readonly MethodInfo ConvertArrayMethod =
typeof(ClickHouseArrayTypeMapping).GetMethod(nameof(ConvertArray), BindingFlags.Static | BindingFlags.NonPublic)!;
typeof(ClickHouseCompositeMaterialization).GetMethod(nameof(ClickHouseCompositeMaterialization.ConvertArray))!;

public RelationalTypeMapping ElementMapping { get; }

Expand Down Expand Up @@ -102,33 +102,6 @@ public override Expression CustomizeDataReaderExpression(Expression expression)
return converted.Type == targetType ? converted : Expression.Convert(converted, targetType);
}

/// <summary>
/// Rebuilds the driver's array as <c>TElement[]</c>, converting each element. Nested composites
/// compose through this: an <c>Array(Array(DateTime64))</c> element mapping is itself a
/// <see cref="ClickHouseArrayTypeMapping"/>, so its own conversion runs per element.
/// </summary>
private static TElement[] ConvertArray<TElement>(
object value,
Func<object, TElement> convertElement,
bool canPassThrough)
{
// The driver often already produces the target type, for example Array(Int32) -> int[].
// See ClickHouseComponentConversion.CanPassThrough for when that proves there is no work
// left to do.
if (canPassThrough && value is TElement[] alreadyTyped)
return alreadyTyped;

var source = (Array)value;
var result = new TElement[source.Length];
for (var i = 0; i < source.Length; i++)
{
var element = source.GetValue(i);
result[i] = element is null or DBNull ? default! : convertElement(element);
}

return result;
}

protected override string GenerateNonNullSqlLiteral(object value)
{
var sb = new StringBuilder("[");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#pragma warning disable EF9100

using System.Collections.Concurrent;
using System.Linq.Expressions;
using Microsoft.EntityFrameworkCore.Query;
using Microsoft.EntityFrameworkCore.Storage;

namespace ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping;
Expand Down Expand Up @@ -38,8 +41,9 @@ internal static class ClickHouseComponentConversion
{
// Keyed by (mapping, target type). The compiled converter is embedded in the materializer as a
// constant, so it is built once per mapping rather than once per row.
private static readonly ConcurrentDictionary<(RelationalTypeMapping Mapping, Type Target), Delegate> ConverterCache = new();
private sealed record ComponentReader(LambdaExpression Quotable, Delegate Compiled);

private static readonly ConcurrentDictionary<(RelationalTypeMapping Mapping, Type Target), ComponentReader> ReaderCache = new();
private static readonly ConcurrentDictionary<RelationalTypeMapping, bool> NeedsConversionCache = new();

/// <summary>
Expand Down Expand Up @@ -119,21 +123,93 @@ public static bool CanPassThrough(RelationalTypeMapping mapping)
/// convert are not usable from precompiled queries.
/// </remarks>
public static Expression CreateConverter(RelationalTypeMapping mapping, Type componentType)
=> Expression.Constant(
GetConverter(mapping, componentType),
typeof(Func<,>).MakeGenericType(typeof(object), componentType));
{
var reader = GetReader(mapping, componentType);
return Lift(reader.Compiled, reader.Quotable, "componentReader");
}

/// <summary>
/// Returns the compiled reader for one component as a <c>Func&lt;object, object?&gt;</c>, for a
/// caller that holds the readers in an array rather than embedding each one separately.
/// Returns an expression of type <c>Func&lt;object, object?&gt;[]</c> holding one reader per mapping.
/// </summary>
public static Func<object, object?> CreateComponentReader(RelationalTypeMapping mapping)
=> (Func<object, object?>)GetConverter(mapping, typeof(object));
public static Expression CreateComponentReaders(IReadOnlyList<RelationalTypeMapping> mappings)
{
var readers = mappings.Select(m => GetReader(m, typeof(object))).ToArray();

var compiled = readers.Select(r => (Func<object, object?>)r.Compiled).ToArray();
var resolver = Expression.NewArrayInit(
typeof(Func<object, object?>),
readers.Select(Expression (r) => r.Quotable));

return Lift(compiled, resolver, "componentReaders");
}


private static Expression Lift(object runtimeValue, Expression resolverBody, string variableName)
{
var context = Expression.Parameter(typeof(MaterializerLiftableConstantContext), "context");

// EF's non-precompiled pipeline only accepts a resolver shaped
// Func<MaterializerLiftableConstantContext, object>, so the body is boxed to object.
// The liftable constant keeps the real type.
var resolver = Expression.Lambda<Func<MaterializerLiftableConstantContext, object>>(
Expression.Convert(resolverBody, typeof(object)),
context);

return new LiftableConstantExpression(
runtimeValue,
resolver,
variableName,
resolverBody.Type);
}

private static ComponentReader BuildReader(RelationalTypeMapping mapping, Type componentType)
{
var parameter = Expression.Parameter(typeof(object), "component");
var body = mapping.CustomizeDataReaderExpression(parameter);

if (mapping.Converter is { } valueConverter)
{
body = Coerce(body, valueConverter.ProviderClrType, mapping, componentType);
body = Expression.Invoke(valueConverter.ConvertFromProviderExpression, body);
}

body = Coerce(body, componentType, mapping, componentType);

var delegateType = typeof(Func<,>).MakeGenericType(typeof(object), componentType);

// A nested composite's own CustomizeDataReaderExpression already returned liftable constants.
// Neither lambda can keep them: the compiled one would not compile, and the quotable one
// must not hold a constant. So each gets the matching replacement.
var runtime = Expression.Lambda(delegateType, new LiftedConstantInliner(useResolvers: false).Visit(body), parameter);
var quotable = Expression.Lambda(delegateType, new LiftedConstantInliner(useResolvers: true).Visit(body), parameter);

return new ComponentReader(quotable, runtime.Compile());
}

private sealed class LiftedConstantInliner(bool useResolvers) : ExpressionVisitor
{
protected override Expression VisitExtension(Expression node)
{
if (node is not LiftableConstantExpression lifted)
return base.VisitExtension(node);

// Runtime path: back to the prebuilt delegate, so a nested reader costs nothing per row.
if (!useResolvers)
return lifted.OriginalExpression;

// Quotable path: the resolver body, without the object box added by Lift().
var body = lifted.ResolverExpression.Body;
if (body is UnaryExpression { NodeType: ExpressionType.Convert } box && body.Type == typeof(object))
body = box.Operand;

return Visit(body);
}
}

private static Delegate GetConverter(RelationalTypeMapping mapping, Type componentType)
=> ConverterCache.GetOrAdd(
private static ComponentReader GetReader(RelationalTypeMapping mapping, Type componentType)
=> ReaderCache.GetOrAdd(
(mapping, componentType),
static key => Compile(key.Mapping, key.Target));
static key => BuildReader(key.Mapping, key.Target));

private static Delegate Compile(RelationalTypeMapping mapping, Type componentType)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
using System.ComponentModel;

namespace ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping;

/// <summary>
/// Helpers that composite type mappings (Array, Map, Tuple) call from materializer expressions.
/// </summary>
/// <remarks>
/// This is a provider-internal API. It is public only because code generated for precompiled
/// queries calls it directly and lives in the user's assembly. It is not intended to be used from
/// application code and may change or be removed in any release without notice.
/// </remarks>
[EditorBrowsable(EditorBrowsableState.Never)]
public static class ClickHouseCompositeMaterialization
{
/// <summary>
/// Rebuilds the driver's array as <c>TElement[]</c>, converting each element. Nested composites
/// compose through this: an <c>Array(Array(DateTime64))</c> element mapping is itself a
/// <see cref="ClickHouseArrayTypeMapping"/>, so its own conversion runs per element.
/// </summary>
public static TElement[] ConvertArray<TElement>(
object value,
Func<object, TElement> convertElement,
bool canPassThrough)
{
// The driver often already produces the target type, for example Array(Int32) -> int[].
// See ClickHouseComponentConversion.CanPassThrough for when that proves there is no work
// left to do.
if (canPassThrough && value is TElement[] alreadyTyped)
return alreadyTyped;

var source = (Array)value;
var result = new TElement[source.Length];
for (var i = 0; i < source.Length; i++)
{
var element = source.GetValue(i);
result[i] = element is null or DBNull ? default! : convertElement(element);
}

return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,10 @@ public override Expression CustomizeDataReaderExpression(Expression expression)
if (!ClrType.IsValueType && !needsComponentConversion)
return Expression.Convert(expression, ClrType);

// Build the delegate array once and embed it as a constant. Expression.NewArrayInit would
// instead make the allocation part of the materializer, so every row read would allocate a
// fresh array of the same delegates — about 40 bytes per row on a two-component tuple.
var componentConverters = Expression.Constant(
ElementMappings.Select(ClickHouseComponentConversion.CreateComponentReader).ToArray(),
typeof(Func<object, object?>[]));
// The readers are lifted: normal compilation resolves the array once, precompiled-query
// generation emits a local initialized from NewArrayInit. Either way the materializer
// itself does not allocate the array per row.
var componentConverters = ClickHouseComponentConversion.CreateComponentReaders(ElementMappings);

return Expression.Call(
ConvertMethod.MakeGenericMethod(ClrType),
Expand Down
38 changes: 30 additions & 8 deletions test/EFCore.ClickHouse.Tests/CompositeElementConversionTests.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
#pragma warning disable EF9100

using System.Linq.Expressions;
using ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Query;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Xunit;
Expand Down Expand Up @@ -400,16 +403,17 @@ public void Converting_components_take_the_rebuild_path(string propertyName)
}

/// <summary>
/// The per-component converter must be embedded as a constant, not as an inline lambda. An inline
/// lambda is rebuilt on every materialization, which allocates a delegate per row for every
/// composite column that converts — including ones where the runtime fast path then returns the
/// driver's array untouched.
/// The per-component converter must be embedded as a liftable constant, not as an inline lambda.
/// An inline lambda is rebuilt on every materialization, which allocates a delegate per row for
/// every composite column that converts — including ones where the runtime fast path then returns
/// the driver's array untouched. A liftable constant keeps the compiled delegate as the runtime
/// value and still lets EF render the converter into precompiled query source.
/// </summary>
[Theory]
[InlineData(nameof(CompositeElementEntity.Offsets))]
[InlineData(nameof(CompositeElementEntity.Ints))]
[InlineData(nameof(CompositeElementEntity.OffsetsByName))]
public void Component_converters_are_embedded_as_constants(string propertyName)
public void Component_converters_are_embedded_as_liftable_constants(string propertyName)
{
using var ctx = new CompositeElementDbContext(_fixture.ConnectionString);
var mapping = ctx.Model
Expand All @@ -420,10 +424,17 @@ public void Component_converters_are_embedded_as_constants(string propertyName)
var read = (MethodCallExpression)mapping.CustomizeDataReaderExpression(
Expression.Parameter(typeof(object), "v"));

// Argument 0 is the raw value; every argument after it is a component converter.
// Argument 0 is the raw value; every argument after it is either the plain bool flag
// or a component converter, which must be lifted.
Assert.All(
read.Arguments.Skip(1),
argument => Assert.Equal(ExpressionType.Constant, argument.NodeType));
argument =>
{
if (argument.Type == typeof(bool))
Assert.Equal(ExpressionType.Constant, argument.NodeType);
else
Assert.IsType<LiftableConstantExpression>(argument);
});
}

// --- components carrying a ValueConverter -------------------------------
Expand Down Expand Up @@ -655,9 +666,20 @@ public async Task An_element_converter_set_on_the_model_is_applied_on_read()
private static T Read<T>(RelationalTypeMapping mapping, object driverValue)
{
var parameter = Expression.Parameter(typeof(object), "value");
var body = mapping.CustomizeDataReaderExpression(parameter);

// EF inlines liftable constants before compiling a materializer; do the same here, since a
// LiftableConstantExpression cannot be compiled on its own.
var body = new InlineLiftables().Visit(mapping.CustomizeDataReaderExpression(parameter));

return Expression.Lambda<Func<object, T>>(Expression.Convert(body, typeof(T)), parameter)
.Compile()(driverValue);
}

private sealed class InlineLiftables : ExpressionVisitor
{
protected override Expression VisitExtension(Expression node)
=> node is LiftableConstantExpression lifted
? lifted.OriginalExpression
: base.VisitExtension(node);
}
}