From 470c17926b0f75f41c008a498852820beab83054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:58:48 -0800 Subject: [PATCH 1/6] Serve the site with a per-request file middleware instead of MapStaticAssets MapStaticAssets needs a manifest of every file of the web root. With the site's ~70,500 files (8.4 GB), building it dominates every StaticHost build and publish, and each instance loads it at startup into one route per file. StaticHost now turns static web assets off and serves the web root with SiteFileMiddleware: - It looks the file up per request, prefers the brotli copy next to it when the client accepts br, and answers single byte ranges. - It computes the ETag (base64 SHA-256, the same value MapStaticAssets sent) the first time a file is served and caches it in memory. - It keeps the existing caching: immutable for /_astro/, the SDK's one-hour default for images, video and fonts, no-cache otherwise. - It steps aside when an endpoint matched, so API, health and download endpoints keep precedence. Site.targets publishes the frontend's output to wwwroot, as the static web assets pipeline did: it asks frontend.esproj for JavaScriptOutputGroup (which runs its BuildCommand) and writes a brotli copy (Optimal, the level used before) of each text file, reusing copies newer than their source. It replaces Caching.targets. The Frontend Build job no longer copies dist into wwwroot and rebuilds StaticHost to regenerate the manifest. Measured on a 4-core VM against the real production build of the site, current setup vs this change: - dotnet publish (Astro build excluded): 213 s -> 55 s (22 s when the brotli copies are reused) - CI-style build with dist in wwwroot: 86 s -> 9 s - Endpoints manifest: 142 MB -> none - First response after start: 13-18 s -> 2.3-5.8 s (the rest is startup both share) - Memory after warm load: 2.1-2.2 GB -> 280 MB RSS - Each of the 33k pages once: 1,450 -> 3,900 req/s, p99 56 -> 9 ms - Warm load: 5,700-5,850 -> 10,100-10,800 req/s, p99 21 -> 9 ms Comparing both servers on every page and file, with and without br (207,786 responses), Content-Type, Cache-Control, ETag, Vary and Content-Length match. The differences: Last-Modified on brotli responses is now the file's own, 4 tiny files no longer get a brotli copy that is not smaller, and StaticHost's own wwwroot (.gitignore, the dev-only Scalar theme) is no longer published. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01871HTuV3jb825jyAUQ3L7d --- .github/workflows/frontend-build.yml | 27 +- src/frontend/playwright.config.mjs | 6 +- .../MarkdownNegotiationMiddleware.cs | 7 +- src/statichost/StaticHost/Caching.targets | 37 -- src/statichost/StaticHost/GlobalUsings.cs | 1 + src/statichost/StaticHost/Program.cs | 10 +- src/statichost/StaticHost/Site.targets | 107 ++++++ .../StaticHost/SiteFiles/SiteFileETags.cs | 45 +++ .../SiteFiles/SiteFileExtensions.cs | 14 + .../SiteFiles/SiteFileMiddleware.cs | 273 +++++++++++++++ src/statichost/StaticHost/StaticHost.csproj | 15 +- .../AgentReadinessTestServer.cs | 7 +- tests/StaticHost.Tests/SiteFileTests.cs | 327 ++++++++++++++++++ 13 files changed, 801 insertions(+), 75 deletions(-) delete mode 100644 src/statichost/StaticHost/Caching.targets create mode 100644 src/statichost/StaticHost/Site.targets create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileETags.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs create mode 100644 tests/StaticHost.Tests/SiteFileTests.cs diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index b5b811da58..9a5512e7c3 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -84,8 +84,9 @@ jobs: node-version: ${{ inputs.node_version }} # The E2E shards serve the site with StaticHost, as production does, - # from the build output this job uploads (statichost-bin). Its restore - # and compile need no site, so they run alongside the Astro build. + # from the build output this job uploads (statichost-bin). StaticHost + # reads the site from its web root per request, so its build needs no + # site and runs alongside the Astro build. - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: global.json @@ -138,16 +139,6 @@ jobs: bun run build:production fi - # StaticHost serves its wwwroot, which holds a copy of dist (hard links) - # as in production. MapStaticAssets lists every file of the web root: - # this incremental build redoes only that, while the site archive is packed - # and uploaded. - - name: Start the StaticHost build - run: | - cp -alf dist/. ../statichost/StaticHost/wwwroot/ - "$SETUP" wait statichost-compile - "$SETUP" start statichost-build dotnet build -c Release --no-restore ../statichost/StaticHost - - name: Save Astro build state if: ${{ success() && (github.event_name == 'push' || steps.astro_cache.outputs.cache-matched-key == '') }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -218,7 +209,7 @@ jobs: - name: Wait for the StaticHost build run: | - "$SETUP" wait statichost-build + "$SETUP" wait statichost-compile "$SETUP" report - name: Upload the StaticHost build @@ -449,17 +440,17 @@ jobs: "$SETUP" start playwright-deps bun run playwright install-deps chromium "$SETUP" start playwright-browsers bun run playwright install chromium - # The Frontend Build job built StaticHost against this run's dist. - # StaticHost reads the web root's files per request, so it starts - # before the site is there, and boots during its download. + # The Frontend Build job built StaticHost. It reads the web root's files + # per request, so it starts before the site is there, and boots during + # its download. - name: Download the StaticHost build uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: statichost-bin path: src/statichost/StaticHost/bin/Release/net10.0 - # Any answer, a 404 included, means StaticHost built its routing table, - # which the first request does. + # Any answer, a 404 included, means StaticHost is up and has built its + # routing table, which the first request does. - name: Start StaticHost run: | mkdir -p ../statichost/StaticHost/wwwroot diff --git a/src/frontend/playwright.config.mjs b/src/frontend/playwright.config.mjs index 6d1b1e0133..827d21acfc 100644 --- a/src/frontend/playwright.config.mjs +++ b/src/frontend/playwright.config.mjs @@ -66,9 +66,9 @@ export default defineConfig({ webServer: isCI ? { // CI serves the built site with StaticHost, as production does (Markdown negotiation, canonical - // redirects, the 404 page, headers). The Frontend Build job builds it against the site's files - // (wwwroot holds a copy of dist), so the tests start its assembly directly: `dotnet run --no-build` - // still evaluates the project, which takes seconds. The E2E jobs start it before dist arrives + // redirects, the 404 page, headers) from its wwwroot, where the E2E jobs unpack dist. The Frontend + // Build job builds it, so the tests start its assembly directly: `dotnet run --no-build` still + // evaluates the project, which takes seconds. The E2E jobs start it before dist arrives // (.github/scripts/start-web-server.mjs), hence reuseExistingServer. command: `dotnet bin/Release/net10.0/StaticHost.dll --urls ${e2eBaseUrl}`, cwd: fileURLToPath(new URL('../statichost/StaticHost/', import.meta.url)), diff --git a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs index 8dc52cde71..1c0eed87c4 100644 --- a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs +++ b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs @@ -14,9 +14,8 @@ namespace StaticHost.AgentReadiness; /// HTML page (via starlight-page-actions). E.g. /get-started/quickstart/ /// has companion /get-started/quickstart.md. This middleware short-circuits /// the pipeline by streaming that file directly when markdown is preferred — -/// it does NOT rewrite Request.Path because rewriting after -/// UseRouting() would not re-trigger endpoint selection in -/// MapStaticAssets(). +/// it does NOT rewrite Request.Path, because the negotiated response needs +/// its own caching headers (below), which the site file middleware would replace. /// /// /// Therefore the middleware MUST run before UseDefaultFiles and @@ -44,7 +43,7 @@ public MarkdownNegotiationMiddleware( { _next = next; _logger = logger; - // Use the same provider MapStaticAssets reads from so wwwroot is the source of truth. + // Use the same provider SiteFileMiddleware reads from so wwwroot is the source of truth. _fileProvider = env.WebRootFileProvider; } diff --git a/src/statichost/StaticHost/Caching.targets b/src/statichost/StaticHost/Caching.targets deleted file mode 100644 index 4d76f64913..0000000000 --- a/src/statichost/StaticHost/Caching.targets +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - <_AstroEndpoints Include="@(StaticWebAssetEndpoint)" - Condition="$([System.String]::Copy('%(Identity)').StartsWith('_astro/'))" /> - - - - - <_CacheHeaderOperation Include="Replace"> - Header - Cache-Control - max-age=31536000, public, immutable - - - - - - - - - - - - - - - diff --git a/src/statichost/StaticHost/GlobalUsings.cs b/src/statichost/StaticHost/GlobalUsings.cs index 9d5d095639..d7de8cf05c 100644 --- a/src/statichost/StaticHost/GlobalUsings.cs +++ b/src/statichost/StaticHost/GlobalUsings.cs @@ -4,6 +4,7 @@ global using StaticHost.AgentReadiness; global using StaticHost.Live; global using StaticHost.Routing; +global using StaticHost.SiteFiles; global using StaticHost.Telemetry; global using OpenTelemetry; diff --git a/src/statichost/StaticHost/Program.cs b/src/statichost/StaticHost/Program.cs index 5e4d3c60d6..3ccf5dc5a0 100644 --- a/src/statichost/StaticHost/Program.cs +++ b/src/statichost/StaticHost/Program.cs @@ -12,6 +12,7 @@ } builder.AddLiveStatus(); +builder.Services.AddSiteFiles(); builder.Services.AddHsts(options => { @@ -43,8 +44,7 @@ // Agent-readiness middlewares MUST run before UseDefaultFiles + UseRouting: // * UseDefaultFiles rewrites /foo/ -> /foo/index.html, breaking .md-companion mapping. -// * MapStaticAssets registers endpoints during UseRouting, so a path rewrite -// after UseRouting wouldn't re-trigger endpoint selection. +// * A path rewrite after UseRouting wouldn't re-trigger endpoint selection. // MarkdownNegotiationMiddleware short-circuits to serve the .md body directly; // LinkHeaderMiddleware attaches a Link header on HTML 2xx responses via OnStarting. app.UseAgentReadiness(); @@ -92,6 +92,10 @@ await next(); }); +// The site's files. Requests that matched an endpoint below (health, downloads, live status) +// pass through to it; a file that does not exist falls through to the 404 page above. +app.UseSiteFiles(); + app.MapGet("/healthz", (HttpContext context) => { // A health probe must always reach the origin, never a cached copy. @@ -139,6 +143,4 @@ return Results.Redirect("https://aka.ms/aspire/get/install.sh"); }); -app.MapStaticAssets(); - await app.RunAsync(); diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets new file mode 100644 index 0000000000..2cba6ca7a7 --- /dev/null +++ b/src/statichost/StaticHost/Site.targets @@ -0,0 +1,107 @@ + + + + + .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + + + + + + + + + + + + + + + + (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + var publishFiles = new ConcurrentBag(); + var compressed = 0; + + Func publish = (path, relativePath) => + { + var item = new TaskItem(path); + item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); + item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); + item.SetMetadata("ExcludeFromSingleFile", "true"); + return item; + }; + + Parallel.ForEach(Files, file => + { + var source = file.GetMetadata("FullPath"); + var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); + publishFiles.Add(publish(source, relativePath)); + + if (!extensions.Contains(Path.GetExtension(source))) + { + return; + } + + // Reuse a copy that is newer than its source, so an unchanged site compresses once. + var target = Path.Combine(OutputDirectory, relativePath + ".br"); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(source)) + using (var output = File.Create(temporary)) + using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) + { + input.CopyTo(brotli); + } + + File.Delete(target); + File.Move(temporary, target); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(source).Length) + { + publishFiles.Add(publish(target, relativePath + ".br")); + } + }); + + PublishFiles = publishFiles.ToArray(); + Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); + ]]> + + + + + + + $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs new file mode 100644 index 0000000000..ef5c00828c --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -0,0 +1,45 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using Microsoft.Extensions.FileProviders; + +namespace StaticHost.SiteFiles; + +/// +/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// +/// +/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the +/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and +/// last-modified time UseStaticFiles uses, stays the same across deployments for files +/// that did not change. An entry is recomputed when the file's length or last-modified time +/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site +/// after StaticHost starts). +/// +internal sealed class SiteFileETags +{ + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + + public int Count => _entries.Count; + + public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + { + var key = file.PhysicalPath ?? file.Name; + var length = file.Length; + var lastModified = file.LastModified; + + if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + { + return entry.ETag; + } + + // Two first requests for the same file may both hash it; the result is the same. + await using var stream = file.CreateReadStream(); + var hash = await SHA256.HashDataAsync(stream, cancellationToken); + var etag = $"\"{Convert.ToBase64String(hash)}\""; + + _entries[key] = new Entry(length, lastModified, etag); + return etag; + } + + private sealed record Entry(long Length, DateTimeOffset LastModified, string ETag); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs new file mode 100644 index 0000000000..6be7e35cab --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs @@ -0,0 +1,14 @@ +namespace StaticHost.SiteFiles; + +internal static class SiteFileExtensions +{ + public static IServiceCollection AddSiteFiles(this IServiceCollection services) => + services.AddSingleton(); + + /// + /// Serves the web root's files, preferring the precompressed brotli copy. Call it after + /// UseRouting: it leaves requests that matched an endpoint to that endpoint. + /// + public static IApplicationBuilder UseSiteFiles(this IApplicationBuilder app) => + app.UseMiddleware(); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs new file mode 100644 index 0000000000..9ba5bb9fd1 --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -0,0 +1,273 @@ +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; + +namespace StaticHost.SiteFiles; + +/// +/// Serves the site's files from the web root, with the brotli copy the publish wrote next to a +/// file (page.html.br) when the client accepts it. +/// +/// +/// +/// This replaces MapStaticAssets, which reads a manifest of every file at startup and builds +/// a route for each one. With about 70,000 files, that manifest took a minute of every build and +/// publish, and each instance seconds and gigabytes of memory to start. This middleware keeps no +/// per-file state up front: it looks the file up on each request, as UseStaticFiles does, +/// and computes the ETag the first time a file is served (). +/// +/// +/// It runs after UseRouting and steps aside when an endpoint matched, so the API, health +/// and download endpoints keep precedence over a file of the same path, as they had over the +/// static asset endpoints. +/// +/// +/// A request for one byte range gets it (video players need ranges); other range requests get the +/// whole file. +/// +/// +internal sealed class SiteFileMiddleware +{ + // Images, video and fonts get the SDK's default media caching, as with MapStaticAssets. + internal const string MediaCacheControl = "max-age=3600, must-revalidate"; + + // Astro hashes the names of the files it emits under /_astro/, so they can be cached forever. + // https://starlight.astro.build/environmental-impact/#caching + internal const string ImmutableCacheControl = "max-age=31536000, public, immutable"; + + internal const string DefaultCacheControl = "no-cache"; + + private const string BrotliExtension = ".br"; + + private const string UnknownContentType = "application/octet-stream"; + + private static readonly PathString s_immutablePrefix = "/_astro"; + + private readonly RequestDelegate _next; + private readonly IFileProvider _fileProvider; + private readonly SiteFileETags _etags; + private readonly FileExtensionContentTypeProvider _contentTypes; + + public SiteFileMiddleware(RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags) + { + _next = next; + _fileProvider = environment.WebRootFileProvider; + _etags = etags; + _contentTypes = CreateContentTypeProvider(); + } + + public async Task InvokeAsync(HttpContext context) + { + var request = context.Request; + if (context.GetEndpoint() is not null + || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) + || !request.Path.HasValue + || request.Path.Value.EndsWith('/')) + { + await _next(context); + return; + } + + var file = _fileProvider.GetFileInfo(request.Path.Value); + if (!file.Exists || file.IsDirectory) + { + await _next(context); + return; + } + + // Every file the site emits is meant to be served, including types this provider does + // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. + if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) + { + contentType = UnknownContentType; + } + + var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var hasCompressed = compressed.Exists && !compressed.IsDirectory; + var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); + var served = useCompressed ? compressed : file; + + var response = context.Response; + var etag = await _etags.GetAsync(served, context.RequestAborted); + var lastModified = TruncateToSeconds(file.LastModified); + + var headers = response.Headers; + headers.CacheControl = GetCacheControl(request.Path, contentType); + headers.ETag = etag; + headers.LastModified = HeaderUtilities.FormatDate(lastModified); + if (hasCompressed) + { + headers.Vary = HeaderNames.AcceptEncoding; + } + + if (IsNotModified(request, etag, lastModified)) + { + response.StatusCode = StatusCodes.Status304NotModified; + return; + } + + headers.AcceptRanges = "bytes"; + var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + if (unsatisfiable) + { + response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; + headers.ContentRange = $"bytes */{served.Length}"; + return; + } + + response.ContentType = contentType; + if (useCompressed) + { + headers.ContentEncoding = "br"; + } + + var (offset, count) = range ?? (0, served.Length); + if (range is not null) + { + response.StatusCode = StatusCodes.Status206PartialContent; + headers.ContentRange = $"bytes {offset}-{offset + count - 1}/{served.Length}"; + } + else + { + response.StatusCode = StatusCodes.Status200OK; + } + + response.ContentLength = count; + if (HttpMethods.IsHead(request.Method)) + { + return; + } + + await response.SendFileAsync(served, offset, count, context.RequestAborted); + } + + internal static string GetCacheControl(PathString path, string contentType) + { + if (path.StartsWithSegments(s_immutablePrefix, StringComparison.Ordinal)) + { + return ImmutableCacheControl; + } + + return contentType.StartsWith("image/", StringComparison.Ordinal) + || contentType.StartsWith("video/", StringComparison.Ordinal) + || contentType.StartsWith("font/", StringComparison.Ordinal) + ? MediaCacheControl + : DefaultCacheControl; + } + + internal static bool AcceptsBrotli(StringValues acceptEncoding) + { + if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) + { + return false; + } + + double? brotli = null; + double? wildcard = null; + foreach (var encoding in encodings) + { + var quality = encoding.Quality ?? 1; + if (StringSegment.Equals(encoding.Value, "br", StringComparison.OrdinalIgnoreCase)) + { + brotli = quality; + } + else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) + { + wildcard = quality; + } + } + + // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. + return (brotli ?? wildcard ?? 0) > 0; + } + + private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + { + var requestHeaders = request.GetTypedHeaders(); + + // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). + var ifNoneMatch = requestHeaders.IfNoneMatch; + if (ifNoneMatch.Count > 0) + { + var current = EntityTagHeaderValue.Parse(etag); + foreach (var candidate in ifNoneMatch) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) + { + return true; + } + } + + return false; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + } + + /// + /// Returns the single byte range to send, or to send the whole file, + /// which a server may do for any range request it does not support, such as several ranges. + /// Safari only plays video from servers that answer ranges. + /// + private static (long Offset, long Count)? GetRange( + HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + { + unsatisfiable = false; + var requestHeaders = request.GetTypedHeaders(); + var range = requestHeaders.Range; + if (range is null + || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) + || range.Ranges.Count != 1) + { + return null; + } + + // If-Range applies the range only to the representation the client has (RFC 9110, 13.1.5). + if (requestHeaders.IfRange is { } ifRange) + { + var matches = ifRange.EntityTag is { } tag + ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) + : ifRange.LastModified == lastModified; + if (!matches) + { + return null; + } + } + + var item = range.Ranges.First(); + long start, end; + if (item.From is { } from) + { + start = from; + end = Math.Min(item.To ?? long.MaxValue, length - 1); + } + else + { + // A suffix range: the last N bytes. + start = Math.Max(0, length - (item.To ?? 0)); + end = length - 1; + } + + if (start >= length || start > end) + { + unsatisfiable = true; + return null; + } + + return (start, end - start + 1); + } + + private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) => + new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset); + + private static FileExtensionContentTypeProvider CreateContentTypeProvider() + { + var provider = new FileExtensionContentTypeProvider(); + + // The SDK's static web assets mapping, which production sent until now, differs here. + provider.Mappings[".md"] = "text/markdown"; + + return provider; + } +} diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index a836de6f0c..8550c3aace 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -8,11 +8,14 @@ true noble-chiseled - - false - brotli - - <_BlazorBrotliCompressionLevel>Optimal + + false + + $(DefaultItemExcludes);wwwroot/** @@ -50,5 +53,5 @@ - + diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index 1750a6d4e9..b395f548d4 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using StaticHost.AgentReadiness; using StaticHost.Routing; +using StaticHost.SiteFiles; namespace StaticHost.Tests; @@ -48,6 +49,7 @@ public static async Task StartAsync( { web.UseTestServer(); web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddSiteFiles()); web.Configure(app => { if (pathBase.HasValue) @@ -60,10 +62,9 @@ public static async Task StartAsync( app.UseCanonicalPathRedirects(); app.UseAgentReadiness(); app.UseDefaultFiles(); - app.UseStaticFiles(); + app.UseSiteFiles(); - // Endpoint that mimics MapStaticAssets fallback: 404 if - // no static asset matched. + // 404 when no file matched, as in production. app.Run(static ctx => { ctx.Response.StatusCode = StatusCodes.Status404NotFound; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs new file mode 100644 index 0000000000..83812ee018 --- /dev/null +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -0,0 +1,327 @@ +using System.IO.Compression; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using StaticHost.SiteFiles; + +namespace StaticHost.Tests; + +public sealed class SiteFileTests +{ + private const string Page = "" + "Hello, brotli. " + ""; + + [Fact] + public async Task Serves_the_brotli_copy_when_the_client_accepts_it() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + var compressed = await File.ReadAllBytesAsync(Path.Combine(server.WebRoot, "docs", "page", "index.html.br")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["br"], response.Content.Headers.ContentEncoding); + Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType); + Assert.Equal(compressed.Length, response.Content.Headers.ContentLength); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(compressed, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal(ETagOf(compressed), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task Serves_the_file_itself_when_the_client_does_not_accept_brotli() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes(Page)), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task File_without_a_brotli_copy_has_no_vary_header() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("robots.txt", "User-agent: *\n")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/robots.txt"); + request.Headers.AcceptEncoding.ParseAdd("br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Empty(response.Headers.Vary); + Assert.Equal("User-agent: *\n", await response.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("", false)] + [InlineData("br", true)] + [InlineData("gzip, deflate, br, zstd", true)] + [InlineData("BR;q=0.5", true)] + [InlineData("gzip", false)] + [InlineData("br;q=0", false)] + [InlineData("*", true)] + [InlineData("*;q=0", false)] + [InlineData("br;q=0, *", false)] + public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => + Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + + [Fact] + public async Task Matching_if_none_match_returns_not_modified() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag; + Assert.NotNull(etag); + + foreach (var ifNoneMatch in new[] { etag.Tag, $"W/{etag.Tag}", "\"other\", " + etag.Tag, "*" }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", ifNoneMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); + Assert.Equal(etag.Tag, response.Headers.ETag?.Tag); + Assert.Equal("no-cache", response.Headers.CacheControl?.ToString()); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + } + + [Fact] + public async Task Other_if_none_match_returns_the_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task If_modified_since_applies_only_without_if_none_match() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified; + Assert.NotNull(lastModified); + + using var conditional = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + conditional.Headers.IfModifiedSince = lastModified; + using var notModified = await server.Client.SendAsync(conditional); + Assert.Equal(HttpStatusCode.NotModified, notModified.StatusCode); + + using var both = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + both.Headers.IfModifiedSince = lastModified; + both.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var modified = await server.Client.SendAsync(both); + Assert.Equal(HttpStatusCode.OK, modified.StatusCode); + } + + [Fact] + public async Task ETag_follows_a_file_that_changes() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + + using var before = await server.Client.GetAsync("/data.json"); + await File.WriteAllTextAsync(Path.Combine(server.WebRoot, "data.json"), "{\"v\":22}"); + using var after = await server.Client.GetAsync("/data.json"); + + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":1}")), before.Headers.ETag?.Tag); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":22}")), after.Headers.ETag?.Tag); + Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/favicon.svg", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/images/hero.webp", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/fonts/site.woff2", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/casts/demo.mp4", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/docs/page/index.html", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/docs/page.md", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/pagefind/pagefind.js", SiteFileMiddleware.DefaultCacheControl)] + public async Task Cache_control_depends_on_the_path_and_the_type(string path, string expected) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // HttpClient reorders the directives it parses; compare what the server sent. + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Markdown_is_served_as_text_markdown() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SamplePages.SeedFolderPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page.md"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task Head_sends_the_headers_without_the_body() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var response = await server.Client.SendAsync(new HttpRequestMessage(HttpMethod.Head, "/docs/page/")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Encoding.UTF8.GetByteCount(Page), response.Content.Headers.ContentLength); + Assert.NotNull(response.Headers.ETag); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + + [Theory] + [InlineData("bytes=2-5", HttpStatusCode.PartialContent, "2345", "bytes 2-5/10")] + [InlineData("bytes=7-", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=-3", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=8-100", HttpStatusCode.PartialContent, "89", "bytes 8-9/10")] + [InlineData("bytes=0-1, 4-5", HttpStatusCode.OK, "0123456789", null)] + [InlineData("items=0-1", HttpStatusCode.OK, "0123456789", null)] + public async Task Answers_a_single_byte_range(string range, HttpStatusCode status, string body, string? contentRange) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + Assert.Equal(body, await response.Content.ReadAsStringAsync()); + Assert.Equal(contentRange, response.Content.Headers.ContentRange?.ToString()); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + + [Fact] + public async Task Range_past_the_end_is_not_satisfiable() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=10-"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.RequestedRangeNotSatisfiable, response.StatusCode); + Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); + } + + [Fact] + public async Task If_range_applies_the_range_only_to_the_current_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + using var first = await server.Client.GetAsync("/casts/demo.mp4"); + var etag = first.Headers.ETag!.Tag; + + foreach (var (ifRange, status) in new[] { (etag, HttpStatusCode.PartialContent), ("\"stale\"", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.TryAddWithoutValidation("If-Range", ifRange); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Theory] + [InlineData("/missing.html")] + [InlineData("/missing/")] + [InlineData("/docs")] + public async Task Missing_files_fall_through(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("/pagefind/fragment/en_1a2b3c4.pf_fragment")] + [InlineData("/pagefind/wasm.en.pagefind")] + [InlineData("/casts/demo.cast")] + public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.MediaType); + Assert.Equal("x", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Post_falls_through() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); + + using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Fact] + public async Task Matched_endpoint_takes_precedence_over_a_file() + { + using var wwwroot = new TempWebRoot(); + wwwroot.WriteFile("api/data.json", "{\"from\":\"file\"}"); + wwwroot.WriteFile("other.json", "{\"from\":\"file\"}"); + + using var host = new HostBuilder() + .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddRouting().AddSiteFiles()); + web.Configure(static app => + { + app.UseRouting(); + app.UseSiteFiles(); + app.UseEndpoints(static endpoints => endpoints.MapGet("/api/data.json", () => Results.Text("endpoint"))); + }); + }) + .Build(); + await host.StartAsync(); + using var client = host.GetTestClient(); + + Assert.Equal("endpoint", await client.GetStringAsync("/api/data.json")); + Assert.Equal("{\"from\":\"file\"}", await client.GetStringAsync("/other.json")); + + await host.StopAsync(); + } + + private static void SeedCompressedPage(TempWebRoot root, string slug) + { + root.WriteFile($"{slug}/index.html", Page); + + using var output = new MemoryStream(); + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + brotli.Write(Encoding.UTF8.GetBytes(Page)); + } + + File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); + } + + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; +} From d4efc16ad7140cbe21820807c42c42120fb4996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:43:26 -0800 Subject: [PATCH 2/6] Compress the site with a file-based app and tighten file preconditions - Write the brotli copies with src/statichost/compress-site.cs, a file-based app that Site.targets runs, instead of an inline MSBuild task that loaded BrotliStream by reflection (which fails under .NET Framework MSBuild). - Reuse a brotli copy only while it carries its source's exact timestamp. A source replaced by new content with an older timestamp (a cache restore or an archive extraction) kept the copy of the old content. - Answer If-Match and If-Unmodified-Since with 412, as MapStaticAssets did. - Apply a date in If-Range only to uncompressed responses: Last-Modified is the original file's, so it does not identify the bytes of its brotli copy. - Key cached ETags by the file's path in the web root, not its physical path or bare name, and parse the ETag once per request. --- src/statichost/StaticHost/Site.targets | 107 +++++------------- .../StaticHost/SiteFiles/SiteFileETags.cs | 12 +- .../SiteFiles/SiteFileMiddleware.cs | 72 ++++++++---- src/statichost/compress-site.cs | 67 +++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 47 ++++++++ 5 files changed, 200 insertions(+), 105 deletions(-) create mode 100644 src/statichost/compress-site.cs diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets index 2cba6ca7a7..3eadbdc080 100644 --- a/src/statichost/StaticHost/Site.targets +++ b/src/statichost/StaticHost/Site.targets @@ -9,99 +9,48 @@ output, and compress it once at publish. Caching and ETags are decided per request. Asking frontend.esproj for JavaScriptOutputGroup runs its BuildCommand and returns dist's - files, as the static web assets pipeline did. + files, as the static web assets pipeline did. The brotli copies are written by a file-based + app (../compress-site.cs), which reuses the copies of files that did not change. --> .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + $(MSBuildThisFileDirectory)../compress-site.cs + + $(DOTNET_HOST_PATH) + dotnet - - - - - - - - - - - - - - (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); - var publishFiles = new ConcurrentBag(); - var compressed = 0; - - Func publish = (path, relativePath) => - { - var item = new TaskItem(path); - item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); - item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); - item.SetMetadata("ExcludeFromSingleFile", "true"); - return item; - }; - - Parallel.ForEach(Files, file => - { - var source = file.GetMetadata("FullPath"); - var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); - publishFiles.Add(publish(source, relativePath)); - - if (!extensions.Contains(Path.GetExtension(source))) - { - return; - } - - // Reuse a copy that is newer than its source, so an unchanged site compresses once. - var target = Path.Combine(OutputDirectory, relativePath + ".br"); - if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) - { - Directory.CreateDirectory(Path.GetDirectoryName(target)); - var temporary = target + ".tmp"; - using (var input = File.OpenRead(source)) - using (var output = File.Create(temporary)) - using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) - { - input.CopyTo(brotli); - } - - File.Delete(target); - File.Move(temporary, target); - Interlocked.Increment(ref compressed); - } - - // A copy that saves nothing (tiny files) is not worth serving. - if (new FileInfo(target).Length < new FileInfo(source).Length) - { - publishFiles.Add(publish(target, relativePath + ".br")); - } - }); - - PublishFiles = publishFiles.ToArray(); - Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); - ]]> - - - - $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + <_SiteIntermediateDirectory>$(MSBuildProjectDirectory)/$(IntermediateOutputPath) + <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br/ + <_SiteFileList>$(_SiteIntermediateDirectory)site-files.txt + <_SiteBrotliList>$(_SiteIntermediateDirectory)site-br.txt - - - + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index ef5c00828c..9c9d887911 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -19,15 +19,15 @@ internal sealed class SiteFileETags { private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - public int Count => _entries.Count; - - public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + /// The file's path in the web root, which identifies it whatever the provider. + /// The file to hash. + /// Cancels the hashing. + public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) { - var key = file.PhysicalPath ?? file.Name; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) { return entry.ETag; } @@ -37,7 +37,7 @@ public async ValueTask GetAsync(IFileInfo file, CancellationToken cancel var hash = await SHA256.HashDataAsync(stream, cancellationToken); var etag = $"\"{Convert.ToBase64String(hash)}\""; - _entries[key] = new Entry(length, lastModified, etag); + _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9ba5bb9fd1..9c39bb203a 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http.Headers; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Primitives; @@ -83,14 +84,17 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } - var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var compressedPath = request.Path.Value + BrotliExtension; + var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); var served = useCompressed ? compressed : file; var response = context.Response; - var etag = await _etags.GetAsync(served, context.RequestAborted); + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + var entityTag = new EntityTagHeaderValue(etag); var lastModified = TruncateToSeconds(file.LastModified); + var requestHeaders = request.GetTypedHeaders(); var headers = response.Headers; headers.CacheControl = GetCacheControl(request.Path, contentType); @@ -101,14 +105,16 @@ public async Task InvokeAsync(HttpContext context) headers.Vary = HeaderNames.AcceptEncoding; } - if (IsNotModified(request, etag, lastModified)) + if (EvaluatePreconditions(requestHeaders, entityTag, lastModified) is { } status) { - response.StatusCode = StatusCodes.Status304NotModified; + response.StatusCode = status; return; } headers.AcceptRanges = "bytes"; - var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes + // of its brotli copy, which can be rewritten while the original keeps its time. + var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -182,27 +188,50 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) return (brotli ?? wildcard ?? 0) > 0; } - private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + /// + /// Evaluates the request's preconditions in the order of RFC 9110, 13.2.2, as + /// MapStaticAssets did. Returns 412 or 304 when one decides the response, otherwise + /// . + /// + private static int? EvaluatePreconditions(RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset lastModified) { - var requestHeaders = request.GetTypedHeaders(); + // If-Match takes precedence over If-Unmodified-Since, and uses the strong comparison. + var ifMatch = requestHeaders.IfMatch; + if (ifMatch.Count > 0) + { + if (!Matches(ifMatch, etag, useStrongComparison: true)) + { + return StatusCodes.Status412PreconditionFailed; + } + } + else if (requestHeaders.IfUnmodifiedSince is { } ifUnmodifiedSince && lastModified > ifUnmodifiedSince) + { + return StatusCodes.Status412PreconditionFailed; + } // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). var ifNoneMatch = requestHeaders.IfNoneMatch; if (ifNoneMatch.Count > 0) { - var current = EntityTagHeaderValue.Parse(etag); - foreach (var candidate in ifNoneMatch) + return Matches(ifNoneMatch, etag, useStrongComparison: false) ? StatusCodes.Status304NotModified : null; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince + ? StatusCodes.Status304NotModified + : null; + } + + private static bool Matches(IList candidates, EntityTagHeaderValue etag, bool useStrongComparison) + { + foreach (var candidate in candidates) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(etag, useStrongComparison)) { - if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) - { - return true; - } + return true; } - - return false; } - return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + return false; } /// @@ -210,11 +239,14 @@ private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffs /// which a server may do for any range request it does not support, such as several ranges. /// Safari only plays video from servers that answer ranges. /// + /// + /// The time a date in If-Range must match, or when a date does + /// not identify the bytes sent, so only an entity tag can. + /// private static (long Offset, long Count)? GetRange( - HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset? lastModified, long length, out bool unsatisfiable) { unsatisfiable = false; - var requestHeaders = request.GetTypedHeaders(); var range = requestHeaders.Range; if (range is null || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) @@ -227,8 +259,8 @@ private static (long Offset, long Count)? GetRange( if (requestHeaders.IfRange is { } ifRange) { var matches = ifRange.EntityTag is { } tag - ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) - : ifRange.LastModified == lastModified; + ? tag.Compare(etag, useStrongComparison: true) + : lastModified is not null && ifRange.LastModified == lastModified; if (!matches) { return null; diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs new file mode 100644 index 0000000000..88cec4f419 --- /dev/null +++ b/src/statichost/compress-site.cs @@ -0,0 +1,67 @@ +// Writes a brotli copy of each text file of the site, for StaticHost to serve to clients that +// accept it. StaticHost's publish runs it (StaticHost/Site.targets); it also runs alone: +// +// dotnet run --file compress-site.cs -- +// +// One line per site file: its full path, a tab, its path in the site. +// Where the copies are written, at their path in the site plus ".br". +// The extensions to compress, separated by semicolons. +// Written with the path in the site of each copy worth publishing. + +using System.Collections.Concurrent; +using System.IO.Compression; + +if (args.Length != 4) +{ + Console.Error.WriteLine("Usage: compress-site.cs "); + return 1; +} + +var (filesPath, outputDirectory, extensionList, publishedPath) = (args[0], Path.GetFullPath(args[1]), args[2], args[3]); +var extensions = new HashSet(extensionList.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + +var files = File.ReadLines(filesPath) + .Where(static line => line.Length > 0) + .Select(static line => line.Split('\t', 2)) + .Select(static parts => (Source: parts[0], SitePath: parts[1].Replace('\\', '/'))) + .Where(file => extensions.Contains(Path.GetExtension(file.Source))) + .ToList(); + +var published = new ConcurrentBag(); +var compressed = 0; + +Parallel.ForEach(files, file => +{ + // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. + // "Newer than the source" is not enough: a cache restore or an archive extraction gives + // new content an older time, and the copy of the old content would be published. + var target = Path.Combine(outputDirectory, file.SitePath + ".br"); + var sourceTime = File.GetLastWriteTimeUtc(file.Source); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(file.Source)) + using (var output = File.Create(temporary)) + // Optimal is brotli quality 4, which the SDK used at publish (_BlazorBrotliCompressionLevel). + // SmallestSize (11) saves about a sixth more but compresses about 100 times slower. + using (var brotli = new BrotliStream(output, CompressionLevel.Optimal)) + { + input.CopyTo(brotli); + } + + File.SetLastWriteTimeUtc(temporary, sourceTime); + File.Move(temporary, target, overwrite: true); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(file.Source).Length) + { + published.Add(file.SitePath + ".br"); + } +}); + +File.WriteAllLines(publishedPath, published.Order(StringComparer.Ordinal)); +Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now, {published.Count} brotli copies published."); +return 0; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index 83812ee018..f207a37af1 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -130,6 +130,53 @@ public async Task If_modified_since_applies_only_without_if_none_match() Assert.Equal(HttpStatusCode.OK, modified.StatusCode); } + [Fact] + public async Task If_match_and_if_unmodified_since_fail_for_another_version() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag!.Tag; + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (ifMatch, status) in new[] { (etag, HttpStatusCode.OK), ("*", HttpStatusCode.OK), ("\"stale\"", HttpStatusCode.PreconditionFailed), ($"W/{etag}", HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-Match", ifMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + + foreach (var (ifUnmodifiedSince, status) in new[] { (lastModified, HttpStatusCode.OK), (lastModified.AddDays(-1), HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.IfUnmodifiedSince = ifUnmodifiedSince; + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Fact] + public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (acceptEncoding, status) in new[] { ("identity", HttpStatusCode.PartialContent), ("br", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd(acceptEncoding); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + [Fact] public async Task ETag_follows_a_file_that_changes() { From 12329a6513e63110ce3dcf54d0d43938f3ca18bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:00:42 -0800 Subject: [PATCH 3/6] Hash ETags with XxHash128 and explain the brotli lookup and key casing - Compute ETags with XxHash128 instead of SHA-256: an ETag only has to tell versions of a file apart, which needs no cryptographic hash. It still hashes the content, so unchanged files keep their ETag across deployments. - Say why the brotli copy is looked up even for clients that do not accept brotli (the Vary header both versions need), and why cached ETags are keyed by ordinal path. --- .../StaticHost/SiteFiles/SiteFileETags.cs | 42 ++++++++++++++----- .../SiteFiles/SiteFileMiddleware.cs | 3 ++ src/statichost/StaticHost/StaticHost.csproj | 1 + tests/StaticHost.Tests/SiteFileTests.cs | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 9c9d887911..772475b9cc 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -1,22 +1,29 @@ +using System.Buffers; using System.Collections.Concurrent; -using System.Security.Cryptography; +using System.IO.Hashing; using Microsoft.Extensions.FileProviders; namespace StaticHost.SiteFiles; /// -/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// Computes a file's ETag from a hash of its content the first time it is served, then keeps it +/// in memory. /// /// -/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the -/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and -/// last-modified time UseStaticFiles uses, stays the same across deployments for files -/// that did not change. An entry is recomputed when the file's length or last-modified time -/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site -/// after StaticHost starts). +/// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no +/// cryptographic hash. A content hash, unlike the length and last-modified time +/// UseStaticFiles uses, stays the same across deployments for files that did not change, +/// though CI writes every file again. An entry is recomputed when the file's length or +/// last-modified time changes, which covers a web root that is replaced while StaticHost runs +/// (CI unpacks the site after StaticHost starts). /// internal sealed class SiteFileETags { + private const int BufferSize = 64 * 1024; + + // Ordinal, as the file system is in production (Linux): two paths that differ only by case are + // two files there, and must not share an entry. On a case-insensitive disk they are one file + // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); /// The file's path in the web root, which identifies it whatever the provider. @@ -33,10 +40,23 @@ public async ValueTask GetAsync(string path, IFileInfo file, Cancellatio } // Two first requests for the same file may both hash it; the result is the same. - await using var stream = file.CreateReadStream(); - var hash = await SHA256.HashDataAsync(stream, cancellationToken); - var etag = $"\"{Convert.ToBase64String(hash)}\""; + var hash = new XxHash128(); + var buffer = ArrayPool.Shared.Rent(BufferSize); + try + { + await using var stream = file.CreateReadStream(); + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + hash.Append(buffer.AsSpan(0, read)); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9c39bb203a..25d78b3952 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -84,6 +84,9 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } + // Look for the brotli copy even when the client does not accept brotli: a file that has one + // must send Vary: Accept-Encoding with both versions, so that a cache does not hand the + // uncompressed one to every client. Nearly every browser accepts brotli anyway. var compressedPath = request.Path.Value + BrotliExtension; var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index 8550c3aace..791917b5e4 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -30,6 +30,7 @@ + diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index f207a37af1..ede50a08f8 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -370,5 +370,5 @@ private static void SeedCompressedPage(TempWebRoot root, string slug) File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); } - private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(System.IO.Hashing.XxHash128.Hash(content))}\""; } From 5e666f1830f774e66ec85c0f30d1961b5eddb716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:22:51 -0800 Subject: [PATCH 4/6] Key cached ETags by physical path so the cache stays bounded Request paths that differ only by empty or dot segments (/a/b.json, /a//b.json, /a/./b.json) name one file, and the file provider resolves them to the same physical path. Keyed by request path, each spelling added an entry and hashed the file again, so a client could grow the cache without bound and force a full read of any file on every request. Keyed by physical path, the cache holds at most one entry per file of the web root. Concurrent first requests for a file may still both hash it; that happens once per file per instance, and serving the file reads it anyway. --- .../StaticHost/SiteFiles/SiteFileETags.cs | 19 +++++++++++++++---- .../AgentReadinessTestServer.cs | 2 ++ tests/StaticHost.Tests/SiteFileTests.cs | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 772475b9cc..84c3502ab2 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -21,25 +21,36 @@ internal sealed class SiteFileETags { private const int BufferSize = 64 * 1024; + // Keyed by the file's physical path, which the provider normalizes: the request paths + // /a/b.json, /a//b.json and /a///b.json are one file, so they share one entry, and the + // dictionary holds at most one entry per file of the web root. Request paths would let any + // client add entries without bound, each one costing a hash of the file. // Ordinal, as the file system is in production (Linux): two paths that differ only by case are // two files there, and must not share an entry. On a case-insensitive disk they are one file // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - /// The file's path in the web root, which identifies it whatever the provider. + internal int Count => _entries.Count; + + /// + /// The file's path in the web root, the key when the provider gives no physical path. + /// /// The file to hash. /// Cancels the hashing. public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) { + var key = file.PhysicalPath ?? path; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) { return entry.ETag; } - // Two first requests for the same file may both hash it; the result is the same. + // Two first requests for the same file may both hash it; the result is the same. That is + // not worth coordinating: it happens once per file per instance, and sending the file + // reads it anyway. var hash = new XxHash128(); var buffer = ArrayPool.Shared.Rent(BufferSize); try @@ -57,7 +68,7 @@ public async ValueTask GetAsync(string path, IFileInfo file, Cancellatio } var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; - _entries[path] = new Entry(length, lastModified, etag); + _entries[key] = new Entry(length, lastModified, etag); return etag; } diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index b395f548d4..c6b4106f47 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -26,6 +26,8 @@ internal sealed class AgentReadinessTestServer : IAsyncDisposable public string WebRoot => _wwwroot.Path; + public IServiceProvider Services => _host.Services; + private AgentReadinessTestServer(IHost host, TestServer testServer, TempWebRoot wwwroot) { _host = host; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index ede50a08f8..860324071b 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -177,6 +177,20 @@ public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() } } + [Fact] + public async Task Spellings_of_one_file_share_one_cached_etag() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/data.json", "{}")); + + foreach (var path in new[] { "/docs/data.json", "/docs//data.json", "/docs///data.json" }) + { + using var response = await server.Client.GetAsync(path); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + Assert.Equal(1, server.Services.GetRequiredService().Count); + } + [Fact] public async Task ETag_follows_a_file_that_changes() { From 59afd5898cd6f614d43b38f8f334378f64f9affa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:05:43 -0800 Subject: [PATCH 5/6] Answer 405 for methods other than GET and HEAD on a file The static asset endpoints allowed only GET and HEAD, so routing matched a file's path and answered 405 for other methods. The middleware stepped aside for them instead, and a POST to a file fell through to the 404 page. It now answers 405 with Allow: GET, HEAD; a path with no file still gets 404. --- .../StaticHost/SiteFiles/SiteFileMiddleware.cs | 12 +++++++++++- tests/StaticHost.Tests/SiteFileTests.cs | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 25d78b3952..c4a7d7c63e 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -43,6 +43,8 @@ internal sealed class SiteFileMiddleware private const string UnknownContentType = "application/octet-stream"; + private const string AllowedMethods = "GET, HEAD"; + private static readonly PathString s_immutablePrefix = "/_astro"; private readonly RequestDelegate _next; @@ -62,7 +64,6 @@ public async Task InvokeAsync(HttpContext context) { var request = context.Request; if (context.GetEndpoint() is not null - || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) || !request.Path.HasValue || request.Path.Value.EndsWith('/')) { @@ -77,6 +78,15 @@ public async Task InvokeAsync(HttpContext context) return; } + // A file only answers GET and HEAD. Other methods get 405, as they did from the static + // asset endpoints, which routing matched by path and then rejected by method. + if (!HttpMethods.IsGet(request.Method) && !HttpMethods.IsHead(request.Method)) + { + context.Response.StatusCode = StatusCodes.Status405MethodNotAllowed; + context.Response.Headers.Allow = AllowedMethods; + return; + } + // Every file the site emits is meant to be served, including types this provider does // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index 860324071b..e162b1d51c 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -331,13 +331,22 @@ public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) } [Fact] - public async Task Post_falls_through() + public async Task Other_methods_on_a_file_are_not_allowed() { await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); - using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + foreach (var method in new[] { HttpMethod.Post, HttpMethod.Put, HttpMethod.Delete, HttpMethod.Options }) + { + using var request = new HttpRequestMessage(method, "/data.json") { Content = new StringContent("{}") }; + using var response = await server.Client.SendAsync(request); - Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + Assert.Equal(HttpStatusCode.MethodNotAllowed, response.StatusCode); + Assert.Equal(["GET", "HEAD"], response.Content.Headers.Allow); + } + + // A path with no file still falls through to the 404 page. + using var missing = await server.Client.PostAsync("/missing.json", new StringContent("{}")); + Assert.Equal(HttpStatusCode.NotFound, missing.StatusCode); } [Fact] From e3a365c238616b0f39e004e92d9d2188935a4849 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:22:09 -0600 Subject: [PATCH 6/6] Address review: HEAD ranges, encoding negotiation, ETag and brotli reuse - Ignore Range on HEAD (RFC 9110, 14.2): HEAD gets the whole file's headers. - Choose brotli or identity by Accept-Encoding quality values; identity when neither is acceptable. - Hash the file again for requests with If-Match, If-None-Match or If-Range, so a file replaced with the same length and time cannot get a stale 304. - Reuse a brotli copy only when its source's SHA-256 matches, not its time. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../StaticHost/SiteFiles/SiteFileETags.cs | 19 +++- .../SiteFiles/SiteFileMiddleware.cs | 31 ++++++- src/statichost/compress-site.cs | 22 +++-- tests/StaticHost.Tests/CompressSiteTests.cs | 86 +++++++++++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 61 +++++++++++++ 5 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 tests/StaticHost.Tests/CompressSiteTests.cs diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 84c3502ab2..195e11d2c5 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -10,12 +10,23 @@ namespace StaticHost.SiteFiles; /// in memory. /// /// +/// /// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no /// cryptographic hash. A content hash, unlike the length and last-modified time /// UseStaticFiles uses, stays the same across deployments for files that did not change, /// though CI writes every file again. An entry is recomputed when the file's length or /// last-modified time changes, which covers a web root that is replaced while StaticHost runs /// (CI unpacks the site after StaticHost starts). +/// +/// +/// Length and time do not identify the bytes, though: a restore can replace a file with one of +/// the same length and time. A cached ETag is therefore only used where a stale one is harmless, +/// on a response that sends the file. A request with a precondition, which a stale ETag could +/// answer with a 304 for bytes the client does not have, hashes the file again +/// (revalidate). Watching the web root for changes would cost an inotify watch per +/// directory on Linux, and .NET does not expose the inode or change time that would tell a +/// replaced file apart. +/// /// internal sealed class SiteFileETags { @@ -36,14 +47,18 @@ internal sealed class SiteFileETags /// The file's path in the web root, the key when the provider gives no physical path. /// /// The file to hash. + /// Hash the file even when an entry matches its length and time. /// Cancels the hashing. - public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) + public async ValueTask GetAsync(string path, IFileInfo file, bool revalidate, CancellationToken cancellationToken) { var key = file.PhysicalPath ?? path; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (!revalidate + && _entries.TryGetValue(key, out var entry) + && entry.Length == length + && entry.LastModified == lastModified) { return entry.ETag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index c4a7d7c63e..844c6b98e7 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -104,7 +104,12 @@ public async Task InvokeAsync(HttpContext context) var served = useCompressed ? compressed : file; var response = context.Response; - var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + // A precondition decides on the ETag, so it must match the bytes on disk now: a 304 for a + // file replaced with one of the same length and time would keep a client on the old bytes. + var revalidate = request.Headers.IfMatch.Count > 0 + || request.Headers.IfNoneMatch.Count > 0 + || request.Headers.IfRange.Count > 0; + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, revalidate, context.RequestAborted); var entityTag = new EntityTagHeaderValue(etag); var lastModified = TruncateToSeconds(file.LastModified); var requestHeaders = request.GetTypedHeaders(); @@ -127,7 +132,11 @@ public async Task InvokeAsync(HttpContext context) headers.AcceptRanges = "bytes"; // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes // of its brotli copy, which can be rewritten while the original keeps its time. - var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); + // Range applies only to GET (RFC 9110, 14.2): HEAD gets the headers of the whole file. + var unsatisfiable = false; + var range = HttpMethods.IsGet(request.Method) + ? GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out unsatisfiable) + : null; if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -175,6 +184,16 @@ internal static string GetCacheControl(PathString path, string contentType) : DefaultCacheControl; } + /// + /// Chooses between the brotli copy and the file itself by the quality values of + /// Accept-Encoding (RFC 9110, 12.5.3), and returns whether to send brotli. + /// + /// + /// * applies to each coding the header does not list, identity included. Unlisted, + /// identity stays acceptable but preferred last, so br;q=0.5 still gets brotli, and a tie + /// goes to brotli. When the client accepts neither, it gets the file itself: RFC 9110, 12.1 lets + /// the server disregard the header, and a 406 could be cached for clients that accept the file. + /// internal static bool AcceptsBrotli(StringValues acceptEncoding) { if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) @@ -183,6 +202,7 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) } double? brotli = null; + double? identity = null; double? wildcard = null; foreach (var encoding in encodings) { @@ -191,6 +211,10 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) { brotli = quality; } + else if (StringSegment.Equals(encoding.Value, "identity", StringComparison.OrdinalIgnoreCase)) + { + identity = quality; + } else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) { wildcard = quality; @@ -198,7 +222,8 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) } // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. - return (brotli ?? wildcard ?? 0) > 0; + var brotliQuality = brotli ?? wildcard ?? 0; + return brotliQuality > 0 && brotliQuality >= (identity ?? wildcard ?? 0); } /// diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs index 88cec4f419..8aa5ac4268 100644 --- a/src/statichost/compress-site.cs +++ b/src/statichost/compress-site.cs @@ -10,6 +10,7 @@ using System.Collections.Concurrent; using System.IO.Compression; +using System.Security.Cryptography; if (args.Length != 4) { @@ -32,14 +33,23 @@ Parallel.ForEach(files, file => { - // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. - // "Newer than the source" is not enough: a cache restore or an archive extraction gives - // new content an older time, and the copy of the old content would be published. + // Reuse a copy whose source has the same content, so an unchanged site compresses once. The + // SHA-256 of the source the copy was written from is kept next to it (".br.sha256", which is + // not published). A time, even an exact one, is not enough: a cache restore or an archive + // extraction can give new content the old time, and the copy of the old content would be published. var target = Path.Combine(outputDirectory, file.SitePath + ".br"); - var sourceTime = File.GetLastWriteTimeUtc(file.Source); - if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + var stamp = target + ".sha256"; + string sourceHash; + using (var source = File.OpenRead(file.Source)) + { + sourceHash = Convert.ToHexString(SHA256.HashData(source)); + } + + if (!File.Exists(target) || !File.Exists(stamp) || File.ReadAllText(stamp) != sourceHash) { Directory.CreateDirectory(Path.GetDirectoryName(target)!); + // Without a stamp, a copy left half-updated by an interrupted run is written again. + File.Delete(stamp); var temporary = target + ".tmp"; using (var input = File.OpenRead(file.Source)) using (var output = File.Create(temporary)) @@ -50,8 +60,8 @@ input.CopyTo(brotli); } - File.SetLastWriteTimeUtc(temporary, sourceTime); File.Move(temporary, target, overwrite: true); + File.WriteAllText(stamp, sourceHash); Interlocked.Increment(ref compressed); } diff --git a/tests/StaticHost.Tests/CompressSiteTests.cs b/tests/StaticHost.Tests/CompressSiteTests.cs new file mode 100644 index 0000000000..d48b4b9c19 --- /dev/null +++ b/tests/StaticHost.Tests/CompressSiteTests.cs @@ -0,0 +1,86 @@ +using System.Diagnostics; +using System.IO.Compression; + +namespace StaticHost.Tests; + +/// +/// Runs src/statichost/compress-site.cs, the file-based app the publish uses to write the site's +/// brotli copies. +/// +public sealed class CompressSiteTests +{ + [Fact] + public async Task Rewrites_a_copy_whose_source_changed_but_kept_its_length_and_time() + { + using var work = new TempWebRoot(); + var source = Path.Combine(work.Path, "site", "page.html"); + var output = Path.Combine(work.Path, "br"); + var copy = Path.Combine(output, "page.html.br"); + var first = new string('a', 2048); + var second = new string('b', 2048); + work.WriteFile("site/page.html", first); + work.WriteFile("files.txt", $"{source}\tpage.html\n"); + var time = File.GetLastWriteTimeUtc(source); + + Assert.Contains("1 compressed now", await RunAsync(work.Path, output)); + Assert.Equal(first, Decompress(copy)); + + Assert.Contains("0 compressed now", await RunAsync(work.Path, output)); + + // A restore: new bytes, same length, the old time. + await File.WriteAllTextAsync(source, second); + File.SetLastWriteTimeUtc(source, time); + + Assert.Contains("1 compressed now", await RunAsync(work.Path, output)); + Assert.Equal(second, Decompress(copy)); + } + + private static async Task RunAsync(string work, string output) + { + var start = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (var argument in new[] + { + "run", "--file", FindCompressSite(), "--", + Path.Combine(work, "files.txt"), output, ".html", Path.Combine(work, "published.txt"), + }) + { + start.ArgumentList.Add(argument); + } + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + Assert.True(process.ExitCode == 0, $"compress-site.cs exited with {process.ExitCode}: {await stdout}{await stderr}"); + return await stdout; + } + + private static string FindCompressSite() + { + for (var directory = new DirectoryInfo(AppContext.BaseDirectory); directory is not null; directory = directory.Parent) + { + var path = Path.Combine(directory.FullName, "src", "statichost", "compress-site.cs"); + if (File.Exists(path)) + { + return path; + } + } + + throw new FileNotFoundException("compress-site.cs was not found above the test's directory."); + } + + private static string Decompress(string path) + { + using var input = File.OpenRead(path); + using var brotli = new BrotliStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(brotli); + return reader.ReadToEnd(); + } +} diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index e162b1d51c..0c3ada8d98 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -71,9 +71,30 @@ public async Task File_without_a_brotli_copy_has_no_vary_header() [InlineData("*", true)] [InlineData("*;q=0", false)] [InlineData("br;q=0, *", false)] + [InlineData("br;q=0.1, identity;q=1", false)] + [InlineData("br;q=0.5, *", false)] + [InlineData("br;q=0.5, identity;q=0.5", true)] + [InlineData("br, identity;q=0.5", true)] + [InlineData("br;q=0.1, identity;q=0", true)] + [InlineData("gzip, br;q=0, identity;q=0", false)] public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + [Fact] + public async Task Client_that_accepts_neither_coding_gets_the_file_itself() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("Accept-Encoding", "gzip, br;q=0, identity;q=0"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + [Fact] public async Task Matching_if_none_match_returns_not_modified() { @@ -205,6 +226,29 @@ public async Task ETag_follows_a_file_that_changes() Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); } + [Fact] + public async Task Precondition_sees_a_file_replaced_with_the_same_length_and_time() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + var path = Path.Combine(server.WebRoot, "data.json"); + var lastWrite = File.GetLastWriteTimeUtc(path); + + using var before = await server.Client.GetAsync("/data.json"); + var staleETag = before.Headers.ETag!.Tag; + + // A restore: new bytes, same length, the old time. + await File.WriteAllTextAsync(path, "{\"v\":2}"); + File.SetLastWriteTimeUtc(path, lastWrite); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/data.json"); + request.Headers.TryAddWithoutValidation("If-None-Match", staleETag); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("{\"v\":2}", await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":2}")), response.Headers.ETag?.Tag); + } + [Theory] [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] @@ -284,6 +328,23 @@ public async Task Range_past_the_end_is_not_satisfiable() Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); } + [Theory] + [InlineData("bytes=2-5")] + [InlineData("bytes=10-")] + public async Task Head_ignores_the_range(string range) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Head, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(10, response.Content.Headers.ContentLength); + Assert.Null(response.Content.Headers.ContentRange); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + [Fact] public async Task If_range_applies_the_range_only_to_the_current_file() {