From 470c17926b0f75f41c008a498852820beab83054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:58:48 -0800 Subject: [PATCH 1/8] Serve the site with a per-request file middleware instead of MapStaticAssets MapStaticAssets needs a manifest of every file of the web root. With the site's ~70,500 files (8.4 GB), building it dominates every StaticHost build and publish, and each instance loads it at startup into one route per file. StaticHost now turns static web assets off and serves the web root with SiteFileMiddleware: - It looks the file up per request, prefers the brotli copy next to it when the client accepts br, and answers single byte ranges. - It computes the ETag (base64 SHA-256, the same value MapStaticAssets sent) the first time a file is served and caches it in memory. - It keeps the existing caching: immutable for /_astro/, the SDK's one-hour default for images, video and fonts, no-cache otherwise. - It steps aside when an endpoint matched, so API, health and download endpoints keep precedence. Site.targets publishes the frontend's output to wwwroot, as the static web assets pipeline did: it asks frontend.esproj for JavaScriptOutputGroup (which runs its BuildCommand) and writes a brotli copy (Optimal, the level used before) of each text file, reusing copies newer than their source. It replaces Caching.targets. The Frontend Build job no longer copies dist into wwwroot and rebuilds StaticHost to regenerate the manifest. Measured on a 4-core VM against the real production build of the site, current setup vs this change: - dotnet publish (Astro build excluded): 213 s -> 55 s (22 s when the brotli copies are reused) - CI-style build with dist in wwwroot: 86 s -> 9 s - Endpoints manifest: 142 MB -> none - First response after start: 13-18 s -> 2.3-5.8 s (the rest is startup both share) - Memory after warm load: 2.1-2.2 GB -> 280 MB RSS - Each of the 33k pages once: 1,450 -> 3,900 req/s, p99 56 -> 9 ms - Warm load: 5,700-5,850 -> 10,100-10,800 req/s, p99 21 -> 9 ms Comparing both servers on every page and file, with and without br (207,786 responses), Content-Type, Cache-Control, ETag, Vary and Content-Length match. The differences: Last-Modified on brotli responses is now the file's own, 4 tiny files no longer get a brotli copy that is not smaller, and StaticHost's own wwwroot (.gitignore, the dev-only Scalar theme) is no longer published. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01871HTuV3jb825jyAUQ3L7d --- .github/workflows/frontend-build.yml | 27 +- src/frontend/playwright.config.mjs | 6 +- .../MarkdownNegotiationMiddleware.cs | 7 +- src/statichost/StaticHost/Caching.targets | 37 -- src/statichost/StaticHost/GlobalUsings.cs | 1 + src/statichost/StaticHost/Program.cs | 10 +- src/statichost/StaticHost/Site.targets | 107 ++++++ .../StaticHost/SiteFiles/SiteFileETags.cs | 45 +++ .../SiteFiles/SiteFileExtensions.cs | 14 + .../SiteFiles/SiteFileMiddleware.cs | 273 +++++++++++++++ src/statichost/StaticHost/StaticHost.csproj | 15 +- .../AgentReadinessTestServer.cs | 7 +- tests/StaticHost.Tests/SiteFileTests.cs | 327 ++++++++++++++++++ 13 files changed, 801 insertions(+), 75 deletions(-) delete mode 100644 src/statichost/StaticHost/Caching.targets create mode 100644 src/statichost/StaticHost/Site.targets create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileETags.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs create mode 100644 tests/StaticHost.Tests/SiteFileTests.cs diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index b5b811da58..9a5512e7c3 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -84,8 +84,9 @@ jobs: node-version: ${{ inputs.node_version }} # The E2E shards serve the site with StaticHost, as production does, - # from the build output this job uploads (statichost-bin). Its restore - # and compile need no site, so they run alongside the Astro build. + # from the build output this job uploads (statichost-bin). StaticHost + # reads the site from its web root per request, so its build needs no + # site and runs alongside the Astro build. - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: global.json @@ -138,16 +139,6 @@ jobs: bun run build:production fi - # StaticHost serves its wwwroot, which holds a copy of dist (hard links) - # as in production. MapStaticAssets lists every file of the web root: - # this incremental build redoes only that, while the site archive is packed - # and uploaded. - - name: Start the StaticHost build - run: | - cp -alf dist/. ../statichost/StaticHost/wwwroot/ - "$SETUP" wait statichost-compile - "$SETUP" start statichost-build dotnet build -c Release --no-restore ../statichost/StaticHost - - name: Save Astro build state if: ${{ success() && (github.event_name == 'push' || steps.astro_cache.outputs.cache-matched-key == '') }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -218,7 +209,7 @@ jobs: - name: Wait for the StaticHost build run: | - "$SETUP" wait statichost-build + "$SETUP" wait statichost-compile "$SETUP" report - name: Upload the StaticHost build @@ -449,17 +440,17 @@ jobs: "$SETUP" start playwright-deps bun run playwright install-deps chromium "$SETUP" start playwright-browsers bun run playwright install chromium - # The Frontend Build job built StaticHost against this run's dist. - # StaticHost reads the web root's files per request, so it starts - # before the site is there, and boots during its download. + # The Frontend Build job built StaticHost. It reads the web root's files + # per request, so it starts before the site is there, and boots during + # its download. - name: Download the StaticHost build uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: statichost-bin path: src/statichost/StaticHost/bin/Release/net10.0 - # Any answer, a 404 included, means StaticHost built its routing table, - # which the first request does. + # Any answer, a 404 included, means StaticHost is up and has built its + # routing table, which the first request does. - name: Start StaticHost run: | mkdir -p ../statichost/StaticHost/wwwroot diff --git a/src/frontend/playwright.config.mjs b/src/frontend/playwright.config.mjs index 6d1b1e0133..827d21acfc 100644 --- a/src/frontend/playwright.config.mjs +++ b/src/frontend/playwright.config.mjs @@ -66,9 +66,9 @@ export default defineConfig({ webServer: isCI ? { // CI serves the built site with StaticHost, as production does (Markdown negotiation, canonical - // redirects, the 404 page, headers). The Frontend Build job builds it against the site's files - // (wwwroot holds a copy of dist), so the tests start its assembly directly: `dotnet run --no-build` - // still evaluates the project, which takes seconds. The E2E jobs start it before dist arrives + // redirects, the 404 page, headers) from its wwwroot, where the E2E jobs unpack dist. The Frontend + // Build job builds it, so the tests start its assembly directly: `dotnet run --no-build` still + // evaluates the project, which takes seconds. The E2E jobs start it before dist arrives // (.github/scripts/start-web-server.mjs), hence reuseExistingServer. command: `dotnet bin/Release/net10.0/StaticHost.dll --urls ${e2eBaseUrl}`, cwd: fileURLToPath(new URL('../statichost/StaticHost/', import.meta.url)), diff --git a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs index 8dc52cde71..1c0eed87c4 100644 --- a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs +++ b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs @@ -14,9 +14,8 @@ namespace StaticHost.AgentReadiness; /// HTML page (via starlight-page-actions). E.g. /get-started/quickstart/ /// has companion /get-started/quickstart.md. This middleware short-circuits /// the pipeline by streaming that file directly when markdown is preferred — -/// it does NOT rewrite Request.Path because rewriting after -/// UseRouting() would not re-trigger endpoint selection in -/// MapStaticAssets(). +/// it does NOT rewrite Request.Path, because the negotiated response needs +/// its own caching headers (below), which the site file middleware would replace. /// /// /// Therefore the middleware MUST run before UseDefaultFiles and @@ -44,7 +43,7 @@ public MarkdownNegotiationMiddleware( { _next = next; _logger = logger; - // Use the same provider MapStaticAssets reads from so wwwroot is the source of truth. + // Use the same provider SiteFileMiddleware reads from so wwwroot is the source of truth. _fileProvider = env.WebRootFileProvider; } diff --git a/src/statichost/StaticHost/Caching.targets b/src/statichost/StaticHost/Caching.targets deleted file mode 100644 index 4d76f64913..0000000000 --- a/src/statichost/StaticHost/Caching.targets +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - <_AstroEndpoints Include="@(StaticWebAssetEndpoint)" - Condition="$([System.String]::Copy('%(Identity)').StartsWith('_astro/'))" /> - - - - - <_CacheHeaderOperation Include="Replace"> - Header - Cache-Control - max-age=31536000, public, immutable - - - - - - - - - - - - - - - diff --git a/src/statichost/StaticHost/GlobalUsings.cs b/src/statichost/StaticHost/GlobalUsings.cs index 9d5d095639..d7de8cf05c 100644 --- a/src/statichost/StaticHost/GlobalUsings.cs +++ b/src/statichost/StaticHost/GlobalUsings.cs @@ -4,6 +4,7 @@ global using StaticHost.AgentReadiness; global using StaticHost.Live; global using StaticHost.Routing; +global using StaticHost.SiteFiles; global using StaticHost.Telemetry; global using OpenTelemetry; diff --git a/src/statichost/StaticHost/Program.cs b/src/statichost/StaticHost/Program.cs index 5e4d3c60d6..3ccf5dc5a0 100644 --- a/src/statichost/StaticHost/Program.cs +++ b/src/statichost/StaticHost/Program.cs @@ -12,6 +12,7 @@ } builder.AddLiveStatus(); +builder.Services.AddSiteFiles(); builder.Services.AddHsts(options => { @@ -43,8 +44,7 @@ // Agent-readiness middlewares MUST run before UseDefaultFiles + UseRouting: // * UseDefaultFiles rewrites /foo/ -> /foo/index.html, breaking .md-companion mapping. -// * MapStaticAssets registers endpoints during UseRouting, so a path rewrite -// after UseRouting wouldn't re-trigger endpoint selection. +// * A path rewrite after UseRouting wouldn't re-trigger endpoint selection. // MarkdownNegotiationMiddleware short-circuits to serve the .md body directly; // LinkHeaderMiddleware attaches a Link header on HTML 2xx responses via OnStarting. app.UseAgentReadiness(); @@ -92,6 +92,10 @@ await next(); }); +// The site's files. Requests that matched an endpoint below (health, downloads, live status) +// pass through to it; a file that does not exist falls through to the 404 page above. +app.UseSiteFiles(); + app.MapGet("/healthz", (HttpContext context) => { // A health probe must always reach the origin, never a cached copy. @@ -139,6 +143,4 @@ return Results.Redirect("https://aka.ms/aspire/get/install.sh"); }); -app.MapStaticAssets(); - await app.RunAsync(); diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets new file mode 100644 index 0000000000..2cba6ca7a7 --- /dev/null +++ b/src/statichost/StaticHost/Site.targets @@ -0,0 +1,107 @@ + + + + + .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + + + + + + + + + + + + + + + + (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + var publishFiles = new ConcurrentBag(); + var compressed = 0; + + Func publish = (path, relativePath) => + { + var item = new TaskItem(path); + item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); + item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); + item.SetMetadata("ExcludeFromSingleFile", "true"); + return item; + }; + + Parallel.ForEach(Files, file => + { + var source = file.GetMetadata("FullPath"); + var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); + publishFiles.Add(publish(source, relativePath)); + + if (!extensions.Contains(Path.GetExtension(source))) + { + return; + } + + // Reuse a copy that is newer than its source, so an unchanged site compresses once. + var target = Path.Combine(OutputDirectory, relativePath + ".br"); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(source)) + using (var output = File.Create(temporary)) + using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) + { + input.CopyTo(brotli); + } + + File.Delete(target); + File.Move(temporary, target); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(source).Length) + { + publishFiles.Add(publish(target, relativePath + ".br")); + } + }); + + PublishFiles = publishFiles.ToArray(); + Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); + ]]> + + + + + + + $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs new file mode 100644 index 0000000000..ef5c00828c --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -0,0 +1,45 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using Microsoft.Extensions.FileProviders; + +namespace StaticHost.SiteFiles; + +/// +/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// +/// +/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the +/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and +/// last-modified time UseStaticFiles uses, stays the same across deployments for files +/// that did not change. An entry is recomputed when the file's length or last-modified time +/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site +/// after StaticHost starts). +/// +internal sealed class SiteFileETags +{ + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + + public int Count => _entries.Count; + + public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + { + var key = file.PhysicalPath ?? file.Name; + var length = file.Length; + var lastModified = file.LastModified; + + if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + { + return entry.ETag; + } + + // Two first requests for the same file may both hash it; the result is the same. + await using var stream = file.CreateReadStream(); + var hash = await SHA256.HashDataAsync(stream, cancellationToken); + var etag = $"\"{Convert.ToBase64String(hash)}\""; + + _entries[key] = new Entry(length, lastModified, etag); + return etag; + } + + private sealed record Entry(long Length, DateTimeOffset LastModified, string ETag); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs new file mode 100644 index 0000000000..6be7e35cab --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs @@ -0,0 +1,14 @@ +namespace StaticHost.SiteFiles; + +internal static class SiteFileExtensions +{ + public static IServiceCollection AddSiteFiles(this IServiceCollection services) => + services.AddSingleton(); + + /// + /// Serves the web root's files, preferring the precompressed brotli copy. Call it after + /// UseRouting: it leaves requests that matched an endpoint to that endpoint. + /// + public static IApplicationBuilder UseSiteFiles(this IApplicationBuilder app) => + app.UseMiddleware(); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs new file mode 100644 index 0000000000..9ba5bb9fd1 --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -0,0 +1,273 @@ +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; + +namespace StaticHost.SiteFiles; + +/// +/// Serves the site's files from the web root, with the brotli copy the publish wrote next to a +/// file (page.html.br) when the client accepts it. +/// +/// +/// +/// This replaces MapStaticAssets, which reads a manifest of every file at startup and builds +/// a route for each one. With about 70,000 files, that manifest took a minute of every build and +/// publish, and each instance seconds and gigabytes of memory to start. This middleware keeps no +/// per-file state up front: it looks the file up on each request, as UseStaticFiles does, +/// and computes the ETag the first time a file is served (). +/// +/// +/// It runs after UseRouting and steps aside when an endpoint matched, so the API, health +/// and download endpoints keep precedence over a file of the same path, as they had over the +/// static asset endpoints. +/// +/// +/// A request for one byte range gets it (video players need ranges); other range requests get the +/// whole file. +/// +/// +internal sealed class SiteFileMiddleware +{ + // Images, video and fonts get the SDK's default media caching, as with MapStaticAssets. + internal const string MediaCacheControl = "max-age=3600, must-revalidate"; + + // Astro hashes the names of the files it emits under /_astro/, so they can be cached forever. + // https://starlight.astro.build/environmental-impact/#caching + internal const string ImmutableCacheControl = "max-age=31536000, public, immutable"; + + internal const string DefaultCacheControl = "no-cache"; + + private const string BrotliExtension = ".br"; + + private const string UnknownContentType = "application/octet-stream"; + + private static readonly PathString s_immutablePrefix = "/_astro"; + + private readonly RequestDelegate _next; + private readonly IFileProvider _fileProvider; + private readonly SiteFileETags _etags; + private readonly FileExtensionContentTypeProvider _contentTypes; + + public SiteFileMiddleware(RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags) + { + _next = next; + _fileProvider = environment.WebRootFileProvider; + _etags = etags; + _contentTypes = CreateContentTypeProvider(); + } + + public async Task InvokeAsync(HttpContext context) + { + var request = context.Request; + if (context.GetEndpoint() is not null + || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) + || !request.Path.HasValue + || request.Path.Value.EndsWith('/')) + { + await _next(context); + return; + } + + var file = _fileProvider.GetFileInfo(request.Path.Value); + if (!file.Exists || file.IsDirectory) + { + await _next(context); + return; + } + + // Every file the site emits is meant to be served, including types this provider does + // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. + if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) + { + contentType = UnknownContentType; + } + + var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var hasCompressed = compressed.Exists && !compressed.IsDirectory; + var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); + var served = useCompressed ? compressed : file; + + var response = context.Response; + var etag = await _etags.GetAsync(served, context.RequestAborted); + var lastModified = TruncateToSeconds(file.LastModified); + + var headers = response.Headers; + headers.CacheControl = GetCacheControl(request.Path, contentType); + headers.ETag = etag; + headers.LastModified = HeaderUtilities.FormatDate(lastModified); + if (hasCompressed) + { + headers.Vary = HeaderNames.AcceptEncoding; + } + + if (IsNotModified(request, etag, lastModified)) + { + response.StatusCode = StatusCodes.Status304NotModified; + return; + } + + headers.AcceptRanges = "bytes"; + var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + if (unsatisfiable) + { + response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; + headers.ContentRange = $"bytes */{served.Length}"; + return; + } + + response.ContentType = contentType; + if (useCompressed) + { + headers.ContentEncoding = "br"; + } + + var (offset, count) = range ?? (0, served.Length); + if (range is not null) + { + response.StatusCode = StatusCodes.Status206PartialContent; + headers.ContentRange = $"bytes {offset}-{offset + count - 1}/{served.Length}"; + } + else + { + response.StatusCode = StatusCodes.Status200OK; + } + + response.ContentLength = count; + if (HttpMethods.IsHead(request.Method)) + { + return; + } + + await response.SendFileAsync(served, offset, count, context.RequestAborted); + } + + internal static string GetCacheControl(PathString path, string contentType) + { + if (path.StartsWithSegments(s_immutablePrefix, StringComparison.Ordinal)) + { + return ImmutableCacheControl; + } + + return contentType.StartsWith("image/", StringComparison.Ordinal) + || contentType.StartsWith("video/", StringComparison.Ordinal) + || contentType.StartsWith("font/", StringComparison.Ordinal) + ? MediaCacheControl + : DefaultCacheControl; + } + + internal static bool AcceptsBrotli(StringValues acceptEncoding) + { + if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) + { + return false; + } + + double? brotli = null; + double? wildcard = null; + foreach (var encoding in encodings) + { + var quality = encoding.Quality ?? 1; + if (StringSegment.Equals(encoding.Value, "br", StringComparison.OrdinalIgnoreCase)) + { + brotli = quality; + } + else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) + { + wildcard = quality; + } + } + + // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. + return (brotli ?? wildcard ?? 0) > 0; + } + + private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + { + var requestHeaders = request.GetTypedHeaders(); + + // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). + var ifNoneMatch = requestHeaders.IfNoneMatch; + if (ifNoneMatch.Count > 0) + { + var current = EntityTagHeaderValue.Parse(etag); + foreach (var candidate in ifNoneMatch) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) + { + return true; + } + } + + return false; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + } + + /// + /// Returns the single byte range to send, or to send the whole file, + /// which a server may do for any range request it does not support, such as several ranges. + /// Safari only plays video from servers that answer ranges. + /// + private static (long Offset, long Count)? GetRange( + HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + { + unsatisfiable = false; + var requestHeaders = request.GetTypedHeaders(); + var range = requestHeaders.Range; + if (range is null + || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) + || range.Ranges.Count != 1) + { + return null; + } + + // If-Range applies the range only to the representation the client has (RFC 9110, 13.1.5). + if (requestHeaders.IfRange is { } ifRange) + { + var matches = ifRange.EntityTag is { } tag + ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) + : ifRange.LastModified == lastModified; + if (!matches) + { + return null; + } + } + + var item = range.Ranges.First(); + long start, end; + if (item.From is { } from) + { + start = from; + end = Math.Min(item.To ?? long.MaxValue, length - 1); + } + else + { + // A suffix range: the last N bytes. + start = Math.Max(0, length - (item.To ?? 0)); + end = length - 1; + } + + if (start >= length || start > end) + { + unsatisfiable = true; + return null; + } + + return (start, end - start + 1); + } + + private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) => + new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset); + + private static FileExtensionContentTypeProvider CreateContentTypeProvider() + { + var provider = new FileExtensionContentTypeProvider(); + + // The SDK's static web assets mapping, which production sent until now, differs here. + provider.Mappings[".md"] = "text/markdown"; + + return provider; + } +} diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index a836de6f0c..8550c3aace 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -8,11 +8,14 @@ true noble-chiseled - - false - brotli - - <_BlazorBrotliCompressionLevel>Optimal + + false + + $(DefaultItemExcludes);wwwroot/** @@ -50,5 +53,5 @@ - + diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index 1750a6d4e9..b395f548d4 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using StaticHost.AgentReadiness; using StaticHost.Routing; +using StaticHost.SiteFiles; namespace StaticHost.Tests; @@ -48,6 +49,7 @@ public static async Task StartAsync( { web.UseTestServer(); web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddSiteFiles()); web.Configure(app => { if (pathBase.HasValue) @@ -60,10 +62,9 @@ public static async Task StartAsync( app.UseCanonicalPathRedirects(); app.UseAgentReadiness(); app.UseDefaultFiles(); - app.UseStaticFiles(); + app.UseSiteFiles(); - // Endpoint that mimics MapStaticAssets fallback: 404 if - // no static asset matched. + // 404 when no file matched, as in production. app.Run(static ctx => { ctx.Response.StatusCode = StatusCodes.Status404NotFound; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs new file mode 100644 index 0000000000..83812ee018 --- /dev/null +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -0,0 +1,327 @@ +using System.IO.Compression; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using StaticHost.SiteFiles; + +namespace StaticHost.Tests; + +public sealed class SiteFileTests +{ + private const string Page = "" + "Hello, brotli. " + ""; + + [Fact] + public async Task Serves_the_brotli_copy_when_the_client_accepts_it() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + var compressed = await File.ReadAllBytesAsync(Path.Combine(server.WebRoot, "docs", "page", "index.html.br")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["br"], response.Content.Headers.ContentEncoding); + Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType); + Assert.Equal(compressed.Length, response.Content.Headers.ContentLength); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(compressed, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal(ETagOf(compressed), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task Serves_the_file_itself_when_the_client_does_not_accept_brotli() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes(Page)), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task File_without_a_brotli_copy_has_no_vary_header() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("robots.txt", "User-agent: *\n")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/robots.txt"); + request.Headers.AcceptEncoding.ParseAdd("br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Empty(response.Headers.Vary); + Assert.Equal("User-agent: *\n", await response.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("", false)] + [InlineData("br", true)] + [InlineData("gzip, deflate, br, zstd", true)] + [InlineData("BR;q=0.5", true)] + [InlineData("gzip", false)] + [InlineData("br;q=0", false)] + [InlineData("*", true)] + [InlineData("*;q=0", false)] + [InlineData("br;q=0, *", false)] + public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => + Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + + [Fact] + public async Task Matching_if_none_match_returns_not_modified() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag; + Assert.NotNull(etag); + + foreach (var ifNoneMatch in new[] { etag.Tag, $"W/{etag.Tag}", "\"other\", " + etag.Tag, "*" }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", ifNoneMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); + Assert.Equal(etag.Tag, response.Headers.ETag?.Tag); + Assert.Equal("no-cache", response.Headers.CacheControl?.ToString()); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + } + + [Fact] + public async Task Other_if_none_match_returns_the_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task If_modified_since_applies_only_without_if_none_match() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified; + Assert.NotNull(lastModified); + + using var conditional = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + conditional.Headers.IfModifiedSince = lastModified; + using var notModified = await server.Client.SendAsync(conditional); + Assert.Equal(HttpStatusCode.NotModified, notModified.StatusCode); + + using var both = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + both.Headers.IfModifiedSince = lastModified; + both.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var modified = await server.Client.SendAsync(both); + Assert.Equal(HttpStatusCode.OK, modified.StatusCode); + } + + [Fact] + public async Task ETag_follows_a_file_that_changes() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + + using var before = await server.Client.GetAsync("/data.json"); + await File.WriteAllTextAsync(Path.Combine(server.WebRoot, "data.json"), "{\"v\":22}"); + using var after = await server.Client.GetAsync("/data.json"); + + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":1}")), before.Headers.ETag?.Tag); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":22}")), after.Headers.ETag?.Tag); + Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/favicon.svg", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/images/hero.webp", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/fonts/site.woff2", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/casts/demo.mp4", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/docs/page/index.html", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/docs/page.md", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/pagefind/pagefind.js", SiteFileMiddleware.DefaultCacheControl)] + public async Task Cache_control_depends_on_the_path_and_the_type(string path, string expected) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // HttpClient reorders the directives it parses; compare what the server sent. + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Markdown_is_served_as_text_markdown() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SamplePages.SeedFolderPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page.md"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task Head_sends_the_headers_without_the_body() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var response = await server.Client.SendAsync(new HttpRequestMessage(HttpMethod.Head, "/docs/page/")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Encoding.UTF8.GetByteCount(Page), response.Content.Headers.ContentLength); + Assert.NotNull(response.Headers.ETag); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + + [Theory] + [InlineData("bytes=2-5", HttpStatusCode.PartialContent, "2345", "bytes 2-5/10")] + [InlineData("bytes=7-", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=-3", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=8-100", HttpStatusCode.PartialContent, "89", "bytes 8-9/10")] + [InlineData("bytes=0-1, 4-5", HttpStatusCode.OK, "0123456789", null)] + [InlineData("items=0-1", HttpStatusCode.OK, "0123456789", null)] + public async Task Answers_a_single_byte_range(string range, HttpStatusCode status, string body, string? contentRange) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + Assert.Equal(body, await response.Content.ReadAsStringAsync()); + Assert.Equal(contentRange, response.Content.Headers.ContentRange?.ToString()); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + + [Fact] + public async Task Range_past_the_end_is_not_satisfiable() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=10-"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.RequestedRangeNotSatisfiable, response.StatusCode); + Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); + } + + [Fact] + public async Task If_range_applies_the_range_only_to_the_current_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + using var first = await server.Client.GetAsync("/casts/demo.mp4"); + var etag = first.Headers.ETag!.Tag; + + foreach (var (ifRange, status) in new[] { (etag, HttpStatusCode.PartialContent), ("\"stale\"", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.TryAddWithoutValidation("If-Range", ifRange); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Theory] + [InlineData("/missing.html")] + [InlineData("/missing/")] + [InlineData("/docs")] + public async Task Missing_files_fall_through(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("/pagefind/fragment/en_1a2b3c4.pf_fragment")] + [InlineData("/pagefind/wasm.en.pagefind")] + [InlineData("/casts/demo.cast")] + public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.MediaType); + Assert.Equal("x", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Post_falls_through() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); + + using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Fact] + public async Task Matched_endpoint_takes_precedence_over_a_file() + { + using var wwwroot = new TempWebRoot(); + wwwroot.WriteFile("api/data.json", "{\"from\":\"file\"}"); + wwwroot.WriteFile("other.json", "{\"from\":\"file\"}"); + + using var host = new HostBuilder() + .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddRouting().AddSiteFiles()); + web.Configure(static app => + { + app.UseRouting(); + app.UseSiteFiles(); + app.UseEndpoints(static endpoints => endpoints.MapGet("/api/data.json", () => Results.Text("endpoint"))); + }); + }) + .Build(); + await host.StartAsync(); + using var client = host.GetTestClient(); + + Assert.Equal("endpoint", await client.GetStringAsync("/api/data.json")); + Assert.Equal("{\"from\":\"file\"}", await client.GetStringAsync("/other.json")); + + await host.StopAsync(); + } + + private static void SeedCompressedPage(TempWebRoot root, string slug) + { + root.WriteFile($"{slug}/index.html", Page); + + using var output = new MemoryStream(); + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + brotli.Write(Encoding.UTF8.GetBytes(Page)); + } + + File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); + } + + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; +} From d4efc16ad7140cbe21820807c42c42120fb4996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:43:26 -0800 Subject: [PATCH 2/8] Compress the site with a file-based app and tighten file preconditions - Write the brotli copies with src/statichost/compress-site.cs, a file-based app that Site.targets runs, instead of an inline MSBuild task that loaded BrotliStream by reflection (which fails under .NET Framework MSBuild). - Reuse a brotli copy only while it carries its source's exact timestamp. A source replaced by new content with an older timestamp (a cache restore or an archive extraction) kept the copy of the old content. - Answer If-Match and If-Unmodified-Since with 412, as MapStaticAssets did. - Apply a date in If-Range only to uncompressed responses: Last-Modified is the original file's, so it does not identify the bytes of its brotli copy. - Key cached ETags by the file's path in the web root, not its physical path or bare name, and parse the ETag once per request. --- src/statichost/StaticHost/Site.targets | 107 +++++------------- .../StaticHost/SiteFiles/SiteFileETags.cs | 12 +- .../SiteFiles/SiteFileMiddleware.cs | 72 ++++++++---- src/statichost/compress-site.cs | 67 +++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 47 ++++++++ 5 files changed, 200 insertions(+), 105 deletions(-) create mode 100644 src/statichost/compress-site.cs diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets index 2cba6ca7a7..3eadbdc080 100644 --- a/src/statichost/StaticHost/Site.targets +++ b/src/statichost/StaticHost/Site.targets @@ -9,99 +9,48 @@ output, and compress it once at publish. Caching and ETags are decided per request. Asking frontend.esproj for JavaScriptOutputGroup runs its BuildCommand and returns dist's - files, as the static web assets pipeline did. + files, as the static web assets pipeline did. The brotli copies are written by a file-based + app (../compress-site.cs), which reuses the copies of files that did not change. --> .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + $(MSBuildThisFileDirectory)../compress-site.cs + + $(DOTNET_HOST_PATH) + dotnet - - - - - - - - - - - - - - (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); - var publishFiles = new ConcurrentBag(); - var compressed = 0; - - Func publish = (path, relativePath) => - { - var item = new TaskItem(path); - item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); - item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); - item.SetMetadata("ExcludeFromSingleFile", "true"); - return item; - }; - - Parallel.ForEach(Files, file => - { - var source = file.GetMetadata("FullPath"); - var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); - publishFiles.Add(publish(source, relativePath)); - - if (!extensions.Contains(Path.GetExtension(source))) - { - return; - } - - // Reuse a copy that is newer than its source, so an unchanged site compresses once. - var target = Path.Combine(OutputDirectory, relativePath + ".br"); - if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) - { - Directory.CreateDirectory(Path.GetDirectoryName(target)); - var temporary = target + ".tmp"; - using (var input = File.OpenRead(source)) - using (var output = File.Create(temporary)) - using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) - { - input.CopyTo(brotli); - } - - File.Delete(target); - File.Move(temporary, target); - Interlocked.Increment(ref compressed); - } - - // A copy that saves nothing (tiny files) is not worth serving. - if (new FileInfo(target).Length < new FileInfo(source).Length) - { - publishFiles.Add(publish(target, relativePath + ".br")); - } - }); - - PublishFiles = publishFiles.ToArray(); - Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); - ]]> - - - - $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + <_SiteIntermediateDirectory>$(MSBuildProjectDirectory)/$(IntermediateOutputPath) + <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br/ + <_SiteFileList>$(_SiteIntermediateDirectory)site-files.txt + <_SiteBrotliList>$(_SiteIntermediateDirectory)site-br.txt - - - + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index ef5c00828c..9c9d887911 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -19,15 +19,15 @@ internal sealed class SiteFileETags { private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - public int Count => _entries.Count; - - public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + /// The file's path in the web root, which identifies it whatever the provider. + /// The file to hash. + /// Cancels the hashing. + public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) { - var key = file.PhysicalPath ?? file.Name; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) { return entry.ETag; } @@ -37,7 +37,7 @@ public async ValueTask GetAsync(IFileInfo file, CancellationToken cancel var hash = await SHA256.HashDataAsync(stream, cancellationToken); var etag = $"\"{Convert.ToBase64String(hash)}\""; - _entries[key] = new Entry(length, lastModified, etag); + _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9ba5bb9fd1..9c39bb203a 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http.Headers; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Primitives; @@ -83,14 +84,17 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } - var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var compressedPath = request.Path.Value + BrotliExtension; + var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); var served = useCompressed ? compressed : file; var response = context.Response; - var etag = await _etags.GetAsync(served, context.RequestAborted); + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + var entityTag = new EntityTagHeaderValue(etag); var lastModified = TruncateToSeconds(file.LastModified); + var requestHeaders = request.GetTypedHeaders(); var headers = response.Headers; headers.CacheControl = GetCacheControl(request.Path, contentType); @@ -101,14 +105,16 @@ public async Task InvokeAsync(HttpContext context) headers.Vary = HeaderNames.AcceptEncoding; } - if (IsNotModified(request, etag, lastModified)) + if (EvaluatePreconditions(requestHeaders, entityTag, lastModified) is { } status) { - response.StatusCode = StatusCodes.Status304NotModified; + response.StatusCode = status; return; } headers.AcceptRanges = "bytes"; - var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes + // of its brotli copy, which can be rewritten while the original keeps its time. + var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -182,27 +188,50 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) return (brotli ?? wildcard ?? 0) > 0; } - private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + /// + /// Evaluates the request's preconditions in the order of RFC 9110, 13.2.2, as + /// MapStaticAssets did. Returns 412 or 304 when one decides the response, otherwise + /// . + /// + private static int? EvaluatePreconditions(RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset lastModified) { - var requestHeaders = request.GetTypedHeaders(); + // If-Match takes precedence over If-Unmodified-Since, and uses the strong comparison. + var ifMatch = requestHeaders.IfMatch; + if (ifMatch.Count > 0) + { + if (!Matches(ifMatch, etag, useStrongComparison: true)) + { + return StatusCodes.Status412PreconditionFailed; + } + } + else if (requestHeaders.IfUnmodifiedSince is { } ifUnmodifiedSince && lastModified > ifUnmodifiedSince) + { + return StatusCodes.Status412PreconditionFailed; + } // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). var ifNoneMatch = requestHeaders.IfNoneMatch; if (ifNoneMatch.Count > 0) { - var current = EntityTagHeaderValue.Parse(etag); - foreach (var candidate in ifNoneMatch) + return Matches(ifNoneMatch, etag, useStrongComparison: false) ? StatusCodes.Status304NotModified : null; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince + ? StatusCodes.Status304NotModified + : null; + } + + private static bool Matches(IList candidates, EntityTagHeaderValue etag, bool useStrongComparison) + { + foreach (var candidate in candidates) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(etag, useStrongComparison)) { - if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) - { - return true; - } + return true; } - - return false; } - return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + return false; } /// @@ -210,11 +239,14 @@ private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffs /// which a server may do for any range request it does not support, such as several ranges. /// Safari only plays video from servers that answer ranges. /// + /// + /// The time a date in If-Range must match, or when a date does + /// not identify the bytes sent, so only an entity tag can. + /// private static (long Offset, long Count)? GetRange( - HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset? lastModified, long length, out bool unsatisfiable) { unsatisfiable = false; - var requestHeaders = request.GetTypedHeaders(); var range = requestHeaders.Range; if (range is null || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) @@ -227,8 +259,8 @@ private static (long Offset, long Count)? GetRange( if (requestHeaders.IfRange is { } ifRange) { var matches = ifRange.EntityTag is { } tag - ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) - : ifRange.LastModified == lastModified; + ? tag.Compare(etag, useStrongComparison: true) + : lastModified is not null && ifRange.LastModified == lastModified; if (!matches) { return null; diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs new file mode 100644 index 0000000000..88cec4f419 --- /dev/null +++ b/src/statichost/compress-site.cs @@ -0,0 +1,67 @@ +// Writes a brotli copy of each text file of the site, for StaticHost to serve to clients that +// accept it. StaticHost's publish runs it (StaticHost/Site.targets); it also runs alone: +// +// dotnet run --file compress-site.cs -- +// +// One line per site file: its full path, a tab, its path in the site. +// Where the copies are written, at their path in the site plus ".br". +// The extensions to compress, separated by semicolons. +// Written with the path in the site of each copy worth publishing. + +using System.Collections.Concurrent; +using System.IO.Compression; + +if (args.Length != 4) +{ + Console.Error.WriteLine("Usage: compress-site.cs "); + return 1; +} + +var (filesPath, outputDirectory, extensionList, publishedPath) = (args[0], Path.GetFullPath(args[1]), args[2], args[3]); +var extensions = new HashSet(extensionList.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + +var files = File.ReadLines(filesPath) + .Where(static line => line.Length > 0) + .Select(static line => line.Split('\t', 2)) + .Select(static parts => (Source: parts[0], SitePath: parts[1].Replace('\\', '/'))) + .Where(file => extensions.Contains(Path.GetExtension(file.Source))) + .ToList(); + +var published = new ConcurrentBag(); +var compressed = 0; + +Parallel.ForEach(files, file => +{ + // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. + // "Newer than the source" is not enough: a cache restore or an archive extraction gives + // new content an older time, and the copy of the old content would be published. + var target = Path.Combine(outputDirectory, file.SitePath + ".br"); + var sourceTime = File.GetLastWriteTimeUtc(file.Source); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(file.Source)) + using (var output = File.Create(temporary)) + // Optimal is brotli quality 4, which the SDK used at publish (_BlazorBrotliCompressionLevel). + // SmallestSize (11) saves about a sixth more but compresses about 100 times slower. + using (var brotli = new BrotliStream(output, CompressionLevel.Optimal)) + { + input.CopyTo(brotli); + } + + File.SetLastWriteTimeUtc(temporary, sourceTime); + File.Move(temporary, target, overwrite: true); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(file.Source).Length) + { + published.Add(file.SitePath + ".br"); + } +}); + +File.WriteAllLines(publishedPath, published.Order(StringComparer.Ordinal)); +Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now, {published.Count} brotli copies published."); +return 0; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index 83812ee018..f207a37af1 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -130,6 +130,53 @@ public async Task If_modified_since_applies_only_without_if_none_match() Assert.Equal(HttpStatusCode.OK, modified.StatusCode); } + [Fact] + public async Task If_match_and_if_unmodified_since_fail_for_another_version() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag!.Tag; + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (ifMatch, status) in new[] { (etag, HttpStatusCode.OK), ("*", HttpStatusCode.OK), ("\"stale\"", HttpStatusCode.PreconditionFailed), ($"W/{etag}", HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-Match", ifMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + + foreach (var (ifUnmodifiedSince, status) in new[] { (lastModified, HttpStatusCode.OK), (lastModified.AddDays(-1), HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.IfUnmodifiedSince = ifUnmodifiedSince; + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Fact] + public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (acceptEncoding, status) in new[] { ("identity", HttpStatusCode.PartialContent), ("br", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd(acceptEncoding); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + [Fact] public async Task ETag_follows_a_file_that_changes() { From 12329a6513e63110ce3dcf54d0d43938f3ca18bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:00:42 -0800 Subject: [PATCH 3/8] Hash ETags with XxHash128 and explain the brotli lookup and key casing - Compute ETags with XxHash128 instead of SHA-256: an ETag only has to tell versions of a file apart, which needs no cryptographic hash. It still hashes the content, so unchanged files keep their ETag across deployments. - Say why the brotli copy is looked up even for clients that do not accept brotli (the Vary header both versions need), and why cached ETags are keyed by ordinal path. --- .../StaticHost/SiteFiles/SiteFileETags.cs | 42 ++++++++++++++----- .../SiteFiles/SiteFileMiddleware.cs | 3 ++ src/statichost/StaticHost/StaticHost.csproj | 1 + tests/StaticHost.Tests/SiteFileTests.cs | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 9c9d887911..772475b9cc 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -1,22 +1,29 @@ +using System.Buffers; using System.Collections.Concurrent; -using System.Security.Cryptography; +using System.IO.Hashing; using Microsoft.Extensions.FileProviders; namespace StaticHost.SiteFiles; /// -/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// Computes a file's ETag from a hash of its content the first time it is served, then keeps it +/// in memory. /// /// -/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the -/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and -/// last-modified time UseStaticFiles uses, stays the same across deployments for files -/// that did not change. An entry is recomputed when the file's length or last-modified time -/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site -/// after StaticHost starts). +/// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no +/// cryptographic hash. A content hash, unlike the length and last-modified time +/// UseStaticFiles uses, stays the same across deployments for files that did not change, +/// though CI writes every file again. An entry is recomputed when the file's length or +/// last-modified time changes, which covers a web root that is replaced while StaticHost runs +/// (CI unpacks the site after StaticHost starts). /// internal sealed class SiteFileETags { + private const int BufferSize = 64 * 1024; + + // Ordinal, as the file system is in production (Linux): two paths that differ only by case are + // two files there, and must not share an entry. On a case-insensitive disk they are one file + // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); /// The file's path in the web root, which identifies it whatever the provider. @@ -33,10 +40,23 @@ public async ValueTask GetAsync(string path, IFileInfo file, Cancellatio } // Two first requests for the same file may both hash it; the result is the same. - await using var stream = file.CreateReadStream(); - var hash = await SHA256.HashDataAsync(stream, cancellationToken); - var etag = $"\"{Convert.ToBase64String(hash)}\""; + var hash = new XxHash128(); + var buffer = ArrayPool.Shared.Rent(BufferSize); + try + { + await using var stream = file.CreateReadStream(); + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + hash.Append(buffer.AsSpan(0, read)); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9c39bb203a..25d78b3952 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -84,6 +84,9 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } + // Look for the brotli copy even when the client does not accept brotli: a file that has one + // must send Vary: Accept-Encoding with both versions, so that a cache does not hand the + // uncompressed one to every client. Nearly every browser accepts brotli anyway. var compressedPath = request.Path.Value + BrotliExtension; var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index 8550c3aace..791917b5e4 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -30,6 +30,7 @@ + diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index f207a37af1..ede50a08f8 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -370,5 +370,5 @@ private static void SeedCompressedPage(TempWebRoot root, string slug) File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); } - private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(System.IO.Hashing.XxHash128.Hash(content))}\""; } From 5e666f1830f774e66ec85c0f30d1961b5eddb716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:22:51 -0800 Subject: [PATCH 4/8] Key cached ETags by physical path so the cache stays bounded Request paths that differ only by empty or dot segments (/a/b.json, /a//b.json, /a/./b.json) name one file, and the file provider resolves them to the same physical path. Keyed by request path, each spelling added an entry and hashed the file again, so a client could grow the cache without bound and force a full read of any file on every request. Keyed by physical path, the cache holds at most one entry per file of the web root. Concurrent first requests for a file may still both hash it; that happens once per file per instance, and serving the file reads it anyway. --- .../StaticHost/SiteFiles/SiteFileETags.cs | 19 +++++++++++++++---- .../AgentReadinessTestServer.cs | 2 ++ tests/StaticHost.Tests/SiteFileTests.cs | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 772475b9cc..84c3502ab2 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -21,25 +21,36 @@ internal sealed class SiteFileETags { private const int BufferSize = 64 * 1024; + // Keyed by the file's physical path, which the provider normalizes: the request paths + // /a/b.json, /a//b.json and /a///b.json are one file, so they share one entry, and the + // dictionary holds at most one entry per file of the web root. Request paths would let any + // client add entries without bound, each one costing a hash of the file. // Ordinal, as the file system is in production (Linux): two paths that differ only by case are // two files there, and must not share an entry. On a case-insensitive disk they are one file // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - /// The file's path in the web root, which identifies it whatever the provider. + internal int Count => _entries.Count; + + /// + /// The file's path in the web root, the key when the provider gives no physical path. + /// /// The file to hash. /// Cancels the hashing. public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) { + var key = file.PhysicalPath ?? path; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) { return entry.ETag; } - // Two first requests for the same file may both hash it; the result is the same. + // Two first requests for the same file may both hash it; the result is the same. That is + // not worth coordinating: it happens once per file per instance, and sending the file + // reads it anyway. var hash = new XxHash128(); var buffer = ArrayPool.Shared.Rent(BufferSize); try @@ -57,7 +68,7 @@ public async ValueTask GetAsync(string path, IFileInfo file, Cancellatio } var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; - _entries[path] = new Entry(length, lastModified, etag); + _entries[key] = new Entry(length, lastModified, etag); return etag; } diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index b395f548d4..c6b4106f47 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -26,6 +26,8 @@ internal sealed class AgentReadinessTestServer : IAsyncDisposable public string WebRoot => _wwwroot.Path; + public IServiceProvider Services => _host.Services; + private AgentReadinessTestServer(IHost host, TestServer testServer, TempWebRoot wwwroot) { _host = host; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index ede50a08f8..860324071b 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -177,6 +177,20 @@ public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() } } + [Fact] + public async Task Spellings_of_one_file_share_one_cached_etag() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/data.json", "{}")); + + foreach (var path in new[] { "/docs/data.json", "/docs//data.json", "/docs///data.json" }) + { + using var response = await server.Client.GetAsync(path); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + Assert.Equal(1, server.Services.GetRequiredService().Count); + } + [Fact] public async Task ETag_follows_a_file_that_changes() { From 59afd5898cd6f614d43b38f8f334378f64f9affa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:05:43 -0800 Subject: [PATCH 5/8] Answer 405 for methods other than GET and HEAD on a file The static asset endpoints allowed only GET and HEAD, so routing matched a file's path and answered 405 for other methods. The middleware stepped aside for them instead, and a POST to a file fell through to the 404 page. It now answers 405 with Allow: GET, HEAD; a path with no file still gets 404. --- .../StaticHost/SiteFiles/SiteFileMiddleware.cs | 12 +++++++++++- tests/StaticHost.Tests/SiteFileTests.cs | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 25d78b3952..c4a7d7c63e 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -43,6 +43,8 @@ internal sealed class SiteFileMiddleware private const string UnknownContentType = "application/octet-stream"; + private const string AllowedMethods = "GET, HEAD"; + private static readonly PathString s_immutablePrefix = "/_astro"; private readonly RequestDelegate _next; @@ -62,7 +64,6 @@ public async Task InvokeAsync(HttpContext context) { var request = context.Request; if (context.GetEndpoint() is not null - || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) || !request.Path.HasValue || request.Path.Value.EndsWith('/')) { @@ -77,6 +78,15 @@ public async Task InvokeAsync(HttpContext context) return; } + // A file only answers GET and HEAD. Other methods get 405, as they did from the static + // asset endpoints, which routing matched by path and then rejected by method. + if (!HttpMethods.IsGet(request.Method) && !HttpMethods.IsHead(request.Method)) + { + context.Response.StatusCode = StatusCodes.Status405MethodNotAllowed; + context.Response.Headers.Allow = AllowedMethods; + return; + } + // Every file the site emits is meant to be served, including types this provider does // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index 860324071b..e162b1d51c 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -331,13 +331,22 @@ public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) } [Fact] - public async Task Post_falls_through() + public async Task Other_methods_on_a_file_are_not_allowed() { await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); - using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + foreach (var method in new[] { HttpMethod.Post, HttpMethod.Put, HttpMethod.Delete, HttpMethod.Options }) + { + using var request = new HttpRequestMessage(method, "/data.json") { Content = new StringContent("{}") }; + using var response = await server.Client.SendAsync(request); - Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + Assert.Equal(HttpStatusCode.MethodNotAllowed, response.StatusCode); + Assert.Equal(["GET", "HEAD"], response.Content.Headers.Allow); + } + + // A path with no file still falls through to the 404 page. + using var missing = await server.Client.PostAsync("/missing.json", new StringContent("{}")); + Assert.Equal(HttpStatusCode.NotFound, missing.StatusCode); } [Fact] From c8ed41bb5ba8b6c218860063164f72f8bd78e53e Mon Sep 17 00:00:00 2001 From: IEvangelist <7679720+IEvangelist@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:43:39 -0600 Subject: [PATCH 6/8] Make StaticHost responses edge-cacheable and configurable - Bind Cache-Control values from SiteFiles:CacheControl (Default, Media, Immutable, ImmutablePathPrefixes, VersionQueryParameter) with validation on start. - Default for non-fingerprinted files becomes public, max-age=0, must-revalidate, s-maxage=3600 so Front Door can cache HTML while browsers revalidate. - Serve ?v= versioned URLs with the immutable Cache-Control. - Send Cache-Control: no-store with every redirect (HTTPS, canonical path, install scripts). - Never serve the brotli copy to a Range request. - Write build-time brotli copies at quality 11 on every core. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/statichost/StaticHost/Program.cs | 3 + .../CanonicalPathRedirectExtensions.cs | 12 ++ .../Routing/NoStoreRedirectMiddleware.cs | 37 +++++ src/statichost/StaticHost/Site.targets | 3 +- .../SiteFiles/SiteFileCacheOptions.cs | 61 ++++++++ .../SiteFiles/SiteFileExtensions.cs | 13 +- .../SiteFiles/SiteFileMiddleware.cs | 65 +++++---- src/statichost/compress-site.cs | 17 ++- .../AgentReadinessTestServer.cs | 16 ++- .../StaticHost.Tests/NoStoreRedirectTests.cs | 115 +++++++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 133 +++++++++++++++--- 11 files changed, 417 insertions(+), 58 deletions(-) create mode 100644 src/statichost/StaticHost/Routing/NoStoreRedirectMiddleware.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs create mode 100644 tests/StaticHost.Tests/NoStoreRedirectTests.cs diff --git a/src/statichost/StaticHost/Program.cs b/src/statichost/StaticHost/Program.cs index 3ccf5dc5a0..4a66c4e46b 100644 --- a/src/statichost/StaticHost/Program.cs +++ b/src/statichost/StaticHost/Program.cs @@ -31,6 +31,9 @@ // Restore the Front Door client IP before telemetry or any middleware reads it. app.UseForwardedClientIp(); +// Before anything that redirects: Front Door must never cache a redirect. +app.UseNoStoreRedirects(); + // Only enable HSTS in production if (!app.Environment.IsDevelopment()) { diff --git a/src/statichost/StaticHost/Routing/CanonicalPathRedirectExtensions.cs b/src/statichost/StaticHost/Routing/CanonicalPathRedirectExtensions.cs index 8edacc3d0e..709ffcc5d3 100644 --- a/src/statichost/StaticHost/Routing/CanonicalPathRedirectExtensions.cs +++ b/src/statichost/StaticHost/Routing/CanonicalPathRedirectExtensions.cs @@ -13,4 +13,16 @@ public static IApplicationBuilder UseCanonicalPathRedirects(this IApplicationBui app.UseMiddleware(); return app; } + + /// + /// Sends Cache-Control: no-store with every redirect, so no cache keeps one. + /// Call first, before any middleware or endpoint that can redirect. + /// + public static IApplicationBuilder UseNoStoreRedirects(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + + app.UseMiddleware(); + return app; + } } diff --git a/src/statichost/StaticHost/Routing/NoStoreRedirectMiddleware.cs b/src/statichost/StaticHost/Routing/NoStoreRedirectMiddleware.cs new file mode 100644 index 0000000000..a8a62a3461 --- /dev/null +++ b/src/statichost/StaticHost/Routing/NoStoreRedirectMiddleware.cs @@ -0,0 +1,37 @@ +namespace StaticHost.Routing; + +/// +/// Marks every redirect the host sends Cache-Control: no-store. +/// +/// +/// A redirect without Cache-Control is heuristically cacheable, and a permanent one is +/// cacheable by default (RFC 9111, 4.2.2). Azure Front Door would keep such a redirect for days, +/// so a canonical path, an HTTPS upgrade or an install script target that changes would go on +/// redirecting from the edge. Run it first, so it sees the redirects of every later middleware +/// and endpoint. 304 Not Modified is not a redirect and keeps the file's caching. +/// +internal sealed class NoStoreRedirectMiddleware(RequestDelegate next) +{ + public Task InvokeAsync(HttpContext context) + { + context.Response.OnStarting(static state => + { + var response = (HttpResponse)state; + if (IsRedirect(response.StatusCode)) + { + response.Headers.CacheControl = "no-store"; + } + + return Task.CompletedTask; + }, context.Response); + + return next(context); + } + + internal static bool IsRedirect(int statusCode) => statusCode + is StatusCodes.Status301MovedPermanently + or StatusCodes.Status302Found + or StatusCodes.Status303SeeOther + or StatusCodes.Status307TemporaryRedirect + or StatusCodes.Status308PermanentRedirect; +} diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets index 3eadbdc080..785b8eb872 100644 --- a/src/statichost/StaticHost/Site.targets +++ b/src/statichost/StaticHost/Site.targets @@ -25,7 +25,8 @@ <_SiteIntermediateDirectory>$(MSBuildProjectDirectory)/$(IntermediateOutputPath) - <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br/ + + <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br-q11/ <_SiteFileList>$(_SiteIntermediateDirectory)site-files.txt <_SiteBrotliList>$(_SiteIntermediateDirectory)site-br.txt diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs new file mode 100644 index 0000000000..c15e39511d --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs @@ -0,0 +1,61 @@ +namespace StaticHost.SiteFiles; + +/// +/// The Cache-Control values sends, bound from the +/// SiteFiles:CacheControl configuration section so they can be tuned with App Service +/// settings (for example SiteFiles__CacheControl__Default) and a restart, without a deploy. +/// +/// +/// +/// Every deployment purges Azure Front Door, so a file that is not fingerprinted can be cached at +/// the edge (s-maxage) while browsers revalidate it on every use (max-age=0): the +/// edge answers those revalidations, and a deployment's purge drops what it cached. Front Door +/// does not cache a response the origin marks no-cache, even with a rule that overrides +/// the cache duration, so the origin must send these values for HTML to be cached at the edge. +/// +/// +internal sealed class SiteFileCacheOptions +{ + public const string SectionName = "SiteFiles:CacheControl"; + + // Browsers revalidate every time; Front Door keeps the response for an hour, or until the + // purge that follows a deployment. An hour bounds how long a missed purge can serve stale HTML. + internal const string DefaultValue = "public, max-age=0, must-revalidate, s-maxage=3600"; + + // The SDK's default for images, video and fonts, as MapStaticAssets sent. + internal const string MediaValue = "max-age=3600, must-revalidate"; + + // https://starlight.astro.build/environmental-impact/#caching + internal const string ImmutableValue = "max-age=31536000, public, immutable"; + + /// The value for files that are neither media nor immutable: HTML, markdown, JSON, XML, text. + public string Default { get; set; } = DefaultValue; + + /// The value for images, video and fonts outside the immutable paths. + public string Media { get; set; } = MediaValue; + + /// The value for fingerprinted files, and for files requested with the version query parameter. + public string Immutable { get; set; } = ImmutableValue; + + /// + /// Paths whose files are fingerprinted, so they can be cached forever, separated by semicolons. + /// Astro hashes the names of the files it emits under /_astro/. A string rather than a + /// list, because configuration adds to a list's default instead of replacing it. + /// + public string ImmutablePathPrefixes { get; set; } = "/_astro"; + + /// + /// The query parameter that marks a URL as versioned (sidebar.json?v=<hash>), so its + /// response can be cached forever: a new version gets a new URL. Empty turns it off. + /// + public string VersionQueryParameter { get; set; } = "v"; + + internal string[] GetImmutablePathPrefixes() => + ImmutablePathPrefixes.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + internal static bool IsValid(SiteFileCacheOptions options) => + !string.IsNullOrWhiteSpace(options.Default) + && !string.IsNullOrWhiteSpace(options.Media) + && !string.IsNullOrWhiteSpace(options.Immutable) + && options.GetImmutablePathPrefixes().All(static prefix => prefix is ['/', _, ..] && !prefix.EndsWith('/')); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs index 6be7e35cab..f8111116a7 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs @@ -2,8 +2,17 @@ namespace StaticHost.SiteFiles; internal static class SiteFileExtensions { - public static IServiceCollection AddSiteFiles(this IServiceCollection services) => - services.AddSingleton(); + public static IServiceCollection AddSiteFiles(this IServiceCollection services) + { + services.AddOptions() + .BindConfiguration(SiteFileCacheOptions.SectionName) + .Validate( + SiteFileCacheOptions.IsValid, + $"{SiteFileCacheOptions.SectionName} needs non-empty values, and path prefixes that start with '/' and do not end with it.") + .ValidateOnStart(); + + return services.AddSingleton(); + } /// /// Serves the web root's files, preferring the precompressed brotli copy. Call it after diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index c4a7d7c63e..b12d31faca 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Http.Headers; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; using Microsoft.Net.Http.Headers; @@ -25,38 +26,37 @@ namespace StaticHost.SiteFiles; /// /// /// A request for one byte range gets it (video players need ranges); other range requests get the -/// whole file. +/// whole file. A range request always gets the file itself, never its brotli copy: a range of +/// compressed bytes is useless to a client that wants to seek, and Azure Front Door answers 503 +/// when the lengths it sees for one URL's ranges disagree across encodings. +/// +/// +/// Cache-Control comes from . /// /// internal sealed class SiteFileMiddleware { - // Images, video and fonts get the SDK's default media caching, as with MapStaticAssets. - internal const string MediaCacheControl = "max-age=3600, must-revalidate"; - - // Astro hashes the names of the files it emits under /_astro/, so they can be cached forever. - // https://starlight.astro.build/environmental-impact/#caching - internal const string ImmutableCacheControl = "max-age=31536000, public, immutable"; - - internal const string DefaultCacheControl = "no-cache"; - private const string BrotliExtension = ".br"; private const string UnknownContentType = "application/octet-stream"; private const string AllowedMethods = "GET, HEAD"; - private static readonly PathString s_immutablePrefix = "/_astro"; - private readonly RequestDelegate _next; private readonly IFileProvider _fileProvider; private readonly SiteFileETags _etags; + private readonly SiteFileCacheOptions _cacheOptions; + private readonly PathString[] _immutablePrefixes; private readonly FileExtensionContentTypeProvider _contentTypes; - public SiteFileMiddleware(RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags) + public SiteFileMiddleware( + RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags, IOptions cacheOptions) { _next = next; _fileProvider = environment.WebRootFileProvider; _etags = etags; + _cacheOptions = cacheOptions.Value; + _immutablePrefixes = [.. _cacheOptions.GetImmutablePathPrefixes().Select(static prefix => new PathString(prefix))]; _contentTypes = CreateContentTypeProvider(); } @@ -100,7 +100,9 @@ public async Task InvokeAsync(HttpContext context) var compressedPath = request.Path.Value + BrotliExtension; var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; - var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); + var useCompressed = hasCompressed + && request.Headers.Range.Count == 0 + && AcceptsBrotli(request.Headers.AcceptEncoding); var served = useCompressed ? compressed : file; var response = context.Response; @@ -110,7 +112,7 @@ public async Task InvokeAsync(HttpContext context) var requestHeaders = request.GetTypedHeaders(); var headers = response.Headers; - headers.CacheControl = GetCacheControl(request.Path, contentType); + headers.CacheControl = GetCacheControl(request, contentType); headers.ETag = etag; headers.LastModified = HeaderUtilities.FormatDate(lastModified); if (hasCompressed) @@ -125,9 +127,7 @@ public async Task InvokeAsync(HttpContext context) } headers.AcceptRanges = "bytes"; - // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes - // of its brotli copy, which can be rewritten while the original keeps its time. - var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); + var range = GetRange(requestHeaders, entityTag, lastModified, served.Length, out var unsatisfiable); if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -161,18 +161,29 @@ public async Task InvokeAsync(HttpContext context) await response.SendFileAsync(served, offset, count, context.RequestAborted); } - internal static string GetCacheControl(PathString path, string contentType) + internal string GetCacheControl(HttpRequest request, string contentType) { - if (path.StartsWithSegments(s_immutablePrefix, StringComparison.Ordinal)) + var path = request.Path; + foreach (var prefix in _immutablePrefixes) + { + if (path.StartsWithSegments(prefix, StringComparison.Ordinal)) + { + return _cacheOptions.Immutable; + } + } + + // A versioned URL changes with its content, so its response never goes stale. + if (_cacheOptions.VersionQueryParameter is { Length: > 0 } version + && !StringValues.IsNullOrEmpty(request.Query[version])) { - return ImmutableCacheControl; + return _cacheOptions.Immutable; } return contentType.StartsWith("image/", StringComparison.Ordinal) || contentType.StartsWith("video/", StringComparison.Ordinal) || contentType.StartsWith("font/", StringComparison.Ordinal) - ? MediaCacheControl - : DefaultCacheControl; + ? _cacheOptions.Media + : _cacheOptions.Default; } internal static bool AcceptsBrotli(StringValues acceptEncoding) @@ -252,12 +263,8 @@ private static bool Matches(IList candidates, EntityTagHea /// which a server may do for any range request it does not support, such as several ranges. /// Safari only plays video from servers that answer ranges. /// - /// - /// The time a date in If-Range must match, or when a date does - /// not identify the bytes sent, so only an entity tag can. - /// private static (long Offset, long Count)? GetRange( - RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset? lastModified, long length, out bool unsatisfiable) + RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) { unsatisfiable = false; var range = requestHeaders.Range; @@ -273,7 +280,7 @@ private static (long Offset, long Count)? GetRange( { var matches = ifRange.EntityTag is { } tag ? tag.Compare(etag, useStrongComparison: true) - : lastModified is not null && ifRange.LastModified == lastModified; + : ifRange.LastModified == lastModified; if (!matches) { return null; diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs index 88cec4f419..0e5af9512a 100644 --- a/src/statichost/compress-site.cs +++ b/src/statichost/compress-site.cs @@ -29,8 +29,11 @@ var published = new ConcurrentBag(); var compressed = 0; +var stopwatch = System.Diagnostics.Stopwatch.StartNew(); -Parallel.ForEach(files, file => +// Brotli is single-threaded per stream, so one file per core keeps every core busy. +var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; +Parallel.ForEach(files, parallelOptions, file => { // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. // "Newer than the source" is not enough: a cache restore or an archive extraction gives @@ -43,9 +46,13 @@ var temporary = target + ".tmp"; using (var input = File.OpenRead(file.Source)) using (var output = File.Create(temporary)) - // Optimal is brotli quality 4, which the SDK used at publish (_BlazorBrotliCompressionLevel). - // SmallestSize (11) saves about a sixth more but compresses about 100 times slower. - using (var brotli = new BrotliStream(output, CompressionLevel.Optimal)) + // SmallestSize is brotli quality 11. On the site's HTML it writes about a tenth less than + // quality 4-5 (16.4% of the original against 18.5%), and on /_astro's JS and CSS 22.8% + // against 24.9%, at about 90 ms a page instead of 4. The copies are written once per + // change and served many times, so the time is spent here rather than on every request. + // Site.targets names the output directory after the quality, so copies written at + // another quality are not reused. + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize)) { input.CopyTo(brotli); } @@ -63,5 +70,5 @@ }); File.WriteAllLines(publishedPath, published.Order(StringComparer.Ordinal)); -Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now, {published.Count} brotli copies published."); +Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now in {stopwatch.Elapsed.TotalSeconds:F0} s on {parallelOptions.MaxDegreeOfParallelism} threads, {published.Count} brotli copies published."); return 0; diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index c6b4106f47..098a7df544 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -38,7 +39,8 @@ private AgentReadinessTestServer(IHost host, TestServer testServer, TempWebRoot public static async Task StartAsync( Action? seed = null, - PathString pathBase = default) + PathString pathBase = default, + IReadOnlyDictionary? configuration = null) { var wwwroot = new TempWebRoot(); try @@ -47,6 +49,13 @@ public static async Task StartAsync( var hostBuilder = Host.CreateDefaultBuilder() .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureAppConfiguration(config => + { + if (configuration is not null) + { + config.AddInMemoryCollection(configuration); + } + }) .ConfigureWebHost(web => { web.UseTestServer(); @@ -59,8 +68,9 @@ public static async Task StartAsync( app.UsePathBase(pathBase); } - // Match production order: canonical redirects and - // agent-readiness BEFORE default files (see Program.cs). + // Match production order: no-store redirects first, then canonical + // redirects and agent-readiness BEFORE default files (see Program.cs). + app.UseNoStoreRedirects(); app.UseCanonicalPathRedirects(); app.UseAgentReadiness(); app.UseDefaultFiles(); diff --git a/tests/StaticHost.Tests/NoStoreRedirectTests.cs b/tests/StaticHost.Tests/NoStoreRedirectTests.cs new file mode 100644 index 0000000000..420865e4f8 --- /dev/null +++ b/tests/StaticHost.Tests/NoStoreRedirectTests.cs @@ -0,0 +1,115 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StaticHost.Routing; + +namespace StaticHost.Tests; + +public sealed class NoStoreRedirectTests +{ + [Theory] + [InlineData("/DIAGNOSTICS/ASPIRE001/")] + [InlineData("/SCRIPTS/SITE.JS?v=1")] + public async Task Canonical_path_redirects_are_not_stored(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => + { + root.WriteFile("diagnostics/aspire001/index.html", SamplePages.Html); + root.WriteFile("scripts/site.js", "console.log('Aspire');"); + }); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.PermanentRedirect, response.StatusCode); + Assert.Equal("no-store", response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Theory] + [InlineData("/install.ps1", HttpStatusCode.Found)] + [InlineData("/permanent", HttpStatusCode.MovedPermanently)] + [InlineData("/cached", HttpStatusCode.PermanentRedirect)] + public async Task Endpoint_redirects_are_not_stored(string path, HttpStatusCode status) + { + using var host = await StartAsync(static app => + { + app.UseNoStoreRedirects(); + app.UseRouting(); + app.UseEndpoints(static endpoints => + { + endpoints.MapGet("/install.ps1", () => Results.Redirect("https://aka.ms/aspire/get/install.ps1")); + endpoints.MapGet("/permanent", () => Results.Redirect("/elsewhere", permanent: true)); + // A redirect that sets its own caching still must not be stored. + endpoints.MapGet("/cached", (HttpContext context) => + { + context.Response.Headers.CacheControl = "public, max-age=86400"; + return Results.Redirect("/elsewhere", permanent: true, preserveMethod: true); + }); + }); + }); + using var client = host.GetTestClient(); + + using var response = await client.GetAsync(path); + + Assert.Equal(status, response.StatusCode); + Assert.Equal("no-store", response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Https_redirects_are_not_stored() + { + using var host = await StartAsync(static app => + { + app.UseNoStoreRedirects(); + app.UseHttpsRedirection(); + app.Run(static context => context.Response.WriteAsync("ok")); + }, static services => services.AddHttpsRedirection(static options => options.HttpsPort = 443)); + using var client = host.GetTestClient(); + + using var response = await client.GetAsync("http://localhost/docs/"); + + Assert.Equal(HttpStatusCode.TemporaryRedirect, response.StatusCode); + Assert.Equal("no-store", response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Other_responses_keep_their_caching() + { + using var host = await StartAsync(static app => + { + app.UseNoStoreRedirects(); + app.Run(static context => + { + context.Response.StatusCode = StatusCodes.Status304NotModified; + context.Response.Headers.CacheControl = "public, max-age=60"; + return Task.CompletedTask; + }); + }); + using var client = host.GetTestClient(); + + using var response = await client.GetAsync("/page/"); + + Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); + Assert.Equal("public, max-age=60", response.Headers.NonValidated["Cache-Control"].ToString()); + } + + private static async Task StartAsync(Action configure, Action? services = null) + { + var host = new HostBuilder() + .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.ConfigureServices(collection => + { + collection.AddRouting(); + services?.Invoke(collection); + }); + web.Configure(configure); + }) + .Build(); + await host.StartAsync(); + return host; + } +} diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index e162b1d51c..e5d9d793fb 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; using StaticHost.SiteFiles; @@ -91,7 +92,7 @@ public async Task Matching_if_none_match_returns_not_modified() Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); Assert.Equal(etag.Tag, response.Headers.ETag?.Tag); - Assert.Equal("no-cache", response.Headers.CacheControl?.ToString()); + Assert.Equal(SiteFileCacheOptions.DefaultValue, response.Headers.NonValidated["Cache-Control"].ToString()); Assert.Empty(await response.Content.ReadAsByteArrayAsync()); } } @@ -159,21 +160,43 @@ public async Task If_match_and_if_unmodified_since_fail_for_another_version() } [Fact] - public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() + public async Task Range_requests_never_get_the_brotli_copy() { await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); using var first = await server.Client.GetAsync("/docs/page/"); var lastModified = first.Content.Headers.LastModified!.Value; - foreach (var (acceptEncoding, status) in new[] { ("identity", HttpStatusCode.PartialContent), ("br", HttpStatusCode.OK) }) + foreach (var range in new[] { "bytes=0-1", "bytes=0-1, 4-5", "bytes=1000-" }) { - using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); - request.Headers.AcceptEncoding.ParseAdd(acceptEncoding); - request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); - request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); - using var response = await server.Client.SendAsync(request); + foreach (var ifRange in new[] { false, true }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + request.Headers.TryAddWithoutValidation("Range", range); + if (ifRange) + { + request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); + } - Assert.Equal(status, response.StatusCode); + using var response = await server.Client.SendAsync(request); + + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes(Page)), response.Headers.ETag?.Tag); + if (response.StatusCode == HttpStatusCode.PartialContent) + { + Assert.Equal($"bytes 0-1/{Encoding.UTF8.GetByteCount(Page)}", response.Content.Headers.ContentRange?.ToString()); + } + else if (response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable) + { + Assert.Equal($"bytes */{Encoding.UTF8.GetByteCount(Page)}", response.Content.Headers.ContentRange?.ToString()); + } + else + { + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + } } } @@ -206,15 +229,19 @@ public async Task ETag_follows_a_file_that_changes() } [Theory] - [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] - [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] - [InlineData("/favicon.svg", SiteFileMiddleware.MediaCacheControl)] - [InlineData("/images/hero.webp", SiteFileMiddleware.MediaCacheControl)] - [InlineData("/fonts/site.woff2", SiteFileMiddleware.MediaCacheControl)] - [InlineData("/casts/demo.mp4", SiteFileMiddleware.MediaCacheControl)] - [InlineData("/docs/page/index.html", SiteFileMiddleware.DefaultCacheControl)] - [InlineData("/docs/page.md", SiteFileMiddleware.DefaultCacheControl)] - [InlineData("/pagefind/pagefind.js", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/_astro/index.Bx1y2z3.js", SiteFileCacheOptions.ImmutableValue)] + [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileCacheOptions.ImmutableValue)] + [InlineData("/favicon.svg", SiteFileCacheOptions.MediaValue)] + [InlineData("/images/hero.webp", SiteFileCacheOptions.MediaValue)] + [InlineData("/fonts/site.woff2", SiteFileCacheOptions.MediaValue)] + [InlineData("/casts/demo.mp4", SiteFileCacheOptions.MediaValue)] + [InlineData("/docs/page/index.html", SiteFileCacheOptions.DefaultValue)] + [InlineData("/docs/page.md", SiteFileCacheOptions.DefaultValue)] + [InlineData("/llms.txt", SiteFileCacheOptions.DefaultValue)] + [InlineData("/sitemap-index.xml", SiteFileCacheOptions.DefaultValue)] + [InlineData("/reference/api/sidebar.json", SiteFileCacheOptions.DefaultValue)] + [InlineData("/pagefind/pagefind.js", SiteFileCacheOptions.DefaultValue)] + [InlineData("/_astroid/page.html", SiteFileCacheOptions.DefaultValue)] public async Task Cache_control_depends_on_the_path_and_the_type(string path, string expected) { await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); @@ -226,6 +253,76 @@ public async Task Cache_control_depends_on_the_path_and_the_type(string path, st Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); } + [Theory] + [InlineData("/reference/api/sidebar.json?v=3f2a9c", SiteFileCacheOptions.ImmutableValue)] + [InlineData("/locale-fallback/fr.json?lang=fr&v=3f2a9c", SiteFileCacheOptions.ImmutableValue)] + [InlineData("/images/hero.webp?v=1", SiteFileCacheOptions.ImmutableValue)] + [InlineData("/reference/api/sidebar.json?v=", SiteFileCacheOptions.DefaultValue)] + [InlineData("/reference/api/sidebar.json?version=3f2a9c", SiteFileCacheOptions.DefaultValue)] + public async Task Versioned_urls_are_immutable(string url, string expected) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => + { + root.WriteFile("reference/api/sidebar.json", "{}"); + root.WriteFile("locale-fallback/fr.json", "{}"); + root.WriteFile("images/hero.webp", "x"); + }); + + using var response = await server.Client.GetAsync(url); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Cache_control_comes_from_configuration() + { + await using var server = await AgentReadinessTestServer.StartAsync( + root => + { + root.WriteFile("docs/page/index.html", Page); + root.WriteFile("images/hero.webp", "x"); + root.WriteFile("_astro/index.Bx1y2z3.js", "x"); + root.WriteFile("pagefind/pagefind.js", "x"); + root.WriteFile("data.json", "{}"); + }, + configuration: new Dictionary + { + ["SiteFiles:CacheControl:Default"] = "no-cache", + ["SiteFiles:CacheControl:Media"] = "public, max-age=60", + ["SiteFiles:CacheControl:Immutable"] = "public, max-age=600, immutable", + // The configured prefixes replace the default: /_astro is no longer immutable. + ["SiteFiles:CacheControl:ImmutablePathPrefixes"] = "/pagefind; /images/fixed", + ["SiteFiles:CacheControl:VersionQueryParameter"] = "build", + }); + + foreach (var (url, expected) in new[] + { + ("/docs/page/", "no-cache"), + ("/images/hero.webp", "public, max-age=60"), + ("/pagefind/pagefind.js", "public, max-age=600, immutable"), + ("/_astro/index.Bx1y2z3.js", "no-cache"), + ("/data.json?build=1", "public, max-age=600, immutable"), + ("/data.json?v=1", "no-cache"), + }) + { + using var response = await server.Client.GetAsync(url); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + } + + [Theory] + [InlineData("SiteFiles:CacheControl:Default", " ")] + [InlineData("SiteFiles:CacheControl:ImmutablePathPrefixes", "_astro")] + [InlineData("SiteFiles:CacheControl:ImmutablePathPrefixes", "/_astro;/pagefind/")] + public async Task Invalid_cache_configuration_fails_at_startup(string key, string value) + { + await Assert.ThrowsAsync(() => AgentReadinessTestServer.StartAsync( + configuration: new Dictionary { [key] = value })); + } + [Fact] public async Task Markdown_is_served_as_text_markdown() { From 8d34dd8bc3804bf9c018187c443e66e71f912b70 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:59:45 -0600 Subject: [PATCH 7/8] Default non-fingerprinted files to no-cache so Front Door never caches them Front Door caches only fingerprinted content (/_astro and ?v= URLs) and deployments no longer purge the edge, so a file whose URL doesn't change with its content must not be cached there. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../StaticHost/SiteFiles/SiteFileCacheOptions.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs index c15e39511d..95fe060d13 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileCacheOptions.cs @@ -7,20 +7,18 @@ namespace StaticHost.SiteFiles; /// /// /// -/// Every deployment purges Azure Front Door, so a file that is not fingerprinted can be cached at -/// the edge (s-maxage) while browsers revalidate it on every use (max-age=0): the -/// edge answers those revalidations, and a deployment's purge drops what it cached. Front Door -/// does not cache a response the origin marks no-cache, even with a rule that overrides -/// the cache duration, so the origin must send these values for HTML to be cached at the edge. +/// Azure Front Door caches only fingerprinted content, and deployments don't purge it. A file +/// that isn't fingerprinted keeps its URL when its content changes, so it defaults to +/// no-cache: browsers revalidate it on every use, and Front Door never caches a response +/// the origin marks no-cache, even on a route or rule that enables caching. /// /// internal sealed class SiteFileCacheOptions { public const string SectionName = "SiteFiles:CacheControl"; - // Browsers revalidate every time; Front Door keeps the response for an hour, or until the - // purge that follows a deployment. An hour bounds how long a missed purge can serve stale HTML. - internal const string DefaultValue = "public, max-age=0, must-revalidate, s-maxage=3600"; + // A file that isn't fingerprinted can change without a new URL, so nothing may reuse it unchecked. + internal const string DefaultValue = "no-cache"; // The SDK's default for images, video and fonts, as MapStaticAssets sent. internal const string MediaValue = "max-age=3600, must-revalidate"; From e3a365c238616b0f39e004e92d9d2188935a4849 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:22:09 -0600 Subject: [PATCH 8/8] Address review: HEAD ranges, encoding negotiation, ETag and brotli reuse - Ignore Range on HEAD (RFC 9110, 14.2): HEAD gets the whole file's headers. - Choose brotli or identity by Accept-Encoding quality values; identity when neither is acceptable. - Hash the file again for requests with If-Match, If-None-Match or If-Range, so a file replaced with the same length and time cannot get a stale 304. - Reuse a brotli copy only when its source's SHA-256 matches, not its time. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../StaticHost/SiteFiles/SiteFileETags.cs | 19 +++- .../SiteFiles/SiteFileMiddleware.cs | 31 ++++++- src/statichost/compress-site.cs | 22 +++-- tests/StaticHost.Tests/CompressSiteTests.cs | 86 +++++++++++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 61 +++++++++++++ 5 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 tests/StaticHost.Tests/CompressSiteTests.cs diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 84c3502ab2..195e11d2c5 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -10,12 +10,23 @@ namespace StaticHost.SiteFiles; /// in memory. /// /// +/// /// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no /// cryptographic hash. A content hash, unlike the length and last-modified time /// UseStaticFiles uses, stays the same across deployments for files that did not change, /// though CI writes every file again. An entry is recomputed when the file's length or /// last-modified time changes, which covers a web root that is replaced while StaticHost runs /// (CI unpacks the site after StaticHost starts). +/// +/// +/// Length and time do not identify the bytes, though: a restore can replace a file with one of +/// the same length and time. A cached ETag is therefore only used where a stale one is harmless, +/// on a response that sends the file. A request with a precondition, which a stale ETag could +/// answer with a 304 for bytes the client does not have, hashes the file again +/// (revalidate). Watching the web root for changes would cost an inotify watch per +/// directory on Linux, and .NET does not expose the inode or change time that would tell a +/// replaced file apart. +/// /// internal sealed class SiteFileETags { @@ -36,14 +47,18 @@ internal sealed class SiteFileETags /// The file's path in the web root, the key when the provider gives no physical path. /// /// The file to hash. + /// Hash the file even when an entry matches its length and time. /// Cancels the hashing. - public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) + public async ValueTask GetAsync(string path, IFileInfo file, bool revalidate, CancellationToken cancellationToken) { var key = file.PhysicalPath ?? path; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (!revalidate + && _entries.TryGetValue(key, out var entry) + && entry.Length == length + && entry.LastModified == lastModified) { return entry.ETag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index c4a7d7c63e..844c6b98e7 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -104,7 +104,12 @@ public async Task InvokeAsync(HttpContext context) var served = useCompressed ? compressed : file; var response = context.Response; - var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + // A precondition decides on the ETag, so it must match the bytes on disk now: a 304 for a + // file replaced with one of the same length and time would keep a client on the old bytes. + var revalidate = request.Headers.IfMatch.Count > 0 + || request.Headers.IfNoneMatch.Count > 0 + || request.Headers.IfRange.Count > 0; + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, revalidate, context.RequestAborted); var entityTag = new EntityTagHeaderValue(etag); var lastModified = TruncateToSeconds(file.LastModified); var requestHeaders = request.GetTypedHeaders(); @@ -127,7 +132,11 @@ public async Task InvokeAsync(HttpContext context) headers.AcceptRanges = "bytes"; // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes // of its brotli copy, which can be rewritten while the original keeps its time. - var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); + // Range applies only to GET (RFC 9110, 14.2): HEAD gets the headers of the whole file. + var unsatisfiable = false; + var range = HttpMethods.IsGet(request.Method) + ? GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out unsatisfiable) + : null; if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -175,6 +184,16 @@ internal static string GetCacheControl(PathString path, string contentType) : DefaultCacheControl; } + /// + /// Chooses between the brotli copy and the file itself by the quality values of + /// Accept-Encoding (RFC 9110, 12.5.3), and returns whether to send brotli. + /// + /// + /// * applies to each coding the header does not list, identity included. Unlisted, + /// identity stays acceptable but preferred last, so br;q=0.5 still gets brotli, and a tie + /// goes to brotli. When the client accepts neither, it gets the file itself: RFC 9110, 12.1 lets + /// the server disregard the header, and a 406 could be cached for clients that accept the file. + /// internal static bool AcceptsBrotli(StringValues acceptEncoding) { if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) @@ -183,6 +202,7 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) } double? brotli = null; + double? identity = null; double? wildcard = null; foreach (var encoding in encodings) { @@ -191,6 +211,10 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) { brotli = quality; } + else if (StringSegment.Equals(encoding.Value, "identity", StringComparison.OrdinalIgnoreCase)) + { + identity = quality; + } else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) { wildcard = quality; @@ -198,7 +222,8 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) } // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. - return (brotli ?? wildcard ?? 0) > 0; + var brotliQuality = brotli ?? wildcard ?? 0; + return brotliQuality > 0 && brotliQuality >= (identity ?? wildcard ?? 0); } /// diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs index 88cec4f419..8aa5ac4268 100644 --- a/src/statichost/compress-site.cs +++ b/src/statichost/compress-site.cs @@ -10,6 +10,7 @@ using System.Collections.Concurrent; using System.IO.Compression; +using System.Security.Cryptography; if (args.Length != 4) { @@ -32,14 +33,23 @@ Parallel.ForEach(files, file => { - // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. - // "Newer than the source" is not enough: a cache restore or an archive extraction gives - // new content an older time, and the copy of the old content would be published. + // Reuse a copy whose source has the same content, so an unchanged site compresses once. The + // SHA-256 of the source the copy was written from is kept next to it (".br.sha256", which is + // not published). A time, even an exact one, is not enough: a cache restore or an archive + // extraction can give new content the old time, and the copy of the old content would be published. var target = Path.Combine(outputDirectory, file.SitePath + ".br"); - var sourceTime = File.GetLastWriteTimeUtc(file.Source); - if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + var stamp = target + ".sha256"; + string sourceHash; + using (var source = File.OpenRead(file.Source)) + { + sourceHash = Convert.ToHexString(SHA256.HashData(source)); + } + + if (!File.Exists(target) || !File.Exists(stamp) || File.ReadAllText(stamp) != sourceHash) { Directory.CreateDirectory(Path.GetDirectoryName(target)!); + // Without a stamp, a copy left half-updated by an interrupted run is written again. + File.Delete(stamp); var temporary = target + ".tmp"; using (var input = File.OpenRead(file.Source)) using (var output = File.Create(temporary)) @@ -50,8 +60,8 @@ input.CopyTo(brotli); } - File.SetLastWriteTimeUtc(temporary, sourceTime); File.Move(temporary, target, overwrite: true); + File.WriteAllText(stamp, sourceHash); Interlocked.Increment(ref compressed); } diff --git a/tests/StaticHost.Tests/CompressSiteTests.cs b/tests/StaticHost.Tests/CompressSiteTests.cs new file mode 100644 index 0000000000..d48b4b9c19 --- /dev/null +++ b/tests/StaticHost.Tests/CompressSiteTests.cs @@ -0,0 +1,86 @@ +using System.Diagnostics; +using System.IO.Compression; + +namespace StaticHost.Tests; + +/// +/// Runs src/statichost/compress-site.cs, the file-based app the publish uses to write the site's +/// brotli copies. +/// +public sealed class CompressSiteTests +{ + [Fact] + public async Task Rewrites_a_copy_whose_source_changed_but_kept_its_length_and_time() + { + using var work = new TempWebRoot(); + var source = Path.Combine(work.Path, "site", "page.html"); + var output = Path.Combine(work.Path, "br"); + var copy = Path.Combine(output, "page.html.br"); + var first = new string('a', 2048); + var second = new string('b', 2048); + work.WriteFile("site/page.html", first); + work.WriteFile("files.txt", $"{source}\tpage.html\n"); + var time = File.GetLastWriteTimeUtc(source); + + Assert.Contains("1 compressed now", await RunAsync(work.Path, output)); + Assert.Equal(first, Decompress(copy)); + + Assert.Contains("0 compressed now", await RunAsync(work.Path, output)); + + // A restore: new bytes, same length, the old time. + await File.WriteAllTextAsync(source, second); + File.SetLastWriteTimeUtc(source, time); + + Assert.Contains("1 compressed now", await RunAsync(work.Path, output)); + Assert.Equal(second, Decompress(copy)); + } + + private static async Task RunAsync(string work, string output) + { + var start = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (var argument in new[] + { + "run", "--file", FindCompressSite(), "--", + Path.Combine(work, "files.txt"), output, ".html", Path.Combine(work, "published.txt"), + }) + { + start.ArgumentList.Add(argument); + } + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + Assert.True(process.ExitCode == 0, $"compress-site.cs exited with {process.ExitCode}: {await stdout}{await stderr}"); + return await stdout; + } + + private static string FindCompressSite() + { + for (var directory = new DirectoryInfo(AppContext.BaseDirectory); directory is not null; directory = directory.Parent) + { + var path = Path.Combine(directory.FullName, "src", "statichost", "compress-site.cs"); + if (File.Exists(path)) + { + return path; + } + } + + throw new FileNotFoundException("compress-site.cs was not found above the test's directory."); + } + + private static string Decompress(string path) + { + using var input = File.OpenRead(path); + using var brotli = new BrotliStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(brotli); + return reader.ReadToEnd(); + } +} diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index e162b1d51c..0c3ada8d98 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -71,9 +71,30 @@ public async Task File_without_a_brotli_copy_has_no_vary_header() [InlineData("*", true)] [InlineData("*;q=0", false)] [InlineData("br;q=0, *", false)] + [InlineData("br;q=0.1, identity;q=1", false)] + [InlineData("br;q=0.5, *", false)] + [InlineData("br;q=0.5, identity;q=0.5", true)] + [InlineData("br, identity;q=0.5", true)] + [InlineData("br;q=0.1, identity;q=0", true)] + [InlineData("gzip, br;q=0, identity;q=0", false)] public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + [Fact] + public async Task Client_that_accepts_neither_coding_gets_the_file_itself() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("Accept-Encoding", "gzip, br;q=0, identity;q=0"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + [Fact] public async Task Matching_if_none_match_returns_not_modified() { @@ -205,6 +226,29 @@ public async Task ETag_follows_a_file_that_changes() Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); } + [Fact] + public async Task Precondition_sees_a_file_replaced_with_the_same_length_and_time() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + var path = Path.Combine(server.WebRoot, "data.json"); + var lastWrite = File.GetLastWriteTimeUtc(path); + + using var before = await server.Client.GetAsync("/data.json"); + var staleETag = before.Headers.ETag!.Tag; + + // A restore: new bytes, same length, the old time. + await File.WriteAllTextAsync(path, "{\"v\":2}"); + File.SetLastWriteTimeUtc(path, lastWrite); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/data.json"); + request.Headers.TryAddWithoutValidation("If-None-Match", staleETag); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("{\"v\":2}", await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":2}")), response.Headers.ETag?.Tag); + } + [Theory] [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] @@ -284,6 +328,23 @@ public async Task Range_past_the_end_is_not_satisfiable() Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); } + [Theory] + [InlineData("bytes=2-5")] + [InlineData("bytes=10-")] + public async Task Head_ignores_the_range(string range) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Head, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(10, response.Content.Headers.ContentLength); + Assert.Null(response.Content.Headers.ContentRange); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + [Fact] public async Task If_range_applies_the_range_only_to_the_current_file() {