From 48488df07962911980faceb322d706d40ba214ac Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 05:39:28 -0700 Subject: [PATCH 1/9] Pick the cheapest seam-safe VAE tile size instead of the nearest smaller one When the requested VAE tile leaves two tiles barely overlapping, #21 took the nearest smaller size that overlaps by half the target, which needs a third tile per axis: 496 px on an 8x VAE decoded 3x3 tiles instead of 2x2. Pick the size that decodes the fewest latents instead, overlapping by the full target up to 8 latents (half the target left a line on Qwen-Image-2.1's 16x VAE), within 25% of the requested size. 496 px gets two 35-latent tiles, Qwen-Image-2.1 480 px two 19-latent tiles. Sizes whose requested tile already overlaps enough are unchanged. --- src/runtime/tiling.cpp | 56 +++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 782a41a8b..29d3f229a 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -62,15 +62,20 @@ static std::vector sd_tiling_plan_axis(int dimension, int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor) { // A tile a little over half the axis (32 of 62 latents) gets two tiles overlapping by 1-2 latents, which shows - // as a seam. Pick the nearest size (smaller first, larger only up to 2x) whose overlaps are >= half the target - // and >= 2 latents. + // as a seam. Keep the requested size when its overlaps are >= half the target and >= 2 latents, else pick the + // cheapest size that overlaps by the full target, up to 8 latents. const int min_tile_size = 4; if (tile_overlap_factor <= 0.f || tile_size >= small_dim) { return tile_size; } - auto overlaps_enough = [&](int size) { - const int min_overlap = std::max(2, static_cast(size * tile_overlap_factor * 0.5f)); - const auto tiles = sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false); + auto overlaps_enough = [&](int size, bool replacement) { + int min_overlap = std::max(2, static_cast(size * tile_overlap_factor * 0.5f)); + if (replacement) { + // Half the target is too little for 16x VAEs: 4 of 16 latents still leaves a line (Qwen-Image-2.1 at + // 480 px), 8 does not, on 8x and 16x VAEs alike. + min_overlap = std::max(min_overlap, std::min(8, static_cast(size * tile_overlap_factor))); + } + const auto tiles = sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false); for (size_t i = 0; i + 1 < tiles.size(); ++i) { if (tiles[i].overlap_after < min_overlap) { return false; @@ -78,13 +83,42 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl } return true; }; - for (int size = tile_size; size >= min_tile_size; --size) { - if (overlaps_enough(size)) { - return size; - } + if (overlaps_enough(tile_size, false)) { + return tile_size; } - for (int size = tile_size + 1; size < small_dim && size <= 2 * tile_size; ++size) { - if (overlaps_enough(size)) { + // Of the sizes that overlap enough, take the one decoding the fewest latents along the axis (tiles * size): a + // 62-latent axis gets two 35-latent tiles instead of three 31-latent ones. Ties go to fewer tiles, then to the + // size nearest the request. Growth stays within a quarter of the request where possible (the most any axis + // needs is 32 -> 36 and 16 -> 20). + auto pick = [&](int lowest, int highest, bool replacement) { + int best_size = 0; + int64_t best_cost = 0; + int best_tiles = 0; + for (int size = lowest; size <= highest; ++size) { + if (!overlaps_enough(size, replacement)) { + continue; + } + const int tiles = static_cast(sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false).size()); + const int64_t cost = static_cast(tiles) * size; + const bool better = best_size == 0 || cost < best_cost || + (cost == best_cost && (tiles < best_tiles || + (tiles == best_tiles && std::abs(size - tile_size) < std::abs(best_size - tile_size)))); + if (better) { + best_size = size; + best_cost = cost; + best_tiles = tiles; + } + } + return best_size; + }; + const int capped = std::min(small_dim - 1, tile_size + tile_size / 4); + const int widest = std::min(small_dim - 1, 2 * tile_size); + for (bool replacement : {true, false}) { + int size = pick(min_tile_size, capped, replacement); + if (size == 0) { + size = pick(capped + 1, widest, replacement); + } + if (size != 0) { return size; } } From 0d71e5692abee644b78cc4d2839bfdaec8ec0f93 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 09:09:48 -0700 Subject: [PATCH 2/9] VAE tiling: never use more tiles than the nearest passing size; document the tile choice --- docs/performance.md | 10 +++++++--- src/runtime/tiling.cpp | 32 ++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/performance.md b/docs/performance.md index 028a6409a..c3218a4c4 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -50,9 +50,13 @@ overlap. For example, `0.5x0.5` uses half the width and height in both encode an decode. The target overlap is clamped to 0 through 0.5 and the actual overlap is adjusted to fit the image. On an axis that does not wrap, a tile size that would leave neighboring tiles overlapping by less than half the target (or by fewer than 2 latent -pixels) is adjusted to the nearest size that does not: smaller first, larger only up to -twice the requested size. Without it, a tile a little over half of the axis gives two -tiles that barely overlap and the tile edge shows as a line. Size and overlap options +pixels) is replaced. The replacement is the size that decodes the fewest latent pixels +along the axis (tiles times size) among sizes whose overlaps reach the full target overlap, +capped at 8 latent pixels, preferring sizes within 25% above the request. It never uses +more tiles than the nearest size that overlaps by half the target (smaller first, larger +only up to twice the request), and falls back to that size when nothing better qualifies. +Without it, a tile a little over half of the axis gives two tiles that barely overlap and +the tile edge shows as a line. Size and overlap options require `--vae-tiling`. **Migration:** `--vae-tile-size` and the C/JSON fields `tile_size_w` and diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 29d3f229a..c72941d9d 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -63,7 +63,8 @@ static std::vector sd_tiling_plan_axis(int dimension, int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor) { // A tile a little over half the axis (32 of 62 latents) gets two tiles overlapping by 1-2 latents, which shows // as a seam. Keep the requested size when its overlaps are >= half the target and >= 2 latents, else pick the - // cheapest size that overlaps by the full target, up to 8 latents. + // cheapest size that overlaps by the full target, up to 8 latents, never using more tiles than the nearest + // passing size would. const int min_tile_size = 4; if (tile_overlap_factor <= 0.f || tile_size >= small_dim) { return tile_size; @@ -86,10 +87,30 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl if (overlaps_enough(tile_size, false)) { return tile_size; } + auto tile_count = [&](int size) { + return static_cast(sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false).size()); + }; + // The nearest passing size (smaller first, larger only up to 2x) bounds the tile count: a cheaper choice made of + // many small tiles would multiply VAE calls (370 latents at 64 with overlap 0.1: 7 tiles, not 16 of 25). + int nearest = 0; + for (int size = tile_size - 1; size >= min_tile_size && nearest == 0; --size) { + if (overlaps_enough(size, false)) { + nearest = size; + } + } + for (int size = tile_size + 1; nearest == 0 && size < small_dim && size <= 2 * tile_size; ++size) { + if (overlaps_enough(size, false)) { + nearest = size; + } + } + if (nearest == 0) { + return tile_size; + } + const int max_tiles = tile_count(nearest); // Of the sizes that overlap enough, take the one decoding the fewest latents along the axis (tiles * size): a // 62-latent axis gets two 35-latent tiles instead of three 31-latent ones. Ties go to fewer tiles, then to the // size nearest the request. Growth stays within a quarter of the request where possible (the most any axis - // needs is 32 -> 36 and 16 -> 20). + // needs is 32 -> 36 and 16 -> 20), and no candidate may need more tiles than the nearest passing size. auto pick = [&](int lowest, int highest, bool replacement) { int best_size = 0; int64_t best_cost = 0; @@ -98,7 +119,10 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl if (!overlaps_enough(size, replacement)) { continue; } - const int tiles = static_cast(sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false).size()); + const int tiles = tile_count(size); + if (tiles > max_tiles) { + continue; + } const int64_t cost = static_cast(tiles) * size; const bool better = best_size == 0 || cost < best_cost || (cost == best_cost && (tiles < best_tiles || @@ -122,7 +146,7 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl return size; } } - return tile_size; + return nearest; } static int64_t sd_tensor_plane_size(const sd::Tensor& tensor) { From 2097dac6ba6b4e63b3b98174399d228c1150c348 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 09:17:52 -0700 Subject: [PATCH 3/9] VAE tiling: trim comments --- src/runtime/tiling.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index c72941d9d..8c9db1dce 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -90,8 +90,7 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl auto tile_count = [&](int size) { return static_cast(sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false).size()); }; - // The nearest passing size (smaller first, larger only up to 2x) bounds the tile count: a cheaper choice made of - // many small tiles would multiply VAE calls (370 latents at 64 with overlap 0.1: 7 tiles, not 16 of 25). + // The nearest passing size (smaller first, then larger up to 2x) caps the tile count so tiny tiles never win. int nearest = 0; for (int size = tile_size - 1; size >= min_tile_size && nearest == 0; --size) { if (overlaps_enough(size, false)) { @@ -107,10 +106,8 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl return tile_size; } const int max_tiles = tile_count(nearest); - // Of the sizes that overlap enough, take the one decoding the fewest latents along the axis (tiles * size): a - // 62-latent axis gets two 35-latent tiles instead of three 31-latent ones. Ties go to fewer tiles, then to the - // size nearest the request. Growth stays within a quarter of the request where possible (the most any axis - // needs is 32 -> 36 and 16 -> 20), and no candidate may need more tiles than the nearest passing size. + // Fewest latents decoded (tiles * size) wins, then fewer tiles, then nearest the request; 62 latents -> 2 x 35. + // Growth stays within a quarter of the request where possible. auto pick = [&](int lowest, int highest, bool replacement) { int best_size = 0; int64_t best_cost = 0; From 312a8b3eb1ff256a346e435922a5d30a2acb48f8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 16:05:29 -0700 Subject: [PATCH 4/9] VAE OOM retry: lower the request until every tile shrinks; drop an impossible tie-break A halved retry request can be widened back to the failed tile size by the seam-safe size search, which made the retry loop give up. Lower the request of an axis that did not shrink one latent at a time instead. First attempts are unchanged. --- src/pipeline/diffusion_engine.cpp | 65 +++++++++++++++++++++---------- src/runtime/tiling.cpp | 7 ++-- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index a39abd936..c32c5fe80 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2895,6 +2895,43 @@ int StableDiffusionGGML::align_video_frames(int frames) { return latent_frames_to_video_frames(video_frames_to_latent_frames(frames)); } +// Tile sizes for a VAE pass; after an out-of-memory failure (last_w/last_h > 0) every tile must come out smaller. +// get_tile_sizes can widen a halved request to keep the seam overlap (back to the failed size), so lower the +// request of an axis that did not shrink one latent at a time. Returns false on invalid parameters or when no +// axis can shrink any further. +static bool sd_vae_tile_sizes(VAE& vae, + sd_tiling_params_t& tiling_params, + int64_t latent_w, + int64_t latent_h, + bool circular_x, + bool circular_y, + int last_w, + int last_h, + int& tile_size_w, + int& tile_size_h) { + const int scale_factor = vae.get_scale_factor(); + for (;;) { + tile_size_w = static_cast(latent_w); + tile_size_h = static_cast(latent_h); + float tile_overlap; + if (tiling_params.enabled && + !vae.get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, + latent_w, latent_h, circular_x, circular_y)) { + return false; + } + if (last_w <= 0 || !tiling_params.enabled) { + return true; + } + const bool lower_w = tile_size_w >= last_w && tiling_params.tile_size_w > 4 * scale_factor; + const bool lower_h = tile_size_h >= last_h && tiling_params.tile_size_h > 4 * scale_factor; + if (!lower_w && !lower_h) { + return tile_size_w < last_w || tile_size_h < last_h; + } + tiling_params.tile_size_w -= lower_w ? scale_factor : 0; + tiling_params.tile_size_h -= lower_h ? scale_factor : 0; + } +} + sd::Tensor StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor& x) { // Same out-of-memory fallback as decode_first_stage, on a copy of the tiling parameters, so the // smaller tiles apply to this call only. @@ -2904,16 +2941,10 @@ sd::Tensor StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor latents; for (int last_w = 0, last_h = 0;;) { - int tile_size_w = static_cast(latent_w); - int tile_size_h = static_cast(latent_h); - float tile_overlap; - if (tiling_params.enabled && - !first_stage_model->get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, - latent_w, latent_h, circular_x, circular_y)) { - return {}; - } - // get_tile_sizes can widen a retry tile to keep the overlap, so stop once a retry no longer shrinks it. - if (tile_size_w == last_w && tile_size_h == last_h) { + int tile_size_w; + int tile_size_h; + if (!sd_vae_tile_sizes(*first_stage_model, tiling_params, latent_w, latent_h, circular_x, circular_y, + last_w, last_h, tile_size_w, tile_size_h)) { return {}; } latents = first_stage_model->encode(n_threads, x, tiling_params, circular_x, circular_y); @@ -2952,16 +2983,10 @@ sd::Tensor StableDiffusionGGML::decode_first_stage(const sd::Tensor 1 && first_stage_model->can_temporal_tile_decode(); for (int last_w = 0, last_h = 0;;) { - int tile_size_w = static_cast(latents.shape()[0]); - int tile_size_h = static_cast(latents.shape()[1]); - float tile_overlap; - if (tiling_params.enabled && - !first_stage_model->get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, - latents.shape()[0], latents.shape()[1], circular_x, circular_y)) { - return {}; - } - // get_tile_sizes can widen a retry tile, so stop once a spatial retry no longer shrinks it. - if (tile_size_w == last_w && tile_size_h == last_h) { + int tile_size_w; + int tile_size_h; + if (!sd_vae_tile_sizes(*first_stage_model, tiling_params, latents.shape()[0], latents.shape()[1], + circular_x, circular_y, last_w, last_h, tile_size_w, tile_size_h)) { return {}; } auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y); diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 8c9db1dce..73beaafae 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -106,7 +106,7 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl return tile_size; } const int max_tiles = tile_count(nearest); - // Fewest latents decoded (tiles * size) wins, then fewer tiles, then nearest the request; 62 latents -> 2 x 35. + // Fewest latents decoded (tiles * size) wins, then fewer tiles; 62 latents -> 2 x 35. // Growth stays within a quarter of the request where possible. auto pick = [&](int lowest, int highest, bool replacement) { int best_size = 0; @@ -121,9 +121,8 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl continue; } const int64_t cost = static_cast(tiles) * size; - const bool better = best_size == 0 || cost < best_cost || - (cost == best_cost && (tiles < best_tiles || - (tiles == best_tiles && std::abs(size - tile_size) < std::abs(best_size - tile_size)))); + // Equal cost and tile count means equal size, so no further tie-break is needed. + const bool better = best_size == 0 || cost < best_cost || (cost == best_cost && tiles < best_tiles); if (better) { best_size = size; best_cost = cost; From b1c2503c472279ae5721a431c55fcb2565d5a0a1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 16:29:54 -0700 Subject: [PATCH 5/9] VAE OOM retry: bound the seam-safe tile below the failed size At the 4-latent minimum request the seam-safe search can still widen the tile (42 latents at 25% overlap: 4 -> 8), and lowering the request cannot go further, so a failed 8-latent tile aborted even when 7 would fit. After an allocation failure the VAE now passes the failed size minus one as an upper bound to the search, which picks the best seam-safe size under it. First attempts carry no bound and are unchanged. --- src/model/vae/vae.hpp | 14 +++++++-- src/pipeline/diffusion_engine.cpp | 51 ++++++++++++++++--------------- src/runtime/tiling.cpp | 11 ++++--- src/runtime/tiling.h | 5 +-- 4 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index 4e5248884..9815216ef 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -166,6 +166,10 @@ struct VAE : public GGMLRunner { } public: + // Upper bound on latent tile sizes while retrying after an out-of-memory failure (0 = none). + int retry_tile_limit_w = 0; + int retry_tile_limit_h = 0; + VAE(SDVersion version, ggml_backend_t backend, const std::string& weight_prefix = "", @@ -249,10 +253,16 @@ struct VAE : public GGMLRunner { } // Circular axes wrap their tiles instead of pinning the last one to the edge, so they keep the requested size. if (!circular_x && !circular_x_enabled) { - tile_size_w = sd_tiling_seam_safe_tile_size(static_cast(latent_w), tile_size_w, tile_overlap); + tile_size_w = sd_tiling_seam_safe_tile_size(static_cast(latent_w), tile_size_w, tile_overlap, + retry_tile_limit_w); + } else if (retry_tile_limit_w >= 4) { + tile_size_w = std::min(tile_size_w, retry_tile_limit_w); } if (!circular_y && !circular_y_enabled) { - tile_size_h = sd_tiling_seam_safe_tile_size(static_cast(latent_h), tile_size_h, tile_overlap); + tile_size_h = sd_tiling_seam_safe_tile_size(static_cast(latent_h), tile_size_h, tile_overlap, + retry_tile_limit_h); + } else if (retry_tile_limit_h >= 4) { + tile_size_h = std::min(tile_size_h, retry_tile_limit_h); } return true; } diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index c32c5fe80..59838d4a5 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2895,12 +2895,12 @@ int StableDiffusionGGML::align_video_frames(int frames) { return latent_frames_to_video_frames(video_frames_to_latent_frames(frames)); } -// Tile sizes for a VAE pass; after an out-of-memory failure (last_w/last_h > 0) every tile must come out smaller. -// get_tile_sizes can widen a halved request to keep the seam overlap (back to the failed size), so lower the -// request of an axis that did not shrink one latent at a time. Returns false on invalid parameters or when no -// axis can shrink any further. +// Tile sizes for a VAE pass; after an out-of-memory failure (last_w/last_h > 0) every axis that can shrink must +// come out smaller. get_tile_sizes can widen a halved request to keep the seam overlap (back to the failed size), +// so bound the seam-safe choice below the failed size instead. Returns false on invalid parameters or when no axis +// can shrink any further. static bool sd_vae_tile_sizes(VAE& vae, - sd_tiling_params_t& tiling_params, + const sd_tiling_params_t& tiling_params, int64_t latent_w, int64_t latent_h, bool circular_x, @@ -2909,29 +2909,28 @@ static bool sd_vae_tile_sizes(VAE& vae, int last_h, int& tile_size_w, int& tile_size_h) { - const int scale_factor = vae.get_scale_factor(); - for (;;) { - tile_size_w = static_cast(latent_w); - tile_size_h = static_cast(latent_h); - float tile_overlap; - if (tiling_params.enabled && - !vae.get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, - latent_w, latent_h, circular_x, circular_y)) { - return false; - } - if (last_w <= 0 || !tiling_params.enabled) { - return true; - } - const bool lower_w = tile_size_w >= last_w && tiling_params.tile_size_w > 4 * scale_factor; - const bool lower_h = tile_size_h >= last_h && tiling_params.tile_size_h > 4 * scale_factor; - if (!lower_w && !lower_h) { - return tile_size_w < last_w || tile_size_h < last_h; - } - tiling_params.tile_size_w -= lower_w ? scale_factor : 0; - tiling_params.tile_size_h -= lower_h ? scale_factor : 0; + vae.retry_tile_limit_w = last_w > 0 ? last_w - 1 : 0; + vae.retry_tile_limit_h = last_h > 0 ? last_h - 1 : 0; + tile_size_w = static_cast(latent_w); + tile_size_h = static_cast(latent_h); + float tile_overlap; + if (tiling_params.enabled && + !vae.get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, + latent_w, latent_h, circular_x, circular_y)) { + return false; } + return last_w <= 0 || !tiling_params.enabled || tile_size_w < last_w || tile_size_h < last_h; } +// Clears the retry bound when a VAE pass ends. +struct VAERetryTileLimitGuard { + VAE& vae; + ~VAERetryTileLimitGuard() { + vae.retry_tile_limit_w = 0; + vae.retry_tile_limit_h = 0; + } +}; + sd::Tensor StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor& x) { // Same out-of-memory fallback as decode_first_stage, on a copy of the tiling parameters, so the // smaller tiles apply to this call only. @@ -2940,6 +2939,7 @@ sd::Tensor StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor latents; + VAERetryTileLimitGuard limit_guard{*first_stage_model}; for (int last_w = 0, last_h = 0;;) { int tile_size_w; int tile_size_h; @@ -2982,6 +2982,7 @@ sd::Tensor StableDiffusionGGML::decode_first_stage(const sd::Tensorresolve_tiling_params(vae_tiling_params); const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 && first_stage_model->can_temporal_tile_decode(); + VAERetryTileLimitGuard limit_guard{*first_stage_model}; for (int last_w = 0, last_h = 0;;) { int tile_size_w; int tile_size_h; diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 73beaafae..86af2b66c 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -1,6 +1,7 @@ #include "runtime/tiling.h" #include +#include #include #include #include @@ -60,12 +61,14 @@ static std::vector sd_tiling_plan_axis(int dimension, return tiles; } -int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor) { +int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size) { // A tile a little over half the axis (32 of 62 latents) gets two tiles overlapping by 1-2 latents, which shows // as a seam. Keep the requested size when its overlaps are >= half the target and >= 2 latents, else pick the // cheapest size that overlaps by the full target, up to 8 latents, never using more tiles than the nearest // passing size would. const int min_tile_size = 4; + const int limit = max_tile_size >= min_tile_size ? max_tile_size : INT_MAX; + tile_size = std::min(tile_size, limit); if (tile_overlap_factor <= 0.f || tile_size >= small_dim) { return tile_size; } @@ -97,7 +100,7 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl nearest = size; } } - for (int size = tile_size + 1; nearest == 0 && size < small_dim && size <= 2 * tile_size; ++size) { + for (int size = tile_size + 1; nearest == 0 && size < small_dim && size <= std::min(2 * tile_size, limit); ++size) { if (overlaps_enough(size, false)) { nearest = size; } @@ -131,8 +134,8 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl } return best_size; }; - const int capped = std::min(small_dim - 1, tile_size + tile_size / 4); - const int widest = std::min(small_dim - 1, 2 * tile_size); + const int capped = std::min({small_dim - 1, tile_size + tile_size / 4, limit}); + const int widest = std::min({small_dim - 1, 2 * tile_size, limit}); for (bool replacement : {true, false}) { int size = pick(min_tile_size, capped, replacement); if (size == 0) { diff --git a/src/runtime/tiling.h b/src/runtime/tiling.h index 2587dab28..ea67af7f5 100644 --- a/src/runtime/tiling.h +++ b/src/runtime/tiling.h @@ -45,7 +45,8 @@ sd::Tensor process_tiles_2d_batched(const sd::Tensor& input, bool silent = false); // Returns tile_size, adjusted if needed so a non-circular axis of small_dim is never split into tiles that -// barely overlap (see the definition for why that shows as a seam). -int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor); +// barely overlap (see the definition for why that shows as a seam). max_tile_size > 0 (an out-of-memory retry) +// bounds the result; it is ignored below the 4-latent minimum. +int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size = 0); #endif // __SD_RUNTIME_TILING_H__ From cde3933acbfd8ff358e93caab7aeb327ebf78d87 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 16:37:29 -0700 Subject: [PATCH 6/9] Drop a comment that restates the retry-bound guard --- src/pipeline/diffusion_engine.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 59838d4a5..fa3f6a341 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2922,7 +2922,6 @@ static bool sd_vae_tile_sizes(VAE& vae, return last_w <= 0 || !tiling_params.enabled || tile_size_w < last_w || tile_size_h < last_h; } -// Clears the retry bound when a VAE pass ends. struct VAERetryTileLimitGuard { VAE& vae; ~VAERetryTileLimitGuard() { From 79f183cb3d76755c2b9bc198a48fc0b5a948af0e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 11 Oct 2026 16:46:03 -0700 Subject: [PATCH 7/9] Trim tiling comments --- src/pipeline/diffusion_engine.cpp | 5 +---- src/runtime/tiling.cpp | 10 ++-------- src/runtime/tiling.h | 4 +--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index fa3f6a341..4755830a6 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2895,10 +2895,7 @@ int StableDiffusionGGML::align_video_frames(int frames) { return latent_frames_to_video_frames(video_frames_to_latent_frames(frames)); } -// Tile sizes for a VAE pass; after an out-of-memory failure (last_w/last_h > 0) every axis that can shrink must -// come out smaller. get_tile_sizes can widen a halved request to keep the seam overlap (back to the failed size), -// so bound the seam-safe choice below the failed size instead. Returns false on invalid parameters or when no axis -// can shrink any further. +// After an OOM (last_w/last_h > 0) cap below the failed size: get_tile_sizes may widen a halved request back to it. static bool sd_vae_tile_sizes(VAE& vae, const sd_tiling_params_t& tiling_params, int64_t latent_w, diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 86af2b66c..4723872ac 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -62,10 +62,7 @@ static std::vector sd_tiling_plan_axis(int dimension, } int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size) { - // A tile a little over half the axis (32 of 62 latents) gets two tiles overlapping by 1-2 latents, which shows - // as a seam. Keep the requested size when its overlaps are >= half the target and >= 2 latents, else pick the - // cheapest size that overlaps by the full target, up to 8 latents, never using more tiles than the nearest - // passing size would. + // Overlaps of 1-2 latents (32-latent tiles on a 62 axis) show as a seam; require >= half the target and >= 2. const int min_tile_size = 4; const int limit = max_tile_size >= min_tile_size ? max_tile_size : INT_MAX; tile_size = std::min(tile_size, limit); @@ -75,8 +72,7 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl auto overlaps_enough = [&](int size, bool replacement) { int min_overlap = std::max(2, static_cast(size * tile_overlap_factor * 0.5f)); if (replacement) { - // Half the target is too little for 16x VAEs: 4 of 16 latents still leaves a line (Qwen-Image-2.1 at - // 480 px), 8 does not, on 8x and 16x VAEs alike. + // Half the target still leaves a line on 16x VAEs (Qwen-Image-2.1 at 480 px); 8 latents does not. min_overlap = std::max(min_overlap, std::min(8, static_cast(size * tile_overlap_factor))); } const auto tiles = sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false); @@ -110,7 +106,6 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl } const int max_tiles = tile_count(nearest); // Fewest latents decoded (tiles * size) wins, then fewer tiles; 62 latents -> 2 x 35. - // Growth stays within a quarter of the request where possible. auto pick = [&](int lowest, int highest, bool replacement) { int best_size = 0; int64_t best_cost = 0; @@ -124,7 +119,6 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl continue; } const int64_t cost = static_cast(tiles) * size; - // Equal cost and tile count means equal size, so no further tie-break is needed. const bool better = best_size == 0 || cost < best_cost || (cost == best_cost && tiles < best_tiles); if (better) { best_size = size; diff --git a/src/runtime/tiling.h b/src/runtime/tiling.h index ea67af7f5..3c83c8083 100644 --- a/src/runtime/tiling.h +++ b/src/runtime/tiling.h @@ -44,9 +44,7 @@ sd::Tensor process_tiles_2d_batched(const sd::Tensor& input, const TileBatchProcessCallback& on_processing, bool silent = false); -// Returns tile_size, adjusted if needed so a non-circular axis of small_dim is never split into tiles that -// barely overlap (see the definition for why that shows as a seam). max_tile_size > 0 (an out-of-memory retry) -// bounds the result; it is ignored below the 4-latent minimum. +// tile_size adjusted so tiles never barely overlap; max_tile_size > 0 (OOM retry) caps it, ignored below 4. int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size = 0); #endif // __SD_RUNTIME_TILING_H__ From 4db928c77bcb3148500aacdf29d46f4166ab0542 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 18:47:11 -0700 Subject: [PATCH 8/9] fix: keep VAE OOM retry caps at the 4-latent minimum so no axis can grow back --- src/model/vae/vae.hpp | 8 ++++---- src/pipeline/diffusion_engine.cpp | 4 ++-- src/runtime/tiling.cpp | 2 +- src/runtime/tiling.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index 9815216ef..2be7eafd7 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -255,14 +255,14 @@ struct VAE : public GGMLRunner { if (!circular_x && !circular_x_enabled) { tile_size_w = sd_tiling_seam_safe_tile_size(static_cast(latent_w), tile_size_w, tile_overlap, retry_tile_limit_w); - } else if (retry_tile_limit_w >= 4) { - tile_size_w = std::min(tile_size_w, retry_tile_limit_w); + } else if (retry_tile_limit_w > 0) { + tile_size_w = std::min(tile_size_w, std::max(4, retry_tile_limit_w)); } if (!circular_y && !circular_y_enabled) { tile_size_h = sd_tiling_seam_safe_tile_size(static_cast(latent_h), tile_size_h, tile_overlap, retry_tile_limit_h); - } else if (retry_tile_limit_h >= 4) { - tile_size_h = std::min(tile_size_h, retry_tile_limit_h); + } else if (retry_tile_limit_h > 0) { + tile_size_h = std::min(tile_size_h, std::max(4, retry_tile_limit_h)); } return true; } diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 4755830a6..12f113e8f 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2906,8 +2906,8 @@ static bool sd_vae_tile_sizes(VAE& vae, int last_h, int& tile_size_w, int& tile_size_h) { - vae.retry_tile_limit_w = last_w > 0 ? last_w - 1 : 0; - vae.retry_tile_limit_h = last_h > 0 ? last_h - 1 : 0; + vae.retry_tile_limit_w = last_w > 0 ? std::max(4, last_w - 1) : 0; + vae.retry_tile_limit_h = last_h > 0 ? std::max(4, last_h - 1) : 0; tile_size_w = static_cast(latent_w); tile_size_h = static_cast(latent_h); float tile_overlap; diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 4723872ac..05b435f17 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -64,7 +64,7 @@ static std::vector sd_tiling_plan_axis(int dimension, int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size) { // Overlaps of 1-2 latents (32-latent tiles on a 62 axis) show as a seam; require >= half the target and >= 2. const int min_tile_size = 4; - const int limit = max_tile_size >= min_tile_size ? max_tile_size : INT_MAX; + const int limit = max_tile_size > 0 ? std::max(max_tile_size, min_tile_size) : INT_MAX; tile_size = std::min(tile_size, limit); if (tile_overlap_factor <= 0.f || tile_size >= small_dim) { return tile_size; diff --git a/src/runtime/tiling.h b/src/runtime/tiling.h index 3c83c8083..029078fd3 100644 --- a/src/runtime/tiling.h +++ b/src/runtime/tiling.h @@ -44,7 +44,7 @@ sd::Tensor process_tiles_2d_batched(const sd::Tensor& input, const TileBatchProcessCallback& on_processing, bool silent = false); -// tile_size adjusted so tiles never barely overlap; max_tile_size > 0 (OOM retry) caps it, ignored below 4. +// tile_size adjusted so tiles never barely overlap; max_tile_size > 0 (OOM retry) caps it, never below 4. int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size = 0); #endif // __SD_RUNTIME_TILING_H__ From 2be3860404d1c7d1e7e71172019d7887e287f50f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 11 Oct 2026 20:43:27 -0700 Subject: [PATCH 9/9] Trim comments --- src/model/vae/vae.hpp | 2 +- src/runtime/tiling.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index 2be7eafd7..a6f9457ba 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -166,7 +166,7 @@ struct VAE : public GGMLRunner { } public: - // Upper bound on latent tile sizes while retrying after an out-of-memory failure (0 = none). + // OOM-retry cap on latent tile sizes (0 = none). int retry_tile_limit_w = 0; int retry_tile_limit_h = 0; diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp index 05b435f17..12c2d6c40 100644 --- a/src/runtime/tiling.cpp +++ b/src/runtime/tiling.cpp @@ -105,7 +105,6 @@ int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overl return tile_size; } const int max_tiles = tile_count(nearest); - // Fewest latents decoded (tiles * size) wins, then fewer tiles; 62 latents -> 2 x 35. auto pick = [&](int lowest, int highest, bool replacement) { int best_size = 0; int64_t best_cost = 0;