diff --git a/examples/server/api.md b/examples/server/api.md index 541e260a5..f8f24596c 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -410,6 +410,20 @@ Field types: | `queue_position` | `integer` | | `result` | `object \| null` | | `error` | `object \| null` | +| `preview` | `object \| null` | + +`preview` sub-fields: + +| Field | Type | Notes | +| --- | --- | --- | +| `preview.pass` | `integer` | Sampling pass number, starting at 1 within the job | +| `preview.step` | `integer` | Positive logical sample step within this pass | +| `preview.total_steps` | `integer` | Actual sample steps in this pass, after schedule and strength adjustments | +| `preview.b64_json` | `string` | Base64-encoded PNG of the current preview frame | + +The preview is updated in place as sampling progresses; poll `GET /sdcpp/v1/jobs/{id}` to retrieve the latest frame. Only the most recent preview is retained. + +`step / total_steps` describes the current sampling pass, not overall job completion. A new batch item, high/low-noise stage, or highres pass increments `pass` and restarts `step`. Samplers with multiple denoiser evaluations per logical step may update a preview more than once at the same step. The frame and its pass/step metadata are published together. ### Endpoints @@ -437,6 +451,7 @@ Top-level fields: | `loras` | `array` | Available LoRA entries | | `upscalers` | `array` | Available highres upscalers, built-in and model-backed | | `upscale` | `boolean` | Whether a compatible RGB ESRGAN model is available for `POST /sdcpp/v1/upscale` | +| `preview_modes` | `array` | Available preview modes, e.g. `["none", "proj", "tae", "vae"]` | | `limits` | `object` | Shared queue and size limits | `model` @@ -598,6 +613,7 @@ Fields returned in `features_by_mode.img_gen`: - `cache` - `cancel_queued` - `cancel_generating` +- `preview` Fields returned in `features_by_mode.vid_gen`: @@ -610,6 +626,7 @@ Fields returned in `features_by_mode.vid_gen`: - `cache` - `cancel_queued` - `cancel_generating` +- `preview` #### `POST /sdcpp/v1/img_gen` @@ -782,6 +799,9 @@ Example: "scm_mask": "", "scm_policy_dynamic": true, + "preview": "none", + "preview_interval": 1, + "output_format": "png", "output_compression": 100 } @@ -847,6 +867,8 @@ Top-level scalar fields: | `control_strength` | `number` | | `ip_adapter_strength` | `number` | | `embed_image_metadata` | `boolean` | +| `preview` | `string` | +| `preview_interval` | `integer` | Image fields: @@ -934,6 +956,10 @@ When omitted, backend defaults apply to these fields: - `sample_params.flow_shift` - `sample_params.guidance.img_cfg` +### Preview Interval Semantics + +`preview_interval` controls the period (in sample steps) at which preview frames are generated. The default is `1` (every step). Any non-positive values will be clamped to `1`. Note that if `preview` is set to `"none"`, no previews are produced regardless of the interval. + ### Completion Result Example completed job: @@ -1130,6 +1156,9 @@ Example: "scm_mask": "", "scm_policy_dynamic": true, + "preview": "none", + "preview_interval": 1, + "output_format": "webm", "output_compression": 100 } @@ -1181,6 +1210,8 @@ Top-level scalar fields: | `fps` | `integer` | | `moe_boundary` | `number` | | `vace_strength` | `number` | +| `preview` | `string` | +| `preview_interval` | `integer` | Image and frame fields: diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp index 287955a84..88408088e 100644 --- a/examples/server/async_jobs.cpp +++ b/examples/server/async_jobs.cpp @@ -10,6 +10,55 @@ #include "common/media_io.h" #include "common/resource_owners.hpp" +static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) { + (void)is_noisy; + if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) { + return; + } + + AsyncGenerationJob* job = static_cast(data); + const sd_image_t& frame = frames[0]; + const sd_preview_info_t info = sd_get_preview_info(); + + auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG, + frame.data, + frame.width, + frame.height, + frame.channel, + "", + 80); + if (image_bytes.empty()) { + return; + } + + std::string b64 = base64_encode(image_bytes); + std::lock_guard lock(job->preview_mutex); + job->preview_b64 = std::move(b64); + job->preview_step = step < 0 ? -step : step; + job->preview_pass = info.sample_pass; + job->preview_total_steps = info.total_steps; +} + +static void clear_preview_callback() { + sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); +} + +static void set_preview_callback(AsyncGenerationJob& job) { + preview_t mode = str_to_preview(job.kind == AsyncJobKind::ImgGen + ? job.img_gen.preview_mode.c_str() + : job.vid_gen.preview_mode.c_str()); + if (mode == PREVIEW_COUNT || mode == PREVIEW_NONE) { + return; + } + int interval = job.kind == AsyncJobKind::ImgGen + ? job.img_gen.preview_interval + : job.vid_gen.preview_interval; + if (interval <= 0) { + interval = 1; + } + sd_set_preview_callback(preview_callback, mode, interval, true, false, &job); +} + const char* async_job_kind_name(AsyncJobKind kind) { switch (kind) { case AsyncJobKind::ImgGen: @@ -161,6 +210,18 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo result["error"] = nullptr; } + if (job.status == AsyncJobStatus::Generating) { + std::lock_guard lock(job.preview_mutex); + if (!job.preview_b64.empty()) { + result["preview"] = { + {"pass", job.preview_pass}, + {"step", job.preview_step}, + {"total_steps", job.preview_total_steps}, + {"b64_json", job.preview_b64}, + }; + } + } + return result; } @@ -174,12 +235,14 @@ bool execute_img_gen_job(ServerRuntime& runtime, { std::lock_guard lock(*runtime.sd_ctx_mutex); + set_preview_callback(job); sd_image_t* raw_results = nullptr; int num_results = 0; if (!generate_image(runtime.sd_ctx, ¶ms, &raw_results, &num_results)) { raw_results = nullptr; num_results = 0; } + clear_preview_callback(); results.adopt(raw_results, num_results); } @@ -247,10 +310,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime, { std::lock_guard lock(*runtime.sd_ctx_mutex); + set_preview_callback(job); sd_image_t* raw_results = nullptr; if (!generate_video(runtime.sd_ctx, ¶ms, &raw_results, &num_results, &generated_audio, &output_fps)) { raw_results = nullptr; } + clear_preview_callback(); results.adopt(raw_results, num_results); } diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h index 89997a3b4..2fe79bb24 100644 --- a/examples/server/async_jobs.h +++ b/examples/server/async_jobs.h @@ -44,6 +44,11 @@ struct AsyncGenerationJob { int result_fps = 0; std::string error_code; std::string error_message; + mutable std::mutex preview_mutex; + std::string preview_b64; + int preview_pass = 0; + int preview_step = 0; + int preview_total_steps = 0; }; struct AsyncJobManager { diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index b8767b629..3139682f9 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -205,6 +205,7 @@ static json make_img_gen_features_json() { {"cache", true}, {"cancel_queued", true}, {"cancel_generating", false}, + {"preview", true}, }; } @@ -219,9 +220,18 @@ static json make_vid_gen_features_json() { {"cache", true}, {"cancel_queued", true}, {"cancel_generating", false}, + {"preview", true}, }; } +static json make_preview_modes_json() { + json modes = json::array(); + for (int i = 0; i < PREVIEW_COUNT; ++i) { + modes.push_back(sd_preview_name((preview_t)i)); + } + return modes; +} + static json make_capabilities_json(ServerRuntime& runtime) { refresh_lora_cache(runtime); refresh_upscaler_cache(runtime); @@ -396,6 +406,7 @@ static json make_capabilities_json(ServerRuntime& runtime) { result["loras"] = available_loras; result["upscalers"] = available_upscalers; result["upscale"] = have_upscaler_models; + result["preview_modes"] = make_preview_modes_json(); return result; } @@ -418,6 +429,8 @@ static bool parse_img_gen_request(const json& body, if (!assign_output_options(request, output_format, output_compression, true, error_message)) { return false; } + request.preview_mode = body.value("preview", std::string("none")); + request.preview_interval = body.value("preview_interval", 1); // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { error_message = "invalid generation parameters"; @@ -445,6 +458,8 @@ static bool parse_vid_gen_request(const json& body, if (!assign_output_options(request, output_format, output_compression, error_message)) { return false; } + request.preview_mode = body.value("preview", std::string("none")); + request.preview_interval = body.value("preview_interval", 1); // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(VID_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { error_message = "invalid generation parameters"; diff --git a/examples/server/runtime.h b/examples/server/runtime.h index b4560f127..5baa633a6 100644 --- a/examples/server/runtime.h +++ b/examples/server/runtime.h @@ -66,6 +66,8 @@ struct ImgGenJobRequest { SDGenerationParams gen_params; std::string output_format = "png"; int output_compression = 100; + std::string preview_mode = "none"; + int preview_interval = 1; sd_img_gen_params_t to_sd_img_gen_params_t() { return gen_params.to_sd_img_gen_params_t(); @@ -76,6 +78,8 @@ struct VidGenJobRequest { SDGenerationParams gen_params; std::string output_format = "webm"; int output_compression = 100; + std::string preview_mode = "none"; + int preview_interval = 1; sd_vid_gen_params_t to_sd_vid_gen_params_t() { return gen_params.to_sd_vid_gen_params_t(); diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index cfedbac1f..a2e157556 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -461,6 +461,11 @@ struct ggml_tensor; typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data); typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data); typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data); + +typedef struct { + int sample_pass; + int total_steps; +} sd_preview_info_t; typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); @@ -469,6 +474,10 @@ SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); // negative interval previews only completed logical step -interval. Zero previews the final // completed step of the first sampling pass (base-resolution or high-noise). SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data); +// Query from a preview callback. Passes are numbered from 1 since the last +// sd_set_preview_callback call; total_steps is the actual count in that pass. +// Both fields are zero before sampling starts. +SD_API sd_preview_info_t sd_get_preview_info(); SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data); SD_API int32_t sd_get_num_physical_cores(); SD_API const char* sd_get_system_info(); diff --git a/src/core/util.cpp b/src/core/util.cpp index 199713d20..cbad4e486 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -360,12 +360,13 @@ int32_t sd_get_num_physical_cores() { static sd_progress_cb_t sd_progress_cb = nullptr; void* sd_progress_cb_data = nullptr; -static sd_preview_cb_t sd_preview_cb = nullptr; -static void* sd_preview_cb_data = nullptr; -preview_t sd_preview_mode = PREVIEW_NONE; -int sd_preview_interval = 1; -bool sd_preview_denoised = true; -bool sd_preview_noisy = false; +static sd_preview_info_t sd_preview_info = {}; +static sd_preview_cb_t sd_preview_cb = nullptr; +static void* sd_preview_cb_data = nullptr; +preview_t sd_preview_mode = PREVIEW_NONE; +int sd_preview_interval = 1; +bool sd_preview_denoised = true; +bool sd_preview_noisy = false; static sd_graph_eval_callback_t sd_backend_eval_cb = nullptr; static void* sd_backend_eval_cb_data = nullptr; @@ -696,6 +697,7 @@ void sd_set_progress_callback(sd_progress_cb_t cb, void* data) { sd_progress_cb_data = data; } void sd_set_preview_callback(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) { + sd_preview_info = {}; sd_preview_cb = cb; sd_preview_cb_data = data; sd_preview_mode = mode; @@ -709,6 +711,17 @@ void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data) { sd_backend_eval_cb_data = data; } +void sd_begin_preview_pass(int total_steps) { + if (sd_preview_cb != nullptr) { + ++sd_preview_info.sample_pass; + sd_preview_info.total_steps = total_steps; + } +} + +sd_preview_info_t sd_get_preview_info() { + return sd_preview_info; +} + sd_preview_cb_t sd_get_preview_callback() { return sd_preview_cb; } diff --git a/src/core/util.h b/src/core/util.h index 334224da2..abe411625 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -101,6 +101,7 @@ std::vector> split_quotation_attention( sd_progress_cb_t sd_get_progress_callback(); void* sd_get_progress_callback_data(); +void sd_begin_preview_pass(int total_steps); sd_preview_cb_t sd_get_preview_callback(); void* sd_get_preview_callback_data(); preview_t sd_get_preview_mode(); diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 4870ccaa6..f9785aabc 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2373,7 +2373,8 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr(steps)); SamplePreviewContext preview = prepare_sample_preview_context(); sd::Tensor processed_init_latent = denoiser->process_latent_in(init_latent);