diff --git a/README.md b/README.md index 89d686dd1..6cb2ad4c8 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ API and command-line option may change frequently.*** - [HunyuanVideo 1.5](./docs/hunyuan_video.md) - [LingBot-Video](./docs/lingbot_video.md) - [PhotoMaker](./docs/photo_maker.md) support. + - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL) - Control Net support with SD 1.5 - [ADetailer](./docs/adetailer.md) - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora) diff --git a/docs/ip_adapter.md b/docs/ip_adapter.md new file mode 100644 index 000000000..23a5e3d0e --- /dev/null +++ b/docs/ip_adapter.md @@ -0,0 +1,55 @@ +# IP-Adapter + +stable-diffusion.cpp supports [IP-Adapter](https://github.com/tencent-ailab/IP-Adapter) +image-prompt conditioning for SD 1.5 and SDXL. Given a reference image, +IP-Adapter transfers the subject and appearance of that image into the +generation, alongside the text prompt. + +IP-Adapter encodes the reference image with a CLIP-Vision (ViT-H/14) +encoder, projects the embedding into a few image tokens, and injects them +through a decoupled cross-attention added to every attn2 layer of the +UNet. It composes with Control Net, so a reference image (appearance) and +an OpenPose hint (pose) can be combined in a single generation. + +## Required weights + +1. A base SD 1.5 or SDXL model. +2. A CLIP-Vision (ViT-H/14) image encoder, passed with `--clip_vision` + (for example `clip_vision_h.safetensors`). +3. An IP-Adapter weight file, passed with `--ip-adapter`. The `vit-h` + variants reuse the same ViT-H encoder as above. From + [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter): + - SD 1.5: `models/ip-adapter_sd15.safetensors` + - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors` + +## Options + +- `--ip-adapter ` path to the IP-Adapter weight file. +- `--ip-adapter-image ` path to the reference image. +- `--ip-adapter-strength ` strength of the IP-Adapter injection + (default 1.0). Lower values let the text prompt dominate; 0.6 to 0.8 is + a good starting range. + +## Example (SD 1.5) + +``` +sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512 +``` + +## Example (SDXL) + +``` +sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --diffusion-fa --vae-tiling +``` + +The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and +`--offload-to-cpu`) on GPUs with limited VRAM. + +## Combining with Control Net + +Add the usual Control Net options to keep the reference appearance while +controlling the pose: + +``` +sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --diffusion-fa --vae-tiling +``` diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 68a3148cc..af62a9a93 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -817,6 +817,16 @@ int main(int argc, const char* argv[]) { } } + if (gen_params.ip_adapter_image_path.size() > 0) { + if (!load_sd_image_from_file(gen_params.ip_adapter_image.put(), + gen_params.ip_adapter_image_path.c_str(), + 0, + 0)) { + LOG_ERROR("load image from '%s' failed", gen_params.ip_adapter_image_path.c_str()); + return 1; + } + } + if (!gen_params.control_video_path.empty()) { gen_params.control_frames.clear(); if (!load_images_from_dir(gen_params.control_video_path, diff --git a/examples/common/common.cpp b/examples/common/common.cpp index a5cb48369..82558e7c2 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -427,6 +427,11 @@ ArgOptions SDContextParams::get_options() { "path to control net model", 0, &control_net_path}, + {"", + "--ip-adapter", + "path to IP-Adapter model (requires --clip_vision)", + 0, + &ip_adapter_path}, {"", "--motion-module", "path to AnimateDiff motion module (SD 1.5); enables video generation on --video-frames > 1", @@ -876,6 +881,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) { sd_ctx_params.audio_vae_path = audio_vae_path.c_str(); sd_ctx_params.taesd_path = taesd_path.c_str(); sd_ctx_params.control_net_path = control_net_path.c_str(); + sd_ctx_params.ip_adapter_path = ip_adapter_path.c_str(); sd_ctx_params.motion_module_path = motion_module_path.c_str(); sd_ctx_params.embeddings = embedding_vec.data(); sd_ctx_params.embedding_count = static_cast(embedding_vec.size()); @@ -966,6 +972,11 @@ ArgOptions SDGenerationParams::get_options() { "path to control image, control net", 0, &control_image_path}, + {"", + "--ip-adapter-image", + "path to the IP-Adapter reference image", + 0, + &ip_adapter_image_path}, {"", "--control-video", "path to control video frames, It must be a directory path. The video frames inside should be stored as images in " @@ -1158,6 +1169,10 @@ ArgOptions SDGenerationParams::get_options() { "--control-strength", "strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image", &control_strength}, + {"", + "--ip-adapter-strength", + "strength to apply IP-Adapter (default: 1.0)", + &ip_adapter_strength}, {"", "--moe-boundary", "timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1", @@ -2479,29 +2494,31 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { LOG_WARN("Notice: --increase-ref-index is deprecated. Use --ref-image-args \"ref_index_mode=increase\" instead."); } - params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); - params.lora_count = static_cast(lora_vec.size()); - params.prompt = prompt.c_str(); - params.negative_prompt = negative_prompt.c_str(); - params.clip_skip = clip_skip; - params.init_image = init_image.get(); - params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); - params.ref_images_count = static_cast(ref_image_views.size()); - params.ref_image_args = ref_image_args.c_str(); - params.mask_image = mask_image.get(); - params.width = get_resolved_width(); - params.height = get_resolved_height(); - params.sample_params = sample_params; - params.strength = strength; - params.seed = seed; - params.batch_count = batch_count; - params.qwen_image_layers = qwen_image_layers; - params.control_image = control_image.get(); - params.control_strength = control_strength; - params.pm_params = pm_params; - params.pulid_params = pulid_params; - params.vae_tiling_params = vae_tiling_params; - params.cache = cache_params; + params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); + params.lora_count = static_cast(lora_vec.size()); + params.prompt = prompt.c_str(); + params.negative_prompt = negative_prompt.c_str(); + params.clip_skip = clip_skip; + params.init_image = init_image.get(); + params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); + params.ref_images_count = static_cast(ref_image_views.size()); + params.ref_image_args = ref_image_args.c_str(); + params.mask_image = mask_image.get(); + params.width = get_resolved_width(); + params.height = get_resolved_height(); + params.sample_params = sample_params; + params.strength = strength; + params.seed = seed; + params.batch_count = batch_count; + params.qwen_image_layers = qwen_image_layers; + params.control_image = control_image.get(); + params.control_strength = control_strength; + params.ip_adapter_image = ip_adapter_image.get(); + params.ip_adapter_strength = ip_adapter_strength; + params.pm_params = pm_params; + params.pulid_params = pulid_params; + params.vae_tiling_params = vae_tiling_params; + params.cache = cache_params; params.hires.enabled = hires_enabled; params.hires.upscaler = resolved_hires_upscaler; diff --git a/examples/common/common.h b/examples/common/common.h index 769651987..ea90c8c1b 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -133,6 +133,7 @@ struct SDContextParams { std::string taesd_path; std::string esrgan_path; std::string control_net_path; + std::string ip_adapter_path; std::string motion_module_path; std::string embedding_dir; std::string photo_maker_path; @@ -200,6 +201,7 @@ struct SDGenerationParams { int64_t seed = 42; float strength = 0.75f; float control_strength = 0.9f; + float ip_adapter_strength = 1.0f; bool auto_resize_ref_image = true; bool increase_ref_index = false; bool embed_image_metadata = true; @@ -208,6 +210,7 @@ struct SDGenerationParams { std::string end_image_path; std::string mask_image_path; std::string control_image_path; + std::string ip_adapter_image_path; std::vector ref_image_paths; std::string control_video_path; @@ -274,6 +277,7 @@ struct SDGenerationParams { std::vector ref_images; SDImageOwner mask_image; SDImageOwner control_image; + SDImageOwner ip_adapter_image; std::vector pm_id_images; std::vector control_frames; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 890050a63..bd568c686 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -200,6 +200,7 @@ typedef struct { const char* audio_vae_path; const char* taesd_path; const char* control_net_path; + const char* ip_adapter_path; const char* motion_module_path; const sd_embedding_t* embeddings; uint32_t embedding_count; @@ -375,6 +376,8 @@ typedef struct { int batch_count; sd_image_t control_image; float control_strength; + sd_image_t ip_adapter_image; + float ip_adapter_strength; sd_pm_params_t pm_params; sd_pulid_params_t pulid_params; sd_tiling_params_t vae_tiling_params; diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index b2127cb73..d8e017795 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1689,6 +1689,8 @@ struct GGMLRunnerContext { bool conv2d_direct_enabled = false; bool circular_x_enabled = false; bool circular_y_enabled = false; + ggml_tensor* ip_context = nullptr; + float ip_scale = 1.0f; std::shared_ptr weight_adapter = nullptr; std::vector>* debug_tensors = nullptr; std::function get_cache_tensor; diff --git a/src/model/adapter/ip_adapter.hpp b/src/model/adapter/ip_adapter.hpp new file mode 100644 index 000000000..35fd1bb88 --- /dev/null +++ b/src/model/adapter/ip_adapter.hpp @@ -0,0 +1,88 @@ +#ifndef __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__ +#define __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__ + +#include "core/ggml_extend.hpp" +#include "model/common/block.hpp" +#include "model_loader.h" + +namespace IPAdapter { + + struct ImageProjModel : public GGMLBlock { + int64_t num_tokens = 4; + int64_t ctx_dim = 768; + int64_t clip_dim = 1024; + + ImageProjModel() {} + ImageProjModel(int64_t num_tokens, int64_t ctx_dim, int64_t clip_dim) + : num_tokens(num_tokens), ctx_dim(ctx_dim), clip_dim(clip_dim) { + blocks["proj"] = std::shared_ptr(new Linear(clip_dim, num_tokens * ctx_dim, true)); + blocks["norm"] = std::shared_ptr(new LayerNorm(ctx_dim)); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) { + auto proj = std::dynamic_pointer_cast(blocks["proj"]); + auto norm = std::dynamic_pointer_cast(blocks["norm"]); + + int64_t n = image_embeds->ne[1]; + auto x = proj->forward(ctx, image_embeds); + x = ggml_reshape_3d(ctx->ggml_ctx, x, ctx_dim, num_tokens, n); + x = norm->forward(ctx, x); + return x; + } + }; + + struct IPAdapterRunner : public GGMLRunner { + ImageProjModel image_proj; + int64_t num_tokens = 4; + std::string prefix; + + IPAdapterRunner(ggml_backend_t backend, + const String2TensorStorage& tensor_storage_map, + const std::string prefix, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), prefix(prefix) { + int64_t ctx_dim = 768; + int64_t clip_dim = 1024; + int64_t out_dim = 3072; + auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight"); + if (norm_iter != tensor_storage_map.end()) { + ctx_dim = norm_iter->second.ne[0]; + } + auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight"); + if (proj_iter != tensor_storage_map.end()) { + clip_dim = proj_iter->second.ne[0]; + out_dim = proj_iter->second.ne[1]; + } + num_tokens = out_dim / ctx_dim; + image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim); + image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); + } + + std::string get_desc() override { + return "ip_adapter"; + } + + void get_param_tensors(std::map& tensors, const std::string = "") { + image_proj.get_param_tensors(tensors, prefix + ".image_proj"); + } + + ggml_cgraph* build_graph(const sd::Tensor& image_embeds_tensor) { + ggml_cgraph* gf = new_graph_custom(1024); + ggml_tensor* embeds = make_input(image_embeds_tensor); + auto runner_ctx = get_context(); + ggml_tensor* out = image_proj.forward(&runner_ctx, embeds); + ggml_build_forward_expand(gf, out); + return gf; + } + + sd::Tensor compute(int n_threads, const sd::Tensor& image_embeds) { + auto get_graph = [&]() -> ggml_cgraph* { + return build_graph(image_embeds); + }; + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true, true, true)); + } + }; + +} // namespace IPAdapter + +#endif // __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__ diff --git a/src/model/common/block.hpp b/src/model/common/block.hpp index 6fb40ec0f..be76dd713 100644 --- a/src/model/common/block.hpp +++ b/src/model/common/block.hpp @@ -310,17 +310,33 @@ class CrossAttention : public GGMLBlock { int64_t context_dim; int64_t n_head; int64_t d_head; - bool xtra_dim = false; + bool xtra_dim = false; + bool enable_ip = false; + bool has_ip = false; + + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + GGMLBlock::init_params(ctx, tensor_storage_map, prefix); + if (enable_ip && + tensor_storage_map.find(prefix + "to_k_ip.weight") != tensor_storage_map.end()) { + has_ip = true; + int64_t inner_dim = d_head * n_head; + int64_t ip_dim = tensor_storage_map.at(prefix + "to_k_ip.weight").ne[0]; + blocks["to_k_ip"] = std::shared_ptr(new Linear(ip_dim, inner_dim, false)); + blocks["to_v_ip"] = std::shared_ptr(new Linear(ip_dim, inner_dim, false)); + } + } public: CrossAttention(int64_t query_dim, int64_t context_dim, int64_t n_head, - int64_t d_head) + int64_t d_head, + bool enable_ip = false) : n_head(n_head), d_head(d_head), query_dim(query_dim), - context_dim(context_dim) { + context_dim(context_dim), + enable_ip(enable_ip) { int64_t inner_dim = d_head * n_head; if (context_dim == 320 && d_head == 320) { // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09"); @@ -363,6 +379,15 @@ class CrossAttention : public GGMLBlock { } x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, inner_dim] + if (has_ip && ctx->ip_context != nullptr && ctx->ip_scale != 0.0f) { + auto to_k_ip = std::dynamic_pointer_cast(blocks["to_k_ip"]); + auto to_v_ip = std::dynamic_pointer_cast(blocks["to_v_ip"]); + auto k_ip = to_k_ip->forward(ctx, ctx->ip_context); + auto v_ip = to_v_ip->forward(ctx, ctx->ip_context); + auto x_ip = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k_ip, v_ip, n_head, nullptr, false, ctx->flash_attn_enabled); + x = ggml_add(ctx->ggml_ctx, x, ggml_scale(ctx->ggml_ctx, x_ip, ctx->ip_scale)); + } + x = to_out_0->forward(ctx, x); // [N, n_token, query_dim] return x; } @@ -387,7 +412,7 @@ class BasicTransformerBlock : public GGMLBlock { // inner_dim is always None or equal to dim // gated_ff is always True blocks["attn1"] = std::shared_ptr(new CrossAttention(dim, dim, n_head, d_head)); - blocks["attn2"] = std::shared_ptr(new CrossAttention(dim, context_dim, n_head, d_head)); + blocks["attn2"] = std::shared_ptr(new CrossAttention(dim, context_dim, n_head, d_head, true)); blocks["ff"] = std::shared_ptr(new FeedForward(dim, dim)); blocks["norm1"] = std::shared_ptr(new LayerNorm(dim)); blocks["norm2"] = std::shared_ptr(new LayerNorm(dim)); diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp index 31c504833..d7a5f4231 100644 --- a/src/model/diffusion/model.hpp +++ b/src/model/diffusion/model.hpp @@ -46,6 +46,8 @@ struct UNetDiffusionExtra { int num_video_frames = -1; const std::vector>* controls = nullptr; float control_strength = 0.f; + const sd::Tensor* ip_context = nullptr; + float ip_scale = 1.f; }; struct SkipLayerDiffusionExtra { diff --git a/src/model/diffusion/unet.hpp b/src/model/diffusion/unet.hpp index 118ed6327..567573049 100644 --- a/src/model/diffusion/unet.hpp +++ b/src/model/diffusion/unet.hpp @@ -775,14 +775,17 @@ struct UNetModelRunner : public DiffusionModelRunner { const sd::Tensor& y_tensor = {}, int num_video_frames = -1, const std::vector>& controls_tensor = {}, - float control_strength = 0.f) { + float control_strength = 0.f, + const sd::Tensor& ip_context_tensor = {}, + float ip_scale = 1.f) { ggml_cgraph* gf = new_graph_custom(UNET_GRAPH_SIZE); - ggml_tensor* x = make_input(x_tensor); - ggml_tensor* timesteps = make_input(timesteps_tensor); - ggml_tensor* context = make_optional_input(context_tensor); - ggml_tensor* c_concat = make_optional_input(c_concat_tensor); - ggml_tensor* y = make_optional_input(y_tensor); + ggml_tensor* x = make_input(x_tensor); + ggml_tensor* timesteps = make_input(timesteps_tensor); + ggml_tensor* context = make_optional_input(context_tensor); + ggml_tensor* c_concat = make_optional_input(c_concat_tensor); + ggml_tensor* y = make_optional_input(y_tensor); + ggml_tensor* ip_context = make_optional_input(ip_context_tensor); std::vector controls; controls.reserve(controls_tensor.size()); for (const auto& control_tensor : controls_tensor) { @@ -793,7 +796,9 @@ struct UNetModelRunner : public DiffusionModelRunner { num_video_frames = static_cast(x->ne[3]); } - auto runner_ctx = get_context(); + auto runner_ctx = get_context(); + runner_ctx.ip_context = ip_context; + runner_ctx.ip_scale = ip_scale; ggml_tensor* out = unet.forward(&runner_ctx, x, @@ -818,14 +823,16 @@ struct UNetModelRunner : public DiffusionModelRunner { const sd::Tensor& y = {}, int num_video_frames = -1, const std::vector>& controls = {}, - float control_strength = 0.f) { + float control_strength = 0.f, + const sd::Tensor& ip_context = {}, + float ip_scale = 1.f) { // x: [N, in_channels, h, w] // timesteps: [N, ] // context: [N, max_position, hidden_size]([N, 77, 768]) or [1, max_position, hidden_size] // c_concat: [N, in_channels, h, w] or [1, in_channels, h, w] // y: [N, adm_in_channels] or [1, adm_in_channels] auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength); + return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength, ip_context, ip_scale); }; return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); @@ -845,7 +852,9 @@ struct UNetModelRunner : public DiffusionModelRunner { tensor_or_empty(diffusion_params.y), extra->num_video_frames, extra->controls ? *extra->controls : empty_controls, - extra->control_strength); + extra->control_strength, + extra->ip_context ? *extra->ip_context : sd::Tensor{}, + extra->ip_scale); } void test() { diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 2b41a5576..63c916edd 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1285,11 +1285,54 @@ static std::string convert_esrgan_tensor_name(std::string name) { return name; } +static const std::map& ip_adapter_index_map(SDVersion version) { + static const std::map sd15_map = { + {1, "input_blocks.1.1.transformer_blocks.0"}, {3, "input_blocks.2.1.transformer_blocks.0"}, {5, "input_blocks.4.1.transformer_blocks.0"}, {7, "input_blocks.5.1.transformer_blocks.0"}, {9, "input_blocks.7.1.transformer_blocks.0"}, {11, "input_blocks.8.1.transformer_blocks.0"}, {13, "output_blocks.3.1.transformer_blocks.0"}, {15, "output_blocks.4.1.transformer_blocks.0"}, {17, "output_blocks.5.1.transformer_blocks.0"}, {19, "output_blocks.6.1.transformer_blocks.0"}, {21, "output_blocks.7.1.transformer_blocks.0"}, {23, "output_blocks.8.1.transformer_blocks.0"}, {25, "output_blocks.9.1.transformer_blocks.0"}, {27, "output_blocks.10.1.transformer_blocks.0"}, {29, "output_blocks.11.1.transformer_blocks.0"}, {31, "middle_block.1.transformer_blocks.0"}}; + + static std::map sdxl_map; + if (sdxl_map.empty()) { + std::vector> order = { + {"input_blocks.4.1", 2}, {"input_blocks.5.1", 2}, {"input_blocks.7.1", 10}, {"input_blocks.8.1", 10}, {"output_blocks.0.1", 10}, {"output_blocks.1.1", 10}, {"output_blocks.2.1", 10}, {"output_blocks.3.1", 2}, {"output_blocks.4.1", 2}, {"output_blocks.5.1", 2}, {"middle_block.1", 10}}; + int idx = 1; + for (const auto& [block, depth] : order) { + for (int m = 0; m < depth; m++) { + sdxl_map[idx] = block + ".transformer_blocks." + std::to_string(m); + idx += 2; + } + } + } + return sd_version_is_sdxl(version) ? sdxl_map : sd15_map; +} + +static std::string convert_ip_adapter_name(std::string name, SDVersion version) { + if (starts_with(name, "image_proj.")) { + return "ip_adapter." + name; + } + if (starts_with(name, "ip_adapter.")) { + auto items = split_string(name, '.'); + if (items.size() < 4) { + return name; + } + int idx = atoi(items[1].c_str()); + const auto& mp = ip_adapter_index_map(version); + auto blk = mp.find(idx); + if (blk == mp.end()) { + return name; + } + return "model.diffusion_model." + blk->second + ".attn2." + items[2] + "." + items[3]; + } + return name; +} + std::string convert_tensor_name(std::string name, SDVersion version) { if (version == VERSION_ESRGAN) { return convert_esrgan_tensor_name(std::move(name)); } + if (starts_with(name, "ip_adapter.") || starts_with(name, "image_proj.")) { + return convert_ip_adapter_name(std::move(name), version); + } + bool is_lora = false; bool is_lycoris_underline = false; bool is_underline = false; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 81f82fbc8..5c24c112d 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -22,6 +22,7 @@ #include "conditioning/conditioner.hpp" #include "core/backend_fit.h" #include "extensions/generation_extension.h" +#include "model/adapter/ip_adapter.hpp" #include "model/adapter/lora.hpp" #include "model/diffusion/anima.hpp" #include "model/diffusion/animatediff.hpp" @@ -221,6 +222,9 @@ class StableDiffusionGGML { std::shared_ptr preview_vae; std::shared_ptr audio_vae_model; std::shared_ptr control_net; + std::shared_ptr ip_adapter; + sd::Tensor ip_adapter_tokens; + float ip_adapter_strength = 1.0f; std::vector> generation_extensions; std::vector> runtime_lora_models; bool apply_lora_immediately = false; @@ -841,6 +845,13 @@ class StableDiffusionGGML { } } + if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) { + if (!model_loader.init_from_file(sd_ctx_params->ip_adapter_path)) { + LOG_ERROR("init ip-adapter model loader from file failed: '%s'", sd_ctx_params->ip_adapter_path); + return false; + } + } + model_loader.convert_tensors_name(); version = model_loader.get_sd_version(); @@ -1294,6 +1305,33 @@ class StableDiffusionGGML { } } + if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0 && clip_vision == nullptr) { + if (!ensure_backend_pair(SDBackendModule::CLIP_VISION)) { + return false; + } + clip_vision = std::make_shared(backend_for(SDBackendModule::CLIP_VISION), + tensor_storage_map, + model_manager); + clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::CLIP_VISION)); + if (!register_runner_params("CLIP vision", + clip_vision, + SDBackendModule::CLIP_VISION)) { + return false; + } + } + + if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0) { + ip_adapter = std::make_shared(backend_for(SDBackendModule::DIFFUSION), + tensor_storage_map, + "ip_adapter", + model_manager); + if (!register_runner_params("IP-Adapter", + ip_adapter, + SDBackendModule::DIFFUSION)) { + return false; + } + } + if (!ensure_backend_pair(SDBackendModule::VAE)) { return false; } @@ -2061,6 +2099,24 @@ class StableDiffusionGGML { return output; } + void compute_ip_adapter_tokens(const sd_image_t& image, float strength) { + ip_adapter_tokens = {}; + ip_adapter_strength = strength; + if (ip_adapter == nullptr || clip_vision == nullptr || image.data == nullptr) { + return; + } + auto image_tensor = sd_image_to_tensor(image); + auto embed = get_clip_vision_output(image_tensor, true, -1); + if (embed.empty()) { + return; + } + ip_adapter_tokens = ip_adapter->compute(n_threads, embed); + if (!ip_adapter_tokens.empty()) { + LOG_INFO("IP-Adapter: %lld image tokens, strength %.2f", + (long long)ip_adapter_tokens.shape()[1], strength); + } + } + std::vector process_timesteps(const std::vector& timesteps, const sd::Tensor& init_latent, const sd::Tensor& denoise_mask, @@ -2549,7 +2605,8 @@ class StableDiffusionGGML { auto run_condition = [&](const SDCondition& condition, const sd::Tensor* c_concat_override = nullptr, const std::vector* local_skip_layers = nullptr, - const std::vector>* ref_latents_override = nullptr) -> sd::Tensor { + const std::vector>* ref_latents_override = nullptr, + bool apply_ip = true) -> sd::Tensor { diffusion_params.context = condition.c_crossattn.empty() ? nullptr : &condition.c_crossattn; diffusion_params.c_concat = c_concat_override != nullptr ? c_concat_override : (condition.c_concat.empty() ? nullptr : &condition.c_concat); diffusion_params.y = condition.c_vector.empty() ? nullptr : &condition.c_vector; @@ -2560,7 +2617,12 @@ class StableDiffusionGGML { if (animatediff_loaded && noised_input.dim() >= 4 && noised_input.shape()[3] > 1) { nvf = static_cast(noised_input.shape()[3]); } - diffusion_params.extra = UNetDiffusionExtra{nvf, &controls, control_strength}; + UNetDiffusionExtra unet_extra{nvf, &controls, control_strength}; + if (apply_ip && !ip_adapter_tokens.empty()) { + unet_extra.ip_context = &ip_adapter_tokens; + unet_extra.ip_scale = ip_adapter_strength; + } + diffusion_params.extra = unet_extra; } else if (sd_version_is_sd3(version)) { diffusion_params.extra = SkipLayerDiffusionExtra{local_skip_layers}; } else if (sd_version_is_flux(version) || sd_version_is_flux2(version) || sd_version_is_longcat(version) || sd_version_is_sefi_image(version)) { @@ -2651,7 +2713,9 @@ class StableDiffusionGGML { } uncond_out = run_condition(uncond, uncond.c_concat.empty() ? nullptr : &uncond.c_concat, - uncond_skip_layers); + uncond_skip_layers, + nullptr, + false); if (uncond_out.empty()) { return {}; } @@ -2660,7 +2724,8 @@ class StableDiffusionGGML { img_uncond_out = run_condition(img_uncond, img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat, nullptr, - uncond_without_ref_latents ? &empty_ref_latents : nullptr); + uncond_without_ref_latents ? &empty_ref_latents : nullptr, + false); if (img_uncond_out.empty()) { return {}; } @@ -4954,6 +5019,7 @@ static std::optional prepare_image_generation_embeds(sd_c request->pulid_params, condition_params, plan->total_steps); + sd_ctx->sd->compute_ip_adapter_tokens(sd_img_gen_params->ip_adapter_image, sd_img_gen_params->ip_adapter_strength); int64_t prepare_start_ms = ggml_time_ms(); condition_params.zero_out_masked = false; auto cond = sd_ctx->sd->cond_stage_model->get_learned_condition(sd_ctx->sd->n_threads,