InstantX/FLUX.1-dev-IP-Adapter

El código aún no está integrado en diffusers; la tarjeta del modelo indica usar los archivos locales del repositorio por ahora.

import os
from PIL import Image
import torch
import torch.nn as nn
from pipeline_flux_ipa import FluxPipeline
from transformer_flux import FluxTransformer2DModel
from attention_processor import IPAFluxAttnProcessor2_0
from transformers import AutoProcessor, SiglipVisionModel
from infer_flux_ipa_siglip import resize_img, MLPProjModel, IPAdapter

image_encoder_path = "google/siglip-so400m-patch14-384"
ipadapter_path = "./ip-adapter.bin"

transformer = FluxTransformer2DModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    torch_dtype=torch.bfloat16
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer,
    torch_dtype=torch.bfloat16
)

ip_model = IPAdapter(pipe, image_encoder_path, ipadapter_path, device="cuda", num_tokens=128)

image_dir = "./assets/images/2.jpg"
image_name = image_dir.split("/")[-1]
image = Image.open(image_dir).convert("RGB")
image = resize_img(image)

prompt = "a young girl"

images = ip_model.generate(
    pil_image=image,
    prompt=prompt,
    scale=0.7,
    width=960,
    height=1280,
    seed=42
)

images[0].save(f"results/{image_name}")

También puede usarse en ComfyUI consultando ComfyUI-IPAdapter-Flux, o probarse en línea a través de Shakker AI según la tarjeta del modelo.