masterful/gligen-1-4-generation-text-box

Instalación y uso básico con Diffusers:

pip install -U diffusers transformers accelerate

import torch
from diffusers import DiffusionPipeline

# switch to "mps" for apple devices
pipe = DiffusionPipeline.from_pretrained(
    "masterful/gligen-1-4-generation-text-box",
    dtype=torch.bfloat16,
    device_map="cuda"
)

prompt = "A high tech solarpunk utopia in the Amazon rainforest"
image = pipe(prompt).images[0]

Ejemplo con cajas delimitadoras usando StableDiffusionGLIGENPipeline:

pip install --upgrade diffusers transformers scipy

import torch
from diffusers import StableDiffusionGLIGENPipeline
from diffusers.utils import load_image

# Generate an image described by the prompt and
# insert objects described by text at the region defined by bounding boxes
pipe = StableDiffusionGLIGENPipeline.from_pretrained(
    "masterful/gligen-1-4-generation-text-box",
    variant="fp16",
    torch_dtype=torch.float16
)
pipe = pipe.to("cuda")

prompt = "a waterfall and a modern high speed train running through the tunnel in a beautiful forest with fall foliage"
boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]]
phrases = ["a waterfall", "a modern high speed train running through the tunnel"]

images = pipe(
    prompt=prompt,
    gligen_phrases=phrases,
    gligen_boxes=boxes,
    gligen_scheduled_sampling_beta=1,
    output_type="pil",
    num_inference_steps=50,
).images

images[0].save("./gligen-1-4-generation-text-box.jpg")