Tune-A-Video - Modern Disney

Cargando con un checkpoint preexistente de Text2Image

import torch
from diffusers import TuneAVideoPipeline, DDIMScheduler, UNet3DConditionModel
from diffusers.utils import export_to_video
from PIL import Image

# Use cualquier checkpoint preentrenado de Text2Image basado en stable diffusion
pretrained_model_path = "nitrosocke/mo-di-diffusion"
unet = UNet3DConditionModel.from_pretrained(
"Tune-A-Video-library/df-cpt-mo-di-bear-guitar", subfolder="unet", torch_dtype=torch.float16
).to("cuda")

pipe = TuneAVideoPipeline.from_pretrained(pretrained_model_path, unet=unet, torch_dtype=torch.float16).to("cuda")

prompt = "A princess playing a guitar, modern disney style"
generator = torch.Generator(device="cuda").manual_seed(42)

video_frames = pipe(prompt, video_length=3, generator=generator, num_inference_steps=50, output_type="np").frames

# Guardando como gif.
pil_frames = [Image.fromarray(frame) for frame in video_frames]
duration = len(pil_frames) / 8
pil_frames[0].save(
"animation.gif",
save_all=True,
append_images=pil_frames[1:], # apendiza el resto de las imágenes
duration=duration * 1000, # en milisegundos
loop=0,
)

# Guardando como video
video_path = export_to_video(video_frames)

Cargando un checkpoint guardado de Tune-A-Video

import torch
from diffusers import DiffusionPipeline, DDIMScheduler
from diffusers.utils import export_to_video
from PIL import Image

pipe = DiffusionPipeline.from_pretrained(
"Tune-A-Video-library/df-cpt-mo-di-bear-guitar", torch_dtype=torch.float16
).to("cuda")

prompt = "A princess playing a guitar, modern disney style"
generator = torch.Generator(device="cuda").manual_seed(42)

video_frames = pipe(prompt, video_length=3, generator=generator, num_inference_steps=50, output_type="np").frames

# Guardando como gif.
pil_frames = [Image.fromarray(frame) for frame in video_frames]
duration = len(pil_frames) / 8
pil_frames[0].save(
"animation.gif",
save_all=True,
append_images=pil_frames[1:], # apendiza el resto de las imágenes
duration=duration * 1000, # en milisegundos
loop=0,
)

# Guardando como video
video_path = export_to_video(video_frames)