krea/krea-realtime-video

Instalación y uso básico con Diffusers:

pip install -U diffusers transformers accelerate

import torch
from diffusers import DiffusionPipeline

# switch to "mps" for apple devices
pipe = DiffusionPipeline.from_pretrained("krea/krea-realtime-video", dtype=torch.bfloat16, device_map="cuda")

prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
image = pipe(prompt).images[0]

Servidor de inferencia de Krea:

sudo apt install ffmpeg

git clone https://github.com/krea-ai/realtime-video
cd realtime-video
uv sync
uv pip install flash_attn --no-build-isolation

huggingface-cli download Wan-AI/Wan2.1-T2V-1.3B --local-dir-use-symlinks False --local-dir wan_models/Wan2.1-T2V-1.3B
huggingface-cli download krea/krea-realtime-video krea-realtime-video-14b.safetensors --local-dir-use-symlinks False --local-dir checkpoints/krea-realtime-video-14b.safetensors

export MODEL_FOLDER=Wan-AI
export CUDA_VISIBLE_DEVICES=0
export DO_COMPILE=true
uvicorn release_server:app --host 0.0.0.0 --port 8000

Después se usa la app web local en http://localhost:8000/.

Ejemplo de texto a video con Modular Diffusers:

import torch
from tqdm import tqdm
from diffusers.utils import export_to_video
from diffusers import ModularPipeline
from diffusers.modular_pipelines import PipelineState

repo_id = "krea/krea-realtime-video"
pipe = ModularPipeline.from_pretrained(repo_id, trust_remote_code=True)
pipe.load_components(
    trust_remote_code=True,
    device_map="cuda",
    torch_dtype={"default": torch.bfloat16, "vae": torch.float16},
)

for block in pipe.transformer.blocks:
    block.self_attn.fuse_projections()

num_blocks = 9
frames = []
state = PipelineState()
prompt = ["a cat sitting on a boat"]
generator = torch.Generator(device=pipe.device).manual_seed(42)

for block_idx in tqdm(range(num_blocks)):
    state = pipe(
        state,
        prompt=prompt,
        num_inference_steps=6,
        num_blocks=num_blocks,
        block_idx=block_idx,
        generator=generator,
    )
    frames.extend(state.values["videos"][0])

export_to_video(frames, "output.mp4", fps=24)

Ejemplo de video a video:

import torch
from tqdm import tqdm
from diffusers.utils import load_video, export_to_video
from diffusers import ModularPipeline
from diffusers.modular_pipelines import PipelineState

repo_id = "krea/krea-realtime-video"
pipe = ModularPipeline.from_pretrained(repo_id, trust_remote_code=True)
pipe.load_components(
    trust_remote_code=True,
    device_map="cuda",
    torch_dtype={"default": torch.bfloat16, "vae": torch.float16},
)

for block in pipe.transformer.blocks:
    block.self_attn.fuse_projections()

num_blocks = 9
video = load_video("https://app-uploads.krea.ai/public/a8218957-1a80-43dc-81b2-da970b5f2221-video.mp4")
frames = []
prompt = ["A car racing down a snowy mountain"]
state = PipelineState()
generator = torch.Generator("cuda").manual_seed(42)

for block_idx in tqdm(range(num_blocks)):
    state = pipe(
        state,
        video=video,
        prompt=prompt,
        num_inference_steps=6,
        strength=0.3,
        block_idx=block_idx,
        generator=generator,
    )
    frames.extend(state.values["videos"][0])

export_to_video(frames, "output-v2v.mp4", fps=24)