videomae-base-finetuned-kinetics-movieshots-scale

Este es el modo de prueba del modelo en un clip de vídeo. El mismo código se utiliza para procesar, transformar y evaluar en el set de prueba de Movienet.

from transformers import VideoMAEImageProcessor, VideoMAEForVideoClassification
from pytorchvideo.transforms import ApplyTransformToKey
from torchvision.transforms import v2
from decord import VideoReader, cpu

## Transformación de evaluación
transform = v2.Compose(
[
ApplyTransformToKey(
key="video",
transform=v2.Compose(
[
v2.Lambda(lambda x: x.permute(0, 3, 1, 2)), # T, H, W, C -> T, C, H, W
v2.UniformTemporalSubsample(16),
v2.Resize(resize_to),
v2.Lambda(lambda x: x / 255.0),
v2.Normalize(img_mean, img_std)
]
),
),
]
)

## Carga del preprocesador y el modelo
image_processor = VideoMAEImageProcessor.from_pretrained("gullalc/videomae-base-finetuned-kinetics-movieshots-scale")
model = VideoMAEForVideoClassification.from_pretrained("gullalc/videomae-base-finetuned-kinetics-movieshots-scale")

img_mean = image_processor.image_mean
img_std = image_processor.image_std
height = width = image_processor.size["shortest_edge"]
resize_to = (height, width)

## Cargar vídeo/clip y predecir
video_path = "random_clip.mp4"
vr = VideoReader(video_path, width=480, height=270, ctx=cpu(0))
frames_tensor = torch.stack([torch.tensor(vr[i].asnumpy()) for i in range(len(vr))]) ## Forma: (T, H, W, C)

frames_tensor = transform({"video": frames_tensor})["video"]

output = model(pixel_values=frames_tensor)
pred = torch.argmax(outputs.logits, axis=1).cpu().numpy()

print(model.config.id2label[pred[0]])