Modelo African-Cross-Lingua-Embeddings

Usando el modelo:

import torch
from transformers import BertModel, BertTokenizerFast

tokenizer = BertTokenizerFast.from_pretrained("sartifyllc/AviLaBSE")
model = BertModel.from_pretrained("sartifyllc/AviLaBSE")
model = model.eval()

english_sentences = [
"dog",
"Puppies are nice.",
"I enjoy taking long walks along the beach with my dog.",
]
english_inputs = tokenizer(english_sentences, return_tensors="pt", padding=True)

with torch.no_grad():
english_outputs = model(**english_inputs)

# Para obtener las incrustaciones de las oraciones, use la salida del pooler:
english_embeddings = english_outputs.pooler_output

# Salida para otros idiomas con pocos recursos:
swahili_sentences = [
"mbwa",
"Mbwa ni mzuri.",
"Ninafurahia kutembea kwa muda mrefu kando ya pwani na mbwa wangu.",
]
zulu_sentences = [
"inja",
"Inja iyavuma.",
"Ngithanda ukubhema izinyawo ezidlula emanzini nabanye nomfana wami.",
]

gbo_sentences = [
"nwa nkịta",
"Nwa nkịta dị ọma.",
"Achọrọ m gaa n'okirikiri na ụzọ nke oke na mgbidi na nwa nkịta m."
]

swahili_inputs = tokenizer(swahili_sentences, return_tensors="pt", padding=True)
zulu_inputs = tokenizer(zulu_sentences, return_tensors="pt", padding=True)
igbo_inputs=tokenizer(igbo_sentences, return_tensors="pt", padding=True)

with torch.no_grad():
swahili_outputs = model(**swahili_inputs)
zulu_outputs = model(**zulu_inputs)
igbo_outputs = model(**igbo_inputs)

swahili_embeddings = swahili_outputs.pooler_output
zulu_embeddings = zulu_outputs.pooler_output
igbo_embeddings=igbo_outputs.pooler_output

# Para la similitud entre oraciones, se recomienda una L2-norm antes de calcular la similitud:
import torch.nn.functional as F

def similarity(embeddings_1, embeddings_2):
normalized_embeddings_1 = F.normalize(embeddings_1, p=2)
normalized_embeddings_2 = F.normalize(embeddings_2, p=2)
return torch.matmul(
normalized_embeddings_1, normalized_embeddings_2.transpose(0, 1)
)

print(similarity(english_embeddings, swahili_embeddings))
print(similarity(english_embeddings, zulu_embeddings))
print(similarity(swahili_embeddings, igbo_embeddings))