vectara/hallucination_evaluation_model

Usando con AutoModel

from transformers import AutoModelForSequenceClassification

pairs = [
    ('The capital of France is Berlin.', 'The capital of France is Paris.'),
    ('I am in California', 'I am in United States.'),
    ('I am in United States', 'I am in California.'),
    ('A person on a horse jumps over a broken down airplane.', 'A person is outdoors, on a horse.'),
    ('A boy is jumping on skateboard in the middle of a red bridge.', 'The boy skates down the sidewalk on a red bridge'),
    ('A man with blond-hair, and a brown shirt drinking out of a public water fountain.', 'A blond man wearing a brown shirt is reading a book.'),
    ('Mark Wahlberg was a fan of Manny.', 'Manny was a fan of Mark Wahlberg.')
]

# Paso 1: Cargar el modelo
model = AutoModelForSequenceClassification.from_pretrained(
    'vectara/hallucination_evaluation_model', trust_remote_code=True
)

# Paso 2: Usar el modelo para predecir
model.predict(pairs) # note the predict() method. Do not do model(pairs).
# tensor([0.0111, 0.6474, 0.1290, 0.8969, 0.1846, 0.0050, 0.0543])

Usando con pipeline

from transformers import pipeline, AutoTokenizer

pairs = [
    ('The capital of France is Berlin.', 'The capital of France is Paris.'),
    ('I am in California', 'I am in United States.'),
    ('I am in United States', 'I am in California.'),
    ('A person on a horse jumps over a broken down airplane.', 'A person is outdoors, on a horse.'),
    ('A boy is jumping on skateboard in the middle of a red bridge.', 'The boy skates down the sidewalk on a red bridge'),
    ('A man with blond-hair, and a brown shirt drinking out of a public water fountain.', 'A blond man wearing a brown shirt is reading a book.'),
    ('Mark Wahlberg was a fan of Manny.', 'Manny was a fan of Mark Wahlberg.')
]

# Preparar los pares
prompt = " Determine if the hypothesis is true given the premise?\n\nPremise: {text1}\n\nHypothesis: {text2}"
input_pairs = [prompt.format(text1=pair[0], text2=pair[1]) for pair in pairs]

# Usar pipeline de clasificación de texto para predecir
classifier = pipeline(
    'text-classification',
    model='vectara/hallucination_evaluation_model',
    tokenizer=AutoTokenizer.from_pretrained('google/flan-t5-base'),
    trust_remote_code=True
)
full_scores = classifier(input_pairs, top_k=None) # List[List[Dict[str, float]]]

# Opcional: extraer las puntuaciones para la etiqueta 'consistent'
simple_scores = [score_dict['score'] for score_for_both_labels in full_scores for score_dict in score_for_both_labels if score_dict['label'] == 'consistent']

print(simple_scores)
# Expected output: [0.011061512865126133, 0.6473632454872131, 0.1290171593427658, 0.8969419002532959, 0.18462494015693665, 0.005031010136008263, 0.05432349815964699]