Source code for fanoutqa.eval.models

from dataclasses import dataclass
from typing import TypedDict

[docs] @dataclass class AccuracyScore: loose: float """Loose accuracy: The mean proportion of reference strings found in the generation.""" strict: float """Strict accuracy: The proportion of questions with a loose accuracy of 1.0."""
[docs] @dataclass class RougeScorePart: precision: float recall: float fscore: float
[docs] @dataclass class RougeScore: rouge1: RougeScorePart rouge2: RougeScorePart rougeL: RougeScorePart
[docs] @dataclass class EvaluationScore: acc: AccuracyScore rouge: RougeScore bleurt: float gpt: float
[docs] class Answer(TypedDict): """A dictionary of the form ``{"id": "...", "answer": "..."}``.""" id: str answer: str