Data

Source code in labicompare/core/data.py

class EvaluationData:
  """
  """

  def __init__(self, data: pd.DataFrame, higher_is_better: bool = True) -> None:
    if not isinstance(data, pd.DataFrame):
      raise TypeError("Input 'data' must be a pandas.DataFrame.")

    self._df = data.copy()
    self.higher_is_better = higher_is_better

    if self._df.isnull().values.any():
      logger.warning(
        "Null values detected. Rows (or datasets) with NaNs will be removed "
        "to ensure the integrity of paired statistical tests and methods."
      )
      self._df = self._df.dropna()

    if self._df.empty:
      raise ValueError(
        "The input DataFrame is empty (or at least become empty after removing NaNs)."
      )

    self.model_names: list[str] = self._df.columns.tolist()
    self.dataset_names: list[str] = self._df.index.tolist()

    self.scores: np.ndarray = self._df.values

    # If higher is better = True, the biggest score/value receive rank 1
    # Otherwise, the lower score receive that rank
    self.ranks_df = self._df.rank(
      axis=1,
      ascending=not self.higher_is_better,
      method="average"
    )

    self.ranks: np.ndarray = self.ranks_df.values

  def __repr__(self) -> str:
    return (
      f"<EvaluationData: {len(self.dataset_names)} datasets, "
      f"{len(self.model_names)} models>"
    )

Source code in labicompare/core/results.py

@dataclass
class ComparisonSummary:
  friedman_stat: float
  friedman_p_value: float
  is_global_sig: bool

  pairwise_results: list[PairwiseResult]

  model_means: dict[str, float]

  alpha: float
  higher_is_better: bool
  n_samples: int

  def to_dataframe(self) -> pd.DataFrame:
    data = []
    for res in self.pairwise_results:
      data.append({
        "Model A": res.model_a,
        "Model B": res.model_b,
        "P-Value": res.p_value,
        "Significant": res.is_significant,
        "Winner": res.winner if res.winner else "Tie",
        "Mean Diff": res.mean_diff
      })
    return pd.DataFrame(data)

  def get_leaderboard(self) -> pd.DataFrame:
    df = pd.Series(self.model_means).to_frame("Mean_Performance")
    df = df.sort_values(by="Mean_Performance", ascending=not self.higher_is_better)
    df["Rank"] = range(1, len(df) + 1)
    return df

  def __repr__(self) -> str:
    sig_str = "REJECTED" if self.is_global_sig else "NOT REJECTED"
    return (
      f"ComparisonSummary(Friedman P-Value={self.friedman_p_value:.4f}, "
      f"H0={sig_str}, Models={len(self.model_means)})"
    )

Source code in labicompare/core/results.py

@dataclass
class PairwiseResult:
  model_a: str
  model_b: str
  p_value: float
  is_significant: bool
  winner: Optional[str]
  mean_diff: float