{ "schema_version": 1, "cohorts": { "v1": { "categories": { "all": { "n_events": 790, "n_species": 79, "models": { "bioclip": { "metrics": { "species_accuracy": { "value": 0.40759493670886077, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.37744358138345485, "high": 0.43668278056331855, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.9999999999999999, "high": 1.0000000000000002, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.40759493670886077, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.3774435813834548, "high": 0.4366827805633185, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.19377658997623626, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.17633500057238008, "high": 0.20292895092335855, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": null, "wrong_species_rate_all_events": { "value": 0.5924050632911393, "unit": "proportion", "denominator": 790, "interval": { "low": 0.5633172194366814, "high": 0.6225564186165452, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.45378062209624936, "unit": "f1", "denominator": null, "interval": { "low": 0.4129363937454471, "high": 0.47521336608634596, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.43356643356643354, "unit": "proportion", "denominator": 429, "interval": { "low": 0.38564615681848224, "high": 0.4819290676218387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 429, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.43356643356643354, "unit": "proportion", "denominator": 429, "interval": { "low": 0.38564615681848224, "high": 0.4819290676218387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5664335664335665, "unit": "proportion", "denominator": 429, "interval": { "low": 0.5180709323781613, "high": 0.6143538431815176, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.22525144798559688, "unit": "f1", "denominator": null, "interval": { "low": 0.18581102096437715, "high": 0.2327245798078507, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4650352474541355, "unit": "f1", "denominator": null, "interval": { "low": 0.3836098497329077, "high": 0.48046364863556285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3767313019390582, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3242506811989101, "high": 0.429730288139379, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 361, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3767313019390582, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3242506811989101, "high": 0.429730288139379, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6232686980609419, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5702697118606209, "high": 0.6757493188010899, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.19503914425228208, "unit": "f1", "denominator": null, "interval": { "low": 0.1471057016719108, "high": 0.20126324698524756, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3768552956739009, "unit": "f1", "denominator": null, "interval": { "low": 0.28423813543386156, "high": 0.3888815280731902, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.5706666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5190208791964778, "high": 0.6215506153351457, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 375, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5706666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5190208791964778, "high": 0.6215506153351457, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.42933333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.3784493846648543, "high": 0.4809791208035223, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3569681825862559, "unit": "f1", "denominator": null, "interval": { "low": 0.3021033359601287, "high": 0.36228625823980637, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.576640910331644, "unit": "f1", "denominator": null, "interval": { "low": 0.48801308116636166, "high": 0.5852316479258409, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2727272727272727, "high": 0.3955579710144927, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 240, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2727272727272727, "high": 0.3955579710144927, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6666666666666666, "unit": "proportion", "denominator": 240, "interval": { "low": 0.6044420289855073, "high": 0.7272727272727273, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.19571272127354958, "unit": "f1", "denominator": null, "interval": { "low": 0.13889177057499402, "high": 0.20008868122703138, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3022398986756082, "unit": "f1", "denominator": null, "interval": { "low": 0.21449108873606673, "high": 0.3089977102493396, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.15517241379310345, "unit": "proportion", "denominator": 174, "interval": { "low": 0.10285328185328185, "high": 0.21142857142857144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 174, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.15517241379310345, "unit": "proportion", "denominator": 174, "interval": { "low": 0.10285328185328185, "high": 0.21142857142857144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.8448275862068966, "unit": "proportion", "denominator": 174, "interval": { "low": 0.7885714285714286, "high": 0.8971467181467181, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.07969720820960491, "unit": "f1", "denominator": null, "interval": { "low": 0.04284947343218079, "high": 0.08733185199203032, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.12523847004366484, "unit": "f1", "denominator": null, "interval": { "low": 0.0673348868219984, "high": 0.13723576741604762, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.5918367346938775, "unit": "proportion", "denominator": 245, "interval": { "low": 0.529143220735149, "high": 0.6536964980544747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 245, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5918367346938775, "unit": "proportion", "denominator": 245, "interval": { "low": 0.529143220735149, "high": 0.6536964980544747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.40816326530612246, "unit": "proportion", "denominator": 245, "interval": { "low": 0.3463035019455253, "high": 0.470856779264851, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3709912983692708, "unit": "f1", "denominator": null, "interval": { "low": 0.29522804535874536, "high": 0.37082372741107344, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5910708821476517, "unit": "f1", "denominator": null, "interval": { "low": 0.47036332650376383, "high": 0.5908039046888289, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5307692307692308, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4428544165757906, "high": 0.6194029850746269, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 130, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5307692307692308, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4428544165757906, "high": 0.6194029850746269, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.46923076923076923, "unit": "proportion", "denominator": 130, "interval": { "low": 0.3805970149253731, "high": 0.5571455834242094, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3512735981966752, "unit": "f1", "denominator": null, "interval": { "low": 0.24627254281666044, "high": 0.35753108003108003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5189269064269064, "unit": "f1", "denominator": null, "interval": { "low": 0.3638117109791575, "high": 0.5281709136822773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.3177570093457944, "unit": "proportion", "denominator": 107, "interval": { "low": 0.23075221238938057, "high": 0.4056673827760134, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 107, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3177570093457944, "unit": "proportion", "denominator": 107, "interval": { "low": 0.23075221238938057, "high": 0.4056673827760134, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6822429906542056, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5943326172239866, "high": 0.7692477876106193, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.19332045732045733, "unit": "f1", "denominator": null, "interval": { "low": 0.11640574795574796, "high": 0.2008696803196803, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2842947901771431, "unit": "f1", "denominator": null, "interval": { "low": 0.1711849234643352, "high": 0.2953965887054122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3458646616541353, "unit": "proportion", "denominator": 133, "interval": { "low": 0.26153846153846155, "high": 0.4296875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 133, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3458646616541353, "unit": "proportion", "denominator": 133, "interval": { "low": 0.26153846153846155, "high": 0.4296875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6541353383458647, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5703125, "high": 0.7384615384615385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.20422734764840028, "unit": "f1", "denominator": null, "interval": { "low": 0.1273655459317224, "high": 0.20972793816543817, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.30842497399962493, "unit": "f1", "denominator": null, "interval": { "low": 0.19234796732545828, "high": 0.3167319882498454, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.07894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.02531645569620253, "high": 0.14492753623188406, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.07894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.02531645569620253, "high": 0.14492753623188406, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.9210526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.855072463768116, "high": 0.9746835443037974, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.048546691403834254, "unit": "f1", "denominator": null, "interval": { "low": 0.011544011544011544, "high": 0.0567099567099567, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.08126293995859213, "unit": "f1", "denominator": null, "interval": { "low": 0.01932367149758454, "high": 0.09492753623188407, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.13541666666666666, "high": 0.3, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 98, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.13541666666666666, "high": 0.3, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 98, "interval": { "low": 0.7, "high": 0.8645833333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.1114969135802469, "unit": "f1", "denominator": null, "interval": { "low": 0.058595792483660133, "high": 0.12577215608465608, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.1745169082125604, "unit": "f1", "denominator": null, "interval": { "low": 0.09171515345268542, "high": 0.19686076604554864, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 790, "population_events": 790 } } }, "speciesnet": { "metrics": { "species_accuracy": { "value": 0.2759493670886076, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.2565859544815241, "high": 0.2972859471751877, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.48481012658227846, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.46199542897327717, "high": 0.5091264133194513, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.5691906005221932, "unit": "proportion", "denominator": 383, "population_events": 790, "interval": { "low": 0.5307347404469154, "high": 0.6091895497121522, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.21071676757461957, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.19353306451633928, "high": 0.22214092042227154, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 0.6272151898734178, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 0.47349599459963393, "high": 0.7919265404850383, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.2088607594936709, "unit": "proportion", "denominator": 790, "interval": { "low": 0.18530846439074286, "high": 0.23283848886696987, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.30940689922349207, "unit": "f1", "denominator": null, "interval": { "low": 0.28417513270753625, "high": 0.3261816046706772, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 1.9753164556962028, "unit": "points_per_event", "denominator": 790, "interval": { "low": 1.8936018240725518, "high": 2.063675550451658, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.26962025316455696, "unit": "proportion", "denominator": 790, "interval": { "low": 0.24453574730473462, "high": 0.29380076410930844, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.20279720279720279, "unit": "proportion", "denominator": 429, "interval": { "low": 0.16173021566427545, "high": 0.24489910570969958, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3613053613053613, "unit": "proportion", "denominator": 429, "interval": { "low": 0.31279568657744744, "high": 0.41067285382830626, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5612903225806452, "unit": "proportion", "denominator": 155, "interval": { "low": 0.47770700636942676, "high": 0.6456733923884513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1585081585081585, "unit": "proportion", "denominator": 429, "interval": { "low": 0.12284904521746627, "high": 0.19529499188210314, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.5407925407925408, "unit": "points_per_event", "denominator": 429, "interval": { "low": 0.27845000459015645, "high": 0.8170738306409038, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5314685314685317, "unit": "points_per_event", "denominator": 429, "interval": { "low": 1.3355511875511876, "high": 1.7330509169404138, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.24428938270401684, "unit": "f1", "denominator": null, "interval": { "low": 0.18414097253697911, "high": 0.2530281419835335, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3230924093827319, "unit": "f1", "denominator": null, "interval": { "low": 0.2435412862585853, "high": 0.3346501232685443, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19813519813519814, "unit": "proportion", "denominator": 429, "interval": { "low": 0.1592479431708823, "high": 0.2375588235294117, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3628808864265928, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3103448275862069, "high": 0.41690705003501666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.631578947368421, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5782122905027933, "high": 0.6840695488721804, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5745614035087719, "unit": "proportion", "denominator": 228, "interval": { "low": 0.5064935064935064, "high": 0.6431785765869863, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.26869806094182824, "unit": "proportion", "denominator": 361, "interval": { "low": 0.22126436781609196, "high": 0.3172062048798318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.7299168975069252, "unit": "points_per_event", "denominator": 361, "interval": { "low": 0.34613874345549744, "high": 1.1223283565823887, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.5027700831024937, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.2835161339787478, "high": 2.724413619378087, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.25608055726816586, "unit": "f1", "denominator": null, "interval": { "low": 0.21684344650954213, "high": 0.2680882591219495, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.35590857111846774, "unit": "f1", "denominator": null, "interval": { "low": 0.3013756375217365, "high": 0.37259724149152307, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3545706371191136, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3019929208987381, "high": 0.4069767441860465, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.3253333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.2761394101876676, "high": 0.3763164968873797, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.512, "unit": "proportion", "denominator": 375, "interval": { "low": 0.4576247968423497, "high": 0.5646437994722955, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6354166666666666, "unit": "proportion", "denominator": 192, "interval": { "low": 0.564591372125328, "high": 0.7055214723926381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18666666666666668, "unit": "proportion", "denominator": 375, "interval": { "low": 0.14758165436115628, "high": 0.2281556816627923, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.0760000000000003, "unit": "points_per_event", "denominator": 375, "interval": { "low": 0.7533688205921247, "high": 1.4008179660894136, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.2226666666666666, "unit": "points_per_event", "denominator": 375, "interval": { "low": 1.9975111750890218, "high": 2.4471649484536084, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2765855604090898, "unit": "f1", "denominator": null, "interval": { "low": 0.22849024069612303, "high": 0.28288112509819346, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3616888097657328, "unit": "f1", "denominator": null, "interval": { "low": 0.298794930141084, "high": 0.36992147128225294, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.22933333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.18591420249244156, "high": 0.27520435967302453, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.2375, "unit": "proportion", "denominator": 240, "interval": { "low": 0.18375850340136055, "high": 0.29338842975206614, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5125, "unit": "proportion", "denominator": 240, "interval": { "low": 0.44680633065922565, "high": 0.5757575757575758, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4634146341463415, "unit": "proportion", "denominator": 123, "interval": { "low": 0.37189534118335404, "high": 0.5564533660589059, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.275, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2155154892066162, "high": 0.3359691699604743, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.13958333333333336, "unit": "points_per_event", "denominator": 240, "interval": { "low": -0.26794018579888085, "high": 0.5576962809917355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.8687500000000001, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.6193415637860082, "high": 2.123377314175187, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.18365873608592054, "unit": "f1", "denominator": null, "interval": { "low": 0.13449501550229706, "high": 0.19284751647425247, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2394537951499977, "unit": "f1", "denominator": null, "interval": { "low": 0.17535426071818475, "high": 0.2514341037575697, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3458333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2833315450643777, "high": 0.4085636948062933, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.22413793103448276, "unit": "proportion", "denominator": 174, "interval": { "low": 0.15950920245398773, "high": 0.29081864564007415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.39080459770114945, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3149171270718232, "high": 0.4662585303646515, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5735294117647058, "unit": "proportion", "denominator": 68, "interval": { "low": 0.44926142697881827, "high": 0.691381766381766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 174, "interval": { "low": 0.11299435028248588, "high": 0.2236024844720497, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.3333333333333333, "unit": "points_per_event", "denominator": 174, "interval": { "low": -0.14534986805211944, "high": 0.7967142857142855, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5977011494252873, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.2959708788898237, "high": 1.9043779020183835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.18976279650436953, "unit": "f1", "denominator": null, "interval": { "low": 0.1136609270878934, "high": 0.18685092593387373, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.21933621933621933, "unit": "f1", "denominator": null, "interval": { "low": 0.13137431832237026, "high": 0.21597055075473715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.25287356321839083, "unit": "proportion", "denominator": 174, "interval": { "low": 0.18888364779874214, "high": 0.31976744186046513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.24897959183673468, "unit": "proportion", "denominator": 245, "interval": { "low": 0.19343028145745098, "high": 0.3080168776371308, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.40816326530612246, "unit": "proportion", "denominator": 245, "interval": { "low": 0.34321723636621515, "high": 0.47388097164778514, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.61, "unit": "proportion", "denominator": 100, "interval": { "low": 0.5094295946890287, "high": 0.7096774193548387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15918367346938775, "unit": "proportion", "denominator": 245, "interval": { "low": 0.11290165683444207, "high": 0.20779348102877515, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.8816326530612246, "unit": "points_per_event", "denominator": 245, "interval": { "low": 0.5365200528139957, "high": 1.2425657505910166, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.8, "unit": "points_per_event", "denominator": 245, "interval": { "low": 1.5376958020050129, "high": 2.0705887965873373, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.27746545691751173, "unit": "f1", "denominator": null, "interval": { "low": 0.19466994371321317, "high": 0.2806427039474985, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3433047178809891, "unit": "f1", "denominator": null, "interval": { "low": 0.24086281171295867, "high": 0.34723588793504045, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1836734693877551, "unit": "proportion", "denominator": 245, "interval": { "low": 0.13461322869955156, "high": 0.23577235772357724, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.46923076923076923, "unit": "proportion", "denominator": 130, "interval": { "low": 0.38297456781914896, "high": 0.5581395348837209, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7076923076923077, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6260851891127608, "high": 0.7867647058823529, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6630434782608695, "unit": "proportion", "denominator": 92, "interval": { "low": 0.5652173913043478, "high": 0.7592681623931623, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.23846153846153847, "unit": "proportion", "denominator": 130, "interval": { "low": 0.16778523489932887, "high": 0.31297709923664124, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.4423076923076925, "unit": "points_per_event", "denominator": 130, "interval": { "low": 0.8418699186991871, "high": 2.059531440781441, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0192307692307696, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.6694182338777903, "high": 3.375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.39531404268246373, "unit": "f1", "denominator": null, "interval": { "low": 0.2764176065162907, "high": 0.3891403382192857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5121113734750098, "unit": "f1", "denominator": null, "interval": { "low": 0.35808644480519486, "high": 0.5041136199658927, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3153846153846154, "unit": "proportion", "denominator": 130, "interval": { "low": 0.23577235772357724, "high": 0.3969465648854962, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.1588785046728972, "unit": "proportion", "denominator": 107, "interval": { "low": 0.09174120795107034, "high": 0.23232323232323232, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.32710280373831774, "unit": "proportion", "denominator": 107, "interval": { "low": 0.2389279045222051, "high": 0.41818181818181815, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4857142857142857, "unit": "proportion", "denominator": 35, "interval": { "low": 0.3142857142857143, "high": 0.6538461538461539, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16822429906542055, "unit": "proportion", "denominator": 107, "interval": { "low": 0.10084033613445378, "high": 0.2413793103448276, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.2897196261682243, "unit": "points_per_event", "denominator": 107, "interval": { "low": -0.20002136752136754, "high": 0.7833376288659794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.3177570093457946, "unit": "points_per_event", "denominator": 107, "interval": { "low": 0.9826086956521739, "high": 1.675, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.15526932084309134, "unit": "f1", "denominator": null, "interval": { "low": 0.07467896174863388, "high": 0.16159250585480095, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.18571428571428572, "unit": "f1", "denominator": null, "interval": { "low": 0.0893218954248366, "high": 0.19327731092436973, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.205607476635514, "unit": "proportion", "denominator": 107, "interval": { "low": 0.13157894736842105, "high": 0.2868870277975766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3007518796992481, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2222222222222222, "high": 0.3821138211382114, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6616541353383458, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5781163194444444, "high": 0.7416666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.45454545454545453, "unit": "proportion", "denominator": 88, "interval": { "low": 0.34615384615384615, "high": 0.5643564356435643, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3609022556390977, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2773109243697479, "high": 0.44696969696969696, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.018796992481203006, "unit": "points_per_event", "denominator": 133, "interval": { "low": -0.5985568010659252, "high": 0.6679961499550355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.31203007518797, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.9830405484934088, "high": 2.6500093984962407, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.21611721611721613, "unit": "f1", "denominator": null, "interval": { "low": 0.14600068232421173, "high": 0.22378513831568128, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.28668610301263364, "unit": "f1", "denominator": null, "interval": { "low": 0.19367437451170944, "high": 0.2968578365412099, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.45864661654135336, "unit": "proportion", "denominator": 133, "interval": { "low": 0.3700787401574803, "high": 0.5447761194029851, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": { "low": 0.05063291139240506, "high": 0.19480519480519481, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.2631578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.16666666666666666, "high": 0.3625, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.45, "unit": "proportion", "denominator": 20, "interval": { "low": 0.22727272727272727, "high": 0.6666666666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.07017543859649122, "high": 0.22727272727272727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.20394736842105263, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.7957815974796146, "high": 0.391304347826087, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 0.9802631578947368, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.6296296296296298, "high": 1.3571669884169884, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.16049382716049382, "unit": "f1", "denominator": null, "interval": { "low": 0.05555555555555555, "high": 0.14814814814814814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.18840579710144925, "unit": "f1", "denominator": null, "interval": { "low": 0.06521739130434782, "high": 0.17391304347826086, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14544721407624633, "high": 0.3333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.30612244897959184, "unit": "proportion", "denominator": 98, "interval": { "low": 0.21176470588235294, "high": 0.40426129572077446, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4897959183673469, "unit": "proportion", "denominator": 98, "interval": { "low": 0.38823529411764707, "high": 0.59, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.625, "unit": "proportion", "denominator": 48, "interval": { "low": 0.4782608695652174, "high": 0.761924939467312, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1836734693877551, "unit": "proportion", "denominator": 98, "interval": { "low": 0.11008944954128441, "high": 0.2631578947368421, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.75, "unit": "points_per_event", "denominator": 98, "interval": { "low": 0.06382717980809345, "high": 1.4224856149604412, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.076530612244898, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.6565476190476194, "high": 2.509031308536259, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.24488536155202822, "unit": "f1", "denominator": null, "interval": { "low": 0.1366547958214625, "high": 0.24438029563029562, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2874741200828157, "unit": "f1", "denominator": null, "interval": { "low": 0.16042084726867334, "high": 0.28688121660947746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2653061224489796, "unit": "proportion", "denominator": 98, "interval": { "low": 0.18018018018018017, "high": 0.35714285714285715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 790, "population_events": 790 } } }, "bioclip-country": { "metrics": { "species_accuracy": { "value": 0.43924050632911393, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.40883941743910096, "high": 0.4685585871582707, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.9999999999999999, "high": 1.0000000000000002, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.43924050632911393, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.4088394174391009, "high": 0.4685585871582707, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.27682104545154496, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.25388708203886007, "high": 0.28975491186793273, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": null, "wrong_species_rate_all_events": { "value": 0.5607594936708861, "unit": "proportion", "denominator": 790, "interval": { "low": 0.5314414128417292, "high": 0.591160582560899, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.47304862197415903, "unit": "f1", "denominator": null, "interval": { "low": 0.4338576718385583, "high": 0.49515079876165696, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.4592074592074592, "unit": "proportion", "denominator": 429, "interval": { "low": 0.4110552226027397, "high": 0.5072824413923751, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 429, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4592074592074592, "unit": "proportion", "denominator": 429, "interval": { "low": 0.4110552226027397, "high": 0.5072824413923751, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5407925407925408, "unit": "proportion", "denominator": 429, "interval": { "low": 0.4927175586076249, "high": 0.5889447773972603, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.298983907576611, "unit": "f1", "denominator": null, "interval": { "low": 0.2477162346035455, "high": 0.30748940186481716, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.49669907226436993, "unit": "f1", "denominator": null, "interval": { "low": 0.4115285832929869, "high": 0.5108291676141319, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.4155124653739612, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3628755309325946, "high": 0.46978356528857956, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 361, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4155124653739612, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3628755309325946, "high": 0.46978356528857956, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5844875346260388, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5302164347114204, "high": 0.6371244690674053, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.27176729202217, "unit": "f1", "denominator": null, "interval": { "low": 0.20811538906329702, "high": 0.2800955122292008, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.39613537481197664, "unit": "f1", "denominator": null, "interval": { "low": 0.3033546349058227, "high": 0.4082748144357842, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.6026666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5524296675191815, "high": 0.6520547945205479, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 375, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6026666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5524296675191815, "high": 0.6520547945205479, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3973333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.34794520547945207, "high": 0.4475703324808184, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.44580929226526594, "unit": "f1", "denominator": null, "interval": { "low": 0.38067166726259244, "high": 0.4522095006908864, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6001278934340117, "unit": "f1", "denominator": null, "interval": { "low": 0.512442629007336, "high": 0.6087435586223469, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.375, "unit": "proportion", "denominator": 240, "interval": { "low": 0.31336257194280176, "high": 0.4393305439330544, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 240, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.375, "unit": "proportion", "denominator": 240, "interval": { "low": 0.31336257194280176, "high": 0.4393305439330544, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.625, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5606694560669456, "high": 0.6866374280571982, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2610571617924559, "unit": "f1", "denominator": null, "interval": { "low": 0.19009548329839487, "high": 0.2654312600239458, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3370611456054494, "unit": "f1", "denominator": null, "interval": { "low": 0.24543973792957313, "high": 0.34270871547395537, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.1724137931034483, "unit": "proportion", "denominator": 174, "interval": { "low": 0.11666666666666667, "high": 0.23115645431458368, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 174, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.1724137931034483, "unit": "proportion", "denominator": 174, "interval": { "low": 0.11666666666666667, "high": 0.23115645431458368, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.8275862068965517, "unit": "proportion", "denominator": 174, "interval": { "low": 0.7688435456854164, "high": 0.8833333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.10125866241293545, "unit": "f1", "denominator": null, "interval": { "low": 0.05421660362698098, "high": 0.10816291556553269, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.1393950417632618, "unit": "f1", "denominator": null, "interval": { "low": 0.07463584395402577, "high": 0.1488995980512528, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.6204081632653061, "unit": "proportion", "denominator": 245, "interval": { "low": 0.5575202410092261, "high": 0.6816326530612244, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 245, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6204081632653061, "unit": "proportion", "denominator": 245, "interval": { "low": 0.5575202410092261, "high": 0.6816326530612244, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3795918367346939, "unit": "proportion", "denominator": 245, "interval": { "low": 0.3183673469387755, "high": 0.44247975899077385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.4624122364563541, "unit": "f1", "denominator": null, "interval": { "low": 0.3679518601190476, "high": 0.46041481782727794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6269996426526834, "unit": "f1", "denominator": null, "interval": { "low": 0.49891777643260693, "high": 0.6242912784098683, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5692307692307692, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4827586206896552, "high": 0.656934306569343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 130, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5692307692307692, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4827586206896552, "high": 0.656934306569343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4307692307692308, "unit": "proportion", "denominator": 130, "interval": { "low": 0.34306569343065696, "high": 0.5172413793103449, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.4355060613989185, "unit": "f1", "denominator": null, "interval": { "low": 0.31235756802721093, "high": 0.4413523158389229, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5542804417804418, "unit": "f1", "denominator": null, "interval": { "low": 0.39754599567099574, "high": 0.5617211292495383, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.34579439252336447, "unit": "proportion", "denominator": 107, "interval": { "low": 0.25742574257425743, "high": 0.4347909060736806, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 107, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.34579439252336447, "unit": "proportion", "denominator": 107, "interval": { "low": 0.25742574257425743, "high": 0.4347909060736806, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6542056074766355, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5652090939263192, "high": 0.7425742574257426, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2116708908375575, "unit": "f1", "denominator": null, "interval": { "low": 0.13088819745069744, "high": 0.2191927208593875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.29882949294714, "unit": "f1", "denominator": null, "interval": { "low": 0.18478333757745521, "high": 0.30944854709560593, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.39849624060150374, "unit": "proportion", "denominator": 133, "interval": { "low": 0.3115912330719886, "high": 0.4838709677419355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 133, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.39849624060150374, "unit": "proportion", "denominator": 133, "interval": { "low": 0.3115912330719886, "high": 0.4838709677419355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6015037593984962, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5161290322580645, "high": 0.6884087669280113, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.29384673706707604, "unit": "f1", "denominator": null, "interval": { "low": 0.19188352960598723, "high": 0.29939263662522686, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.35381545891749977, "unit": "f1", "denominator": null, "interval": { "low": 0.23104343360720914, "high": 0.3604931747120078, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.03125, "high": 0.16216216216216217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.03125, "high": 0.16216216216216217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.9078947368421053, "unit": "proportion", "denominator": 76, "interval": { "low": 0.8378378378378378, "high": 0.96875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.05592020592020591, "unit": "f1", "denominator": null, "interval": { "low": 0.014414414414414415, "high": 0.06516545391545392, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.08995859213250516, "unit": "f1", "denominator": null, "interval": { "low": 0.02318840579710145, "high": 0.10483138238573021, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.23469387755102042, "unit": "proportion", "denominator": 98, "interval": { "low": 0.15217391304347827, "high": 0.32222222222222224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 98, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.23469387755102042, "unit": "proportion", "denominator": 98, "interval": { "low": 0.15217391304347827, "high": 0.32222222222222224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.7653061224489796, "unit": "proportion", "denominator": 98, "interval": { "low": 0.6777777777777778, "high": 0.8478260869565217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.14702956424267902, "unit": "f1", "denominator": null, "interval": { "low": 0.07613816738816738, "high": 0.16141058167697508, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.19497398736529173, "unit": "f1", "denominator": null, "interval": { "low": 0.10096583066691761, "high": 0.2140444670064235, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 790, "population_events": 790 } } }, "speciesnet-country": { "metrics": { "species_accuracy": { "value": 0.28354430379746837, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.26437227593082024, "high": 0.3049054286079602, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.34810126582278483, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.3270305678849983, "high": 0.37001442430635473, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.8145454545454546, "unit": "proportion", "denominator": 275, "population_events": 790, "interval": { "low": 0.7802622064846542, "high": 0.8515058971710102, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.2996057929727042, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.27668206816782753, "high": 0.31512404802767724, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 1.1753164556962028, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 1.0471834244342153, "high": 1.3128998221363726, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.06455696202531645, "unit": "proportion", "denominator": 790, "interval": { "low": 0.05072203288975441, "high": 0.07814962463696641, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.3185681849330019, "unit": "f1", "denominator": null, "interval": { "low": 0.29419359146958873, "high": 0.33506860802942895, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 1.979113924050633, "unit": "points_per_event", "denominator": 790, "interval": { "low": 1.898304604590206, "high": 2.067126918440209, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.16075949367088607, "unit": "proportion", "denominator": 790, "interval": { "low": 0.14020044126056783, "high": 0.18097017785308925, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.21212121212121213, "unit": "proportion", "denominator": 429, "interval": { "low": 0.17084282460136674, "high": 0.2548555587283457, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.28438228438228436, "unit": "proportion", "denominator": 429, "interval": { "low": 0.2399976247030879, "high": 0.33049834700954095, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7459016393442623, "unit": "proportion", "denominator": 122, "interval": { "low": 0.6607142857142857, "high": 0.8260869565217391, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07226107226107226, "unit": "proportion", "denominator": 429, "interval": { "low": 0.04741379310344827, "high": 0.09887640449438202, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.8752913752913752, "unit": "points_per_event", "denominator": 429, "interval": { "low": 0.620719965409601, "high": 1.134664338013414, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5396270396270397, "unit": "points_per_event", "denominator": 429, "interval": { "low": 1.3418575314700236, "high": 1.744530675738047, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3155633928361201, "unit": "f1", "denominator": null, "interval": { "low": 0.24246509857023862, "high": 0.32642063674260646, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3359223214061924, "unit": "f1", "denominator": null, "interval": { "low": 0.25810800815541524, "high": 0.34748003266148425, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13286713286713286, "unit": "proportion", "denominator": 429, "interval": { "low": 0.10048962341946124, "high": 0.16625916870415647, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 361, "interval": { "low": 0.31666666666666665, "high": 0.4231807883120434, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.42382271468144045, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3693170332026464, "high": 0.47863475177304965, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.869281045751634, "unit": "proportion", "denominator": 153, "interval": { "low": 0.8092105263157895, "high": 0.9246575342465754, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.055401662049861494, "unit": "proportion", "denominator": 361, "interval": { "low": 0.03154506340784395, "high": 0.08256948391112554, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5318559556786706, "unit": "points_per_event", "denominator": 361, "interval": { "low": 1.1976325290878316, "high": 1.8682243316686251, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.501385041551247, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.279096309641562, "high": 2.7249301575931235, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.34954589419670884, "unit": "f1", "denominator": null, "interval": { "low": 0.29738036619515024, "high": 0.36543287850737854, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.36139490755930914, "unit": "f1", "denominator": null, "interval": { "low": 0.30746105657464684, "high": 0.3778204337110185, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19390581717451524, "unit": "proportion", "denominator": 361, "interval": { "low": 0.15193282747045248, "high": 0.23782234957020057, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.33866666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.28895026754800124, "high": 0.39071126048295335, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4106666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.35883700043230027, "high": 0.46329422661315217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8246753246753247, "unit": "proportion", "denominator": 154, "interval": { "low": 0.7579580513535032, "high": 0.8875828912466842, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.072, "unit": "proportion", "denominator": 375, "interval": { "low": 0.04497354497354497, "high": 0.10084033613445378, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.568, "unit": "points_per_event", "denominator": 375, "interval": { "low": 1.2716515448259325, "high": 1.8679269227085529, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.2346666666666666, "unit": "points_per_event", "denominator": 375, "interval": { "low": 2.0076918158567776, "high": 2.4622674357491987, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3580125159393452, "unit": "f1", "denominator": null, "interval": { "low": 0.2972066855908319, "high": 0.36557131269329507, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3763721321413629, "unit": "f1", "denominator": null, "interval": { "low": 0.3124480540826695, "high": 0.3843185594980794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13333333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.09755784865540963, "high": 0.17187594582324453, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.24166666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.18749377490039842, "high": 0.298702645901816, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.32083333333333336, "unit": "proportion", "denominator": 240, "interval": { "low": 0.25957446808510637, "high": 0.38376795598313124, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7532467532467533, "unit": "proportion", "denominator": 77, "interval": { "low": 0.6385542168674698, "high": 0.8589743589743589, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07916666666666666, "unit": "proportion", "denominator": 240, "interval": { "low": 0.042550682456844645, "high": 0.12174187066130797, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.8291666666666668, "unit": "points_per_event", "denominator": 240, "interval": { "low": 0.4566050111504657, "high": 1.2069019621988168, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.8708333333333333, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.624987449799197, "high": 2.1270955504115228, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.23777415999638224, "unit": "f1", "denominator": null, "interval": { "low": 0.17657195428028763, "high": 0.24916219068996848, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.24379375898363242, "unit": "f1", "denominator": null, "interval": { "low": 0.1810421303380164, "high": 0.25547009425173983, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.20833333333333334, "unit": "proportion", "denominator": 240, "interval": { "low": 0.15596155858652808, "high": 0.26337448559670784, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.22413793103448276, "unit": "proportion", "denominator": 174, "interval": { "low": 0.15950920245398773, "high": 0.29081864564007415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.25287356321839083, "unit": "proportion", "denominator": 174, "interval": { "low": 0.18681318681318682, "high": 0.32085891595695515, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8863636363636364, "unit": "proportion", "denominator": 44, "interval": { "low": 0.78, "high": 0.9736842105263158, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.028735632183908046, "unit": "proportion", "denominator": 174, "interval": { "low": 0.005952380952380952, "high": 0.056179775280898875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.8103448275862069, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.37878509358288776, "high": 1.2380952380952381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5862068965517242, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.2837701140446807, "high": 1.8932630318659056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2171589310829817, "unit": "f1", "denominator": null, "interval": { "low": 0.129957805907173, "high": 0.21377347160575008, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2227994227994228, "unit": "f1", "denominator": null, "interval": { "low": 0.13333333333333333, "high": 0.21932602930979556, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.15517241379310345, "unit": "proportion", "denominator": 174, "interval": { "low": 0.10344827586206896, "high": 0.21142857142857144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.2653061224489796, "unit": "proportion", "denominator": 245, "interval": { "low": 0.20832908163265307, "high": 0.3254014396456256, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3551020408163265, "unit": "proportion", "denominator": 245, "interval": { "low": 0.2923680860710051, "high": 0.4184123458610583, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7471264367816092, "unit": "proportion", "denominator": 87, "interval": { "low": 0.6451612903225806, "high": 0.845360824742268, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08979591836734693, "unit": "proportion", "denominator": 245, "interval": { "low": 0.052862988431646434, "high": 0.13008130081300814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.189795918367347, "unit": "points_per_event", "denominator": 245, "interval": { "low": 0.849994635193133, "high": 1.5381638687101344, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.822448979591837, "unit": "points_per_event", "denominator": 245, "interval": { "low": 1.5576900921658987, "high": 2.10148708230198, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.34928537071394217, "unit": "f1", "denominator": null, "interval": { "low": 0.2458757112422526, "high": 0.34844482237339375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3729657348301416, "unit": "f1", "denominator": null, "interval": { "low": 0.26254525098749004, "high": 0.3720682001614205, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.12653061224489795, "unit": "proportion", "denominator": 245, "interval": { "low": 0.08366533864541832, "high": 0.17269076305220885, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.47692307692307695, "unit": "proportion", "denominator": 130, "interval": { "low": 0.39090198863636366, "high": 0.5666666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5153846153846153, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4271715210355987, "high": 0.6043205733920326, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9253731343283582, "unit": "proportion", "denominator": 67, "interval": { "low": 0.8571428571428571, "high": 0.9838709677419355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.038461538461538464, "unit": "proportion", "denominator": 130, "interval": { "low": 0.008064516129032258, "high": 0.07518796992481203, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2807692307692307, "unit": "points_per_event", "denominator": 130, "interval": { "low": 1.7434601449275364, "high": 2.792614734299517, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0115384615384615, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.6638655462184873, "high": 3.367660427807487, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5118422318422318, "unit": "f1", "denominator": null, "interval": { "low": 0.3574269680936347, "high": 0.5041812169312169, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5234750098386461, "unit": "f1", "denominator": null, "interval": { "low": 0.365550308277581, "high": 0.5156398809523809, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14615384615384616, "unit": "proportion", "denominator": 130, "interval": { "low": 0.08526719445818902, "high": 0.21311475409836064, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.1588785046728972, "unit": "proportion", "denominator": 107, "interval": { "low": 0.09174120795107034, "high": 0.23232323232323232, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.21495327102803738, "unit": "proportion", "denominator": 107, "interval": { "low": 0.14, "high": 0.29565882874889077, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7391304347826086, "unit": "proportion", "denominator": 23, "interval": { "low": 0.5416666666666666, "high": 0.9090909090909091, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.056074766355140186, "unit": "proportion", "denominator": 107, "interval": { "low": 0.018018018018018018, "high": 0.10281096444744184, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.6962616822429908, "unit": "points_per_event", "denominator": 107, "interval": { "low": 0.2340248226950355, "high": 1.1434792673107892, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.3037383177570094, "unit": "points_per_event", "denominator": 107, "interval": { "low": 0.9714285714285714, "high": 1.655970470183486, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.18214285714285713, "unit": "f1", "denominator": null, "interval": { "low": 0.08760416666666666, "high": 0.18956043956043958, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.18571428571428572, "unit": "f1", "denominator": null, "interval": { "low": 0.0893218954248366, "high": 0.19327731092436973, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.12149532710280374, "unit": "proportion", "denominator": 107, "interval": { "low": 0.06306306306306306, "high": 0.18691588785046728, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3082706766917293, "unit": "proportion", "denominator": 133, "interval": { "low": 0.22949965846994536, "high": 0.3902480788506515, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.40601503759398494, "unit": "proportion", "denominator": 133, "interval": { "low": 0.32025490196078427, "high": 0.49296019170579025, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7592592592592593, "unit": "proportion", "denominator": 54, "interval": { "low": 0.6271123197571464, "high": 0.8805970149253731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09774436090225563, "unit": "proportion", "denominator": 133, "interval": { "low": 0.045454545454545456, "high": 0.15909452808338154, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.9360902255639098, "unit": "points_per_event", "denominator": 133, "interval": { "low": 0.3780420054200542, "high": 1.5069121078558463, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.327067669172932, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.9959652426019727, "high": 2.6723097178060415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2878095238095238, "unit": "f1", "denominator": null, "interval": { "low": 0.19799642857142855, "high": 0.2964254604872252, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2936831875607386, "unit": "f1", "denominator": null, "interval": { "low": 0.20203717201166183, "high": 0.30247495968084204, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2781954887218045, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2, "high": 0.358974358974359, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": { "low": 0.05063291139240506, "high": 0.19480519480519481, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.15789473684210525, "unit": "proportion", "denominator": 76, "interval": { "low": 0.08, "high": 0.24324324324324326, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.75, "unit": "proportion", "denominator": 12, "interval": { "low": 0.4666666666666667, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.039473684210526314, "unit": "proportion", "denominator": 76, "interval": { "low": 0.0, "high": 0.08823529411764706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.11842105263157894, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.45394736842105265, "high": 0.6866686746987951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 0.9736842105263158, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.6216216216216216, "high": 1.3513513513513513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.18055555555555555, "unit": "f1", "denominator": null, "interval": { "low": 0.0625, "high": 0.16666666666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.18840579710144925, "unit": "f1", "denominator": null, "interval": { "low": 0.06521739130434782, "high": 0.17391304347826086, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.17105263157894737, "unit": "proportion", "denominator": 76, "interval": { "low": 0.09090909090909091, "high": 0.25758797653958926, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.30612244897959184, "unit": "proportion", "denominator": 98, "interval": { "low": 0.21176470588235294, "high": 0.40426129572077446, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.32653061224489793, "unit": "proportion", "denominator": 98, "interval": { "low": 0.23076923076923078, "high": 0.425531914893617, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9375, "unit": "proportion", "denominator": 32, "interval": { "low": 0.8387096774193549, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.02040816326530612, "unit": "proportion", "denominator": 98, "interval": { "low": 0.0, "high": 0.05263157894736842, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.3469387755102042, "unit": "points_per_event", "denominator": 98, "interval": { "low": 0.7403750000000001, "high": 1.9255417651694247, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.061224489795918, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.641282975085491, "high": 2.495499412455934, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2932712215320911, "unit": "f1", "denominator": null, "interval": { "low": 0.16500862663906143, "high": 0.2916230176828003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2932712215320911, "unit": "f1", "denominator": null, "interval": { "low": 0.16500862663906143, "high": 0.2916230176828003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 98, "interval": { "low": 0.07692307692307693, "high": 0.21505376344086022, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 790, "population_events": 790 } } }, "gemini-minimal": { "metrics": { "species_accuracy": { "value": 0.4531645569620253, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.4263783860302847, "high": 0.4795359848484848, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.610126582278481, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.5833888957385793, "high": 0.6359636372769284, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.7427385892116183, "unit": "proportion", "denominator": 482, "population_events": 790, "interval": { "low": 0.7097350943188843, "high": 0.7751527001066967, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.384620059106388, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.3583504031464412, "high": 0.39855282226614575, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 1.8594936708860759, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 1.6622813978391506, "high": 2.051830783053093, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.1569620253164557, "unit": "proportion", "denominator": 790, "interval": { "low": 0.13510063336803843, "high": 0.1789345991561181, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.5160724843705965, "unit": "f1", "denominator": null, "interval": { "low": 0.48082459156357926, "high": 0.5347670779773601, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.09367088607595, "unit": "points_per_event", "denominator": 790, "interval": { "low": 2.9782408967614664, "high": 3.1986251512200887, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.2468354430379747, "unit": "proportion", "denominator": 790, "interval": { "low": 0.2210091385407841, "high": 0.27222614410905555, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 429, "interval": { "low": 0.48853078890626983, "high": 0.5871559633027523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6806526806526807, "unit": "proportion", "denominator": 429, "interval": { "low": 0.6353699270862545, "high": 0.7245667370567246, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.791095890410959, "unit": "proportion", "denominator": 292, "interval": { "low": 0.743201123618756, "high": 0.8371215205803968, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14219114219114218, "unit": "proportion", "denominator": 429, "interval": { "low": 0.1100244498777506, "high": 0.17592875577288777, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.312354312354312, "unit": "points_per_event", "denominator": 429, "interval": { "low": 1.9736765960038989, "high": 2.6432040333213473, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.384615384615385, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.201609435682016, "high": 3.5649213964452002, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4429138522252915, "unit": "f1", "denominator": null, "interval": { "low": 0.37837072021640095, "high": 0.45170270382921246, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5643579729967425, "unit": "f1", "denominator": null, "interval": { "low": 0.4821175305983174, "high": 0.57555667100819, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21445221445221446, "unit": "proportion", "denominator": 429, "interval": { "low": 0.1755536858974359, "high": 0.25547772577002387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3518005540166205, "unit": "proportion", "denominator": 361, "interval": { "low": 0.30057762875884914, "high": 0.40298507462686567, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5263157894736842, "unit": "proportion", "denominator": 361, "interval": { "low": 0.47246156617823687, "high": 0.5795454545454546, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6684210526315789, "unit": "proportion", "denominator": 190, "interval": { "low": 0.599009900990099, "high": 0.736559795436664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1745152354570637, "unit": "proportion", "denominator": 361, "interval": { "low": 0.1352112676056338, "high": 0.21487603305785125, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.3213296398891967, "unit": "points_per_event", "denominator": 361, "interval": { "low": 0.9622798503929553, "high": 1.6893758066829196, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.7479224376731306, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.5482899912587413, "high": 2.9529590313669583, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.28475413397310384, "unit": "f1", "denominator": null, "interval": { "low": 0.21815943209231833, "high": 0.29202158641477943, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3716282765411694, "unit": "f1", "denominator": null, "interval": { "low": 0.28471654696794085, "high": 0.3811129178633562, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2853185595567867, "unit": "proportion", "denominator": 361, "interval": { "low": 0.2371967654986523, "high": 0.33246897163120537, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.6293333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5785090462363189, "high": 0.680628272251309, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7893333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.745945945945946, "high": 0.8310258742367175, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7972972972972973, "unit": "proportion", "denominator": 296, "interval": { "low": 0.7507984700819382, "high": 0.8421094139133322, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16, "unit": "proportion", "denominator": 375, "interval": { "low": 0.12403049641038227, "high": 0.19726027397260273, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.8226666666666667, "unit": "points_per_event", "denominator": 375, "interval": { "low": 2.4706623913927337, "high": 3.1644222445862926, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.916, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.7564086294416246, "high": 4.072605273273694, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5094660158435669, "unit": "f1", "denominator": null, "interval": { "low": 0.4416274509778075, "high": 0.5124802425132557, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6400983275983275, "unit": "f1", "denominator": null, "interval": { "low": 0.554865258920835, "high": 0.64388543290127, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21866666666666668, "unit": "proportion", "denominator": 375, "interval": { "low": 0.17595123712403662, "high": 0.26285787923416787, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.4125, "unit": "proportion", "denominator": 240, "interval": { "low": 0.35019455252918286, "high": 0.4756583180715735, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5791666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5145203279604212, "high": 0.6428571428571429, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7122302158273381, "unit": "proportion", "denominator": 139, "interval": { "low": 0.6381459763539283, "high": 0.7862595419847328, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 240, "interval": { "low": 0.12083333333333333, "high": 0.21428571428571427, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.666666666666667, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.2301170657883025, "high": 2.0983800876201246, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.020833333333334, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.786035558863949, "high": 3.256637168141593, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4520707900018244, "unit": "f1", "denominator": null, "interval": { "low": 0.32700141493244944, "high": 0.4337621462190427, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.49785011050833833, "unit": "f1", "denominator": null, "interval": { "low": 0.36011548226738105, "high": 0.47768742684881915, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2708333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.21518987341772153, "high": 0.32644628099173556, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.12643678160919541, "unit": "proportion", "denominator": 174, "interval": { "low": 0.07975460122699386, "high": 0.1783783783783784, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.26436781609195403, "unit": "proportion", "denominator": 174, "interval": { "low": 0.2010575192276223, "high": 0.3333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4782608695652174, "unit": "proportion", "denominator": 46, "interval": { "low": 0.3333333333333333, "high": 0.6222222222222222, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13793103448275862, "unit": "proportion", "denominator": 174, "interval": { "low": 0.08928571428571429, "high": 0.19047619047619047, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.031609195402298854, "unit": "points_per_event", "denominator": 174, "interval": { "low": -0.40238122171945706, "high": 0.46428571428571436, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.4109195402298853, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.1697477650063859, "high": 1.6630449314112998, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.12717480931766648, "unit": "f1", "denominator": null, "interval": { "low": 0.06375661375661375, "high": 0.12991307634164775, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.1387361556192725, "unit": "f1", "denominator": null, "interval": { "low": 0.06955266955266956, "high": 0.1417233560090703, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27586206896551724, "unit": "proportion", "denominator": 174, "interval": { "low": 0.21052631578947367, "high": 0.3423913043478261, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.689795918367347, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6282051282051282, "high": 0.75, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8448979591836735, "unit": "proportion", "denominator": 245, "interval": { "low": 0.7972350230414746, "high": 0.8906882591093117, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8164251207729468, "unit": "proportion", "denominator": 207, "interval": { "low": 0.7625539241655945, "high": 0.8679245283018868, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15510204081632653, "unit": "proportion", "denominator": 245, "interval": { "low": 0.1111111111111111, "high": 0.2008201008299609, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.148979591836735, "unit": "points_per_event", "denominator": 245, "interval": { "low": 2.7312179691163667, "high": 3.5514410856935372, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.108163265306122, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.923828125, "high": 4.286766130870954, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5430831128747795, "unit": "f1", "denominator": null, "interval": { "low": 0.44114724877745715, "high": 0.5395518275864477, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.662745493677697, "unit": "f1", "denominator": null, "interval": { "low": 0.5383491849487613, "high": 0.6584361285800718, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19183673469387755, "unit": "proportion", "denominator": 245, "interval": { "low": 0.1405605894652992, "high": 0.2446351931330472, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5153846153846153, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4295774647887324, "high": 0.6050420168067226, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6846153846153846, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6027397260273972, "high": 0.7647058823529411, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7528089887640449, "unit": "proportion", "denominator": 89, "interval": { "low": 0.6623332748332749, "high": 0.8421052631578947, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16923076923076924, "unit": "proportion", "denominator": 130, "interval": { "low": 0.10686240770867227, "high": 0.23448803659394785, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2076923076923083, "unit": "points_per_event", "denominator": 130, "interval": { "low": 1.5952193475815524, "high": 2.8177974313493626, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5538461538461537, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.2703502415458936, "high": 3.8396281692216983, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.42257495590828925, "unit": "f1", "denominator": null, "interval": { "low": 0.30799402757736094, "high": 0.4257711372989151, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5186147186147186, "unit": "f1", "denominator": null, "interval": { "low": 0.37799267020857935, "high": 0.5225373048668502, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2692307692307692, "unit": "proportion", "denominator": 130, "interval": { "low": 0.19117412888722368, "high": 0.34814814814814815, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.5046728971962616, "unit": "proportion", "denominator": 107, "interval": { "low": 0.4090909090909091, "high": 0.5979381443298969, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6355140186915887, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5416666666666666, "high": 0.7254901960784313, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7941176470588235, "unit": "proportion", "denominator": 68, "interval": { "low": 0.6935483870967742, "high": 0.8852459016393442, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1308411214953271, "unit": "proportion", "denominator": 107, "interval": { "low": 0.07079646017699115, "high": 0.2, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.032710280373832, "unit": "points_per_event", "denominator": 107, "interval": { "low": 1.3715448357502222, "high": 2.6698160377358495, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2009345794392523, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.8246252291175704, "high": 3.559833333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5196428571428571, "unit": "f1", "denominator": null, "interval": { "low": 0.3259757653061225, "high": 0.4816921768707482, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5705882352941177, "unit": "f1", "denominator": null, "interval": { "low": 0.3579341736694677, "high": 0.5289169000933706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2336448598130841, "unit": "proportion", "denominator": 107, "interval": { "low": 0.15730337078651685, "high": 0.315219839404622, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3383458646616541, "unit": "proportion", "denominator": 133, "interval": { "low": 0.25862068965517243, "high": 0.4198473282442748, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5338345864661654, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4473684210526316, "high": 0.6197183098591549, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6338028169014085, "unit": "proportion", "denominator": 71, "interval": { "low": 0.5185185185185185, "high": 0.746031746031746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19548872180451127, "unit": "proportion", "denominator": 133, "interval": { "low": 0.12799548872180452, "high": 0.26515151515151514, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.3721804511278197, "unit": "points_per_event", "denominator": 133, "interval": { "low": 0.7933870523415978, "high": 1.943609022556391, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.8759398496240607, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.577512289657781, "high": 3.1796953125, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3334750566893424, "unit": "f1", "denominator": null, "interval": { "low": 0.20898224914965985, "high": 0.3308555581323438, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.38111435050210557, "unit": "f1", "denominator": null, "interval": { "low": 0.23883685617103984, "high": 0.3781206378655358, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3007518796992481, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2222222222222222, "high": 0.3800041322314049, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.03076923076923077, "high": 0.16304347826086957, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.21052631578947367, "unit": "proportion", "denominator": 76, "interval": { "low": 0.12194297956493079, "high": 0.3055555555555556, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4375, "unit": "proportion", "denominator": 16, "interval": { "low": 0.19047619047619047, "high": 0.6923076923076923, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": { "low": 0.05063291139240506, "high": 0.1969750320102432, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.02631578947368421, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.685513440860215, "high": 0.6214333976833977, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.2894736842105265, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.9615259740259741, "high": 1.6353046218487393, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.10690943043884221, "unit": "f1", "denominator": null, "interval": { "low": 0.027450980392156862, "high": 0.1134453781512605, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.11853002070393373, "unit": "f1", "denominator": null, "interval": { "low": 0.03043478260869565, "high": 0.12577639751552794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2631578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.16666666666666666, "high": 0.36619718309859156, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.15306122448979592, "unit": "proportion", "denominator": 98, "interval": { "low": 0.08602150537634409, "high": 0.22916666666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.30612244897959184, "unit": "proportion", "denominator": 98, "interval": { "low": 0.21739130434782608, "high": 0.40404040404040403, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 30, "interval": { "low": 0.32, "high": 0.6785714285714286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15306122448979592, "unit": "proportion", "denominator": 98, "interval": { "low": 0.08510136491368929, "high": 0.2289156626506024, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.07653061224489797, "unit": "points_per_event", "denominator": 98, "interval": { "low": -0.5284131493506494, "high": 0.678954816285998, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5051020408163265, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.161454133064516, "high": 1.8722327044025153, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.12275641025641025, "unit": "f1", "denominator": null, "interval": { "low": 0.06046932234432234, "high": 0.14026556776556773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.13876811594202898, "unit": "f1", "denominator": null, "interval": { "low": 0.06835662525879917, "high": 0.1585610766045548, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 98, "interval": { "low": 0.2, "high": 0.37777777777777777, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 779, "estimated_events": 11, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "779 saved telemetry estimates plus corrected selected-response result tokens at retained prices; no provider invoice. Earlier replaced calls are separate.", "value": 0.0017474293114182412 } }, "gemini-low": { "metrics": { "species_accuracy": { "value": 0.46582278481012657, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.4380837982341147, "high": 0.49218202595576016, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.6278481012658228, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6001950563501196, "high": 0.6533056386300059, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.7419354838709677, "unit": "proportion", "denominator": 496, "population_events": 790, "interval": { "low": 0.7082568164755375, "high": 0.7739269137725746, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.38942685947078304, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.36217189284084783, "high": 0.40367993488144427, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 1.8443037974683545, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 1.6361009507370263, "high": 2.0410591856280145, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.1620253164556962, "unit": "proportion", "denominator": 790, "interval": { "low": 0.14017642255775167, "high": 0.18462976281805393, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.52745156915663, "unit": "f1", "denominator": null, "interval": { "low": 0.4905366143540598, "high": 0.5467563674976524, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.1544303797468354, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.0410697618134326, "high": 3.2579031385281385, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.2620253164556962, "unit": "proportion", "denominator": 790, "interval": { "low": 0.23668882102426408, "high": 0.2884082057183323, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.5501165501165501, "unit": "proportion", "denominator": 429, "interval": { "low": 0.5, "high": 0.598086689967989, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6993006993006993, "unit": "proportion", "denominator": 429, "interval": { "low": 0.6536547924689774, "high": 0.743878874905425, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7866666666666666, "unit": "proportion", "denominator": 300, "interval": { "low": 0.738093336376084, "high": 0.8327759197324415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14918414918414918, "unit": "proportion", "denominator": 429, "interval": { "low": 0.11566265060240964, "high": 0.1848341232227488, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3578088578088576, "unit": "points_per_event", "denominator": 429, "interval": { "low": 2.015617629716981, "high": 2.6878038069687973, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.453379953379953, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.2727143300890456, "high": 3.631114173228346, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.41656594752569986, "unit": "f1", "denominator": null, "interval": { "low": 0.3575201952296675, "high": 0.42504995181813016, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5643796708412706, "unit": "f1", "denominator": null, "interval": { "low": 0.48438219998858173, "high": 0.5758741282697247, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2191142191142191, "unit": "proportion", "denominator": 429, "interval": { "low": 0.18013747359434346, "high": 0.26054884574224124, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3656509695290859, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3131868131868132, "high": 0.418283937486606, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5429362880886427, "unit": "proportion", "denominator": 361, "interval": { "low": 0.4888165452130524, "high": 0.5966862119318952, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.673469387755102, "unit": "proportion", "denominator": 196, "interval": { "low": 0.6038624555646706, "high": 0.7417582417582418, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1772853185595568, "unit": "proportion", "denominator": 361, "interval": { "low": 0.13662790697674418, "high": 0.21846448087431689, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.234072022160665, "unit": "points_per_event", "denominator": 361, "interval": { "low": 0.854408613445378, "high": 1.6187522644927537, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.799168975069252, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.592685761102194, "high": 3.001535218413762, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.29717896697986373, "unit": "f1", "denominator": null, "interval": { "low": 0.22896800683550192, "high": 0.3051930029136122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3727329416357613, "unit": "f1", "denominator": null, "interval": { "low": 0.28718021196317195, "high": 0.38278444433232717, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.31301939058171746, "unit": "proportion", "denominator": 361, "interval": { "low": 0.2631542898341745, "high": 0.36193029490616624, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.6426666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5922077922077922, "high": 0.6926832340111814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7973333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.7542833200319234, "high": 0.8388906926406926, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8060200668896321, "unit": "proportion", "denominator": 299, "interval": { "low": 0.7593220338983051, "high": 0.8491803278688524, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15466666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.11968028277868704, "high": 0.19200263157894734, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.7933333333333334, "unit": "points_per_event", "denominator": 375, "interval": { "low": 2.437844279661017, "high": 3.140320568835269, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.94, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.7804500125989784, "high": 4.0960092178770955, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5238805312334723, "unit": "f1", "denominator": null, "interval": { "low": 0.454377981352297, "high": 0.5283396577396728, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6514924555082925, "unit": "f1", "denominator": null, "interval": { "low": 0.5650597973227283, "high": 0.6570377794967727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.22933333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.18701245908562983, "high": 0.2727327624066754, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.4041666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.34156378600823045, "high": 0.4669421487603306, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5377299528301887, "high": 0.6625514403292181, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6736111111111112, "unit": "proportion", "denominator": 144, "interval": { "low": 0.5961506622516556, "high": 0.75, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19583333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.14601532530669606, "high": 0.24778857252789532, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5291666666666668, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.0787335369513789, "high": 1.982075475874281, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0708333333333333, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.8427887206169284, "high": 3.304624328272081, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.41970945720945724, "unit": "f1", "denominator": null, "interval": { "low": 0.30556971910665087, "high": 0.40520989269568813, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4675244586636991, "unit": "f1", "denominator": null, "interval": { "low": 0.34038145925804153, "high": 0.4513730450281083, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.30833333333333335, "unit": "proportion", "denominator": 240, "interval": { "low": 0.25, "high": 0.36681222707423583, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 174, "interval": { "low": 0.11309271589991929, "high": 0.2251655629139073, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.2988505747126437, "unit": "proportion", "denominator": 174, "interval": { "low": 0.23214285714285715, "high": 0.3697954299132948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5576923076923077, "unit": "proportion", "denominator": 52, "interval": { "low": 0.42371251629726203, "high": 0.693556616194865, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13218390804597702, "unit": "proportion", "denominator": 174, "interval": { "low": 0.08333333333333333, "high": 0.18391157182145118, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.21551724137931033, "unit": "points_per_event", "denominator": 174, "interval": { "low": -0.24450844854070658, "high": 0.6833477011494251, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.5660919540229887, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.3060077145612343, "high": 1.836436225551394, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1611013129531648, "unit": "f1", "denominator": null, "interval": { "low": 0.08885768038545817, "high": 0.16342025530751023, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.16947021232735518, "unit": "f1", "denominator": null, "interval": { "low": 0.09347366378210534, "high": 0.17190961921958867, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27011494252873564, "unit": "proportion", "denominator": 174, "interval": { "low": 0.20555555555555555, "high": 0.33689984887235525, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.7061224489795919, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6459123702983139, "high": 0.7647058823529411, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8408163265306122, "unit": "proportion", "denominator": 245, "interval": { "low": 0.7890295358649789, "high": 0.8870967741935484, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8398058252427184, "unit": "proportion", "denominator": 206, "interval": { "low": 0.7887323943661971, "high": 0.8883495145631068, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1346938775510204, "unit": "proportion", "denominator": 245, "interval": { "low": 0.0931174089068826, "high": 0.17857142857142858, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.2306122448979595, "unit": "points_per_event", "denominator": 245, "interval": { "low": 2.8220330508474576, "high": 3.620264076822348, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.148979591836735, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.9650643790321136, "high": 4.326447437399778, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5688163971718239, "unit": "f1", "denominator": null, "interval": { "low": 0.4616772500720305, "high": 0.5615022397967091, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6845078677830421, "unit": "f1", "denominator": null, "interval": { "low": 0.5555777077137993, "high": 0.6757060851790907, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1836734693877551, "unit": "proportion", "denominator": 245, "interval": { "low": 0.13692946058091288, "high": 0.23423423423423423, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5230769230769231, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4351145038167939, "high": 0.6120689655172413, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7153846153846154, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6349206349206349, "high": 0.7948717948717948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7311827956989247, "unit": "proportion", "denominator": 93, "interval": { "low": 0.6363636363636364, "high": 0.8192771084337349, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 130, "interval": { "low": 0.12666455696202533, "high": 0.2631578947368421, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.9692307692307693, "unit": "points_per_event", "denominator": 130, "interval": { "low": 1.3157793522267207, "high": 2.6260523128170186, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.546153846153846, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.2584710451977403, "high": 3.8387130098640365, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4262057387057387, "unit": "f1", "denominator": null, "interval": { "low": 0.3120593295593296, "high": 0.43502816627816626, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5036976911976911, "unit": "f1", "denominator": null, "interval": { "low": 0.36879738947920765, "high": 0.5141241965105601, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3153846153846154, "unit": "proportion", "denominator": 130, "interval": { "low": 0.23529411764705882, "high": 0.3969465648854962, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.4953271028037383, "unit": "proportion", "denominator": 107, "interval": { "low": 0.40186915887850466, "high": 0.5887850467289719, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7009345794392523, "unit": "proportion", "denominator": 107, "interval": { "low": 0.6105263157894737, "high": 0.7857142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7066666666666667, "unit": "proportion", "denominator": 75, "interval": { "low": 0.6025641025641025, "high": 0.8076923076923077, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.205607476635514, "unit": "proportion", "denominator": 107, "interval": { "low": 0.13043478260869565, "high": 0.2833333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.925233644859813, "unit": "points_per_event", "denominator": 107, "interval": { "low": 1.2473184697855753, "high": 2.5840718302869905, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3271028037383177, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.9770075031928482, "high": 3.6584158415841586, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.46988304093567246, "unit": "f1", "denominator": null, "interval": { "low": 0.2997446267183109, "high": 0.4413140490114174, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5251633986928105, "unit": "f1", "denominator": null, "interval": { "low": 0.335008700449877, "high": 0.4932333488951136, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2803738317757009, "unit": "proportion", "denominator": 107, "interval": { "low": 0.19801980198019803, "high": 0.36666666666666664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3308270676691729, "unit": "proportion", "denominator": 133, "interval": { "low": 0.25, "high": 0.4146341463414634, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.518796992481203, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4297520661157025, "high": 0.6083347902097901, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6376811594202898, "unit": "proportion", "denominator": 69, "interval": { "low": 0.5147058823529411, "high": 0.7534246575342466, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18796992481203006, "unit": "proportion", "denominator": 133, "interval": { "low": 0.12194297956493079, "high": 0.2597402597402597, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.2105263157894737, "unit": "points_per_event", "denominator": 133, "interval": { "low": 0.5952193475815524, "high": 1.8134343165126747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.8646616541353382, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.563380281690141, "high": 3.1614017125837677, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3162954217499672, "unit": "f1", "denominator": null, "interval": { "low": 0.20259188523894406, "high": 0.3204784244276222, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3550254733928203, "unit": "f1", "denominator": null, "interval": { "low": 0.22739905486003925, "high": 0.3597206804799842, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3308270676691729, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2517482517482518, "high": 0.4132231404958678, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": { "low": 0.05061708860759495, "high": 0.19444444444444445, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14492276887871855, "high": 0.33766233766233766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 18, "interval": { "low": 0.2631578947368421, "high": 0.7333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": { "low": 0.05128205128205128, "high": 0.19753086419753085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.11842105263157894, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.5571635610766046, "high": 0.76, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.368421052631579, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.0208261986301368, "high": 1.7333544303797468, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.13647798742138365, "unit": "f1", "denominator": null, "interval": { "low": 0.0440251572327044, "high": 0.13836477987421383, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.15724637681159417, "unit": "f1", "denominator": null, "interval": { "low": 0.05072463768115942, "high": 0.15942028985507245, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.25, "unit": "proportion", "denominator": 76, "interval": { "low": 0.15714285714285714, "high": 0.35366322701688546, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.20408163265306123, "unit": "proportion", "denominator": 98, "interval": { "low": 0.12675610625779996, "high": 0.2882882882882883, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3469387755102041, "unit": "proportion", "denominator": 98, "interval": { "low": 0.25263157894736843, "high": 0.44554455445544555, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5882352941176471, "unit": "proportion", "denominator": 34, "interval": { "low": 0.42105263157894735, "high": 0.75, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 98, "interval": { "low": 0.07692307692307693, "high": 0.2127708825847123, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.29081632653061223, "unit": "points_per_event", "denominator": 98, "interval": { "low": -0.36046791144901613, "high": 0.9495527522935778, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.7193877551020411, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.3490541470125788, "high": 2.1087010331467932, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1511175898931001, "unit": "f1", "denominator": null, "interval": { "low": 0.08830911152339725, "high": 0.17303759166004062, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.16097308488612835, "unit": "f1", "denominator": null, "interval": { "low": 0.09406840140535792, "high": 0.18432265198569542, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 98, "interval": { "low": 0.19801980198019803, "high": 0.375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.0022434227848101267 } }, "gemini-medium": { "metrics": { "species_accuracy": { "value": 0.5050632911392405, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.47745408072939727, "high": 0.53048139624089, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.689873417721519, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6645643596909419, "high": 0.7147793372239575, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.7321100917431193, "unit": "proportion", "denominator": 545, "population_events": 790, "interval": { "low": 0.7008204108738859, "high": 0.7612962443950506, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.39410302408254455, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.369012504140656, "high": 0.40790367039280523, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 1.9759493670886077, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 1.7623803135672547, "high": 2.16761077881331, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.1848101265822785, "unit": "proportion", "denominator": 790, "interval": { "low": 0.16303734108718287, "high": 0.208183674623548, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.5587283379398098, "unit": "f1", "denominator": null, "interval": { "low": 0.5231569678956136, "high": 0.5782938111897998, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.311392405063291, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.1984617193636176, "high": 3.4113239195755023, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.2670886075949367, "unit": "proportion", "denominator": 790, "interval": { "low": 0.2428527291976659, "high": 0.2938827847609493, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.6013986013986014, "unit": "proportion", "denominator": 429, "interval": { "low": 0.5514012835827887, "high": 0.6498830935251798, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7808857808857809, "unit": "proportion", "denominator": 429, "interval": { "low": 0.7410698227132579, "high": 0.8198198198198198, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7701492537313432, "unit": "proportion", "denominator": 335, "interval": { "low": 0.7226858731297396, "high": 0.8164585049809003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1794871794871795, "unit": "proportion", "denominator": 429, "interval": { "low": 0.14285714285714285, "high": 0.21695819413729825, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.375291375291375, "unit": "points_per_event", "denominator": 429, "interval": { "low": 2.019091143288857, "high": 2.728666281755196, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.6107226107226107, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.4311163895486936, "high": 3.788964060133415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4654919191330071, "unit": "f1", "denominator": null, "interval": { "low": 0.3976692922665775, "high": 0.4713525415265455, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6306664710834289, "unit": "f1", "denominator": null, "interval": { "low": 0.5387777508127825, "high": 0.6386066691649969, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.24708624708624707, "unit": "proportion", "denominator": 429, "interval": { "low": 0.205377349019052, "high": 0.2885106086214535, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.39058171745152354, "unit": "proportion", "denominator": 361, "interval": { "low": 0.338623297087864, "high": 0.44385026737967914, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5817174515235457, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5283018867924528, "high": 0.6332475813544415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6714285714285714, "unit": "proportion", "denominator": 210, "interval": { "low": 0.6048780487804878, "high": 0.7355795704845816, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19113573407202217, "unit": "proportion", "denominator": 361, "interval": { "low": 0.15056818181818182, "high": 0.23351648351648352, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5013850415512469, "unit": "points_per_event", "denominator": 361, "interval": { "low": 1.118985631200821, "high": 1.8689475743348982, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.9556786703601112, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.7528397187658715, "high": 3.154173259493671, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2888468366428893, "unit": "f1", "denominator": null, "interval": { "low": 0.22631365892797076, "high": 0.29702985247806746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.39165672765137527, "unit": "f1", "denominator": null, "interval": { "low": 0.3068659782074179, "high": 0.4027523423431422, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.29085872576177285, "unit": "proportion", "denominator": 361, "interval": { "low": 0.24331550802139038, "high": 0.3408450704225352, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.6826666666666666, "unit": "proportion", "denominator": 375, "interval": { "low": 0.6322713627765442, "high": 0.7336818168918152, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.856, "unit": "proportion", "denominator": 375, "interval": { "low": 0.8179271708683473, "high": 0.8922662558436039, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7975077881619937, "unit": "proportion", "denominator": 321, "interval": { "low": 0.7507148093841642, "high": 0.8433734939759037, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.17333333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.13350785340314136, "high": 0.21448467966573817, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.9173333333333336, "unit": "points_per_event", "denominator": 375, "interval": { "low": 2.559053099131839, "high": 3.2723582434034064, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.0906666666666665, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.9360546552904845, "high": 4.243351063829787, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5561552774710669, "unit": "f1", "denominator": null, "interval": { "low": 0.49417766617328757, "high": 0.5620633601445499, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6773686071762993, "unit": "f1", "denominator": null, "interval": { "low": 0.6018830549546451, "high": 0.6845643488940031, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.23466666666666666, "unit": "proportion", "denominator": 375, "interval": { "low": 0.18994413407821228, "high": 0.27979274611398963, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.44583333333333336, "unit": "proportion", "denominator": 240, "interval": { "low": 0.38235294117647056, "high": 0.5084033613445378, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6291666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5670961850649351, "high": 0.6901978136387298, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7086092715231788, "unit": "proportion", "denominator": 151, "interval": { "low": 0.6350259685569904, "high": 0.781021897810219, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18333333333333332, "unit": "proportion", "denominator": 240, "interval": { "low": 0.13565891472868216, "high": 0.23236742010628228, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.8687500000000001, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.4132825552628197, "high": 2.3114820468573707, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2229166666666673, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.997965326581232, "high": 3.4480593270366002, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.428970606687998, "unit": "f1", "denominator": null, "interval": { "low": 0.31851245222169133, "high": 0.41735012252595366, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.49956070652273177, "unit": "f1", "denominator": null, "interval": { "low": 0.3709258937265267, "high": 0.4860279907897182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2708333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.21338772663877267, "high": 0.3292181069958848, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.20114942528735633, "unit": "proportion", "denominator": 174, "interval": { "low": 0.143646408839779, "high": 0.2648648648648649, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.41379310344827586, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3413173652694611, "high": 0.4913294797687861, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4861111111111111, "unit": "proportion", "denominator": 72, "interval": { "low": 0.3723267866884888, "high": 0.6027397260273972, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.21264367816091953, "unit": "proportion", "denominator": 174, "interval": { "low": 0.15300546448087432, "high": 0.2752808988764045, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.07758620689655173, "unit": "points_per_event", "denominator": 174, "interval": { "low": -0.4342857142857143, "high": 0.5925171604046238, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.7442528735632186, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.4666620111731845, "high": 2.0287397847319113, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1834154351395731, "unit": "f1", "denominator": null, "interval": { "low": 0.10519978106185005, "high": 0.18124816514902722, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.20723562152133582, "unit": "f1", "denominator": null, "interval": { "low": 0.1188620902906617, "high": 0.2047868878956541, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 174, "interval": { "low": 0.26282051282051283, "high": 0.4037274370862858, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.7428571428571429, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6818139257745999, "high": 0.803347280334728, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9102040816326531, "unit": "proportion", "denominator": 245, "interval": { "low": 0.8710905997983871, "high": 0.9444444444444444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8161434977578476, "unit": "proportion", "denominator": 223, "interval": { "low": 0.7592592592592593, "high": 0.870590770791075, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1673469387755102, "unit": "proportion", "denominator": 245, "interval": { "low": 0.11764705882352941, "high": 0.21900924832743013, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.1938775510204085, "unit": "points_per_event", "denominator": 245, "interval": { "low": 2.7397476408596226, "high": 3.63292194092827, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.255102040816326, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.064514336917562, "high": 4.435743893003205, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.605582264957265, "unit": "f1", "denominator": null, "interval": { "low": 0.4907888170970279, "high": 0.593548859619793, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7390156453715775, "unit": "f1", "denominator": null, "interval": { "low": 0.5989287259489153, "high": 0.7243308117394084, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21224489795918366, "unit": "proportion", "denominator": 245, "interval": { "low": 0.1582966207966208, "high": 0.2681992337164751, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5692307692307692, "unit": "proportion", "denominator": 130, "interval": { "low": 0.48333333333333334, "high": 0.6571526418786692, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7538461538461538, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6744186046511628, "high": 0.8321180644173345, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7551020408163265, "unit": "proportion", "denominator": 98, "interval": { "low": 0.6666666666666666, "high": 0.8387096774193549, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18461538461538463, "unit": "proportion", "denominator": 130, "interval": { "low": 0.12, "high": 0.2540983606557377, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.396153846153846, "unit": "points_per_event", "denominator": 130, "interval": { "low": 1.7829299161525076, "high": 2.9803149606299217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.7807692307692307, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.5090791536050165, "high": 4.050789091012773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.45083943833943835, "unit": "f1", "denominator": null, "interval": { "low": 0.34779137529137527, "high": 0.45883793290043284, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5328102453102453, "unit": "f1", "denominator": null, "interval": { "low": 0.411026170798898, "high": 0.5422630116096026, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27692307692307694, "unit": "proportion", "denominator": 130, "interval": { "low": 0.2, "high": 0.3577981651376147, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.5700934579439252, "unit": "proportion", "denominator": 107, "interval": { "low": 0.47572019735795, "high": 0.66087324981577, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7476635514018691, "unit": "proportion", "denominator": 107, "interval": { "low": 0.6630434782608695, "high": 0.8285714285714286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7625, "unit": "proportion", "denominator": 80, "interval": { "low": 0.6626506024096386, "high": 0.8536653877774731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.17757009345794392, "unit": "proportion", "denominator": 107, "interval": { "low": 0.10678578806031812, "high": 0.25555764411027565, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.1588785046728973, "unit": "points_per_event", "denominator": 107, "interval": { "low": 1.4380803571428573, "high": 2.8539425373794565, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5140186915887854, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.1666666666666665, "high": 3.8482245249017035, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5385204081632653, "unit": "f1", "denominator": null, "interval": { "low": 0.34609539270253564, "high": 0.502640306122449, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5913165266106443, "unit": "f1", "denominator": null, "interval": { "low": 0.38002631355572525, "high": 0.5519187675070027, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27102803738317754, "unit": "proportion", "denominator": 107, "interval": { "low": 0.18867924528301888, "high": 0.35714285714285715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3458646616541353, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2635556915544676, "high": 0.42857142857142855, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5338345864661654, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4485294117647059, "high": 0.6166666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.647887323943662, "unit": "proportion", "denominator": 71, "interval": { "low": 0.5342465753424658, "high": 0.7567567567567568, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18796992481203006, "unit": "proportion", "denominator": 133, "interval": { "low": 0.125, "high": 0.2565833926031294, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.635338345864662, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.0679806722689076, "high": 2.184435138987883, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.9887218045112784, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.6951219512195124, "high": 3.277372262773723, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.31372074253430193, "unit": "f1", "denominator": null, "interval": { "low": 0.20401790300095382, "high": 0.31341736467517023, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.37774538386783285, "unit": "f1", "denominator": null, "interval": { "low": 0.2456542097358424, "high": 0.3773800921598988, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2706766917293233, "unit": "proportion", "denominator": 133, "interval": { "low": 0.19548872180451127, "high": 0.34965909090909075, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.18421052631578946, "unit": "proportion", "denominator": 76, "interval": { "low": 0.10168877425693934, "high": 0.2753623188405797, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.40789473684210525, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2987012987012987, "high": 0.5185185185185185, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.45161290322580644, "unit": "proportion", "denominator": 31, "interval": { "low": 0.27586206896551724, "high": 0.6296296296296297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.13333333333333333, "high": 0.32, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.006578947368421052, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.7619360902255639, "high": 0.7800312499999996, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.6513157894736843, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.25, "high": 2.0641233766233764, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1812121212121212, "unit": "f1", "denominator": null, "interval": { "low": 0.07454329004329004, "high": 0.1777215007215007, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.21666666666666667, "unit": "f1", "denominator": null, "interval": { "low": 0.08912784679089027, "high": 0.21249309868875085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.32894736842105265, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2261904761904762, "high": 0.43478260869565216, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.13592233009708737, "high": 0.3020893895348836, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.41836734693877553, "unit": "proportion", "denominator": 98, "interval": { "low": 0.31868131868131866, "high": 0.5252525252525253, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5121951219512195, "unit": "proportion", "denominator": 41, "interval": { "low": 0.358974358974359, "high": 0.6666666666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.20408163265306123, "unit": "proportion", "denominator": 98, "interval": { "low": 0.125, "high": 0.28846153846153844, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.1326530612244898, "unit": "points_per_event", "denominator": 98, "interval": { "low": -0.5543589934484813, "high": 0.8275909961685823, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.816326530612245, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.4471153846153848, "high": 2.2209302325581395, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.15296495956873316, "unit": "f1", "denominator": null, "interval": { "low": 0.08513426039369436, "high": 0.17103250915750914, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.17624223602484473, "unit": "f1", "denominator": null, "interval": { "low": 0.09808947393186523, "high": 0.19705919533365185, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.336734693877551, "unit": "proportion", "denominator": 98, "interval": { "low": 0.24443798449612403, "high": 0.4326923076923077, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.003248393670886076 } }, "gemini-high": { "metrics": { "species_accuracy": { "value": 0.5139240506329114, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.48570456969191145, "high": 0.5386299776173193, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.7316455696202532, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.7062302566456048, "high": 0.7552009716724906, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.7024221453287197, "unit": "proportion", "denominator": 578, "population_events": 790, "interval": { "low": 0.6708708952491154, "high": 0.7324712048388243, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.3771972779892321, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.3529444026478563, "high": 0.38958075995382013, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 1.8740506329113924, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 1.6584591735620213, "high": 2.0725644858833143, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.21772151898734177, "unit": "proportion", "denominator": 790, "interval": { "low": 0.19377996739547376, "high": 0.24249152482538558, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.5681832415280837, "unit": "f1", "denominator": null, "interval": { "low": 0.5316504293049987, "high": 0.5868368409430961, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.3424050632911397, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.229050540981337, "high": 3.4410532211283797, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.2936708860759494, "unit": "proportion", "denominator": 790, "interval": { "low": 0.26745416942410616, "high": 0.32023772992127425, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.6060606060606061, "unit": "proportion", "denominator": 429, "interval": { "low": 0.5576459745188398, "high": 0.654320987654321, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8135198135198135, "unit": "proportion", "denominator": 429, "interval": { "low": 0.7748799387214119, "high": 0.8495151249550521, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7449856733524355, "unit": "proportion", "denominator": 349, "interval": { "low": 0.6974063400576369, "high": 0.7916666666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.20745920745920746, "unit": "proportion", "denominator": 429, "interval": { "low": 0.16904761904761906, "high": 0.24708624708624707, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.358974358974359, "unit": "points_per_event", "denominator": 429, "interval": { "low": 1.9977703682602166, "high": 2.7055291931293306, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.641025641025641, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.461084547009136, "high": 3.8148159067085956, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4296899965310341, "unit": "f1", "denominator": null, "interval": { "low": 0.37659909514081835, "high": 0.44071373632432786, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.602952091906451, "unit": "f1", "denominator": null, "interval": { "low": 0.5284535689879226, "high": 0.6184208880680085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2564102564102564, "unit": "proportion", "denominator": 429, "interval": { "low": 0.21461852125483066, "high": 0.29904306220095694, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.40443213296398894, "unit": "proportion", "denominator": 361, "interval": { "low": 0.352112676056338, "high": 0.4568245125348189, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6343490304709142, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5826233626609914, "high": 0.685016980080085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6375545851528385, "unit": "proportion", "denominator": 229, "interval": { "low": 0.5714285714285714, "high": 0.7021276595744681, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2299168975069252, "unit": "proportion", "denominator": 361, "interval": { "low": 0.18579234972677597, "high": 0.2755681818181818, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.2977839335180055, "unit": "points_per_event", "denominator": 361, "interval": { "low": 0.8977095651638335, "high": 1.6852459287700254, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.987534626038781, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.7849311242060155, "high": 3.186638303536293, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2868774052367834, "unit": "f1", "denominator": null, "interval": { "low": 0.2262036091715442, "high": 0.2938424611300952, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4084356277947424, "unit": "f1", "denominator": null, "interval": { "low": 0.3220525961086392, "high": 0.4183519785581017, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3379501385041551, "unit": "proportion", "denominator": 361, "interval": { "low": 0.28653216194242503, "high": 0.38875035521454954, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.6906666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.6416181349327075, "high": 0.7393787036321076, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8746666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.8404255319148937, "high": 0.9076527298558961, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7896341463414634, "unit": "proportion", "denominator": 328, "interval": { "low": 0.743202416918429, "high": 0.833846394984326, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.184, "unit": "proportion", "denominator": 375, "interval": { "low": 0.14477123606603642, "high": 0.22493224932249323, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.9493333333333336, "unit": "points_per_event", "denominator": 375, "interval": { "low": 2.6005215453290926, "high": 3.29264715531882, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.1226666666666665, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.9755128232212273, "high": 4.269241192411925, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5346079846079846, "unit": "f1", "denominator": null, "interval": { "low": 0.4735553460474888, "high": 0.5418435870400683, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6785409035409036, "unit": "f1", "denominator": null, "interval": { "low": 0.6010510161371972, "high": 0.687724552781625, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.23466666666666666, "unit": "proportion", "denominator": 375, "interval": { "low": 0.19137466307277629, "high": 0.27823691460055094, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.475, "unit": "proportion", "denominator": 240, "interval": { "low": 0.4126984126984127, "high": 0.5381526104417671, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7041666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.643936799501868, "high": 0.7615070807853235, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6745562130177515, "unit": "proportion", "denominator": 169, "interval": { "low": 0.6031746031746031, "high": 0.7430205757548005, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.22916666666666666, "unit": "proportion", "denominator": 240, "interval": { "low": 0.17721394398485, "high": 0.28270042194092826, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.7041666666666666, "unit": "points_per_event", "denominator": 240, "interval": { "low": 1.2284370689655173, "high": 2.173439161298393, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2875, "unit": "points_per_event", "denominator": 240, "interval": { "low": 3.0572033898305087, "high": 3.5150230779191953, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.452971341681019, "unit": "f1", "denominator": null, "interval": { "low": 0.3382683254267695, "high": 0.43860563197256747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5332447440042375, "unit": "f1", "denominator": null, "interval": { "low": 0.3982146109454374, "high": 0.5163332123221365, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.31666666666666665, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2562999166608792, "high": 0.3782612766650382, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.1839080459770115, "unit": "proportion", "denominator": 174, "interval": { "low": 0.12727272727272726, "high": 0.2441893498942917, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.45977011494252873, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3864995817066369, "high": 0.534893980848153, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4, "unit": "proportion", "denominator": 80, "interval": { "low": 0.29333333333333333, "high": 0.5068493150684932, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.27586206896551724, "unit": "proportion", "denominator": 174, "interval": { "low": 0.21176470588235294, "high": 0.34355828220858897, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.22701149425287356, "unit": "points_per_event", "denominator": 174, "interval": { "low": -0.7452244321595962, "high": 0.29697601010101005, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.7270114942528736, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.4580775196594764, "high": 2.006373589775613, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.16573100812231245, "unit": "f1", "denominator": null, "interval": { "low": 0.09024741034931252, "high": 0.16307320383687074, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.19801626944484085, "unit": "f1", "denominator": null, "interval": { "low": 0.10782807470307469, "high": 0.19484071107781958, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.39080459770114945, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3172043010752688, "high": 0.46703296703296704, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.7551020408163265, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6956521739130435, "high": 0.8122275198288923, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9306122448979591, "unit": "proportion", "denominator": 245, "interval": { "low": 0.8966942148760331, "high": 0.9610894941634242, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8114035087719298, "unit": "proportion", "denominator": 228, "interval": { "low": 0.7567567567567568, "high": 0.863854398319743, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.17551020408163265, "unit": "proportion", "denominator": 245, "interval": { "low": 0.12652940227474388, "high": 0.22608695652173913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.3000000000000003, "unit": "points_per_event", "denominator": 245, "interval": { "low": 2.872339468850452, "high": 3.7107553790253633, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.3, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.121716212527863, "high": 4.469433595752096, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5714178464178464, "unit": "f1", "denominator": null, "interval": { "low": 0.47556384306384314, "high": 0.5685184446581505, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7166935700834006, "unit": "f1", "denominator": null, "interval": { "low": 0.5964699048597354, "high": 0.713057032283104, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2, "unit": "proportion", "denominator": 245, "interval": { "low": 0.15017857142857144, "high": 0.252, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.5692307692307692, "unit": "proportion", "denominator": 130, "interval": { "low": 0.48360655737704916, "high": 0.6549356483729966, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7692307692307693, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6949152542372882, "high": 0.8409090909090909, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.74, "unit": "proportion", "denominator": 100, "interval": { "low": 0.6511627906976745, "high": 0.8240766178266178, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2, "unit": "proportion", "denominator": 130, "interval": { "low": 0.13380281690140844, "high": 0.271189725397451, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2884615384615383, "unit": "points_per_event", "denominator": 130, "interval": { "low": 1.6850615993595448, "high": 2.8949600082034457, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.7884615384615388, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.5269078947368424, "high": 4.046524050884516, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4380217519106408, "unit": "f1", "denominator": null, "interval": { "low": 0.3333241809630698, "high": 0.4464432209040053, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5375721500721501, "unit": "f1", "denominator": null, "interval": { "low": 0.40907967663649475, "high": 0.5479075892912791, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3, "unit": "proportion", "denominator": 130, "interval": { "low": 0.22033898305084745, "high": 0.3787878787878788, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.5794392523364486, "unit": "proportion", "denominator": 107, "interval": { "low": 0.48514851485148514, "high": 0.6698113207547169, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7850467289719626, "unit": "proportion", "denominator": 107, "interval": { "low": 0.7064220183486238, "high": 0.8571428571428571, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7380952380952381, "unit": "proportion", "denominator": 84, "interval": { "low": 0.6395187338501293, "high": 0.8300127118644065, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.205607476635514, "unit": "proportion", "denominator": 107, "interval": { "low": 0.1308411214953271, "high": 0.2857142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.107476635514019, "unit": "points_per_event", "denominator": 107, "interval": { "low": 1.3818157380254155, "high": 2.8136385918003564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.55607476635514, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.211108562691132, "high": 3.890476190476191, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5098466505246166, "unit": "f1", "denominator": null, "interval": { "low": 0.3329161163695062, "high": 0.4811956122973072, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5898225957049487, "unit": "f1", "denominator": null, "interval": { "low": 0.3851382522706052, "high": 0.5566772769713946, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2897196261682243, "unit": "proportion", "denominator": 107, "interval": { "low": 0.20407667921537548, "high": 0.3786407766990291, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.39097744360902253, "unit": "proportion", "denominator": 133, "interval": { "low": 0.30767716535433076, "high": 0.47580784204671855, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6390977443609023, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5555555555555556, "high": 0.7196969696969697, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.611764705882353, "unit": "proportion", "denominator": 85, "interval": { "low": 0.5046428571428573, "high": 0.7142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.24812030075187969, "unit": "proportion", "denominator": 133, "interval": { "low": 0.17518248175182483, "high": 0.32608695652173914, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.3796992481203008, "unit": "points_per_event", "denominator": 133, "interval": { "low": 0.7605633802816901, "high": 1.9843780281007755, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0714285714285716, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.762403483184626, "high": 3.371212121212121, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.35121693121693115, "unit": "f1", "denominator": null, "interval": { "low": 0.2299664953102453, "high": 0.34595628908128906, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4300615484288954, "unit": "f1", "denominator": null, "interval": { "low": 0.2815916269105045, "high": 0.4236199458138234, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3383458646616541, "unit": "proportion", "denominator": 133, "interval": { "low": 0.2569444444444444, "high": 0.4227655917498772, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.15789473684210525, "unit": "proportion", "denominator": 76, "interval": { "low": 0.078125, "high": 0.24358974358974358, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.47368421052631576, "unit": "proportion", "denominator": 76, "interval": { "low": 0.36, "high": 0.5853658536585366, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 36, "interval": { "low": 0.18181818181818182, "high": 0.48717948717948717, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3157894736842105, "unit": "proportion", "denominator": 76, "interval": { "low": 0.21621621621621623, "high": 0.4225352112676056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.35526315789473684, "unit": "points_per_event", "denominator": 76, "interval": { "low": -1.081085304054054, "high": 0.39103915662650585, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.6184210526315792, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.2272727272727275, "high": 2.019743421052632, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1596045197740113, "unit": "f1", "denominator": null, "interval": { "low": 0.057908931934355666, "high": 0.1556383447061413, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.20471014492753623, "unit": "f1", "denominator": null, "interval": { "low": 0.07427449965493443, "high": 0.1996230942970073, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.39473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2857142857142857, "high": 0.506333223738287, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.20408163265306123, "unit": "proportion", "denominator": 98, "interval": { "low": 0.12631578947368421, "high": 0.2894798327594687, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4489795918367347, "unit": "proportion", "denominator": 98, "interval": { "low": 0.3490566037735849, "high": 0.550561797752809, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.45454545454545453, "unit": "proportion", "denominator": 44, "interval": { "low": 0.3055555555555556, "high": 0.6052631578947368, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.24489795918367346, "unit": "proportion", "denominator": 98, "interval": { "low": 0.1619047619047619, "high": 0.3333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.12755102040816327, "unit": "points_per_event", "denominator": 98, "interval": { "low": -0.8724489795918369, "high": 0.6044327731092431, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.8112244897959184, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.4444314641744551, "high": 2.2070801096067054, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.14779183081069872, "unit": "f1", "denominator": null, "interval": { "low": 0.08113157282968603, "high": 0.16378640135299402, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.17028189202102245, "unit": "f1", "denominator": null, "interval": { "low": 0.09347768173855131, "high": 0.18871041895018875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3877551020408163, "unit": "proportion", "denominator": 98, "interval": { "low": 0.28888888888888886, "high": 0.49, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.004065537974683544 } }, "gemini-medium37": { "metrics": { "species_accuracy": { "value": 0.5569620253164557, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.5289582736180838, "high": 0.5833146679270096, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.669620253164557, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6436389208541109, "high": 0.6939144382340586, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.831758034026465, "unit": "proportion", "denominator": 529, "population_events": 790, "interval": { "low": 0.8035102363854232, "high": 0.8593842856522088, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.5158212687832463, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.485934472060884, "high": 0.5317084254079105, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 2.7278481012658227, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 2.535525864782194, "high": 2.9083559706819213, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.11265822784810127, "unit": "proportion", "denominator": 790, "interval": { "low": 0.0935086880419159, "high": 0.13236580929618905, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.6268207823188815, "unit": "f1", "denominator": null, "interval": { "low": 0.5905026495929728, "high": 0.6461266941665748, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.5189873417721524, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.4080927322501697, "high": 3.623406529300516, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.15822784810126583, "unit": "proportion", "denominator": 790, "interval": { "low": 0.1360821597530458, "high": 0.18104793413337716, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.6503496503496503, "unit": "proportion", "denominator": 429, "interval": { "low": 0.6017073642589098, "high": 0.6977912871355494, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7482517482517482, "unit": "proportion", "denominator": 429, "interval": { "low": 0.7034266356168403, "high": 0.7906976744186046, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8691588785046729, "unit": "proportion", "denominator": 321, "interval": { "low": 0.8292008880447617, "high": 0.9060791360474888, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.0979020979020979, "unit": "proportion", "denominator": 429, "interval": { "low": 0.06994773361353672, "high": 0.12809276865456637, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.1573426573426575, "unit": "points_per_event", "denominator": 429, "interval": { "low": 2.851803304457175, "high": 3.4441782769806046, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.7983682983682985, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.623852675126468, "high": 3.96946045310016, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5693708656290797, "unit": "f1", "denominator": null, "interval": { "low": 0.4885538818334084, "high": 0.5733599868573677, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7071218815070828, "unit": "f1", "denominator": null, "interval": { "low": 0.6067524016318138, "high": 0.7120761127099566, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1282051282051282, "unit": "proportion", "denominator": 429, "interval": { "low": 0.09579187720788802, "high": 0.16312056737588654, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.44598337950138506, "unit": "proportion", "denominator": 361, "interval": { "low": 0.39322749732905987, "high": 0.498567335243553, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5761772853185596, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5220994475138122, "high": 0.6293103448275862, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7740384615384616, "unit": "proportion", "denominator": 208, "interval": { "low": 0.7162790697674418, "high": 0.8287068318318318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13019390581717452, "unit": "proportion", "denominator": 361, "interval": { "low": 0.09710886208476019, "high": 0.1657612328528249, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2174515235457064, "unit": "points_per_event", "denominator": 361, "interval": { "low": 1.8729848545894077, "high": 2.553152482500938, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.186980609418283, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.9888252463749923, "high": 3.3839804083459364, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3922731053912909, "unit": "f1", "denominator": null, "interval": { "low": 0.31622015203762194, "high": 0.4006330134517594, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.458760072406764, "unit": "f1", "denominator": null, "interval": { "low": 0.3698167879762019, "high": 0.4685369140368034, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19390581717451524, "unit": "proportion", "denominator": 361, "interval": { "low": 0.15363881401617252, "high": 0.23697916666666666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.7386666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.6916395027521984, "high": 0.7849506798187149, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8453333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.8061776354695136, "high": 0.8841322275825756, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8738170347003155, "unit": "proportion", "denominator": 317, "interval": { "low": 0.8360128617363344, "high": 0.909375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.10666666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.07646722047597666, "high": 0.13920544447640965, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.6106666666666665, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.3203246099556925, "high": 3.883692024575624, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.264, "unit": "points_per_event", "denominator": 375, "interval": { "low": 4.122714801904145, "high": 4.401832460732984, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6917675788264023, "unit": "f1", "denominator": null, "interval": { "low": 0.6092684192932306, "high": 0.6934404355584225, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7538492846185153, "unit": "f1", "denominator": null, "interval": { "low": 0.6639463543580076, "high": 0.7556722695187938, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13066666666666665, "unit": "proportion", "denominator": 375, "interval": { "low": 0.09714210723659543, "high": 0.1671018276762402, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.5083333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.4435146443514644, "high": 0.572, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6083333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5450368050977807, "high": 0.6709429824561403, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8356164383561644, "unit": "proportion", "denominator": 146, "interval": { "low": 0.7741880247459125, "high": 0.891724048889654, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1, "unit": "proportion", "denominator": 240, "interval": { "low": 0.06493506493506493, "high": 0.13924310022826308, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.583333333333334, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.1633674461626033, "high": 2.9716656098076517, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.458333333333334, "unit": "points_per_event", "denominator": 240, "interval": { "low": 3.232063695125236, "high": 3.677928882272103, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5349987267634326, "unit": "f1", "denominator": null, "interval": { "low": 0.403278626602156, "high": 0.5148592693144843, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5756315414543262, "unit": "f1", "denominator": null, "interval": { "low": 0.4339073830529527, "high": 0.5539625049586224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.175, "unit": "proportion", "denominator": 240, "interval": { "low": 0.12704918032786885, "high": 0.22666801619433194, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.22988505747126436, "unit": "proportion", "denominator": 174, "interval": { "low": 0.16867294760437096, "high": 0.2947383632491633, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3735632183908046, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3016759776536313, "high": 0.449438202247191, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6153846153846154, "unit": "proportion", "denominator": 65, "interval": { "low": 0.5, "high": 0.734375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14367816091954022, "unit": "proportion", "denominator": 174, "interval": { "low": 0.09259259259259259, "high": 0.19753086419753085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.0114942528735633, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.5570423302720298, "high": 1.4634932896379524, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.988505747126437, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.7085710466004587, "high": 2.27457044400686, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.23909191583610187, "unit": "f1", "denominator": null, "interval": { "low": 0.13719663411523877, "high": 0.22759589247961343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2670377241805813, "unit": "f1", "denominator": null, "interval": { "low": 0.15323260433650043, "high": 0.2541980097824254, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19540229885057472, "unit": "proportion", "denominator": 174, "interval": { "low": 0.13736263736263737, "high": 0.2561599445812808, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.8, "unit": "proportion", "denominator": 245, "interval": { "low": 0.746720363028998, "high": 0.8512412055669913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8938775510204081, "unit": "proportion", "denominator": 245, "interval": { "low": 0.8493123505109806, "high": 0.9356223175965666, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8949771689497716, "unit": "proportion", "denominator": 219, "interval": { "low": 0.8530612244897959, "high": 0.9339622641509434, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09387755102040816, "unit": "proportion", "denominator": 245, "interval": { "low": 0.05859375, "high": 0.13158165475416933, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.8877551020408165, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.555752551020408, "high": 4.199579831932773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.438775510204081, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.283329646017699, "high": 4.586618240385241, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7142061642061642, "unit": "f1", "denominator": null, "interval": { "low": 0.5822732846372024, "high": 0.695857211341837, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7989424887729972, "unit": "f1", "denominator": null, "interval": { "low": 0.6513565556958536, "high": 0.778416541501038, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.11020408163265306, "unit": "proportion", "denominator": 245, "interval": { "low": 0.07199924242424242, "high": 0.15126050420168066, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.6230769230769231, "unit": "proportion", "denominator": 130, "interval": { "low": 0.5371862565044383, "high": 0.7086614173228346, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7538461538461538, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6759243295019157, "high": 0.8297872340425532, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.826530612244898, "unit": "proportion", "denominator": 98, "interval": { "low": 0.75, "high": 0.8977272727272727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13076923076923078, "unit": "proportion", "denominator": 130, "interval": { "low": 0.07627118644067797, "high": 0.1896551724137931, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.088461538461539, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.546589281451421, "high": 3.5990611047097794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.934615384615385, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.6607142857142856, "high": 4.203547481279783, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5803902116402117, "unit": "f1", "denominator": null, "interval": { "low": 0.43799546807359313, "high": 0.5790610166514771, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6331529581529581, "unit": "f1", "denominator": null, "interval": { "low": 0.4778132378984651, "high": 0.6317029272561568, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.16923076923076924, "unit": "proportion", "denominator": 130, "interval": { "low": 0.10526315789473684, "high": 0.23741744309470444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.5981308411214953, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5042735042735043, "high": 0.6885245901639344, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6822429906542056, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5904761904761905, "high": 0.7663551401869159, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8767123287671232, "unit": "proportion", "denominator": 73, "interval": { "low": 0.7966101694915254, "high": 0.9473684210526315, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08411214953271028, "unit": "proportion", "denominator": 107, "interval": { "low": 0.03571428571428571, "high": 0.14130434782608695, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.9392523364485985, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.361319933378757, "high": 3.4854403939507095, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.640186915887851, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.2889858826320792, "high": 3.961166094743891, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5605442176870749, "unit": "f1", "denominator": null, "interval": { "low": 0.36844954648526074, "high": 0.5252251317193544, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6154995331465919, "unit": "f1", "denominator": null, "interval": { "low": 0.40457205104263927, "high": 0.5767177916918402, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14018691588785046, "unit": "proportion", "denominator": 107, "interval": { "low": 0.07563025210084033, "high": 0.2087987012987012, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.43609022556390975, "unit": "proportion", "denominator": 133, "interval": { "low": 0.3515625, "high": 0.5220604502863092, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5488721804511278, "unit": "proportion", "denominator": 133, "interval": { "low": 0.46218487394957986, "high": 0.6349206349206349, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7945205479452054, "unit": "proportion", "denominator": 73, "interval": { "low": 0.6973684210526315, "high": 0.8815982972136219, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11278195488721804, "unit": "proportion", "denominator": 133, "interval": { "low": 0.06299074940807224, "high": 0.1680016806722689, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2969924812030076, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.7519056510516782, "high": 2.8104869257087, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3120300751879705, "unit": "points_per_event", "denominator": 133, "interval": { "low": 3.0147031987977675, "high": 3.597744360902256, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.46521578421578424, "unit": "f1", "denominator": null, "interval": { "low": 0.30725842490842487, "high": 0.4542333638583639, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47470998389365743, "unit": "f1", "denominator": null, "interval": { "low": 0.3135290050085969, "high": 0.46350343250853454, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.20300751879699247, "unit": "proportion", "denominator": 133, "interval": { "low": 0.1366906474820144, "high": 0.2765957446808511, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14473684210526316, "high": 0.33783783783783783, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2602739726027397, "high": 0.4794520547945205, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6428571428571429, "unit": "proportion", "denominator": 28, "interval": { "low": 0.4583333333333333, "high": 0.8181818181818182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13157894736842105, "unit": "proportion", "denominator": 76, "interval": { "low": 0.060596933187294644, "high": 0.2112676056338028, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.0855263157894737, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.4090909090909091, "high": 1.7711254107338439, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.9407894736842106, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.5249922839506174, "high": 2.3852666528325375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2759259259259259, "unit": "f1", "denominator": null, "interval": { "low": 0.11724046015712682, "high": 0.25044091710758376, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3239130434782609, "unit": "f1", "denominator": null, "interval": { "low": 0.1376301054018445, "high": 0.2939958592132505, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.17105263157894737, "unit": "proportion", "denominator": 76, "interval": { "low": 0.09090909090909091, "high": 0.25925925925925924, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.22448979591836735, "unit": "proportion", "denominator": 98, "interval": { "low": 0.14457496653279786, "high": 0.31314616755793206, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.37755102040816324, "unit": "proportion", "denominator": 98, "interval": { "low": 0.2803738317757009, "high": 0.47959183673469385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5945945945945946, "unit": "proportion", "denominator": 37, "interval": { "low": 0.4358974358974359, "high": 0.75, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15306122448979592, "unit": "proportion", "denominator": 98, "interval": { "low": 0.08602150537634409, "high": 0.22772277227722773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.9540816326530612, "unit": "points_per_event", "denominator": 98, "interval": { "low": 0.33683941997851774, "high": 1.5693069306930694, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.0255102040816326, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.6553279920955408, "high": 2.421081871345029, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.18898225957049486, "unit": "f1", "denominator": null, "interval": { "low": 0.10469771241830067, "high": 0.20046724245253655, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.20952380952380947, "unit": "f1", "denominator": null, "interval": { "low": 0.11607789855072465, "high": 0.22225716011042096, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.1346153846153846, "high": 0.2990887850467286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Saved provider-token cost estimate for each selected response; not an invoice.", "value": 0.0032213569260759493 } }, "gemini-medium38": { "metrics": { "species_accuracy": { "value": 0.5670886075949367, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.5389654798436445, "high": 0.593053275838086, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.660759493670886, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6348663748347293, "high": 0.6849344765642235, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.8582375478927203, "unit": "proportion", "denominator": 522, "population_events": 790, "interval": { "low": 0.8315573625250614, "high": 0.883581261189724, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.5717006119944849, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.5400427581382938, "high": 0.5877039707227594, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 2.8784810126582276, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 2.695389768020306, "high": 3.0551292879061864, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.09367088607594937, "unit": "proportion", "denominator": 790, "interval": { "low": 0.07666521681553327, "high": 0.1117573029993916, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.6368310614622108, "unit": "f1", "denominator": null, "interval": { "low": 0.6015666166603778, "high": 0.6546575876405419, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.568354430379747, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.4558900259374945, "high": 3.66975747529545, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1379746835443038, "unit": "proportion", "denominator": 790, "interval": { "low": 0.11743710491337073, "high": 0.15924142225566273, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.6783216783216783, "unit": "proportion", "denominator": 429, "interval": { "low": 0.631336405529954, "high": 0.7246393861892583, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7575757575757576, "unit": "proportion", "denominator": 429, "interval": { "low": 0.7136038186157518, "high": 0.8, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8953846153846153, "unit": "proportion", "denominator": 325, "interval": { "low": 0.8610271903323263, "high": 0.926984126984127, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07925407925407925, "unit": "proportion", "denominator": 429, "interval": { "low": 0.055023325358851675, "high": 0.10552989667438312, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.371794871794872, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.0942477422003285, "high": 3.6412313254587243, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.8962703962703964, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.7297603165396094, "high": 4.059288663908255, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6164839736143914, "unit": "f1", "denominator": null, "interval": { "low": 0.5357398042384097, "high": 0.6193322491825886, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7159168725844545, "unit": "f1", "denominator": null, "interval": { "low": 0.6221494500833143, "high": 0.7192245474378447, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1048951048951049, "unit": "proportion", "denominator": 429, "interval": { "low": 0.07586206896551724, "high": 0.1358590827513136, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.43490304709141275, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3808187961364721, "high": 0.488890423572744, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5457063711911357, "unit": "proportion", "denominator": 361, "interval": { "low": 0.4913282195187525, "high": 0.5988712048137431, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7969543147208121, "unit": "proportion", "denominator": 197, "interval": { "low": 0.7357512953367875, "high": 0.8523829039812646, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11080332409972299, "unit": "proportion", "denominator": 361, "interval": { "low": 0.07926605750661612, "high": 0.14641059061551626, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2922437673130194, "unit": "points_per_event", "denominator": 361, "interval": { "low": 1.952243848778736, "high": 2.621520630144567, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1786703601108037, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.9796682209469156, "high": 3.3761107563178534, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.41804116678889525, "unit": "f1", "denominator": null, "interval": { "low": 0.3363835465957599, "high": 0.4298271630594536, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4534683843133779, "unit": "f1", "denominator": null, "interval": { "low": 0.3648906268157397, "high": 0.4662531938272039, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1772853185595568, "unit": "proportion", "denominator": 361, "interval": { "low": 0.13829630788485608, "high": 0.21957671957671956, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.7626666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.7164128680919726, "high": 0.8079118644067796, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8426666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.802138331945875, "high": 0.8820375335120644, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9050632911392406, "unit": "proportion", "denominator": 316, "interval": { "low": 0.8706624605678234, "high": 0.9367088607594937, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08, "unit": "proportion", "denominator": 375, "interval": { "low": 0.05316225868043278, "high": 0.109375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.88, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.6189169604386997, "high": 4.124301675977654, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.346666666666667, "unit": "points_per_event", "denominator": 375, "interval": { "low": 4.210327206029347, "high": 4.477721032449445, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7303325857542724, "unit": "f1", "denominator": null, "interval": { "low": 0.6490451096466334, "high": 0.7319890034241242, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7771487771487771, "unit": "f1", "denominator": null, "interval": { "low": 0.6906505653932123, "high": 0.7789113754384911, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.09333333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.06417112299465241, "high": 0.12430939226519337, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.49583333333333335, "unit": "proportion", "denominator": 240, "interval": { "low": 0.4297872340425532, "high": 0.5603448275862069, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6083333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5447154471544715, "high": 0.6708860759493671, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.815068493150685, "unit": "proportion", "denominator": 146, "interval": { "low": 0.7481481481481481, "high": 0.875, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1125, "unit": "proportion", "denominator": 240, "interval": { "low": 0.07441764366711513, "high": 0.15451282444308187, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.54375, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.1189447486506845, "high": 2.938818565400844, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.439583333333333, "unit": "points_per_event", "denominator": 240, "interval": { "low": 3.2120573105585972, "high": 3.6611603572380695, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5312174100409395, "unit": "f1", "denominator": null, "interval": { "low": 0.39787418300653593, "high": 0.5096462758997816, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5715630361199983, "unit": "f1", "denominator": null, "interval": { "low": 0.4280924753867792, "high": 0.5483535879934358, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.17916666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.13022877670637173, "high": 0.23333333333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.2413793103448276, "unit": "proportion", "denominator": 174, "interval": { "low": 0.17901234567901234, "high": 0.3072639324158605, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3390804597701149, "unit": "proportion", "denominator": 174, "interval": { "low": 0.269836811128946, "high": 0.4114285714285714, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.711864406779661, "unit": "proportion", "denominator": 59, "interval": { "low": 0.5909090909090909, "high": 0.8245614035087719, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09770114942528736, "unit": "proportion", "denominator": 174, "interval": { "low": 0.055865921787709494, "high": 0.14450867052023122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.1695402298850575, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.7028532818532819, "high": 1.6114369747899158, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.060344827586207, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.776532842916132, "high": 2.3493589743589745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.294006919006919, "unit": "f1", "denominator": null, "interval": { "low": 0.1657100075850076, "high": 0.2729249835132188, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2978251906823335, "unit": "f1", "denominator": null, "interval": { "low": 0.16786208560559213, "high": 0.2764694638185852, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1781609195402299, "unit": "proportion", "denominator": 174, "interval": { "low": 0.12209302325581395, "high": 0.23837209302325582, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.8448979591836735, "unit": "proportion", "denominator": 245, "interval": { "low": 0.7940074906367042, "high": 0.8933345588235294, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9061224489795918, "unit": "proportion", "denominator": 245, "interval": { "low": 0.8619399081995879, "high": 0.9458333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9324324324324325, "unit": "proportion", "denominator": 222, "interval": { "low": 0.8976715825297787, "high": 0.9641255605381166, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.061224489795918366, "unit": "proportion", "denominator": 245, "interval": { "low": 0.032520325203252036, "high": 0.09282700421940929, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 4.259183673469388, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.9860382059800674, "high": 4.512346954392409, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.585714285714285, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.44924659208998, "high": 4.71401380597015, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7884168741311597, "unit": "f1", "denominator": null, "interval": { "low": 0.6433944250002739, "high": 0.764112325064706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.841868865597679, "unit": "f1", "denominator": null, "interval": { "low": 0.6870143860172415, "high": 0.8159165504928217, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.0653061224489796, "unit": "proportion", "denominator": 245, "interval": { "low": 0.03539562727745966, "high": 0.09767616716209125, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.6076923076923076, "unit": "proportion", "denominator": 130, "interval": { "low": 0.5217313664596274, "high": 0.695659518213866, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7230769230769231, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6408395686619719, "high": 0.8017260795278036, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8404255319148937, "unit": "proportion", "denominator": 94, "interval": { "low": 0.7582417582417582, "high": 0.9146341463414634, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 130, "interval": { "low": 0.06086809635722679, "high": 0.1760014084507042, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.1653846153846157, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.649604459489293, "high": 3.653508771929825, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.896153846153846, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.6222154471544714, "high": 4.171433901918976, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5672949735449735, "unit": "f1", "denominator": null, "interval": { "low": 0.4270411532911533, "high": 0.5654266826923077, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6188672438672438, "unit": "f1", "denominator": null, "interval": { "low": 0.46586307631762175, "high": 0.6168291083916084, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14615384615384616, "unit": "proportion", "denominator": 130, "interval": { "low": 0.08546511627906977, "high": 0.21088568239795918, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.5794392523364486, "unit": "proportion", "denominator": 107, "interval": { "low": 0.48484848484848486, "high": 0.6704614745011085, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7009345794392523, "unit": "proportion", "denominator": 107, "interval": { "low": 0.6131999391357273, "high": 0.7840909090909091, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8266666666666667, "unit": "proportion", "denominator": 75, "interval": { "low": 0.734375, "high": 0.9090909090909091, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12149532710280374, "unit": "proportion", "denominator": 107, "interval": { "low": 0.0625, "high": 0.18811881188118812, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.7570093457943927, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.1176395558223287, "high": 3.3617042899152403, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5981308411214954, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.252332385209264, "high": 3.9272880244755246, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5518796992481203, "unit": "f1", "denominator": null, "interval": { "low": 0.3559276663881927, "high": 0.5115302144249513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6168067226890757, "unit": "f1", "denominator": null, "interval": { "low": 0.3978015094926859, "high": 0.5717102396514161, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.16822429906542055, "unit": "proportion", "denominator": 107, "interval": { "low": 0.09900990099009901, "high": 0.24347826086956523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.42857142857142855, "unit": "proportion", "denominator": 133, "interval": { "low": 0.34285714285714286, "high": 0.514290890269151, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5338345864661654, "unit": "proportion", "denominator": 133, "interval": { "low": 0.44881166049096805, "high": 0.6201550387596899, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8028169014084507, "unit": "proportion", "denominator": 71, "interval": { "low": 0.704225352112676, "high": 0.8902480788506515, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.10526315789473684, "unit": "proportion", "denominator": 133, "interval": { "low": 0.05714285714285714, "high": 0.16058394160583941, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3721804511278197, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.8257386363636365, "high": 2.871570642201835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3120300751879705, "unit": "points_per_event", "denominator": 133, "interval": { "low": 3.02082902892562, "high": 3.595238095238096, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4678095238095238, "unit": "f1", "denominator": null, "interval": { "low": 0.3112497308247308, "high": 0.4589404952890247, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47735665694849366, "unit": "f1", "denominator": null, "interval": { "low": 0.31760176614768454, "high": 0.46830662784594357, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.18796992481203006, "unit": "proportion", "denominator": 133, "interval": { "low": 0.12295081967213115, "high": 0.25925925925925924, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.27631578947368424, "unit": "proportion", "denominator": 76, "interval": { "low": 0.17855919765166342, "high": 0.38095238095238093, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.35526315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.25, "high": 0.4657534246575342, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7777777777777778, "unit": "proportion", "denominator": 27, "interval": { "low": 0.6071428571428571, "high": 0.9259259259259259, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.025, "high": 0.14516129032258066, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.3552631578947372, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.6551621510673235, "high": 2.0357250430292595, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.0789473684210527, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.6428571428571428, "high": 2.5294117647058822, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.35763888888888884, "unit": "f1", "denominator": null, "interval": { "low": 0.1581345390720391, "high": 0.3180654761904761, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3731884057971015, "unit": "f1", "denominator": null, "interval": { "low": 0.1650099538143016, "high": 0.33189440993788805, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.07042253521126761, "high": 0.22857142857142856, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.13541666666666666, "high": 0.3023332313341492, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.32653061224489793, "unit": "proportion", "denominator": 98, "interval": { "low": 0.23333333333333334, "high": 0.4226903251387786, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.65625, "unit": "proportion", "denominator": 32, "interval": { "low": 0.4827586206896552, "high": 0.8181818181818182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11224489795918367, "unit": "proportion", "denominator": 98, "interval": { "low": 0.05319148936170213, "high": 0.1794943820224718, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.0255102040816326, "unit": "points_per_event", "denominator": 98, "interval": { "low": 0.40525657894736844, "high": 1.6443298969072164, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.045918367346939, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.6768277285045574, "high": 2.4350000000000005, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.19630192502532925, "unit": "f1", "denominator": null, "interval": { "low": 0.10437310030395137, "high": 0.2128021092382794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.20056935817805382, "unit": "f1", "denominator": null, "interval": { "low": 0.10664208074534162, "high": 0.2174282420478072, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.20408163265306123, "unit": "proportion", "denominator": 98, "interval": { "low": 0.125, "high": 0.28846153846153844, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "verified_selected_result_token_reconstruction", "coverage": { "observed_events": 0, "estimated_events": 790, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Selected-response tokens repriced with the run's retained list prices and GBP buffers; no provider invoice. Earlier failed/uncertain attempts are separate.", "value": 0.0075642658227848105 } }, "gemini38-low": { "metrics": { "species_accuracy": { "value": 0.569620253164557, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.5417234620161835, "high": 0.5961324408713649, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.6670886075949367, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6418739268818383, "high": 0.6918905716294957, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.8538899430740038, "unit": "proportion", "denominator": 527, "population_events": 790, "interval": { "low": 0.8265536540508284, "high": 0.8789170821302109, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.5374228127338921, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.5076246402887419, "high": 0.5529720878074124, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 2.867721518987342, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 2.6761928204531054, "high": 3.0395548118372484, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.09746835443037975, "unit": "proportion", "denominator": 790, "interval": { "low": 0.08026443229766014, "high": 0.11624285700235067, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.6394651189491882, "unit": "f1", "denominator": null, "interval": { "low": 0.604009065660022, "high": 0.6579667880240097, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.5702531645569624, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.4587884623288105, "high": 3.6734464502621154, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.14050632911392405, "unit": "proportion", "denominator": 790, "interval": { "low": 0.12054258680840958, "high": 0.16327144638062357, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.6876456876456877, "unit": "proportion", "denominator": 429, "interval": { "low": 0.6402873628698851, "high": 0.7333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7645687645687645, "unit": "proportion", "denominator": 429, "interval": { "low": 0.7213086424416408, "high": 0.8069309615343313, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.899390243902439, "unit": "proportion", "denominator": 328, "interval": { "low": 0.8660428501618715, "high": 0.9300911854103343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 429, "interval": { "low": 0.052995391705069124, "high": 0.10267884430667132, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.4149184149184153, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.1325187542623323, "high": 3.6757502194426155, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.9160839160839163, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.746691479009378, "high": 4.0813108796543265, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5889353821931136, "unit": "f1", "denominator": null, "interval": { "low": 0.5126891394994124, "high": 0.5943557395231674, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7314197488527379, "unit": "f1", "denominator": null, "interval": { "low": 0.6367268345395928, "high": 0.7381514829561918, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.10023310023310024, "unit": "proportion", "denominator": 429, "interval": { "low": 0.07242990654205607, "high": 0.13013908989936385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.4293628808864266, "unit": "proportion", "denominator": 361, "interval": { "low": 0.3771917293233083, "high": 0.48265915221579964, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5512465373961218, "unit": "proportion", "denominator": 361, "interval": { "low": 0.49865210848873986, "high": 0.6034493507610285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7788944723618091, "unit": "proportion", "denominator": 199, "interval": { "low": 0.7184452070081464, "high": 0.835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12188365650969529, "unit": "proportion", "denominator": 361, "interval": { "low": 0.08963376395334254, "high": 0.15691764111130643, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2174515235457064, "unit": "points_per_event", "denominator": 361, "interval": { "low": 1.8790728689931353, "high": 2.540733877016808, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1592797783933517, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.964001595474383, "high": 3.357340720221607, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4066446751628938, "unit": "f1", "denominator": null, "interval": { "low": 0.32916461914704476, "high": 0.4191355405619619, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4479983709421711, "unit": "f1", "denominator": null, "interval": { "low": 0.3626389871958967, "high": 0.46175949383944964, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1883656509695291, "unit": "proportion", "denominator": 361, "interval": { "low": 0.14763002876843692, "high": 0.2305393379906853, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.768, "unit": "proportion", "denominator": 375, "interval": { "low": 0.7222222222222222, "high": 0.8128491620111732, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8426666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.8016991232373748, "high": 0.8810810810810811, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9113924050632911, "unit": "proportion", "denominator": 316, "interval": { "low": 0.8786861677054417, "high": 0.9409838611139395, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07466666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.049479166666666664, "high": 0.10236220472440945, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.9133333333333336, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.6608060845819916, "high": 4.152521610448182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.366666666666666, "unit": "points_per_event", "denominator": 375, "interval": { "low": 4.233070701079844, "high": 4.494318984334874, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7129652754652754, "unit": "f1", "denominator": null, "interval": { "low": 0.6376155725448074, "high": 0.7178825557336174, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7678087581933735, "unit": "f1", "denominator": null, "interval": { "low": 0.6866629242790233, "high": 0.7731042907900493, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.09066666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.06284117243155093, "high": 0.12096774193548387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.5208333333333334, "unit": "proportion", "denominator": 240, "interval": { "low": 0.4570106490872211, "high": 0.5833333333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6166666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5542168674698795, "high": 0.6785714285714286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8445945945945946, "unit": "proportion", "denominator": 148, "interval": { "low": 0.7832167832167832, "high": 0.9, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09583333333333334, "unit": "proportion", "denominator": 240, "interval": { "low": 0.060603005865102644, "high": 0.13524821274497717, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.7604166666666665, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.3496228654262774, "high": 3.135757412721186, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5104166666666665, "unit": "points_per_event", "denominator": 240, "interval": { "low": 3.2865711554361305, "high": 3.727277384500745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5414579606440071, "unit": "f1", "denominator": null, "interval": { "low": 0.40863227857413903, "high": 0.5186205636978279, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5894352482960077, "unit": "f1", "denominator": null, "interval": { "low": 0.44484020199210067, "high": 0.5645742845318128, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.15, "unit": "proportion", "denominator": 240, "interval": { "low": 0.10384615384615385, "high": 0.2, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.20689655172413793, "unit": "proportion", "denominator": 174, "interval": { "low": 0.14917127071823205, "high": 0.26993865030674846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3563218390804598, "unit": "proportion", "denominator": 174, "interval": { "low": 0.2857142857142857, "high": 0.4301675977653631, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5806451612903226, "unit": "proportion", "denominator": 62, "interval": { "low": 0.45651220442410373, "high": 0.704225352112676, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14942528735632185, "unit": "proportion", "denominator": 174, "interval": { "low": 0.09815950920245399, "high": 0.2057142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.75, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.2813984865154173, "high": 1.216388192703982, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.9281609195402298, "unit": "points_per_event", "denominator": 174, "interval": { "low": 1.6583261494252879, "high": 2.2088302139037435, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.24082892416225748, "unit": "f1", "denominator": null, "interval": { "low": 0.13113870697204033, "high": 0.22645930201485756, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2533395176252319, "unit": "f1", "denominator": null, "interval": { "low": 0.13795110733422425, "high": 0.23822342160004498, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.23563218390804597, "unit": "proportion", "denominator": 174, "interval": { "low": 0.1728395061728395, "high": 0.30120481927710846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.8448979591836735, "unit": "proportion", "denominator": 245, "interval": { "low": 0.7934262600395763, "high": 0.8931623931623932, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8938775510204081, "unit": "proportion", "denominator": 245, "interval": { "low": 0.847925829230257, "high": 0.9354838709677419, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9452054794520548, "unit": "proportion", "denominator": 219, "interval": { "low": 0.91324200913242, "high": 0.9734513274336283, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.04897959183673469, "unit": "proportion", "denominator": 245, "interval": { "low": 0.023622047244094488, "high": 0.07751937984496124, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 4.295918367346939, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.025087889703338, "high": 4.535417076771654, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.581632653061225, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.441803392658509, "high": 4.7126193843253, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7860630551106741, "unit": "f1", "denominator": null, "interval": { "low": 0.6448308171493854, "high": 0.765472030563067, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.8393554656266521, "unit": "f1", "denominator": null, "interval": { "low": 0.6885481606849368, "high": 0.8173684394148003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.05714285714285714, "unit": "proportion", "denominator": 245, "interval": { "low": 0.029661016949152543, "high": 0.08823529411764706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.6230769230769231, "unit": "proportion", "denominator": 130, "interval": { "low": 0.5352112676056338, "high": 0.7096835754740212, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7461538461538462, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6666666666666666, "high": 0.8222222222222222, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8350515463917526, "unit": "proportion", "denominator": 97, "interval": { "low": 0.7582330827067669, "high": 0.9052667140825036, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12307692307692308, "unit": "proportion", "denominator": 130, "interval": { "low": 0.06956521739130435, "high": 0.1810344827586207, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.192307692307693, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.6790911513859275, "high": 3.672947865531598, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.9615384615384617, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.690826164976644, "high": 4.217062689585441, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5785545423843296, "unit": "f1", "denominator": null, "interval": { "low": 0.4417746231310061, "high": 0.58267972275951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.618001443001443, "unit": "f1", "denominator": null, "interval": { "low": 0.4718956201626657, "high": 0.6224078856749311, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.15384615384615385, "unit": "proportion", "denominator": 130, "interval": { "low": 0.0948905109489051, "high": 0.21768844634628043, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.6448598130841121, "unit": "proportion", "denominator": 107, "interval": { "low": 0.5517241379310345, "high": 0.7327586206896551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7383177570093458, "unit": "proportion", "denominator": 107, "interval": { "low": 0.6526315789473685, "high": 0.8198243243243242, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8734177215189873, "unit": "proportion", "denominator": 79, "interval": { "low": 0.7948717948717948, "high": 0.9431818181818182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09345794392523364, "unit": "proportion", "denominator": 107, "interval": { "low": 0.041237113402061855, "high": 0.15315315315315314, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.116822429906542, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.510989010989011, "high": 3.659309734513274, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.771028037383178, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.4215557275541792, "high": 4.087962962962963, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5935139573070608, "unit": "f1", "denominator": null, "interval": { "low": 0.3889114668856049, "high": 0.549341133004926, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6749766573295986, "unit": "f1", "denominator": null, "interval": { "low": 0.4422914721444134, "high": 0.6247408963585432, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1308411214953271, "unit": "proportion", "denominator": 107, "interval": { "low": 0.0707070707070707, "high": 0.2, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 133, "interval": { "low": 0.33793103448275863, "high": 0.5038167938931297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.518796992481203, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4338235294117647, "high": 0.6033057851239669, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8115942028985508, "unit": "proportion", "denominator": 69, "interval": { "low": 0.7160493827160493, "high": 0.8983050847457628, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09774436090225563, "unit": "proportion", "denominator": 133, "interval": { "low": 0.05042016806722689, "high": 0.15151515151515152, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.473684210526316, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.9599814814814818, "high": 2.9500088652482273, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.300751879699248, "unit": "points_per_event", "denominator": 133, "interval": { "low": 3.012597293015567, "high": 3.57936507936508, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4639014047177312, "unit": "f1", "denominator": null, "interval": { "low": 0.3063629737609329, "high": 0.4534023980405559, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4639014047177312, "unit": "f1", "denominator": null, "interval": { "low": 0.3063629737609329, "high": 0.4534023980405559, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.16541353383458646, "unit": "proportion", "denominator": 133, "interval": { "low": 0.10317460317460317, "high": 0.23484975594874924, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14492753623188406, "high": 0.33707865168539325, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3815789473684211, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2714285714285714, "high": 0.49230769230769234, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6206896551724138, "unit": "proportion", "denominator": 29, "interval": { "low": 0.43586956521739134, "high": 0.8, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.07042253521126761, "high": 0.22917520491803264, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.9736842105263158, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.25609393147502907, "high": 1.6667134831460668, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.9605263157894737, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.5460376794258375, "high": 2.3947368421052633, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.29084967320261434, "unit": "f1", "denominator": null, "interval": { "low": 0.1222222222222222, "high": 0.2619047619047619, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.322463768115942, "unit": "f1", "denominator": null, "interval": { "low": 0.1355072463768116, "high": 0.2903726708074534, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.19736842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.1111111111111111, "high": 0.2916666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.1836734693877551, "unit": "proportion", "denominator": 98, "interval": { "low": 0.1111111111111111, "high": 0.26732673267326734, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.336734693877551, "unit": "proportion", "denominator": 98, "interval": { "low": 0.24489795918367346, "high": 0.4380952380952381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5454545454545454, "unit": "proportion", "denominator": 33, "interval": { "low": 0.37140211640211646, "high": 0.7142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15306122448979592, "unit": "proportion", "denominator": 98, "interval": { "low": 0.08571428571428572, "high": 0.23, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.576530612244898, "unit": "points_per_event", "denominator": 98, "interval": { "low": -0.05617977528089888, "high": 1.2143049155145926, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.903061224489796, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.5494419642857142, "high": 2.284727822580645, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.15170068027210884, "unit": "f1", "denominator": null, "interval": { "low": 0.08027210884353742, "high": 0.17067077480342782, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.1615942028985507, "unit": "f1", "denominator": null, "interval": { "low": 0.0855072463768116, "high": 0.18180147750799922, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2653061224489796, "unit": "proportion", "denominator": 98, "interval": { "low": 0.17755792445482865, "high": 0.3592233009708738, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 789, "estimated_events": 1, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.006292627848101266 } }, "gemini38-high": { "metrics": { "species_accuracy": { "value": 0.640506329113924, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.6124181760574166, "high": 0.6668709305813737, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.7734177215189874, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.7483349430105759, "high": 0.7955571450349932, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.8281505728314239, "unit": "proportion", "denominator": 611, "population_events": 790, "interval": { "low": 0.8019380435410597, "high": 0.8543367724636198, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.5700173160569032, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.5417594480316914, "high": 0.5854840138287108, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 2.945569620253165, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 2.7489428758653442, "high": 3.1343632299486726, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.13291139240506328, "unit": "proportion", "denominator": 790, "interval": { "low": 0.1122995894752224, "high": 0.15348890121833156, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.6998946792091089, "unit": "f1", "denominator": null, "interval": { "low": 0.6651983096085324, "high": 0.7188854347010752, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 3.8063291139240505, "unit": "points_per_event", "denominator": 790, "interval": { "low": 3.6953067172806087, "high": 3.907151573373883, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.17215189873417722, "unit": "proportion", "denominator": 790, "interval": { "low": 0.14883463143747955, "high": 0.19550607708044415, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.7319347319347319, "unit": "proportion", "denominator": 429, "interval": { "low": 0.6877931616850425, "high": 0.7739745180962888, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8531468531468531, "unit": "proportion", "denominator": 429, "interval": { "low": 0.8173076923076923, "high": 0.8868384938549561, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8579234972677595, "unit": "proportion", "denominator": 366, "interval": { "low": 0.8205128205128205, "high": 0.8931512946628186, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12121212121212122, "unit": "proportion", "denominator": 429, "interval": { "low": 0.09070294784580499, "high": 0.15328510191433686, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.4055944055944054, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.1085524416555246, "high": 3.6841507177033495, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.093240093240094, "unit": "points_per_event", "denominator": 429, "interval": { "low": 3.9378910168718657, "high": 4.241252165841584, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6111803656842256, "unit": "f1", "denominator": null, "interval": { "low": 0.5328345661557514, "high": 0.6144362888196777, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7787620788557069, "unit": "f1", "denominator": null, "interval": { "low": 0.6789343665532962, "high": 0.7829107551089441, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13752913752913754, "unit": "proportion", "denominator": 429, "interval": { "low": 0.1047613110081938, "high": 0.1721870147976805, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.5318559556786704, "unit": "proportion", "denominator": 361, "interval": { "low": 0.47780523897699717, "high": 0.5849596220689877, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6786703601108033, "unit": "proportion", "denominator": 361, "interval": { "low": 0.6287243104526535, "high": 0.7277936962750716, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7836734693877551, "unit": "proportion", "denominator": 245, "interval": { "low": 0.7288135593220338, "high": 0.8347835791925465, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14681440443213298, "unit": "proportion", "denominator": 361, "interval": { "low": 0.11080332409972299, "high": 0.18571623731459794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.398891966759003, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.0397243781287595, "high": 2.75220023020918, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.465373961218837, "unit": "points_per_event", "denominator": 361, "interval": { "low": 3.266196099674973, "high": 3.661727993332388, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4679993612043887, "unit": "f1", "denominator": null, "interval": { "low": 0.38971527884421053, "high": 0.47986115462039364, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5314569017066787, "unit": "f1", "denominator": null, "interval": { "low": 0.44255802851800186, "high": 0.5449270738909553, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21329639889196675, "unit": "proportion", "denominator": 361, "interval": { "low": 0.17109047508150907, "high": 0.25806451612903225, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.8133333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.7716535433070866, "high": 0.8532345068137572, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9066666666666666, "unit": "proportion", "denominator": 375, "interval": { "low": 0.8743311855293534, "high": 0.9375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8970588235294118, "unit": "proportion", "denominator": 340, "interval": { "low": 0.8643067846607669, "high": 0.927377255312326, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09333333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.06557377049180328, "high": 0.12335958005249344, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.9280000000000004, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.6716412868091974, "high": 4.169231266149871, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.488, "unit": "points_per_event", "denominator": 375, "interval": { "low": 4.3692523209549075, "high": 4.60340784183275, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7348382989559459, "unit": "f1", "denominator": null, "interval": { "low": 0.6601597557648675, "high": 0.7394745897377476, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.8007853257853257, "unit": "f1", "denominator": null, "interval": { "low": 0.7194048620514581, "high": 0.8058376939449814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.112, "unit": "proportion", "denominator": 375, "interval": { "low": 0.082010582010582, "high": 0.1447730335926398, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.6083333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5439983263598327, "high": 0.6707818930041153, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7416666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.6848739495798319, "high": 0.7974151571366216, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8202247191011236, "unit": "proportion", "denominator": 178, "interval": { "low": 0.7597765363128491, "high": 0.8757070504462943, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13333333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.09130434782608696, "high": 0.1799163179916318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.8625000000000003, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.4336278595821246, "high": 3.2680866077556625, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.7791666666666672, "unit": "points_per_event", "denominator": 240, "interval": { "low": 3.5635946937014666, "high": 3.991562067865121, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.604353086249638, "unit": "f1", "denominator": null, "interval": { "low": 0.4608287727316632, "high": 0.5789137339352856, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6655533987812469, "unit": "f1", "denominator": null, "interval": { "low": 0.5074949775652493, "high": 0.637537909523669, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.18333333333333332, "unit": "proportion", "denominator": 240, "interval": { "low": 0.13274336283185842, "high": 0.2384937238493724, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.3103448275862069, "unit": "proportion", "denominator": 174, "interval": { "low": 0.2422360248447205, "high": 0.3801232375417069, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5287356321839081, "unit": "proportion", "denominator": 174, "interval": { "low": 0.45454545454545453, "high": 0.603448275862069, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5869565217391305, "unit": "proportion", "denominator": 92, "interval": { "low": 0.4842105263157895, "high": 0.6888888888888889, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.21839080459770116, "unit": "proportion", "denominator": 174, "interval": { "low": 0.15789473684210525, "high": 0.28160919540229884, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.9310344827586207, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.4084854631507775, "high": 1.4542986279257464, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.367816091954023, "unit": "points_per_event", "denominator": 174, "interval": { "low": 2.0602826198410664, "high": 2.6705202312138727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3339663643235071, "unit": "f1", "denominator": null, "interval": { "low": 0.19873754950698228, "high": 0.30663355989248847, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3643269428983714, "unit": "f1", "denominator": null, "interval": { "low": 0.21680459946216246, "high": 0.3345093380645328, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.28735632183908044, "unit": "proportion", "denominator": 174, "interval": { "low": 0.22042782941498087, "high": 0.35542168674698793, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.8653061224489796, "unit": "proportion", "denominator": 245, "interval": { "low": 0.8189646303391515, "high": 0.9082578822963824, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9510204081632653, "unit": "proportion", "denominator": 245, "interval": { "low": 0.9182128092552236, "high": 0.9795081967213115, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9098712446351931, "unit": "proportion", "denominator": 233, "interval": { "low": 0.8709677419354839, "high": 0.9444444444444444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08571428571428572, "unit": "proportion", "denominator": 245, "interval": { "low": 0.05263157894736842, "high": 0.12244897959183673, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 4.183673469387755, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.8799105553160786, "high": 4.456528448738593, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.63265306122449, "unit": "points_per_event", "denominator": 245, "interval": { "low": 4.5042359584426634, "high": 4.752233678735256, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.7473849382940291, "unit": "f1", "denominator": null, "interval": { "low": 0.622920330821473, "high": 0.7324318183425803, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.8360577275831513, "unit": "f1", "denominator": null, "interval": { "low": 0.6968261327833427, "high": 0.8193305086544119, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.08979591836734693, "unit": "proportion", "denominator": 245, "interval": { "low": 0.05622489959839357, "high": 0.12766191770550187, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.7153846153846154, "unit": "proportion", "denominator": 130, "interval": { "low": 0.6363636363636364, "high": 0.7950868077965664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.823076923076923, "unit": "proportion", "denominator": 130, "interval": { "low": 0.7534223744292238, "high": 0.8888888888888888, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8691588785046729, "unit": "proportion", "denominator": 107, "interval": { "low": 0.8021978021978022, "high": 0.9304347826086956, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1076923076923077, "unit": "proportion", "denominator": 130, "interval": { "low": 0.05737704918032787, "high": 0.16393442622950818, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.446153846153847, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.9541227470882947, "high": 3.9178540851272015, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.2153846153846155, "unit": "points_per_event", "denominator": 130, "interval": { "low": 3.976557221283784, "high": 4.446979665071771, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6379794973544973, "unit": "f1", "denominator": null, "interval": { "low": 0.4953498151154401, "high": 0.638349511721222, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6959776334776334, "unit": "f1", "denominator": null, "interval": { "low": 0.5403816164895711, "high": 0.6963812855140604, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.15384615384615385, "unit": "proportion", "denominator": 130, "interval": { "low": 0.09401709401709402, "high": 0.21804511278195488, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.6915887850467289, "unit": "proportion", "denominator": 107, "interval": { "low": 0.603448275862069, "high": 0.7764705882352941, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.822429906542056, "unit": "proportion", "denominator": 107, "interval": { "low": 0.7474720893141946, "high": 0.8910891089108911, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8409090909090909, "unit": "proportion", "denominator": 88, "interval": { "low": 0.759493670886076, "high": 0.9139784946236559, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1308411214953271, "unit": "proportion", "denominator": 107, "interval": { "low": 0.06930693069306931, "high": 0.19820013979496737, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.2242990654205608, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.617297880690738, "high": 3.7857142857142856, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.9719626168224305, "unit": "points_per_event", "denominator": 107, "interval": { "low": 3.656514945652174, "high": 4.268915643180349, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6417898193760262, "unit": "f1", "denominator": null, "interval": { "low": 0.4235570607553366, "high": 0.590068076081007, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7298786181139122, "unit": "f1", "denominator": null, "interval": { "low": 0.48169234360410823, "high": 0.6710578120136943, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.14953271028037382, "unit": "proportion", "denominator": 107, "interval": { "low": 0.0847457627118644, "high": 0.22, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.5413533834586466, "unit": "proportion", "denominator": 133, "interval": { "low": 0.45384615384615384, "high": 0.6283249051833122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6766917293233082, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5948261775974758, "high": 0.7575757575757576, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8, "unit": "proportion", "denominator": 90, "interval": { "low": 0.7093023255813954, "high": 0.8829787234042553, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13533834586466165, "unit": "proportion", "denominator": 133, "interval": { "low": 0.0782608695652174, "high": 0.2, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.5714285714285716, "unit": "points_per_event", "denominator": 133, "interval": { "low": 1.9797007722007725, "high": 3.1229524460057245, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.62406015037594, "unit": "points_per_event", "denominator": 133, "interval": { "low": 3.326441152335737, "high": 3.9029879671410392, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5442528060175119, "unit": "f1", "denominator": null, "interval": { "low": 0.37526196842373316, "high": 0.5285456547572916, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5664672062631246, "unit": "f1", "denominator": null, "interval": { "low": 0.3905787834614366, "high": 0.5501189467882014, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21052631578947367, "unit": "proportion", "denominator": 133, "interval": { "low": 0.1399965034965035, "high": 0.2868429436126152, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.35526315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.25, "high": 0.4642857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5789473684210527, "unit": "proportion", "denominator": 76, "interval": { "low": 0.46875, "high": 0.6896673142302087, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6136363636363636, "unit": "proportion", "denominator": 44, "interval": { "low": 0.4666666666666667, "high": 0.7560975609756098, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.13414634146341464, "high": 0.3230769230769231, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.131578947368421, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.3209853510226645, "high": 1.925931186868687, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.5131578947368425, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.057142857142857, "high": 2.975, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3902964959568733, "unit": "f1", "denominator": null, "interval": { "low": 0.18658243486073675, "high": 0.34380091344713987, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4496894409937888, "unit": "f1", "denominator": null, "interval": { "low": 0.21497541407867496, "high": 0.3961184437543133, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27631578947368424, "unit": "proportion", "denominator": 76, "interval": { "low": 0.17857142857142858, "high": 0.38095238095238093, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.2755102040816326, "unit": "proportion", "denominator": 98, "interval": { "low": 0.18811669628501312, "high": 0.3695652173913043, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4897959183673469, "unit": "proportion", "denominator": 98, "interval": { "low": 0.38823529411764707, "high": 0.5904761904761905, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5625, "unit": "proportion", "denominator": 48, "interval": { "low": 0.4166666666666667, "high": 0.7045454545454546, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 98, "interval": { "low": 0.13592233009708737, "high": 0.29896907216494845, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.7755102040816326, "unit": "points_per_event", "denominator": 98, "interval": { "low": 0.07605978260869568, "high": 1.474375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.2551020408163267, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.8520408163265305, "high": 2.6770917338709683, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.23257142857142857, "unit": "f1", "denominator": null, "interval": { "low": 0.13551338383838382, "high": 0.2434307692307692, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.25279503105590057, "unit": "f1", "denominator": null, "interval": { "low": 0.14729715634606938, "high": 0.26459866220735784, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.29591836734693877, "unit": "proportion", "denominator": 98, "interval": { "low": 0.20731432652164358, "high": 0.38636363636363635, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 789, "estimated_events": 1, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.013143683544303798 } }, "qwen-repaired": { "metrics": { "species_accuracy": { "value": 0.4050632911392405, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.37932481460171324, "high": 0.42949030839062485, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.8607594936708861, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.8362663136792884, "high": 0.8822261686976877, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.47058823529411764, "unit": "proportion", "denominator": 680, "population_events": 790, "interval": { "low": 0.4443503349531054, "high": 0.4971062481667023, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.2028763468703074, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.1880233773912816, "high": 0.21117139483899539, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 0.4044303797468355, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 0.20845360774949376, "high": 0.59785657204961, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.45569620253164556, "unit": "proportion", "denominator": 790, "interval": { "low": 0.4297494839899903, "high": 0.48085025997209546, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.4365693540247121, "unit": "f1", "denominator": null, "interval": { "low": 0.4046072678040237, "high": 0.4544194572484709, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 2.809493670886076, "unit": "points_per_event", "denominator": 790, "interval": { "low": 2.6910758979944105, "high": 2.9168447357846095, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.4810126582278481, "unit": "proportion", "denominator": 790, "interval": { "low": 0.4549480003119244, "high": 0.5062528926769433, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.48951048951048953, "unit": "proportion", "denominator": 429, "interval": { "low": 0.4397313015972721, "high": 0.5395348837209303, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8741258741258742, "unit": "proportion", "denominator": 429, "interval": { "low": 0.8426952033308618, "high": 0.9047619047619048, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.56, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5062034739454094, "high": 0.6129055039183524, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.38461538461538464, "unit": "proportion", "denominator": 429, "interval": { "low": 0.33647058823529413, "high": 0.43387470997679817, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.1002331002331003, "unit": "points_per_event", "denominator": 429, "interval": { "low": 0.7157678752693073, "high": 1.4771219614096984, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.174825174825175, "unit": "points_per_event", "denominator": 429, "interval": { "low": 2.979155585106383, "high": 3.3698033785759662, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.27320079576369943, "unit": "f1", "denominator": null, "interval": { "low": 0.23530682696226463, "high": 0.28262942481471504, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.46708523146696995, "unit": "f1", "denominator": null, "interval": { "low": 0.40229876867742026, "high": 0.4832051456509644, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4149184149184149, "unit": "proportion", "denominator": 429, "interval": { "low": 0.36658157397767716, "high": 0.46454902957786093, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.3047091412742382, "unit": "proportion", "denominator": 361, "interval": { "low": 0.2553191489361702, "high": 0.355747044580544, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8448753462603878, "unit": "proportion", "denominator": 361, "interval": { "low": 0.8033696193470073, "high": 0.8834701820029564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.36065573770491804, "unit": "proportion", "denominator": 305, "interval": { "low": 0.30475875118259227, "high": 0.41818478906714196, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5401662049861495, "unit": "proportion", "denominator": 361, "interval": { "low": 0.48633879781420764, "high": 0.5945223991114402, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.422437673130194, "unit": "points_per_event", "denominator": 361, "interval": { "low": -0.8128314781896204, "high": -0.007040778188685206, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.3753462603878117, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.166655027932961, "high": 2.589780907198335, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.147244129838632, "unit": "f1", "denominator": null, "interval": { "low": 0.11118732279369133, "high": 0.15303127974848377, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.30946223898288755, "unit": "f1", "denominator": null, "interval": { "low": 0.23368183095623263, "high": 0.3216250625222371, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5595567867036011, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5051175548589342, "high": 0.6135709975864843, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.5893333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5376344086021505, "high": 0.6413043478260869, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9893333333333333, "unit": "proportion", "denominator": 375, "interval": { "low": 0.9778393351800554, "high": 0.9974620895714195, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5956873315363881, "unit": "proportion", "denominator": 371, "interval": { "low": 0.5437113413209914, "high": 0.6472222222222223, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4, "unit": "proportion", "denominator": 375, "interval": { "low": 0.34908136482939633, "high": 0.4520547945205479, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5613333333333335, "unit": "points_per_event", "denominator": 375, "interval": { "low": 1.1409053184281845, "high": 1.9761252675227394, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.708, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.527214921811124, "high": 3.883602531102531, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3341092241092241, "unit": "f1", "denominator": null, "interval": { "low": 0.28877395047234, "high": 0.3394712910434869, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5568487068487068, "unit": "f1", "denominator": null, "interval": { "low": 0.48128991745389993, "high": 0.5657854850724782, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.42933333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.3775201932053416, "high": 0.48128591136648075, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.32083333333333336, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2616033755274262, "high": 0.38257658102766795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8916666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.8508754244482173, "high": 0.9297520661157025, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3598130841121495, "unit": "proportion", "denominator": 214, "interval": { "low": 0.2949293985584308, "high": 0.42660550458715596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5708333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5061728395061729, "high": 0.6347860593512767, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.3875, "unit": "points_per_event", "denominator": 240, "interval": { "low": -0.867782395432465, "high": 0.11163256795211454, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.591666666666667, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.3473420599803343, "high": 2.8394313604833297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2151797535797536, "unit": "f1", "denominator": null, "interval": { "low": 0.15615385336885335, "high": 0.21379193192772142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3404742936388506, "unit": "f1", "denominator": null, "interval": { "low": 0.24707888191274266, "high": 0.33827837330335664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5958333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5321078320603787, "high": 0.6600790513833992, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.1206896551724138, "unit": "proportion", "denominator": 174, "interval": { "low": 0.07262569832402235, "high": 0.17318435754189945, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5402298850574713, "unit": "proportion", "denominator": 174, "interval": { "low": 0.4634146341463415, "high": 0.6170243161094224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.22340425531914893, "unit": "proportion", "denominator": 94, "interval": { "low": 0.13829787234042554, "high": 0.3113207547169811, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.41954022988505746, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3465909090909091, "high": 0.49404761904761907, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.0229885057471264, "unit": "points_per_event", "denominator": 174, "interval": { "low": -1.4401252891943386, "high": -0.5952255639097747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.1609195402298853, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.90625, "high": 1.4273352129448025, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.07115812536373284, "unit": "f1", "denominator": null, "interval": { "low": 0.03657070538495772, "high": 0.08156063303259564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.09888207031064172, "unit": "f1", "denominator": null, "interval": { "low": 0.05081903215831787, "high": 0.11333750304529523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4367816091954023, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3619631901840491, "high": 0.5126590310351474, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.6857142857142857, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6234309623430963, "high": 0.7449412874816523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9918367346938776, "unit": "proportion", "denominator": 245, "interval": { "low": 0.9789029535864979, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.691358024691358, "unit": "proportion", "denominator": 243, "interval": { "low": 0.6290322580645161, "high": 0.7510377820963986, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.30612244897959184, "unit": "proportion", "denominator": 245, "interval": { "low": 0.24686192468619247, "high": 0.368, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.369387755102041, "unit": "points_per_event", "denominator": 245, "interval": { "low": 1.8681965593940053, "high": 2.84439949285385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.083673469387755, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.882128891155832, "high": 4.27451258912656, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4293459552495697, "unit": "f1", "denominator": null, "interval": { "low": 0.3572486670288907, "high": 0.43041251452611723, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6039951573849879, "unit": "f1", "denominator": null, "interval": { "low": 0.5025701587016598, "high": 0.6054955712825039, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.34285714285714286, "unit": "proportion", "denominator": 245, "interval": { "low": 0.2813658507604563, "high": 0.40613326149425283, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.4076923076923077, "unit": "proportion", "denominator": 130, "interval": { "low": 0.32231404958677684, "high": 0.4957282913165266, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9846153846153847, "unit": "proportion", "denominator": 130, "interval": { "low": 0.96, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4140625, "unit": "proportion", "denominator": 128, "interval": { "low": 0.3274296057924377, "high": 0.5035460992907801, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5769230769230769, "unit": "proportion", "denominator": 130, "interval": { "low": 0.48836751510712323, "high": 0.6635535259133389, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.038461538461538464, "unit": "points_per_event", "denominator": 130, "interval": { "low": -0.6760057377049181, "high": 0.7698660714285711, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.6773041586073507, "high": 3.3248220802919715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2205654761904762, "unit": "f1", "denominator": null, "interval": { "low": 0.15291540230602732, "high": 0.229375744047619, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4010281385281385, "unit": "f1", "denominator": null, "interval": { "low": 0.27802800419277696, "high": 0.4170468073593073, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5923076923076923, "unit": "proportion", "denominator": 130, "interval": { "low": 0.5038744896147701, "high": 0.6779661016949152, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.3364485981308411, "unit": "proportion", "denominator": 107, "interval": { "low": 0.24770642201834864, "high": 0.42609369733737773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.897196261682243, "unit": "proportion", "denominator": 107, "interval": { "low": 0.8365384615384616, "high": 0.9516206575682381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.375, "unit": "proportion", "denominator": 96, "interval": { "low": 0.2792740414833438, "high": 0.4717034131863472, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5607476635514018, "unit": "proportion", "denominator": 107, "interval": { "low": 0.46609963797926607, "high": 0.6551724137931034, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.17289719626168223, "unit": "points_per_event", "denominator": 107, "interval": { "low": -0.8701967592592592, "high": 0.5244118009361909, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.7242990654205608, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.375, "high": 3.0809634551495018, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2309206349206349, "unit": "f1", "denominator": null, "interval": { "low": 0.14162012987012987, "high": 0.22733333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.33958916900093367, "unit": "f1", "denominator": null, "interval": { "low": 0.20826489686783803, "high": 0.3343137254901961, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5794392523364486, "unit": "proportion", "denominator": 107, "interval": { "low": 0.4891304347826087, "high": 0.6728971962616822, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3082706766917293, "unit": "proportion", "denominator": 133, "interval": { "low": 0.22880750605326877, "high": 0.3944954128440367, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8872180451127819, "unit": "proportion", "denominator": 133, "interval": { "low": 0.8309859154929577, "high": 0.9396551724137931, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3474576271186441, "unit": "proportion", "denominator": 118, "interval": { "low": 0.2601626016260163, "high": 0.44118309485956536, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5789473684210527, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4895067698259188, "high": 0.6642335766423357, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.5601503759398496, "unit": "points_per_event", "denominator": 133, "interval": { "low": -1.1965811965811965, "high": 0.11539115646258497, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.484962406015038, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.1547579887218045, "high": 2.828, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.19558103076621594, "unit": "f1", "denominator": null, "interval": { "low": 0.12479618571938833, "high": 0.19909069631528667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3233074182053774, "unit": "f1", "denominator": null, "interval": { "low": 0.20629573557694805, "high": 0.3291091102354738, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.6090225563909775, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5220588235294118, "high": 0.6936936936936937, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.06578947368421052, "unit": "proportion", "denominator": 76, "interval": { "low": 0.013513513513513514, "high": 0.12698412698412698, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4605263157894737, "unit": "proportion", "denominator": 76, "interval": { "low": 0.34882840028188866, "high": 0.5731747967479673, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 35, "interval": { "low": 0.03125, "high": 0.2702702702702703, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.39473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.287496408045977, "high": 0.5061728395061729, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.25, "unit": "points_per_event", "denominator": 76, "interval": { "low": -1.7608954451345757, "high": -0.7465584998877164, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 0.8552631578947368, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.5416666666666666, "high": 1.2025316455696204, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.048205128205128206, "unit": "f1", "denominator": null, "interval": { "low": 0.010256410256410256, "high": 0.05846153846153846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.06811594202898551, "unit": "f1", "denominator": null, "interval": { "low": 0.014492753623188404, "high": 0.08260869565217391, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": { "low": 0.3125, "high": 0.5316455696202531, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.16326530612244897, "unit": "proportion", "denominator": 98, "interval": { "low": 0.09090909090909091, "high": 0.24691721132897596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6020408163265306, "unit": "proportion", "denominator": 98, "interval": { "low": 0.5, "high": 0.7040816326530612, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.2711864406779661, "unit": "proportion", "denominator": 59, "interval": { "low": 0.15623263888888891, "high": 0.39285714285714285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4387755102040816, "unit": "proportion", "denominator": 98, "interval": { "low": 0.33962264150943394, "high": 0.5421686746987951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.8469387755102041, "unit": "points_per_event", "denominator": 98, "interval": { "low": -1.470977628918099, "high": -0.21126369327073558, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.3979591836734695, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.0287315270935962, "high": 1.7973116733755032, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.08222912353347137, "unit": "f1", "denominator": null, "interval": { "low": 0.039195709684840124, "high": 0.09879227053140097, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.12334368530020705, "unit": "f1", "denominator": null, "interval": { "low": 0.058793564527260186, "high": 0.14818840579710144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4489795918367347, "unit": "proportion", "denominator": 98, "interval": { "low": 0.3474553969669938, "high": 0.5505732630130702, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "Selected evaluated response attempt only: 786 first-pass outputs, four successful repair outputs. Four superseded first-pass length charges are excluded from this comparison and retained in the audit below.", "value": 0.0019741087746835443 } }, "qwen-first-pass": { "metrics": { "species_accuracy": { "value": 0.40379746835443037, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.3781239382979889, "high": 0.42818381600818317, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "answer_coverage": { "value": 0.8556962025316456, "unit": "proportion", "denominator": 790, "population_events": 790, "interval": { "low": 0.8312612492992241, "high": 0.8774901250296819, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "accuracy_among_answers": { "value": 0.4718934911242604, "unit": "proportion", "denominator": 676, "population_events": 790, "interval": { "low": 0.445274581992246, "high": 0.49831855704556355, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "proportion", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "macro_f1": { "value": 0.20385000853147844, "unit": "f1", "denominator": null, "population_events": 790, "interval": { "low": 0.18882257406561237, "high": 0.2121418537359752, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "f1", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "adjusted_tis": { "value": 0.4151898734177215, "unit": "points_per_event", "denominator": 790, "population_events": 790, "interval": { "low": 0.2174646349606793, "high": 0.6079605744993402, "level": 0.95, "method": "species_balanced_source_cluster", "unit": "points_per_event", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval" }, "wrong_species_rate_all_events": { "value": 0.4518987341772152, "unit": "proportion", "denominator": 790, "interval": { "low": 0.4261005757603859, "high": 0.47705611723016794, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "reference_species_macro_f1": { "value": 0.4360841954660741, "unit": "f1", "denominator": null, "interval": { "low": 0.40393689895048723, "high": 0.45382244659974436, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "raw_tis": { "value": 2.80126582278481, "unit": "points_per_event", "denominator": 790, "interval": { "low": 2.6832318911749287, "high": 2.90948593556584, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.4772151898734177, "unit": "proportion", "denominator": 790, "interval": { "low": 0.4512202951759915, "high": 0.5025796215036722, "level": 0.95, "method": "species_balanced_source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_headline_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 429, "n_species": 62, "metrics": { "species_accuracy": { "value": 0.48951048951048953, "unit": "proportion", "denominator": 429, "interval": { "low": 0.4397313015972721, "high": 0.5395348837209303, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8717948717948718, "unit": "proportion", "denominator": 429, "interval": { "low": 0.8400954653937948, "high": 0.9026144159139113, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5614973262032086, "unit": "proportion", "denominator": 374, "interval": { "low": 0.5079787234042553, "high": 0.6143617021276596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3822843822843823, "unit": "proportion", "denominator": 429, "interval": { "low": 0.3348309624056404, "high": 0.43137254901960786, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.1118881118881119, "unit": "points_per_event", "denominator": 429, "interval": { "low": 0.7304206372051449, "high": 1.4860597783674105, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.174825174825175, "unit": "points_per_event", "denominator": 429, "interval": { "low": 2.979155585106383, "high": 3.3698033785759662, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.27580270810430607, "unit": "f1", "denominator": null, "interval": { "low": 0.23754784436190526, "high": 0.2853211336224742, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.46708523146696995, "unit": "f1", "denominator": null, "interval": { "low": 0.40229876867742026, "high": 0.4832051456509644, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4125874125874126, "unit": "proportion", "denominator": 429, "interval": { "low": 0.3642852999668545, "high": 0.46153846153846156, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 361, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.30193905817174516, "unit": "proportion", "denominator": 361, "interval": { "low": 0.2532964485624485, "high": 0.35278514588859416, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8365650969529086, "unit": "proportion", "denominator": 361, "interval": { "low": 0.7946416505791505, "high": 0.8761329305135952, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3609271523178808, "unit": "proportion", "denominator": 302, "interval": { "low": 0.30491803278688523, "high": 0.41883116883116883, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5346260387811634, "unit": "proportion", "denominator": 361, "interval": { "low": 0.4807692307692308, "high": 0.5894751341162824, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.4127423822714682, "unit": "points_per_event", "denominator": 361, "interval": { "low": -0.7982748540217572, "high": 0.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.357340720221607, "unit": "points_per_event", "denominator": 361, "interval": { "low": 2.1464444503964, "high": 2.5723926468229417, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1464895552300676, "unit": "f1", "denominator": null, "interval": { "low": 0.11064952326137786, "high": 0.15225922164258712, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3078763533648877, "unit": "f1", "denominator": null, "interval": { "low": 0.2325515404137433, "high": 0.3200024319267933, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.554016620498615, "unit": "proportion", "denominator": 361, "interval": { "low": 0.5, "high": 0.6073328432769768, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 375, "n_species": 78, "metrics": { "species_accuracy": { "value": 0.5866666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.5345744680851063, "high": 0.6381782945736434, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9786666666666667, "unit": "proportion", "denominator": 375, "interval": { "low": 0.9629629629629629, "high": 0.9920429666119321, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5994550408719346, "unit": "proportion", "denominator": 367, "interval": { "low": 0.5474254742547425, "high": 0.6511643748811863, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.392, "unit": "proportion", "denominator": 375, "interval": { "low": 0.3410841510955774, "high": 0.4432724478415798, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.584, "unit": "points_per_event", "denominator": 375, "interval": { "low": 1.1693301829268294, "high": 1.9973200959205475, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.690666666666667, "unit": "points_per_event", "denominator": 375, "interval": { "low": 3.5081075617396373, "high": 3.8676101996068355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.33875402284119394, "unit": "f1", "denominator": null, "interval": { "low": 0.2927426374648299, "high": 0.3443524435912997, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5559040374829848, "unit": "f1", "denominator": null, "interval": { "low": 0.4803981743012593, "high": 0.5650911894831584, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.42133333333333334, "unit": "proportion", "denominator": 375, "interval": { "low": 0.36923076923076925, "high": 0.47342071859243323, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium": { "n_events": 240, "n_species": 79, "metrics": { "species_accuracy": { "value": 0.32083333333333336, "unit": "proportion", "denominator": 240, "interval": { "low": 0.2616033755274262, "high": 0.38257658102766795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8916666666666667, "unit": "proportion", "denominator": 240, "interval": { "low": 0.8508754244482173, "high": 0.9297520661157025, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3598130841121495, "unit": "proportion", "denominator": 214, "interval": { "low": 0.2949293985584308, "high": 0.42660550458715596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5708333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5061728395061729, "high": 0.6347860593512767, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.3875, "unit": "points_per_event", "denominator": 240, "interval": { "low": -0.867782395432465, "high": 0.11163256795211454, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.591666666666667, "unit": "points_per_event", "denominator": 240, "interval": { "low": 2.3473420599803343, "high": 2.8394313604833297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2151797535797536, "unit": "f1", "denominator": null, "interval": { "low": 0.15615385336885335, "high": 0.21379193192772142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3404742936388506, "unit": "f1", "denominator": null, "interval": { "low": 0.24707888191274266, "high": 0.33827837330335664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5958333333333333, "unit": "proportion", "denominator": 240, "interval": { "low": 0.5321078320603787, "high": 0.6600790513833992, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard": { "n_events": 174, "n_species": 77, "metrics": { "species_accuracy": { "value": 0.1206896551724138, "unit": "proportion", "denominator": 174, "interval": { "low": 0.07262569832402235, "high": 0.17318435754189945, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5402298850574713, "unit": "proportion", "denominator": 174, "interval": { "low": 0.4634146341463415, "high": 0.6170243161094224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.22340425531914893, "unit": "proportion", "denominator": 94, "interval": { "low": 0.13829787234042554, "high": 0.3113207547169811, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.41954022988505746, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3465909090909091, "high": 0.49404761904761907, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.0229885057471264, "unit": "points_per_event", "denominator": 174, "interval": { "low": -1.4401252891943386, "high": -0.5952255639097747, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.1609195402298853, "unit": "points_per_event", "denominator": 174, "interval": { "low": 0.90625, "high": 1.4273352129448025, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.07115812536373284, "unit": "f1", "denominator": null, "interval": { "low": 0.03657070538495772, "high": 0.08156063303259564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.09888207031064172, "unit": "f1", "denominator": null, "interval": { "low": 0.05081903215831787, "high": 0.11333750304529523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4367816091954023, "unit": "proportion", "denominator": 174, "interval": { "low": 0.3619631901840491, "high": 0.5126590310351474, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | day": { "n_events": 245, "n_species": 59, "metrics": { "species_accuracy": { "value": 0.6857142857142857, "unit": "proportion", "denominator": 245, "interval": { "low": 0.6234309623430963, "high": 0.7449412874816523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9877551020408163, "unit": "proportion", "denominator": 245, "interval": { "low": 0.97265625, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6942148760330579, "unit": "proportion", "denominator": 242, "interval": { "low": 0.6322314049586777, "high": 0.7543103448275862, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.3020408163265306, "unit": "proportion", "denominator": 245, "interval": { "low": 0.24267782426778242, "high": 0.3632657047193878, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3897959183673474, "unit": "points_per_event", "denominator": 245, "interval": { "low": 1.8948178982022512, "high": 2.8605840032726237, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.083673469387755, "unit": "points_per_event", "denominator": 245, "interval": { "low": 3.882128891155832, "high": 4.27451258912656, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.43458188153310107, "unit": "f1", "denominator": null, "interval": { "low": 0.36160535809021865, "high": 0.4356614476300943, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6039951573849879, "unit": "f1", "denominator": null, "interval": { "low": 0.5025701587016598, "high": 0.6054955712825039, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.33877551020408164, "unit": "proportion", "denominator": 245, "interval": { "low": 0.2767830674760023, "high": 0.4016745342059149, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy | night_ir": { "n_events": 130, "n_species": 44, "metrics": { "species_accuracy": { "value": 0.4, "unit": "proportion", "denominator": 130, "interval": { "low": 0.3153846153846154, "high": 0.48717948717948717, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.9615384615384616, "unit": "proportion", "denominator": 130, "interval": { "low": 0.9252336448598131, "high": 0.9920634920634921, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.416, "unit": "proportion", "denominator": 125, "interval": { "low": 0.3283582089552239, "high": 0.5042735042735043, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5615384615384615, "unit": "proportion", "denominator": 130, "interval": { "low": 0.4732824427480916, "high": 0.648854961832061, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.06538461538461539, "unit": "points_per_event", "denominator": 130, "interval": { "low": -0.6318213383838385, "high": 0.7879231974921623, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.95, "unit": "points_per_event", "denominator": 130, "interval": { "low": 2.6277971370734527, "high": 3.2810105929424114, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.22110709262607994, "unit": "f1", "denominator": null, "interval": { "low": 0.15301787084223792, "high": 0.2298774362065501, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3969877344877345, "unit": "f1", "denominator": null, "interval": { "low": 0.2747366317394726, "high": 0.4127344877344877, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5769230769230769, "unit": "proportion", "denominator": 130, "interval": { "low": 0.48836751510712323, "high": 0.6623608534322817, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | day": { "n_events": 107, "n_species": 51, "metrics": { "species_accuracy": { "value": 0.3364485981308411, "unit": "proportion", "denominator": 107, "interval": { "low": 0.24770642201834864, "high": 0.42609369733737773, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.897196261682243, "unit": "proportion", "denominator": 107, "interval": { "low": 0.8365384615384616, "high": 0.9516206575682381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.375, "unit": "proportion", "denominator": 96, "interval": { "low": 0.2792740414833438, "high": 0.4717034131863472, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5607476635514018, "unit": "proportion", "denominator": 107, "interval": { "low": 0.46609963797926607, "high": 0.6551724137931034, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.17289719626168223, "unit": "points_per_event", "denominator": 107, "interval": { "low": -0.8701967592592592, "high": 0.5244118009361909, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.7242990654205608, "unit": "points_per_event", "denominator": 107, "interval": { "low": 2.375, "high": 3.0809634551495018, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2309206349206349, "unit": "f1", "denominator": null, "interval": { "low": 0.14162012987012987, "high": 0.22733333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.33958916900093367, "unit": "f1", "denominator": null, "interval": { "low": 0.20826489686783803, "high": 0.3343137254901961, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5794392523364486, "unit": "proportion", "denominator": 107, "interval": { "low": 0.4891304347826087, "high": 0.6728971962616822, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "medium | night_ir": { "n_events": 133, "n_species": 49, "metrics": { "species_accuracy": { "value": 0.3082706766917293, "unit": "proportion", "denominator": 133, "interval": { "low": 0.22880750605326877, "high": 0.3944954128440367, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8872180451127819, "unit": "proportion", "denominator": 133, "interval": { "low": 0.8309859154929577, "high": 0.9396551724137931, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3474576271186441, "unit": "proportion", "denominator": 118, "interval": { "low": 0.2601626016260163, "high": 0.44118309485956536, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5789473684210527, "unit": "proportion", "denominator": 133, "interval": { "low": 0.4895067698259188, "high": 0.6642335766423357, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.5601503759398496, "unit": "points_per_event", "denominator": 133, "interval": { "low": -1.1965811965811965, "high": 0.11539115646258497, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.484962406015038, "unit": "points_per_event", "denominator": 133, "interval": { "low": 2.1547579887218045, "high": 2.828, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.19558103076621594, "unit": "f1", "denominator": null, "interval": { "low": 0.12479618571938833, "high": 0.19909069631528667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3233074182053774, "unit": "f1", "denominator": null, "interval": { "low": 0.20629573557694805, "high": 0.3291091102354738, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.6090225563909775, "unit": "proportion", "denominator": 133, "interval": { "low": 0.5220588235294118, "high": 0.6936936936936937, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | day": { "n_events": 76, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.06578947368421052, "unit": "proportion", "denominator": 76, "interval": { "low": 0.013513513513513514, "high": 0.12698412698412698, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.4605263157894737, "unit": "proportion", "denominator": 76, "interval": { "low": 0.34882840028188866, "high": 0.5731747967479673, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 35, "interval": { "low": 0.03125, "high": 0.2702702702702703, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.39473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.287496408045977, "high": 0.5061728395061729, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.25, "unit": "points_per_event", "denominator": 76, "interval": { "low": -1.7608954451345757, "high": -0.7465584998877164, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 0.8552631578947368, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.5416666666666666, "high": 1.2025316455696204, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.048205128205128206, "unit": "f1", "denominator": null, "interval": { "low": 0.010256410256410256, "high": 0.05846153846153846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.06811594202898551, "unit": "f1", "denominator": null, "interval": { "low": 0.014492753623188404, "high": 0.08260869565217391, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": { "low": 0.3125, "high": 0.5316455696202531, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "hard | night_ir": { "n_events": 98, "n_species": 46, "metrics": { "species_accuracy": { "value": 0.16326530612244897, "unit": "proportion", "denominator": 98, "interval": { "low": 0.09090909090909091, "high": 0.24691721132897596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6020408163265306, "unit": "proportion", "denominator": 98, "interval": { "low": 0.5, "high": 0.7040816326530612, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.2711864406779661, "unit": "proportion", "denominator": 59, "interval": { "low": 0.15623263888888891, "high": 0.39285714285714285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4387755102040816, "unit": "proportion", "denominator": 98, "interval": { "low": 0.33962264150943394, "high": 0.5421686746987951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.8469387755102041, "unit": "points_per_event", "denominator": 98, "interval": { "low": -1.470977628918099, "high": -0.21126369327073558, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.3979591836734695, "unit": "points_per_event", "denominator": 98, "interval": { "low": 1.0287315270935962, "high": 1.7973116733755032, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.08222912353347137, "unit": "f1", "denominator": null, "interval": { "low": 0.039195709684840124, "high": 0.09879227053140097, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.12334368530020705, "unit": "f1", "denominator": null, "interval": { "low": 0.058793564527260186, "high": 0.14818840579710144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4489795918367347, "unit": "proportion", "denominator": 98, "interval": { "low": 0.3474553969669938, "high": 0.5505732630130702, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 790, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 790, "estimated_events": 0, "missing_events": 0, "population_events": 790, "population_attempts": 790 }, "note": "First pass includes 786 model outputs and four charged HTTP 200 length nonresponses. 785 charges came from result usage and five exact charges from hash-bound provider generation metadata.", "value": 0.0019925871011392403 } } } }, "birds": { "n_events": 400, "n_species": 40, "models": { "bioclip": { "metrics": { "species_accuracy": { "value": 0.3875, "unit": "proportion", "denominator": 400, "interval": { "low": 0.33886158349595763, "high": 0.43596059113300495, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 400, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3875, "unit": "proportion", "denominator": 400, "interval": { "low": 0.33886158349595763, "high": 0.43596059113300495, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6125, "unit": "proportion", "denominator": 400, "interval": { "low": 0.5640394088669951, "high": 0.6611384165040424, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.15161317024192555, "unit": "f1", "denominator": null, "interval": { "low": 0.13137522302392565, "high": 0.16129822046561862, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4889524740302099, "unit": "f1", "denominator": null, "interval": { "low": 0.4236850942521602, "high": 0.52018676100162, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": null, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.41975308641975306, "unit": "proportion", "denominator": 324, "interval": { "low": 0.36569402305825244, "high": 0.4741648691048142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 324, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.41975308641975306, "unit": "proportion", "denominator": 324, "interval": { "low": 0.36569402305825244, "high": 0.4741648691048142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5802469135802469, "unit": "proportion", "denominator": 324, "interval": { "low": 0.5258351308951859, "high": 0.6343059769417475, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.1783428019138341, "unit": "f1", "denominator": null, "interval": { "low": 0.15185341092813434, "high": 0.1896237204681453, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.49748255270701086, "unit": "f1", "denominator": null, "interval": { "low": 0.42359109364163783, "high": 0.5289503781479845, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 76, "interval": { "low": 0.15, "high": 0.3563218390804598, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.25, "unit": "proportion", "denominator": 76, "interval": { "low": 0.15, "high": 0.3563218390804598, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.75, "unit": "proportion", "denominator": 76, "interval": { "low": 0.6436781609195402, "high": 0.85, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.12016272418058131, "unit": "f1", "denominator": null, "interval": { "low": 0.05414349489795918, "high": 0.13376677713246035, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.33645562770562776, "unit": "f1", "denominator": null, "interval": { "low": 0.1516017857142857, "high": 0.37454697597088904, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.5409090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5409090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4590909090909091, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.25742574257425743, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.25742574257425743, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7425742574257426, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8846153846153846, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5833333333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5833333333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4166666666666667, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.25, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.75, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.24, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.24, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.76, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.3076923076923077, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3076923076923077, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6923076923076923, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.9107142857142857, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 400, "population_events": 400 } } }, "speciesnet": { "metrics": { "species_accuracy": { "value": 0.12, "unit": "proportion", "denominator": 400, "interval": { "low": 0.08644859813084112, "high": 0.15625, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.2675, "unit": "proportion", "denominator": 400, "interval": { "low": 0.2210796915167095, "high": 0.31368040140584535, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4485981308411215, "unit": "proportion", "denominator": 107, "interval": { "low": 0.3461448598130841, "high": 0.5545454545454546, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1475, "unit": "proportion", "denominator": 400, "interval": { "low": 0.11056378603548415, "high": 0.18518518518518517, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.10598496391563618, "unit": "f1", "denominator": null, "interval": { "low": 0.08181920280091934, "high": 0.12321767899743628, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.14837894948189065, "unit": "f1", "denominator": null, "interval": { "low": 0.11454688392128708, "high": 0.1725047505964108, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.11375, "unit": "points_per_event", "denominator": 400, "interval": { "low": -0.11410714285714285, "high": 0.3510692853246044, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.13875, "unit": "points_per_event", "denominator": 400, "interval": { "low": 0.9665684624924107, "high": 1.3190677147034429, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.205, "unit": "proportion", "denominator": 400, "interval": { "low": 0.1628498727735369, "high": 0.24880382775119617, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.10493827160493827, "unit": "proportion", "denominator": 324, "interval": { "low": 0.0707395498392283, "high": 0.14285714285714285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.24074074074074073, "unit": "proportion", "denominator": 324, "interval": { "low": 0.19314534321624235, "high": 0.29070026223173, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4358974358974359, "unit": "proportion", "denominator": 78, "interval": { "low": 0.3188405797101449, "high": 0.5569620253164557, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.13580246913580246, "unit": "proportion", "denominator": 324, "interval": { "low": 0.09771940169980156, "high": 0.1760797342192691, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.12808641975308643, "unit": "points_per_event", "denominator": 324, "interval": { "low": -0.10691869342349221, "high": 0.37199057859344353, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.007716049382716, "unit": "points_per_event", "denominator": 324, "interval": { "low": 0.831817480359147, "high": 1.197641925501764, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.12773670773670773, "unit": "f1", "denominator": null, "interval": { "low": 0.08511994134176797, "high": 0.1475071928071928, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.16807461544303648, "unit": "f1", "denominator": null, "interval": { "low": 0.11199992281811574, "high": 0.19408841158841156, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.17592592592592593, "unit": "proportion", "denominator": 324, "interval": { "low": 0.13354037267080746, "high": 0.22115384615384615, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.18421052631578946, "unit": "proportion", "denominator": 76, "interval": { "low": 0.09722222222222222, "high": 0.2835820895522388, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.3815789473684211, "unit": "proportion", "denominator": 76, "interval": { "low": 0.26582278481012656, "high": 0.5, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4827586206896552, "unit": "proportion", "denominator": 29, "interval": { "low": 0.2857142857142857, "high": 0.68, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19736842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.10588235294117647, "high": 0.2962962962962963, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.05263157894736842, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.5915492957746479, "high": 0.7105448480355816, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.6973684210526316, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.235612591431557, "high": 2.1538732394366193, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.12527472527472527, "unit": "f1", "denominator": null, "interval": { "low": 0.0699538602423218, "high": 0.15511197136197138, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.16285714285714287, "unit": "f1", "denominator": null, "interval": { "low": 0.09094001831501831, "high": 0.2016455627705628, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.32894736842105265, "unit": "proportion", "denominator": 76, "interval": { "low": 0.21516726943942135, "high": 0.44871794871794873, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.17272727272727273, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3409090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5066666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.16818181818181818, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.4772727272727273, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.5, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.0594059405940594, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2079207920792079, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 21, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1485148514851485, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.2722772277227723, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.7673267326732673, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.05128205128205128, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.14102564102564102, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.36363636363636365, "unit": "proportion", "denominator": 11, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08974358974358974, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.4166666666666667, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.6089743589743589, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.15104166666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3020833333333333, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 58, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15104166666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.4244791666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.3098958333333333, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6071428571428571, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5294117647058824, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.8392857142857143, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.8035714285714284, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.04, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.17333333333333334, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.23076923076923078, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.13333333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.14666666666666667, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.6533333333333333, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3076923076923077, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.375, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6346153846153846, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0961538461538463, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.03571428571428571, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.125, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 7, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5267857142857143, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.45535714285714285, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 4, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.13636363636363635, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 400, "population_events": 400 } } }, "bioclip-country": { "metrics": { "species_accuracy": { "value": 0.4175, "unit": "proportion", "denominator": 400, "interval": { "low": 0.3686746987951807, "high": 0.46594142718893056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 400, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4175, "unit": "proportion", "denominator": 400, "interval": { "low": 0.3686746987951807, "high": 0.46594142718893056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5825, "unit": "proportion", "denominator": 400, "interval": { "low": 0.5340585728110694, "high": 0.6313253012048192, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2078819215356367, "unit": "f1", "denominator": null, "interval": { "low": 0.18175755511458577, "high": 0.2205133767698909, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5197048038390918, "unit": "f1", "denominator": null, "interval": { "low": 0.4543938877864644, "high": 0.5512834419247273, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": null, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.4444444444444444, "unit": "proportion", "denominator": 324, "interval": { "low": 0.39062127976190475, "high": 0.49847140996147343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 324, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4444444444444444, "unit": "proportion", "denominator": 324, "interval": { "low": 0.39062127976190475, "high": 0.49847140996147343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5555555555555556, "unit": "proportion", "denominator": 324, "interval": { "low": 0.5015285900385266, "high": 0.6093787202380951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2387020889204657, "unit": "f1", "denominator": null, "interval": { "low": 0.2043271972044104, "high": 0.2530410745582378, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5276572491926084, "unit": "f1", "denominator": null, "interval": { "low": 0.4516706464518546, "high": 0.559353954286631, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.3026315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.19480519480519481, "high": 0.4142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3026315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.19480519480519481, "high": 0.4142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6973684210526315, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5857142857142857, "high": 0.8051948051948052, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.16010989756790428, "unit": "f1", "denominator": null, "interval": { "low": 0.07980530090824207, "high": 0.17705178686561662, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3762582592845751, "unit": "f1", "denominator": null, "interval": { "low": 0.18754245713436893, "high": 0.41607169913419906, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.5727272727272728, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5727272727272728, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.42727272727272725, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.2871287128712871, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2871287128712871, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7128712871287128, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.14102564102564102, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.14102564102564102, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8589743589743589, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.609375, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.609375, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.390625, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6785714285714286, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.26666666666666666, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.26666666666666666, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7333333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.10714285714285714, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.10714285714285714, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.22727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.22727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 400, "population_events": 400 } } }, "speciesnet-country": { "metrics": { "species_accuracy": { "value": 0.1225, "unit": "proportion", "denominator": 400, "interval": { "low": 0.08844842347228982, "high": 0.15931545818841672, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.2025, "unit": "proportion", "denominator": 400, "interval": { "low": 0.16058394160583941, "high": 0.24611498605021917, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6049382716049383, "unit": "proportion", "denominator": 81, "interval": { "low": 0.4861111111111111, "high": 0.7222222222222222, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08, "unit": "proportion", "denominator": 400, "interval": { "low": 0.05223880597014925, "high": 0.11027659075746919, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.13656107706909845, "unit": "f1", "denominator": null, "interval": { "low": 0.10622364253569397, "high": 0.1583469879089808, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.15021718477600832, "unit": "f1", "denominator": null, "interval": { "low": 0.11684600678926339, "high": 0.17418168669987888, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.315, "unit": "points_per_event", "denominator": 400, "interval": { "low": 0.09007572808365832, "high": 0.5449877720025769, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.1025, "unit": "points_per_event", "denominator": 400, "interval": { "low": 0.9310322779786578, "high": 1.2804919585299395, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1575, "unit": "proportion", "denominator": 400, "interval": { "low": 0.11893171083705065, "high": 0.19845360824742267, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.10802469135802469, "unit": "proportion", "denominator": 324, "interval": { "low": 0.0732484076433121, "high": 0.14679158243922663, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.18209876543209877, "unit": "proportion", "denominator": 324, "interval": { "low": 0.1387057909055577, "high": 0.22769306930693067, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5932203389830508, "unit": "proportion", "denominator": 59, "interval": { "low": 0.4528301886792453, "high": 0.7288135593220338, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07407407407407407, "unit": "proportion", "denominator": 324, "interval": { "low": 0.045317220543806644, "high": 0.10632183908045977, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.3179012345679013, "unit": "points_per_event", "denominator": 324, "interval": { "low": 0.08882289438637557, "high": 0.5504439248304542, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 0.9814814814814815, "unit": "points_per_event", "denominator": 324, "interval": { "low": 0.8074624548067117, "high": 1.1682246664315525, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.15916400672498232, "unit": "f1", "denominator": null, "interval": { "low": 0.10907171247539746, "high": 0.18310944622715564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.17172958620327042, "unit": "f1", "denominator": null, "interval": { "low": 0.11768263714450776, "high": 0.19756545513982582, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13271604938271606, "unit": "proportion", "denominator": 324, "interval": { "low": 0.09523444347063979, "high": 0.17365269461077845, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.18421052631578946, "unit": "proportion", "denominator": 76, "interval": { "low": 0.09722222222222222, "high": 0.2835820895522388, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.2894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.18643082380764683, "high": 0.4025974025974026, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 22, "interval": { "low": 0.4166666666666667, "high": 0.8421052631578947, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.10526315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.04054054054054054, "high": 0.17978909559771578, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.3026315789473684, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.3382939045183291, "high": 0.9500176056338026, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 1.6184210526315792, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.174223098330241, "high": 2.0696403102157244, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.14805194805194805, "unit": "f1", "denominator": null, "interval": { "low": 0.08267274392274392, "high": 0.18331414797323886, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.16285714285714287, "unit": "f1", "denominator": null, "interval": { "low": 0.09094001831501831, "high": 0.2016455627705628, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2631578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.15625, "high": 0.3780487804878049, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.17727272727272728, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2818181818181818, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6290322580645161, "unit": "proportion", "denominator": 62, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.10454545454545454, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.6818181818181818, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.4545454545454546, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.0594059405940594, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.10891089108910891, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5454545454545454, "unit": "proportion", "denominator": 11, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.04950495049504951, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.01485148514851485, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.7277227722772277, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.05128205128205128, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.10256410256410256, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05128205128205128, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.2948717948717949, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.6025641025641025, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.15625, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2552083333333333, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6122448979591837, "unit": "proportion", "denominator": 49, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09895833333333333, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5807291666666666, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.2838541666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4642857142857143, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6923076923076923, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.375, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.625, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.04, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.08, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 6, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.04, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.14666666666666667, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.6133333333333333, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6, "unit": "proportion", "denominator": 5, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.4807692307692308, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0576923076923077, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.03571428571428571, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.07142857142857142, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 4, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.03571428571428571, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.35714285714285715, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.44642857142857145, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 4, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.13636363636363635, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 400, "population_events": 400 } } }, "gemini-minimal": { "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 400, "interval": { "low": 0.44801980198019803, "high": 0.551219512195122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.665, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6159600997506235, "high": 0.7113924050632912, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7518796992481203, "unit": "proportion", "denominator": 266, "interval": { "low": 0.6966280130605974, "high": 0.8042704626334519, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.165, "unit": "proportion", "denominator": 400, "interval": { "low": 0.1287122076818942, "high": 0.2034313725490196, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3868238089527153, "unit": "f1", "denominator": null, "interval": { "low": 0.3507570427397413, "high": 0.4062130086997508, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.560894522981437, "unit": "f1", "denominator": null, "interval": { "low": 0.5085977119726248, "high": 0.5890088626146385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.9987500000000002, "unit": "points_per_event", "denominator": 400, "interval": { "low": 1.6232044158201147, "high": 2.360165420536899, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.22375, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.023929471032746, "high": 3.4178276909722225, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.245, "unit": "proportion", "denominator": 400, "interval": { "low": 0.20191587193460492, "high": 0.28967372113279505, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5493827160493827, "unit": "proportion", "denominator": 324, "interval": { "low": 0.4939015038156516, "high": 0.6046511627906976, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6975308641975309, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6470588235294118, "high": 0.7461785249036835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7876106194690266, "unit": "proportion", "denominator": 226, "interval": { "low": 0.7324561403508771, "high": 0.8407079646017699, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14814814814814814, "unit": "proportion", "denominator": 324, "interval": { "low": 0.11038961038961038, "high": 0.18688601195036003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3425925925925926, "unit": "points_per_event", "denominator": 324, "interval": { "low": 1.9472764788459676, "high": 2.721859417608103, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3919753086419755, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.176378373346573, "high": 3.6021732440598306, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4111462295345362, "unit": "f1", "denominator": null, "interval": { "low": 0.37239530279133876, "high": 0.4320253382883027, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5626211562051548, "unit": "f1", "denominator": null, "interval": { "low": 0.5095935722407794, "high": 0.5911925681839932, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.20987654320987653, "unit": "proportion", "denominator": 324, "interval": { "low": 0.1651376146788991, "high": 0.256797583081571, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.2894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.18292682926829268, "high": 0.4024429838454228, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5263157894736842, "unit": "proportion", "denominator": 76, "interval": { "low": 0.4074074074074074, "high": 0.6406350160256409, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.55, "unit": "proportion", "denominator": 40, "interval": { "low": 0.3870967741935484, "high": 0.7058823529411765, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14285714285714285, "high": 0.33784259611724393, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.5328947368421053, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.3354498191681736, "high": 1.4109790491539078, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.5065789473684212, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.097697645079899, "high": 2.9262355351976854, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.21617186059376728, "unit": "f1", "denominator": null, "interval": { "low": 0.10451746529332735, "high": 0.24018622182415286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.31344919786096254, "unit": "f1", "denominator": null, "interval": { "low": 0.1515503246753247, "high": 0.34827002164502163, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.39473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2816791373239437, "high": 0.5087884806355509, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6409090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8136363636363636, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7877094972067039, "unit": "proportion", "denominator": 179, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.17272727272727273, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.9045454545454548, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.9727272727272727, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.504950495049505, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6435643564356436, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7846153846153846, "unit": "proportion", "denominator": 65, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.13861386138613863, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.9702970297029703, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.1584158415841586, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.08974358974358974, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2692307692307692, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 21, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1794871794871795, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5576923076923077, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.1730769230769231, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6875, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8385416666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8198757763975155, "unit": "proportion", "denominator": 161, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15104166666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.2005208333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.111979166666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6428571428571429, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 18, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.875, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.017857142857143, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.5333333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.68, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7843137254901961, "unit": "proportion", "denominator": 51, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14666666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.14, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2066666666666666, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.4230769230769231, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 14, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.4807692307692308, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.019230769230769, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.23214285714285715, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.38461538461538464, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.375, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.1428571428571428, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.36363636363636365, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.25, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.0227272727272727, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.25, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 390, "estimated_events": 10, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "779 saved telemetry estimates plus corrected selected-response result tokens at retained prices; no provider invoice. Earlier replaced calls are separate.", "value": 0.0017722417305069645 } }, "gemini-low": { "metrics": { "species_accuracy": { "value": 0.5075, "unit": "proportion", "denominator": 400, "interval": { "low": 0.4556054298813513, "high": 0.5578425920861516, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6825, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6346141975308641, "high": 0.728, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7435897435897436, "unit": "proportion", "denominator": 273, "interval": { "low": 0.6885783204684589, "high": 0.7952231866921071, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.175, "unit": "proportion", "denominator": 400, "interval": { "low": 0.13793103448275862, "high": 0.21445783132530122, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.37121324271570644, "unit": "f1", "denominator": null, "interval": { "low": 0.3358122897889391, "high": 0.3900793684897925, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5661001951414523, "unit": "f1", "denominator": null, "interval": { "low": 0.5121137419281322, "high": 0.5948710369469336, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.9912500000000002, "unit": "points_per_event", "denominator": 400, "interval": { "low": 1.620342585646412, "high": 2.3522058396093573, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2662500000000003, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.07001959168418, "high": 3.455266093383064, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.255, "unit": "proportion", "denominator": 400, "interval": { "low": 0.21148711403816428, "high": 0.30025445292620867, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5555555555555556, "unit": "proportion", "denominator": 324, "interval": { "low": 0.4984423676012461, "high": 0.6117656192911948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7160493827160493, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6655390377031837, "high": 0.7645262161297205, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7758620689655172, "unit": "proportion", "denominator": 232, "interval": { "low": 0.7194547779586142, "high": 0.8305785123966942, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16049382716049382, "unit": "proportion", "denominator": 324, "interval": { "low": 0.12084334926529047, "high": 0.20327868852459016, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.358024691358025, "unit": "points_per_event", "denominator": 324, "interval": { "low": 1.966961977082823, "high": 2.7375443480579142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.4382716049382718, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.222213541666667, "high": 3.647846747571789, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3806293538084315, "unit": "f1", "denominator": null, "interval": { "low": 0.3422577334462904, "high": 0.3995077819102162, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5709440307126472, "unit": "f1", "denominator": null, "interval": { "low": 0.5133866001694357, "high": 0.5992616728653243, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.21604938271604937, "unit": "proportion", "denominator": 324, "interval": { "low": 0.1710914454277286, "high": 0.2625080128205127, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.3026315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.19696969696969696, "high": 0.41333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5394736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.4230769230769231, "high": 0.6538461538461539, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5609756097560976, "unit": "proportion", "denominator": 41, "interval": { "low": 0.4, "high": 0.717948717948718, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14285714285714285, "high": 0.33766233766233766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.4276315789473684, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.4294117647058824, "high": 1.2945293291183704, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.5328947368421058, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.114940587138863, "high": 2.9629697712418297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2020428861199652, "unit": "f1", "denominator": null, "interval": { "low": 0.10126468333304835, "high": 0.22335180145524974, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.29296218487394954, "unit": "f1", "denominator": null, "interval": { "low": 0.1468337908329201, "high": 0.3238601121101121, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": { "low": 0.3076923076923077, "high": 0.5333333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6545454545454545, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8181818181818182, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8, "unit": "proportion", "denominator": 180, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.16363636363636364, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.8136363636363635, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.9727272727272727, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.4752475247524752, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6633663366336634, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7164179104477612, "unit": "proportion", "denominator": 67, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18811881188118812, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.8663366336633664, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.202970297029703, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.1282051282051282, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.32051282051282054, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4, "unit": "proportion", "denominator": 25, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.20512820512820512, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.3333333333333333, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.703125, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8489583333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8282208588957055, "unit": "proportion", "denominator": 163, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14583333333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.2005208333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.138020833333333, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6071428571428571, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5294117647058824, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.16071428571428573, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.8392857142857144, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.49333333333333335, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7066666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6981132075471698, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.21333333333333335, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.9933333333333334, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.26, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.4230769230769231, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 14, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.5, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.0384615384615383, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.125, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.26785714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4666666666666667, "unit": "proportion", "denominator": 15, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.08928571428571429, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.25, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.13636363636363635, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.45454545454545453, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3, "unit": "proportion", "denominator": 10, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.5454545454545454, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.0022678025 } }, "gemini-medium": { "metrics": { "species_accuracy": { "value": 0.55, "unit": "proportion", "denominator": 400, "interval": { "low": 0.4987593052109181, "high": 0.601467446561282, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7675, "unit": "proportion", "denominator": 400, "interval": { "low": 0.7243086776218898, "high": 0.8108761853987947, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7166123778501629, "unit": "proportion", "denominator": 307, "interval": { "low": 0.6643829691425321, "high": 0.7682119205298014, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2175, "unit": "proportion", "denominator": 400, "interval": { "low": 0.17585301837270342, "high": 0.2595641551071879, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.37299546410126566, "unit": "f1", "denominator": null, "interval": { "low": 0.3405976362492402, "high": 0.3908888539057059, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5967927425620252, "unit": "f1", "denominator": null, "interval": { "low": 0.5449562179987842, "high": 0.6254221662491294, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.04625, "unit": "points_per_event", "denominator": 400, "interval": { "low": 1.6640897097625331, "high": 2.425840659404469, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.44625, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.255233578021267, "high": 3.6349624035989714, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.28, "unit": "proportion", "denominator": 400, "interval": { "low": 0.2347412100967664, "high": 0.32673267326732675, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5987654320987654, "unit": "proportion", "denominator": 324, "interval": { "low": 0.541795665634675, "high": 0.6547619047619048, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7901234567901234, "unit": "proportion", "denominator": 324, "interval": { "low": 0.7460802607354332, "high": 0.8333333333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7578125, "unit": "proportion", "denominator": 256, "interval": { "low": 0.7017543859649122, "high": 0.8125, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19135802469135801, "unit": "proportion", "denominator": 324, "interval": { "low": 0.14733475716429706, "high": 0.2367601246105919, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3796296296296298, "unit": "points_per_event", "denominator": 324, "interval": { "low": 1.9649620472040952, "high": 2.7861490835135414, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5987654320987654, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.3887239225229786, "high": 3.80791864538731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.414646816983449, "unit": "f1", "denominator": null, "interval": { "low": 0.372733505192438, "high": 0.4361010420009951, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6110584671335039, "unit": "f1", "denominator": null, "interval": { "low": 0.5492914813362246, "high": 0.6426752197909402, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.24382716049382716, "unit": "proportion", "denominator": 324, "interval": { "low": 0.1962025316455696, "high": 0.2920661216293746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.34210526315789475, "unit": "proportion", "denominator": 76, "interval": { "low": 0.22972972972972974, "high": 0.4597837985769018, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6710526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.548376049491825, "high": 0.7875094696969696, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5098039215686274, "unit": "proportion", "denominator": 51, "interval": { "low": 0.36363636363636365, "high": 0.6491228070175439, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.32894736842105265, "unit": "proportion", "denominator": 76, "interval": { "low": 0.22093023255813954, "high": 0.4444444444444444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.625, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.29733437705998683, "high": 1.5367715141612202, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.7960526315789473, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.3714125200642058, "high": 3.2228962001853576, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.19913419913419914, "unit": "f1", "denominator": null, "interval": { "low": 0.1034880018970928, "high": 0.21906589080573632, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3285714285714286, "unit": "f1", "denominator": null, "interval": { "low": 0.17075520313020315, "high": 0.361458719829465, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4342105263157895, "unit": "proportion", "denominator": 76, "interval": { "low": 0.3176470588235294, "high": 0.5584415584415584, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6909090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7676767676767676, "unit": "proportion", "denominator": 198, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.20909090909090908, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.8727272727272726, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.122727272727273, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.5346534653465347, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7128712871287128, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.75, "unit": "proportion", "denominator": 72, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1782178217821782, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.198019801980198, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.4356435643564356, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.46153846153846156, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3611111111111111, "unit": "proportion", "denominator": 36, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2948717948717949, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5192307692307693, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.5320512820512822, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7395833333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.921875, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8022598870056498, "unit": "proportion", "denominator": 177, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18229166666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.1927083333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.260416666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.35714285714285715, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.75, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.47619047619047616, "unit": "proportion", "denominator": 21, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.39285714285714285, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.6785714285714286, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.1785714285714284, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.56, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7333333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7636363636363637, "unit": "proportion", "denominator": 55, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.17333333333333334, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.2733333333333334, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.473333333333333, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.46153846153846156, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7058823529411765, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.9807692307692308, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.326923076923077, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.16071428571428573, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4107142857142857, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.391304347826087, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.25, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.3125, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.4732142857142858, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5909090909090909, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3076923076923077, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.0454545454545454, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.6818181818181819, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.0032434425 } }, "gemini-high": { "metrics": { "species_accuracy": { "value": 0.56, "unit": "proportion", "denominator": 400, "interval": { "low": 0.5099502487562189, "high": 0.6092575112467866, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8075, "unit": "proportion", "denominator": 400, "interval": { "low": 0.7692307692307693, "high": 0.8445040214477212, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6934984520123839, "unit": "proportion", "denominator": 323, "interval": { "low": 0.6426424072975797, "high": 0.7435897435897436, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2475, "unit": "proportion", "denominator": 400, "interval": { "low": 0.206266318537859, "high": 0.2889541525518862, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.36069291156813826, "unit": "f1", "denominator": null, "interval": { "low": 0.32860588250455736, "high": 0.3773509533405161, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6131779496658349, "unit": "f1", "denominator": null, "interval": { "low": 0.5586300002577476, "high": 0.6414966206788774, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.9475000000000002, "unit": "points_per_event", "denominator": 400, "interval": { "low": 1.5561621431314814, "high": 2.3280870179689077, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.4725, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.2828629605918334, "high": 3.6583794557289324, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.305, "unit": "proportion", "denominator": 400, "interval": { "low": 0.25925925925925924, "high": 0.35189980951210514, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6018518518518519, "unit": "proportion", "denominator": 324, "interval": { "low": 0.5471698113207547, "high": 0.6570605187319885, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8240740740740741, "unit": "proportion", "denominator": 324, "interval": { "low": 0.7835341579928539, "high": 0.8626229405500219, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7303370786516854, "unit": "proportion", "denominator": 267, "interval": { "low": 0.67578125, "high": 0.7844548105375047, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2222222222222222, "unit": "proportion", "denominator": 324, "interval": { "low": 0.17717717717717718, "high": 0.26726797552269255, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.2978395061728394, "unit": "points_per_event", "denominator": 324, "interval": { "low": 1.8764981353987378, "high": 2.7043360690850324, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.625, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.4187080344117824, "high": 3.8276935975609754, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.39320527755125284, "unit": "f1", "denominator": null, "interval": { "low": 0.35413933447537865, "high": 0.4131148520925158, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6208504382388202, "unit": "f1", "denominator": null, "interval": { "low": 0.559167370224282, "high": 0.6522866085671303, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2654320987654321, "unit": "proportion", "denominator": 324, "interval": { "low": 0.21751105379513633, "high": 0.31446540880503143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.3815789473684211, "unit": "proportion", "denominator": 76, "interval": { "low": 0.2682926829268293, "high": 0.5, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7368421052631579, "unit": "proportion", "denominator": 76, "interval": { "low": 0.6363636363636364, "high": 0.8333333333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5178571428571429, "unit": "proportion", "denominator": 56, "interval": { "low": 0.38, "high": 0.6538461538461539, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.35526315789473684, "unit": "proportion", "denominator": 76, "interval": { "low": 0.24675324675324675, "high": 0.4677572478562676, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.45394736842105265, "unit": "points_per_event", "denominator": 76, "interval": { "low": -0.46380182002022247, "high": 1.382820109577922, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.822368421052632, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.3903933319903303, "high": 3.2605969148222664, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.22140615948665487, "unit": "f1", "denominator": null, "interval": { "low": 0.12458978311251037, "high": 0.23829817755676638, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3653201631529805, "unit": "f1", "denominator": null, "interval": { "low": 0.20557314213564212, "high": 0.39319199296866447, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.47368421052631576, "unit": "proportion", "denominator": 76, "interval": { "low": 0.35896739130434785, "high": 0.5869651483781918, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.7136363636363636, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9181818181818182, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7772277227722773, "unit": "proportion", "denominator": 202, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.20454545454545456, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.9136363636363636, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.163636363636364, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.5544554455445545, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7623762376237624, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7272727272727273, "unit": "proportion", "denominator": 77, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2079207920792079, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.108910891089109, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.495049504950495, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.1282051282051282, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5512820512820513, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.23255813953488372, "unit": "proportion", "denominator": 43, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4230769230769231, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.0256410256410255, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.4743589743589745, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7552083333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9322916666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8100558659217877, "unit": "proportion", "denominator": 179, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.17708333333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.2786458333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.294270833333333, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.42857142857142855, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8214285714285714, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5217391304347826, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.39285714285714285, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.4107142857142857, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.267857142857143, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.56, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7733333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7241379310344828, "unit": "proportion", "denominator": 58, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.21333333333333335, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.1466666666666665, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.48, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7307692307692307, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7368421052631579, "unit": "proportion", "denominator": 19, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.0, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.5384615384615383, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.125, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5178571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2413793103448276, "unit": "proportion", "denominator": 29, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.39285714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.9107142857142857, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.5, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.13636363636363635, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 14, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.3181818181818181, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.4090909090909092, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.0040265475 } }, "gemini-medium37": { "metrics": { "species_accuracy": { "value": 0.625, "unit": "proportion", "denominator": 400, "interval": { "low": 0.5748651987247069, "high": 0.6742424242424242, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7425, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6974342105263157, "high": 0.7862237156180149, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8417508417508418, "unit": "proportion", "denominator": 297, "interval": { "low": 0.7979443105341033, "high": 0.8819708783153405, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1175, "unit": "proportion", "denominator": 400, "interval": { "low": 0.08730109250682816, "high": 0.15, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5250346685202693, "unit": "f1", "denominator": null, "interval": { "low": 0.4828035179866669, "high": 0.5469169879771919, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6956709357893567, "unit": "f1", "denominator": null, "interval": { "low": 0.6397146613323337, "high": 0.7246650090697794, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.93375, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.6071387043189374, "high": 3.2452200988333395, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.68375, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.4959038639978797, "high": 3.864797642029785, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.15, "unit": "proportion", "denominator": 400, "interval": { "low": 0.11491442542787286, "high": 0.1876433031047829, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6728395061728395, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6191948214414286, "high": 0.7256944444444444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7654320987654321, "unit": "proportion", "denominator": 324, "interval": { "low": 0.717391304347826, "high": 0.8117293028322441, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8790322580645161, "unit": "proportion", "denominator": 248, "interval": { "low": 0.8352919145186165, "high": 0.9181826630618453, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.09259259259259259, "unit": "proportion", "denominator": 324, "interval": { "low": 0.0625, "high": 0.1261003672293995, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.2623456790123457, "unit": "points_per_event", "denominator": 324, "interval": { "low": 2.920601379981677, "high": 3.588061856371637, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.8487654320987654, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.6464968152866244, "high": 4.045180722891566, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5778212733813252, "unit": "f1", "denominator": null, "interval": { "low": 0.5153238895506652, "high": 0.6001641279245447, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7298795032185161, "unit": "f1", "denominator": null, "interval": { "low": 0.6509354394324193, "high": 0.7581020563257408, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.11728395061728394, "unit": "proportion", "denominator": 324, "interval": { "low": 0.08231707317073171, "high": 0.15540746539222144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": { "low": 0.3, "high": 0.5405516171909613, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6447368421052632, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5277701465201465, "high": 0.7571428571428571, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6530612244897959, "unit": "proportion", "denominator": 49, "interval": { "low": 0.5094253859348199, "high": 0.7857142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.13333333333333333, "high": 0.32222222222222224, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5328947368421053, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.6217350746268657, "high": 2.3904583772391983, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.9802631578947367, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.513513513513514, "high": 3.426670731707317, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2977832512315271, "unit": "f1", "denominator": null, "interval": { "low": 0.17351796909986567, "high": 0.3246398716226302, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.43178571428571433, "unit": "f1", "denominator": null, "interval": { "low": 0.2516010551948052, "high": 0.47072781385281376, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.18518518518518517, "high": 0.40229885057471265, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.7772727272727272, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8909090909090909, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8724489795918368, "unit": "proportion", "denominator": 196, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11363636363636363, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.6977272727272728, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.3568181818181815, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6237623762376238, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7029702970297029, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8873239436619719, "unit": "proportion", "denominator": 71, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07920792079207921, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.0693069306930694, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.712871287128713, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3717948717948718, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5172413793103449, "unit": "proportion", "denominator": 29, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1794871794871795, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5769230769230769, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.7307692307692308, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.8177083333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9010416666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9075144508670521, "unit": "proportion", "denominator": 173, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08333333333333333, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.9973958333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.4921875, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8214285714285714, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6086956521739131, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.6428571428571428, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.4285714285714284, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.64, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.72, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8888888888888888, "unit": "proportion", "denominator": 54, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.2066666666666666, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.74, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.5769230769230769, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8823529411764706, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.673076923076923, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.6346153846153846, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.35714285714285715, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.7857142857142857, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.7678571428571428, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.13636363636363635, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 9, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.045454545454545456, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.6363636363636365, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Saved provider-token cost estimate for each selected response; not an invoice.", "value": 0.0032643324735 } }, "gemini-medium38": { "metrics": { "species_accuracy": { "value": 0.66, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6116745108668868, "high": 0.7079213704080688, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.76, "unit": "proportion", "denominator": 400, "interval": { "low": 0.7164547515586625, "high": 0.8026370483810477, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.868421052631579, "unit": "proportion", "denominator": 304, "interval": { "low": 0.8278145695364238, "high": 0.9057239057239057, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1, "unit": "proportion", "denominator": 400, "interval": { "low": 0.07105263157894737, "high": 0.13101604278074866, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.6194118479870407, "unit": "f1", "denominator": null, "interval": { "low": 0.5731193928964311, "high": 0.6434837705102915, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7278089213847727, "unit": "f1", "denominator": null, "interval": { "low": 0.6734152866533064, "high": 0.7560934303495924, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 3.16125, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.8486029573438874, "high": 3.457500606796117, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.79875, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.6180435231023105, "high": 3.975128621597893, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1275, "unit": "proportion", "denominator": 400, "interval": { "low": 0.09571788413098237, "high": 0.1618798955613577, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7098765432098766, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6594410313467491, "high": 0.7603557385106906, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7870370370370371, "unit": "proportion", "denominator": 324, "interval": { "low": 0.7405243200354283, "high": 0.8317167226204518, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9019607843137255, "unit": "proportion", "denominator": 255, "interval": { "low": 0.8634682642007142, "high": 0.9367088607594937, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07716049382716049, "unit": "proportion", "denominator": 324, "interval": { "low": 0.04985314404217027, "high": 0.10714285714285714, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.483024691358025, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.161762504402959, "high": 3.7821341190348763, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.9614197530864197, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.7663339149219013, "high": 4.150772865853659, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.6462027392780747, "unit": "f1", "denominator": null, "interval": { "low": 0.5791572090196322, "high": 0.6688190750504688, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7652400859871937, "unit": "f1", "denominator": null, "interval": { "low": 0.6858440633127222, "high": 0.7920225888755551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.09567901234567901, "unit": "proportion", "denominator": 324, "interval": { "low": 0.06432748538011696, "high": 0.1298741506171846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.4473684210526316, "unit": "proportion", "denominator": 76, "interval": { "low": 0.32529367469879517, "high": 0.5692307692307692, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6447368421052632, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5238095238095238, "high": 0.7600140845070421, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6938775510204082, "unit": "proportion", "denominator": 49, "interval": { "low": 0.5476078167115903, "high": 0.8292682926829268, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19736842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.10666666666666667, "high": 0.3, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.7894736842105265, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.8818716397849464, "high": 2.6323573706591072, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1052631578947367, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.6384246575342467, "high": 3.5507432864450124, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.34795198781266895, "unit": "f1", "denominator": null, "interval": { "low": 0.20830328272351295, "high": 0.3856887374321584, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4175423853752028, "unit": "f1", "denominator": null, "interval": { "low": 0.24996393926821556, "high": 0.46282648491859013, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2631578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.16216216216216217, "high": 0.3736607142857138, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.8181818181818182, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9090909090909091, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9, "unit": "proportion", "denominator": 200, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.038636363636364, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.493181818181818, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6237623762376238, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7326732673267327, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8513513513513513, "unit": "proportion", "denominator": 74, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.10891089108910891, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.9356435643564356, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.6782178217821784, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.2564102564102564, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3717948717948718, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6896551724137931, "unit": "proportion", "denominator": 29, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11538461538461539, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.9551282051282052, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9807692307692308, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.8697916666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9270833333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9382022471910112, "unit": "proportion", "denominator": 178, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.057291666666666664, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.359375, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.645833333333333, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.4642857142857143, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5909090909090909, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.32142857142857145, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.8392857142857142, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.4464285714285716, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.6266666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7466666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8392857142857143, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.0, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.6666666666666665, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.6153846153846154, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6923076923076923, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8888888888888888, "unit": "proportion", "denominator": 18, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.75, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.7115384615384617, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.26785714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.35714285714285715, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.75, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.0982142857142858, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9910714285714286, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.22727272727272727, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5555555555555556, "unit": "proportion", "denominator": 9, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5909090909090909, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9545454545454546, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "verified_selected_result_token_reconstruction", "coverage": { "observed_events": 0, "estimated_events": 400, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Selected-response tokens repriced with the run's retained list prices and GBP buffers; no provider invoice. Earlier failed/uncertain attempts are separate.", "value": 0.0076179075 } }, "gemini38-low": { "metrics": { "species_accuracy": { "value": 0.66, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6120481927710844, "high": 0.707808564231738, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.765, "unit": "proportion", "denominator": 400, "interval": { "low": 0.7220750394465348, "high": 0.8070175438596491, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8627450980392157, "unit": "proportion", "denominator": 306, "interval": { "low": 0.8228218725054168, "high": 0.8993294038176589, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.105, "unit": "proportion", "denominator": 400, "interval": { "low": 0.07673267326732673, "high": 0.1358974358974359, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5410575647262954, "unit": "f1", "denominator": null, "interval": { "low": 0.5022617742238907, "high": 0.5616383998065115, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7304277123804986, "unit": "f1", "denominator": null, "interval": { "low": 0.6780533952022526, "high": 0.7582118397387907, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 3.145, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.8400485131048385, "high": 3.4411764705882355, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.7825000000000006, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.604438241350774, "high": 3.9582335176085177, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1275, "unit": "proportion", "denominator": 400, "interval": { "low": 0.09660574412532637, "high": 0.1605287019361876, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7129629629629629, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6615372670807453, "high": 0.7628578792341679, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7870370370370371, "unit": "proportion", "denominator": 324, "interval": { "low": 0.7412082962342219, "high": 0.8317460317460318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9058823529411765, "unit": "proportion", "denominator": 255, "interval": { "low": 0.8686411135120535, "high": 0.9395161290322581, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.07407407407407407, "unit": "proportion", "denominator": 324, "interval": { "low": 0.04745461839334069, "high": 0.10377440099236142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.4984567901234573, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.180839280904929, "high": 3.7940342874044415, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.9614197530864197, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.766561383625463, "high": 4.1507468427095295, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5983094983387549, "unit": "f1", "denominator": null, "interval": { "low": 0.5386304688811944, "high": 0.6184960786156588, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.771504353121026, "unit": "f1", "denominator": null, "interval": { "low": 0.6945498151362771, "high": 0.7975344171622969, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.09259259259259259, "unit": "proportion", "denominator": 324, "interval": { "low": 0.0626956908248725, "high": 0.12538226299694188, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.4342105263157895, "unit": "proportion", "denominator": 76, "interval": { "low": 0.313953488372093, "high": 0.5555555555555556, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6710526315789473, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5569620253164557, "high": 0.7812589798850573, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6470588235294118, "unit": "proportion", "denominator": 51, "interval": { "low": 0.5068661971830988, "high": 0.7777777777777778, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": { "low": 0.14544721407624633, "high": 0.3384615384615385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.6381578947368423, "unit": "points_per_event", "denominator": 76, "interval": { "low": 0.7714199655765922, "high": 2.4687690548780488, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0197368421052637, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.5579640468227427, "high": 3.466688311688312, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3142857142857143, "unit": "f1", "denominator": null, "interval": { "low": 0.19118086797008363, "high": 0.34836276120998344, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.42428571428571427, "unit": "f1", "denominator": null, "interval": { "low": 0.2580941717596129, "high": 0.4702897276334776, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.27631578947368424, "unit": "proportion", "denominator": 76, "interval": { "low": 0.1794871794871795, "high": 0.3823620188816266, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.8227272727272728, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9045454545454545, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9095477386934674, "unit": "proportion", "denominator": 199, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08181818181818182, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.056818181818182, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.488636363636363, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6633663366336634, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7425742574257426, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8933333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07920792079207921, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.3069306930693068, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.801980198019802, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3974358974358974, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4838709677419355, "unit": "proportion", "denominator": 31, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.20512820512820512, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.33974358974358976, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.75, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.8645833333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9166666666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9431818181818182, "unit": "proportion", "denominator": 176, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.052083333333333336, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.333333333333333, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.619791666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.5357142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8214285714285714, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6521739130434783, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.1607142857142856, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.5892857142857144, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.68, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.76, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8947368421052632, "unit": "proportion", "denominator": 57, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.36, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.8266666666666667, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.6153846153846154, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6923076923076923, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8888888888888888, "unit": "proportion", "denominator": 18, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.1538461538461537, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.730769230769231, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.23214285714285715, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.375, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6190476190476191, "unit": "proportion", "denominator": 21, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.7946428571428571, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.8660714285714286, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.45454545454545453, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2, "unit": "proportion", "denominator": 10, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.36363636363636365, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.8181818181818182, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.4545454545454546, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 399, "estimated_events": 1, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.00642868 } }, "gemini38-high": { "metrics": { "species_accuracy": { "value": 0.7, "unit": "proportion", "denominator": 400, "interval": { "low": 0.6542546575632476, "high": 0.744899247577819, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8475, "unit": "proportion", "denominator": 400, "interval": { "low": 0.8102564102564103, "high": 0.8836633663366337, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8259587020648967, "unit": "proportion", "denominator": 339, "interval": { "low": 0.7840218423551757, "high": 0.8647906357061286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1475, "unit": "proportion", "denominator": 400, "interval": { "low": 0.11369509043927649, "high": 0.18357607796597977, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5473418155271824, "unit": "f1", "denominator": null, "interval": { "low": 0.5095120118424594, "high": 0.56817361341227, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7525949963498759, "unit": "f1", "denominator": null, "interval": { "low": 0.7005790162833817, "high": 0.7812387184418713, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 3.1887500000000006, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.8758883770375623, "high": 3.4824826208607362, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.97625, "unit": "points_per_event", "denominator": 400, "interval": { "low": 3.8129860786695318, "high": 4.135311963058419, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1575, "unit": "proportion", "denominator": 400, "interval": { "low": 0.12339067507131035, "high": 0.19444444444444445, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7376543209876543, "unit": "proportion", "denominator": 324, "interval": { "low": 0.6881347757370975, "high": 0.7850746268656716, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8611111111111112, "unit": "proportion", "denominator": 324, "interval": { "low": 0.8207540563395169, "high": 0.8993906949054697, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8566308243727598, "unit": "proportion", "denominator": 279, "interval": { "low": 0.8136185529433206, "high": 0.896797153024911, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12345679012345678, "unit": "proportion", "denominator": 324, "interval": { "low": 0.0884124701854274, "high": 0.16088418724391745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.4413580246913584, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.10647095959596, "high": 3.749222217627436, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.104938271604938, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.9262788461538465, "high": 4.274697185856551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5835279762023525, "unit": "f1", "denominator": null, "interval": { "low": 0.5304215943957684, "high": 0.6049760598081002, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7831559680610518, "unit": "f1", "denominator": null, "interval": { "low": 0.7118816135311629, "high": 0.8119415539529766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13271604938271606, "unit": "proportion", "denominator": 324, "interval": { "low": 0.09687477742165243, "high": 0.17251485342749845, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.5394736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.41975308641975306, "high": 0.654320987654321, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7894736842105263, "unit": "proportion", "denominator": 76, "interval": { "low": 0.6933333333333334, "high": 0.8809523809523809, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6833333333333333, "unit": "proportion", "denominator": 60, "interval": { "low": 0.5535714285714286, "high": 0.8028169014084507, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.25, "unit": "proportion", "denominator": 76, "interval": { "low": 0.15384615384615385, "high": 0.35714285714285715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.111842105263158, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.2222222222222223, "high": 2.91783176369863, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.4276315789473686, "unit": "points_per_event", "denominator": 76, "interval": { "low": 2.983328651685394, "high": 3.845085637280473, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3525867136978248, "unit": "f1", "denominator": null, "interval": { "low": 0.23130200817700816, "high": 0.3825755571588904, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47599206349206347, "unit": "f1", "denominator": null, "interval": { "low": 0.312257711038961, "high": 0.5164770021645022, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2631578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.1684925093632959, "high": 0.36709506587445095, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.85, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9363636363636364, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9077669902912622, "unit": "proportion", "denominator": 206, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08636363636363636, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.156818181818182, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.611363636363636, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.7128712871287128, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8415841584158416, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8470588235294118, "unit": "proportion", "denominator": 85, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12871287128712872, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.366336633663366, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.00990099009901, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.2564102564102564, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6025641025641025, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.425531914893617, "unit": "proportion", "denominator": 47, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.20512820512820512, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.128205128205128, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.8802083333333334, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9479166666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9285714285714286, "unit": "proportion", "denominator": 182, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.06770833333333333, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.3515625, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.690104166666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.6428571428571429, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8571428571428571, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.75, "unit": "proportion", "denominator": 24, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.8214285714285716, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.071428571428571, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.6933333333333334, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.84, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8253968253968254, "unit": "proportion", "denominator": 63, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14666666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.2, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.933333333333333, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.7692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8461538461538461, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.8461538461538463, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.230769230769231, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.30357142857142855, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5892857142857143, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5151515151515151, "unit": "proportion", "denominator": 33, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2857142857142857, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.6160714285714286, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.3125, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.13636363636363635, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 14, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.8409090909090909, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.6590909090909092, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 399, "estimated_events": 1, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.012317345 } }, "qwen-repaired": { "metrics": { "species_accuracy": { "value": 0.455, "unit": "proportion", "denominator": 400, "interval": { "low": 0.40540540540540543, "high": 0.50495111078272, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.875, "unit": "proportion", "denominator": 400, "interval": { "low": 0.8414322250639387, "high": 0.9066994942608523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.52, "unit": "proportion", "denominator": 350, "interval": { "low": 0.4668587896253602, "high": 0.5738185781760929, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.42, "unit": "proportion", "denominator": 400, "interval": { "low": 0.36939313984168864, "high": 0.469334693877551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2042805257464443, "unit": "f1", "denominator": null, "interval": { "low": 0.1858802338328159, "high": 0.21486251008167706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4953802749351276, "unit": "f1", "denominator": null, "interval": { "low": 0.4507595670445786, "high": 0.5210415869480669, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.7425000000000002, "unit": "points_per_event", "denominator": 400, "interval": { "low": 0.3414286390077459, "high": 1.1350294478970948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.005, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.7986644549763033, "high": 3.2057441653481007, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.4525, "unit": "proportion", "denominator": 400, "interval": { "low": 0.40245962672855623, "high": 0.5024878731343284, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5092592592592593, "unit": "proportion", "denominator": 324, "interval": { "low": 0.45364238410596025, "high": 0.5641891891891891, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8734567901234568, "unit": "proportion", "denominator": 324, "interval": { "low": 0.8368564371968766, "high": 0.9093569459903535, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5830388692579506, "unit": "proportion", "denominator": 283, "interval": { "low": 0.5243055555555556, "high": 0.6411149825783972, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.36419753086419754, "unit": "proportion", "denominator": 324, "interval": { "low": 0.3111111111111111, "high": 0.4171789298338276, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.2098765432098766, "unit": "points_per_event", "denominator": 324, "interval": { "low": 0.7786121836338644, "high": 1.6238953927492443, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.231481481481482, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.0063086522627622, "high": 3.4471302330599913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2564240868637153, "unit": "f1", "denominator": null, "interval": { "low": 0.2322738089112476, "high": 0.2707035669158167, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5061001714415434, "unit": "f1", "denominator": null, "interval": { "low": 0.4584351491669362, "high": 0.5342833557549014, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.404320987654321, "unit": "proportion", "denominator": 324, "interval": { "low": 0.3501577287066246, "high": 0.4591564188757925, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.1323529411764706, "high": 0.32142857142857145, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.881578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.8028169014084507, "high": 0.95, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.2537313432835821, "unit": "proportion", "denominator": 67, "interval": { "low": 0.1506849315068493, "high": 0.36363636363636365, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6578947368421053, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5443037974683544, "high": 0.7666666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.25, "unit": "points_per_event", "denominator": 76, "interval": { "low": -2.073346590909091, "high": -0.3856728778467915, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.039473684210526, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.6249663978494624, "high": 2.4743589743589745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.10329004329004329, "unit": "f1", "denominator": null, "interval": { "low": 0.05166590909090908, "high": 0.11227152406417111, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2582251082251082, "unit": "f1", "denominator": null, "interval": { "low": 0.12916477272727273, "high": 0.28067881016042784, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.6578947368421053, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5443037974683544, "high": 0.7666666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6272727272727273, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9954545454545455, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6301369863013698, "unit": "proportion", "denominator": 219, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.36818181818181817, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.7954545454545454, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.840909090909091, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.3564356435643564, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.900990099009901, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3956043956043956, "unit": "proportion", "denominator": 91, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5445544554455446, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.07920792079207921, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.742574257425743, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.08974358974358974, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.1794871794871795, "unit": "proportion", "denominator": 39, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.41025641025641024, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.2179487179487178, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.9615384615384616, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6875, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9947916666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6910994764397905, "unit": "proportion", "denominator": 191, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3072916666666667, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.3020833333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.072916666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.6785714285714286, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.25, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.36, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8666666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4153846153846154, "unit": "proportion", "denominator": 65, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5066666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.12, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.7866666666666666, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6538461538461539, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.6153846153846154, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4642857142857143, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.375, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.1428571428571428, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.9107142857142857, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5909090909090909, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.15384615384615385, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.4090909090909092, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0909090909090908, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "Selected evaluated response attempt only: 786 first-pass outputs, four successful repair outputs. Four superseded first-pass length charges are excluded from this comparison and retained in the audit below.", "value": 0.0021851117630625 } }, "qwen-first-pass": { "metrics": { "species_accuracy": { "value": 0.455, "unit": "proportion", "denominator": 400, "interval": { "low": 0.40540540540540543, "high": 0.50495111078272, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.875, "unit": "proportion", "denominator": 400, "interval": { "low": 0.8414322250639387, "high": 0.9066994942608523, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.52, "unit": "proportion", "denominator": 350, "interval": { "low": 0.4668587896253602, "high": 0.5738185781760929, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.42, "unit": "proportion", "denominator": 400, "interval": { "low": 0.36939313984168864, "high": 0.469334693877551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2042805257464443, "unit": "f1", "denominator": null, "interval": { "low": 0.1858802338328159, "high": 0.21486251008167706, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4953802749351276, "unit": "f1", "denominator": null, "interval": { "low": 0.4507595670445786, "high": 0.5210415869480669, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.7425000000000002, "unit": "points_per_event", "denominator": 400, "interval": { "low": 0.3414286390077459, "high": 1.1350294478970948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.005, "unit": "points_per_event", "denominator": 400, "interval": { "low": 2.7986644549763033, "high": 3.2057441653481007, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.4525, "unit": "proportion", "denominator": 400, "interval": { "low": 0.40245962672855623, "high": 0.5024878731343284, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 324, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5092592592592593, "unit": "proportion", "denominator": 324, "interval": { "low": 0.45364238410596025, "high": 0.5641891891891891, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8734567901234568, "unit": "proportion", "denominator": 324, "interval": { "low": 0.8368564371968766, "high": 0.9093569459903535, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5830388692579506, "unit": "proportion", "denominator": 283, "interval": { "low": 0.5243055555555556, "high": 0.6411149825783972, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.36419753086419754, "unit": "proportion", "denominator": 324, "interval": { "low": 0.3111111111111111, "high": 0.4171789298338276, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.2098765432098766, "unit": "points_per_event", "denominator": 324, "interval": { "low": 0.7786121836338644, "high": 1.6238953927492443, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.231481481481482, "unit": "points_per_event", "denominator": 324, "interval": { "low": 3.0063086522627622, "high": 3.4471302330599913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2564240868637153, "unit": "f1", "denominator": null, "interval": { "low": 0.2322738089112476, "high": 0.2707035669158167, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5061001714415434, "unit": "f1", "denominator": null, "interval": { "low": 0.4584351491669362, "high": 0.5342833557549014, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.404320987654321, "unit": "proportion", "denominator": 324, "interval": { "low": 0.3501577287066246, "high": 0.4591564188757925, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 76, "n_species": 20, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": { "low": 0.1323529411764706, "high": 0.32142857142857145, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.881578947368421, "unit": "proportion", "denominator": 76, "interval": { "low": 0.8028169014084507, "high": 0.95, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.2537313432835821, "unit": "proportion", "denominator": 67, "interval": { "low": 0.1506849315068493, "high": 0.36363636363636365, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.6578947368421053, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5443037974683544, "high": 0.7666666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -1.25, "unit": "points_per_event", "denominator": 76, "interval": { "low": -2.073346590909091, "high": -0.3856728778467915, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.039473684210526, "unit": "points_per_event", "denominator": 76, "interval": { "low": 1.6249663978494624, "high": 2.4743589743589745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.10329004329004329, "unit": "f1", "denominator": null, "interval": { "low": 0.05166590909090908, "high": 0.11227152406417111, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.2582251082251082, "unit": "f1", "denominator": null, "interval": { "low": 0.12916477272727273, "high": 0.28067881016042784, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.6578947368421053, "unit": "proportion", "denominator": 76, "interval": { "low": 0.5443037974683544, "high": 0.7666666666666667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 220, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.6272727272727273, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9954545454545455, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6301369863013698, "unit": "proportion", "denominator": 219, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.36818181818181817, "unit": "proportion", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.7954545454545454, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.840909090909091, "unit": "points_per_event", "denominator": 220, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 101, "n_species": 40, "metrics": { "species_accuracy": { "value": 0.3564356435643564, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.900990099009901, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3956043956043956, "unit": "proportion", "denominator": 91, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5445544554455446, "unit": "proportion", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.07920792079207921, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.742574257425743, "unit": "points_per_event", "denominator": 101, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 78, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.08974358974358974, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.1794871794871795, "unit": "proportion", "denominator": 39, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.41025641025641024, "unit": "proportion", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.2179487179487178, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.9615384615384616, "unit": "points_per_event", "denominator": 78, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 192, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6875, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9947916666666666, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6910994764397905, "unit": "proportion", "denominator": 191, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3072916666666667, "unit": "proportion", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.3020833333333335, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.072916666666667, "unit": "points_per_event", "denominator": 192, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 28, "n_species": 11, "metrics": { "species_accuracy": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.21428571428571427, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7857142857142857, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.6785714285714286, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.25, "unit": "points_per_event", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 75, "n_species": 34, "metrics": { "species_accuracy": { "value": 0.36, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8666666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4153846153846154, "unit": "proportion", "denominator": 65, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5066666666666667, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.12, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.7866666666666666, "unit": "points_per_event", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 26, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.34615384615384615, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6538461538461539, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.6153846153846154, "unit": "points_per_event", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 56, "n_species": 31, "metrics": { "species_accuracy": { "value": 0.08928571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4642857142857143, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.19230769230769232, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.375, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.1428571428571428, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.9107142857142857, "unit": "points_per_event", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 22, "n_species": 14, "metrics": { "species_accuracy": { "value": 0.09090909090909091, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5909090909090909, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.15384615384615385, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.4090909090909092, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.0909090909090908, "unit": "points_per_event", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 400, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 400, "estimated_events": 0, "missing_events": 0, "population_events": 400, "population_attempts": 400 }, "note": "First pass includes 786 model outputs and four charged HTTP 200 length nonresponses. 785 charges came from result usage and five exact charges from hash-bound provider generation metadata.", "value": 0.0021851117630625 } } } }, "mammals": { "n_events": 390, "n_species": 39, "models": { "bioclip": { "metrics": { "species_accuracy": { "value": 0.4282051282051282, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3753351206434316, "high": 0.48052038239538236, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 390, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4282051282051282, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3753351206434316, "high": 0.48052038239538236, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5194796176046176, "high": 0.6246648793565683, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.21273344644357361, "unit": "f1", "denominator": null, "interval": { "low": 0.18372988850400115, "high": 0.22799181345311462, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.45274041166196427, "unit": "f1", "denominator": null, "interval": { "low": 0.39101489091877173, "high": 0.4852133465797054, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": null, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.47619047619047616, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3734820233314209, "high": 0.5798319327731093, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 105, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.47619047619047616, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3734820233314209, "high": 0.5798319327731093, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5238095238095238, "unit": "proportion", "denominator": 105, "interval": { "low": 0.42016806722689076, "high": 0.6265179766685789, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.29517698525795694, "unit": "f1", "denominator": null, "interval": { "low": 0.1912797510223981, "high": 0.3149832404437667, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47966260104417996, "unit": "f1", "denominator": null, "interval": { "low": 0.31082959541139693, "high": 0.511847765721121, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4105263157894737, "unit": "proportion", "denominator": 285, "interval": { "low": 0.35116993030202454, "high": 0.4709946780817955, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 285, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4105263157894737, "unit": "proportion", "denominator": 285, "interval": { "low": 0.35116993030202454, "high": 0.4709946780817955, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5894736842105263, "unit": "proportion", "denominator": 285, "interval": { "low": 0.5290053219182044, "high": 0.6488300696979754, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.22266362223470063, "unit": "f1", "denominator": null, "interval": { "low": 0.17962296574783346, "high": 0.23753944284421868, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.41107130258713964, "unit": "f1", "denominator": null, "interval": { "low": 0.3316116290729233, "high": 0.43853435602009605, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6129032258064516, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6129032258064516, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3870967741935484, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.38848920863309355, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.38848920863309355, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6115107913669064, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.1875, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.1875, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8125, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.6226415094339622, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6226415094339622, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.37735849056603776, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6078431372549019, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6078431372549019, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.39215686274509803, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.35514018691588783, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.35514018691588783, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6448598130841121, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.95, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7763157894736842, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 390, "population_events": 390 } } }, "speciesnet": { "metrics": { "species_accuracy": { "value": 0.4358974358974359, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3842239185750636, "high": 0.48894365862597905, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7076923076923077, "unit": "proportion", "denominator": 390, "interval": { "low": 0.6601458407736052, "high": 0.753695500330858, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6159420289855072, "unit": "proportion", "denominator": 276, "interval": { "low": 0.5546875, "high": 0.6772911898505269, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.2717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.22538860103626943, "high": 0.318769888140068, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2985159203125846, "unit": "f1", "denominator": null, "interval": { "low": 0.2703997825889327, "high": 0.3164523007682581, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4745637707533396, "unit": "f1", "denominator": null, "interval": { "low": 0.429866321038816, "high": 0.5030780166059488, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.1538461538461537, "unit": "points_per_event", "denominator": 390, "interval": { "low": 0.7627304656491979, "high": 1.5520725201349208, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.833333333333334, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.622361909755108, "high": 3.0442858573717952, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.33589743589743587, "unit": "proportion", "denominator": 390, "interval": { "low": 0.28690591868022713, "high": 0.38536646802236285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.5047619047619047, "unit": "proportion", "denominator": 105, "interval": { "low": 0.4077669902912621, "high": 0.6018540992448759, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7333333333333333, "unit": "proportion", "denominator": 105, "interval": { "low": 0.6463287601626017, "high": 0.8151283339422725, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6883116883116883, "unit": "proportion", "denominator": 77, "interval": { "low": 0.5769230769230769, "high": 0.7957044107965766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.22857142857142856, "unit": "proportion", "denominator": 105, "interval": { "low": 0.14678305915849416, "high": 0.3148148148148148, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.8142857142857143, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.068133899233297, "high": 2.5385135135135126, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1476190476190475, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.7521195652173915, "high": 3.5309803650442477, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4264029373404373, "unit": "f1", "denominator": null, "interval": { "low": 0.3022962184873949, "high": 0.4499582747350258, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5685372497872497, "unit": "f1", "denominator": null, "interval": { "low": 0.4030616246498599, "high": 0.5999443663133679, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.26666666666666666, "unit": "proportion", "denominator": 105, "interval": { "low": 0.18, "high": 0.354846246608381, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4105263157894737, "unit": "proportion", "denominator": 285, "interval": { "low": 0.35018050541516244, "high": 0.4725274725274725, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6982456140350877, "unit": "proportion", "denominator": 285, "interval": { "low": 0.6420620574448987, "high": 0.7526501766784452, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5879396984924623, "unit": "proportion", "denominator": 199, "interval": { "low": 0.5149962871287129, "high": 0.6614583333333334, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.28771929824561404, "unit": "proportion", "denominator": 285, "interval": { "low": 0.23371548761701624, "high": 0.3448303142732431, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.9105263157894739, "unit": "points_per_event", "denominator": 285, "interval": { "low": 0.4514911951036187, "high": 1.3721405388087147, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.7175438596491226, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.476114513814917, "high": 2.966105898574119, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.31125373401485507, "unit": "f1", "denominator": null, "interval": { "low": 0.26469666469774217, "high": 0.32501670601740834, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.45490930356017284, "unit": "f1", "denominator": null, "interval": { "low": 0.3868643560967001, "high": 0.47502441648698135, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.36140350877192984, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3028161999887773, "high": 0.4204946996466431, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5419354838709678, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7548387096774194, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.717948717948718, "unit": "proportion", "denominator": 117, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2129032258064516, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.9258064516129032, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2483870967741937, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3669064748201439, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7338129496402878, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3669064748201439, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.43884892086330934, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.6690647482014387, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3645833333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.59375, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6140350877192983, "unit": "proportion", "denominator": 57, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.22916666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.9427083333333334, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.4010416666666665, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.6037735849056604, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7924528301886793, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7619047619047619, "unit": "proportion", "denominator": 42, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18867924528301888, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.5377358490566038, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.5754716981132075, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.5098039215686274, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7352941176470589, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6933333333333334, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.22549019607843138, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.607843137254902, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.0784313725490198, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.4375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.25, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.3125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.875, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.34579439252336447, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7476635514018691, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4625, "unit": "proportion", "denominator": 80, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.40186915887850466, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.17757009345794392, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.607476635514019, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.35, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.65, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 13, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.45, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5789473684210527, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 44, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.21052631578947367, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.006578947368421, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.388157894736842, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 390, "population_events": 390 } } }, "bioclip-country": { "metrics": { "species_accuracy": { "value": 0.46153846153846156, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4083742576132689, "high": 0.5144375999833795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 390, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.46153846153846156, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4083742576132689, "high": 0.5144375999833795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5384615384615384, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4855624000166206, "high": 0.591625742386731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.2874866981663751, "unit": "f1", "denominator": null, "interval": { "low": 0.2507405535490119, "high": 0.3077505863215063, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4644015893456829, "unit": "f1", "denominator": null, "interval": { "low": 0.40504243265609624, "high": 0.4971355625193564, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": null, "raw_tis": null, "any_rank_conflict_rate": null }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.5047619047619047, "unit": "proportion", "denominator": 105, "interval": { "low": 0.4019607843137255, "high": 0.6068376068376068, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 105, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.5047619047619047, "unit": "proportion", "denominator": 105, "interval": { "low": 0.4019607843137255, "high": 0.6068376068376068, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.49523809523809526, "unit": "proportion", "denominator": 105, "interval": { "low": 0.39316239316239315, "high": 0.5980392156862745, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3514661528947243, "unit": "f1", "denominator": null, "interval": { "low": 0.22981918280132563, "high": 0.37307821741136643, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5125548063048063, "unit": "f1", "denominator": null, "interval": { "low": 0.33515297491859997, "high": 0.544072400391576, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4456140350877193, "unit": "proportion", "denominator": 285, "interval": { "low": 0.38571428571428573, "high": 0.5076335877862596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 285, "interval": { "low": 1.0, "high": 1.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4456140350877193, "unit": "proportion", "denominator": 285, "interval": { "low": 0.38571428571428573, "high": 0.5076335877862596, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5543859649122806, "unit": "proportion", "denominator": 285, "interval": { "low": 0.49236641221374045, "high": 0.6142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3042640295820698, "unit": "f1", "denominator": null, "interval": { "low": 0.2486817262463597, "high": 0.32480684543414234, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4212886563444043, "unit": "f1", "denominator": null, "interval": { "low": 0.344328544033421, "high": 0.4497325552165048, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6451612903225806, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6451612903225806, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3548387096774194, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.43884892086330934, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.43884892086330934, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5611510791366906, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.19791666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.19791666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.8020833333333334, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.660377358490566, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.660377358490566, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.33962264150943394, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6372549019607843, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6372549019607843, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3627450980392157, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.53125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.53125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.46875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.411214953271028, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.411214953271028, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5887850467289719, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.95, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 1.0, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.23684210526315788, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.7631578947368421, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 390, "population_events": 390 } } }, "speciesnet-country": { "metrics": { "species_accuracy": { "value": 0.44871794871794873, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3969839656296751, "high": 0.502591010902418, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.49743589743589745, "unit": "proportion", "denominator": 390, "interval": { "low": 0.44472953216374267, "high": 0.551076441959617, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9020618556701031, "unit": "proportion", "denominator": 194, "interval": { "low": 0.8502355072463769, "high": 0.9476792406039898, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.04871794871794872, "unit": "proportion", "denominator": 390, "interval": { "low": 0.02564102564102564, "high": 0.07653061224489796, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.4789549804666704, "unit": "f1", "denominator": null, "interval": { "low": 0.43602614790514854, "high": 0.5059200301212662, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4912358774017132, "unit": "f1", "denominator": null, "interval": { "low": 0.4472063055437421, "high": 0.518892338585914, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.0576923076923075, "unit": "points_per_event", "denominator": 390, "interval": { "low": 1.7278996532573665, "high": 2.3867222222222213, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.878205128205128, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.6675257731958766, "high": 3.0911420094936712, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.1641025641025641, "unit": "proportion", "denominator": 390, "interval": { "low": 0.12698412698412698, "high": 0.20308483290488433, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.5333333333333333, "unit": "proportion", "denominator": 105, "interval": { "low": 0.43636363636363634, "high": 0.6296296296296297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5081967213114754, "high": 0.6907263355201498, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8888888888888888, "unit": "proportion", "denominator": 63, "interval": { "low": 0.8059597844112769, "high": 0.9622641509433962, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.06666666666666667, "unit": "proportion", "denominator": 105, "interval": { "low": 0.021505376344086023, "high": 0.11764705882352941, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.5952380952380953, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.968412653975364, "high": 3.178579619921363, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2619047619047623, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.8734004602991945, "high": 3.6416789215686274, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5720583860583861, "unit": "f1", "denominator": null, "interval": { "low": 0.4137630670771644, "high": 0.6001817644229408, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5958941521441522, "unit": "f1", "denominator": null, "interval": { "low": 0.4310031948720463, "high": 0.6251893379405634, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13333333333333333, "unit": "proportion", "denominator": 105, "interval": { "low": 0.07272727272727272, "high": 0.2, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.41754385964912283, "unit": "proportion", "denominator": 285, "interval": { "low": 0.35815602836879434, "high": 0.48, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.45964912280701753, "unit": "proportion", "denominator": 285, "interval": { "low": 0.39855072463768115, "high": 0.5226480836236934, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.9083969465648855, "unit": "proportion", "denominator": 131, "interval": { "low": 0.847995652173913, "high": 0.9600066225165561, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.042105263157894736, "unit": "proportion", "denominator": 285, "interval": { "low": 0.01818181818181818, "high": 0.07063197026022305, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.8596491228070176, "unit": "points_per_event", "denominator": 285, "interval": { "low": 1.477178362177449, "high": 2.2422693606840016, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.736842105263158, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.49119640546459, "high": 2.989547038327526, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4632091458681123, "unit": "f1", "denominator": null, "interval": { "low": 0.39594358076506175, "high": 0.48295268414449993, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4632091458681123, "unit": "f1", "denominator": null, "interval": { "low": 0.39594358076506175, "high": 0.48295268414449993, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.17543859649122806, "unit": "proportion", "denominator": 285, "interval": { "low": 0.1310344827586207, "high": 0.22183098591549297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.567741935483871, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5935483870967742, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9565217391304348, "unit": "proportion", "denominator": 92, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.025806451612903226, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.825806451612903, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.3419354838709676, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.37410071942446044, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4748201438848921, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7878787878787878, "unit": "proportion", "denominator": 66, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.10071942446043165, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.4424460431654675, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.7014388489208634, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3645833333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.375, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9722222222222222, "unit": "proportion", "denominator": 36, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.010416666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.7083333333333333, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.3854166666666665, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.660377358490566, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7169811320754716, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9210526315789473, "unit": "proportion", "denominator": 38, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05660377358490566, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.3962264150943398, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.7735849056603774, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.5196078431372549, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5294117647058824, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9814814814814815, "unit": "proportion", "denominator": 54, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.00980392156862745, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.5294117647058822, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.1176470588235294, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.4375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.53125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8235294117647058, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.984375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.921875, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.35514018691588783, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.45794392523364486, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7755102040816326, "unit": "proportion", "denominator": 49, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.102803738317757, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.280373831775701, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.635514018691589, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.35, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.875, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.45, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.45, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 1.0, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.0, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.7763157894736843, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.3684210526315788, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "unavailable", "reason": "Local compute cost was not measured; zero is not assumed", "coverage": { "observed_events": 0, "estimated_events": 0, "missing_events": 390, "population_events": 390 } } }, "gemini-minimal": { "metrics": { "species_accuracy": { "value": 0.40512820512820513, "unit": "proportion", "denominator": 390, "interval": { "low": 0.35426873953098825, "high": 0.45641439205955325, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5538461538461539, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5013546502757252, "high": 0.6058201058201058, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7314814814814815, "unit": "proportion", "denominator": 216, "interval": { "low": 0.6698113207547169, "high": 0.7903950925348304, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14871794871794872, "unit": "proportion", "denominator": 390, "interval": { "low": 0.11398963730569948, "high": 0.18414322250639387, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.37517134480551423, "unit": "f1", "denominator": null, "interval": { "low": 0.33174762079806003, "high": 0.39839433383733197, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47136912552487686, "unit": "f1", "denominator": null, "interval": { "low": 0.4168111133103831, "high": 0.5005467271289555, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.7166666666666666, "unit": "points_per_event", "denominator": 390, "interval": { "low": 1.3637564484126985, "high": 2.060849787069299, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.9602564102564104, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.766413892865665, "high": 3.1554586862609466, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.24871794871794872, "unit": "proportion", "denominator": 390, "interval": { "low": 0.2040289735203078, "high": 0.2948717948717949, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.5047619047619047, "unit": "proportion", "denominator": 105, "interval": { "low": 0.39795644064077246, "high": 0.6100097402597401, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6285714285714286, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5306122448979592, "high": 0.7265151515151512, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.803030303030303, "unit": "proportion", "denominator": 66, "interval": { "low": 0.6956521739130435, "high": 0.8947368421052632, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.12380952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.0660377358490566, "high": 0.18919630156472253, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.219047619047619, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.491430817610063, "high": 2.914288931788932, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3619047619047624, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.988764044943821, "high": 3.730477484472049, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5040959403704502, "unit": "f1", "denominator": null, "interval": { "low": 0.348370453769799, "high": 0.5282053308562971, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5671079329167564, "unit": "f1", "denominator": null, "interval": { "low": 0.39191676049102386, "high": 0.5942309972133342, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.22857142857142856, "unit": "proportion", "denominator": 105, "interval": { "low": 0.14034677816035415, "high": 0.32075471698113206, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3103448275862069, "high": 0.42758620689655175, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5263157894736842, "unit": "proportion", "denominator": 285, "interval": { "low": 0.4659854802135908, "high": 0.5860927152317881, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7, "unit": "proportion", "denominator": 150, "interval": { "low": 0.625, "high": 0.7730496453900709, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15789473684210525, "unit": "proportion", "denominator": 285, "interval": { "low": 0.11610079791564892, "high": 0.20074384645713642, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.5315789473684214, "unit": "points_per_event", "denominator": 285, "interval": { "low": 1.1333281733746128, "high": 1.9199306049822065, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.812280701754386, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.581039623536856, "high": 3.0439212980586223, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.32005562008879124, "unit": "f1", "denominator": null, "interval": { "low": 0.256300135050008, "high": 0.33979984290336984, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.40212116370130185, "unit": "f1", "denominator": null, "interval": { "low": 0.3220181183961639, "high": 0.4269280077503878, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.256140350877193, "unit": "proportion", "denominator": 285, "interval": { "low": 0.20512820512820512, "high": 0.30827577418121566, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6129032258064516, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7548387096774194, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.811965811965812, "unit": "proportion", "denominator": 117, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14193548387096774, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.706451612903226, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.835483870967742, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.34532374100719426, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5323741007194245, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6486486486486487, "unit": "proportion", "denominator": 74, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18705035971223022, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.4460431654676258, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.920863309352518, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.15625, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2604166666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6, "unit": "proportion", "denominator": 25, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.10416666666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5104166666666666, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.6041666666666667, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.6981132075471698, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8679245283018868, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8043478260869565, "unit": "proportion", "denominator": 46, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.16981132075471697, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.9622641509433962, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.09433962264151, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.5686274509803921, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.696078431372549, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8169014084507042, "unit": "proportion", "denominator": 71, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12745098039215685, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.573529411764706, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.700980392156863, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.4375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.53125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8235294117647058, "unit": "proportion", "denominator": 17, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.78125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.1875, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.3177570093457944, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5327102803738317, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5964912280701754, "unit": "proportion", "denominator": 57, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.21495327102803738, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.3457943925233644, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.8411214953271027, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.1, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.15, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6666666666666666, "unit": "proportion", "denominator": 3, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.95, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.17105263157894737, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.2894736842105263, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5909090909090909, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.39473684210526316, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.5789473684210527, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 389, "estimated_events": 1, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "779 saved telemetry estimates plus corrected selected-response result tokens at retained prices; no provider invoice. Earlier replaced calls are separate.", "value": 0.0017219806764554482 } }, "gemini-low": { "metrics": { "species_accuracy": { "value": 0.4230769230769231, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3700980392156863, "high": 0.4755784061696658, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5187032418952618, "high": 0.6246913580246913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7399103139013453, "unit": "proportion", "denominator": 223, "interval": { "low": 0.6778242677824268, "high": 0.8, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14871794871794872, "unit": "proportion", "denominator": 390, "interval": { "low": 0.11315789473684211, "high": 0.18666751918158567, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.40579330224116467, "unit": "f1", "denominator": null, "interval": { "low": 0.3570427674279906, "high": 0.43146331137058513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.48903295398294216, "unit": "f1", "denominator": null, "interval": { "low": 0.4302823094645015, "high": 0.519968606010705, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.6935897435897436, "unit": "points_per_event", "denominator": 390, "interval": { "low": 1.3286006978496425, "high": 2.056044911012236, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.03974358974359, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.839549072522505, "high": 3.2370486506030867, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.2692307692307692, "unit": "proportion", "denominator": 390, "interval": { "low": 0.225, "high": 0.3153877005347593, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.5333333333333333, "unit": "proportion", "denominator": 105, "interval": { "low": 0.42341476091476093, "high": 0.6403508771929824, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6476190476190476, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5392156862745098, "high": 0.75, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8235294117647058, "unit": "proportion", "denominator": 68, "interval": { "low": 0.7236842105263158, "high": 0.9102564102564102, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11428571428571428, "unit": "proportion", "denominator": 105, "interval": { "low": 0.05714285714285714, "high": 0.18085106382978725, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.357142857142857, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.623808345120226, "high": 3.0416776315789473, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.5, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.132181197137281, "high": 3.857142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4924320898177107, "unit": "f1", "denominator": null, "interval": { "low": 0.345452240660574, "high": 0.5191738970905637, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5539861010449245, "unit": "f1", "denominator": null, "interval": { "low": 0.38863377074314576, "high": 0.5840706342268842, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.22857142857142856, "unit": "proportion", "denominator": 105, "interval": { "low": 0.14423076923076922, "high": 0.3191489361702128, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3824561403508772, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3231292517006803, "high": 0.4416961130742049, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.543859649122807, "unit": "proportion", "denominator": 285, "interval": { "low": 0.48288940830251575, "high": 0.6043997827267789, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7032258064516129, "unit": "proportion", "denominator": 155, "interval": { "low": 0.625, "high": 0.7777777777777778, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.16140350877192983, "unit": "proportion", "denominator": 285, "interval": { "low": 0.11787038294405214, "high": 0.2080568099598313, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.449122807017544, "unit": "points_per_event", "denominator": 285, "interval": { "low": 1.0272598054119795, "high": 1.8647270937941791, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.8701754385964913, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.637863336415842, "high": 3.0970721857647656, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.351231095671801, "unit": "f1", "denominator": null, "interval": { "low": 0.2783211840908452, "high": 0.3743689624053126, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.41427257438212434, "unit": "f1", "denominator": null, "interval": { "low": 0.3282762684148431, "high": 0.44156339155498414, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.28421052631578947, "unit": "proportion", "denominator": 285, "interval": { "low": 0.2318840579710145, "high": 0.3381308044473512, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6258064516129033, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7677419354838709, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8151260504201681, "unit": "proportion", "denominator": 119, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14193548387096774, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.764516129032258, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.893548387096774, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.35251798561151076, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5539568345323741, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6363636363636364, "unit": "proportion", "denominator": 77, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2014388489208633, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.2841726618705036, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.9748201438848922, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.19791666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.28125, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7037037037037037, "unit": "proportion", "denominator": 27, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08333333333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5572916666666666, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.7552083333333333, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7169811320754716, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8113207547169812, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8837209302325582, "unit": "proportion", "denominator": 43, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09433962264150944, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.339622641509434, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.188679245283019, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.5784313725490197, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7450980392156863, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7763157894736842, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.465686274509804, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.7401960784313726, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7272727272727273, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.765625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.484375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.308411214953271, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.514018691588785, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6, "unit": "proportion", "denominator": 55, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.205607476635514, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.1401869158878504, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.822429906542056, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.1, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.15, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6666666666666666, "unit": "proportion", "denominator": 3, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3157894736842105, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7083333333333334, "unit": "proportion", "denominator": 24, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5197368421052632, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.769736842105263, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.0022184179487179487 } }, "gemini-medium": { "metrics": { "species_accuracy": { "value": 0.45897435897435895, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4067335979221605, "high": 0.5117508891675746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6102564102564103, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5604113110539846, "high": 0.6595758715544826, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7521008403361344, "unit": "proportion", "denominator": 238, "interval": { "low": 0.6945577638307764, "high": 0.807088821016207, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.15128205128205127, "unit": "proportion", "denominator": 390, "interval": { "low": 0.11622259668869839, "high": 0.18932038834951456, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.4142105923113062, "unit": "f1", "denominator": null, "interval": { "low": 0.3694420747216249, "high": 0.4390089536226359, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5204184364936925, "unit": "f1", "denominator": null, "interval": { "low": 0.46417081182973385, "high": 0.5515753519874144, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.9038461538461537, "unit": "points_per_event", "denominator": 390, "interval": { "low": 1.5347924554245225, "high": 2.258190606929866, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1730769230769234, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.975539606330255, "high": 3.366754617414248, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.25384615384615383, "unit": "proportion", "denominator": 390, "interval": { "low": 0.20971763880867916, "high": 0.3007528716249727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.6095238095238096, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5045871559633027, "high": 0.7118838885641922, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.7523809523809524, "unit": "proportion", "denominator": 105, "interval": { "low": 0.6630404250122129, "high": 0.8372093023255814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.810126582278481, "unit": "proportion", "denominator": 79, "interval": { "low": 0.7162162162162162, "high": 0.8939438502673795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.14285714285714285, "unit": "proportion", "denominator": 105, "interval": { "low": 0.08, "high": 0.21052631578947367, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.361904761904762, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.6136363636363638, "high": 3.083372641509434, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.6476190476190475, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.2823485730926034, "high": 4.0, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5671821234321234, "unit": "f1", "denominator": null, "interval": { "low": 0.401583014438673, "high": 0.5820668544807199, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6617124773374773, "unit": "f1", "denominator": null, "interval": { "low": 0.4685135168451185, "high": 0.6790779968941733, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2571428571428571, "unit": "proportion", "denominator": 105, "interval": { "low": 0.17391304347826086, "high": 0.34375, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.40350877192982454, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3444776238254499, "high": 0.462688030436055, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5578947368421052, "unit": "proportion", "denominator": 285, "interval": { "low": 0.5, "high": 0.6167280761477565, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7232704402515723, "unit": "proportion", "denominator": 159, "interval": { "low": 0.6511588887154675, "high": 0.7922077922077922, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1543859649122807, "unit": "proportion", "denominator": 285, "interval": { "low": 0.11347517730496454, "high": 0.19798657718120805, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.7350877192982457, "unit": "points_per_event", "denominator": 285, "interval": { "low": 1.3286675418444371, "high": 2.1210690789473685, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.9982456140350884, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.7706206453978734, "high": 3.2218635025754234, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3451476581410792, "unit": "f1", "denominator": null, "interval": { "low": 0.2822105819473091, "high": 0.36753220941886483, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4247971177120975, "unit": "f1", "denominator": null, "interval": { "low": 0.3473361008582265, "high": 0.45234733466937216, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.25263157894736843, "unit": "proportion", "denominator": 285, "interval": { "low": 0.20149185391722704, "high": 0.3055555555555556, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6709677419354839, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7935483870967742, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8455284552845529, "unit": "proportion", "denominator": 123, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12258064516129032, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.9806451612903224, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.04516129032258, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.381294964028777, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5683453237410072, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6708860759493671, "unit": "proportion", "denominator": 79, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18705035971223022, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.629496402877698, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.068345323741007, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.22916666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.375, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6111111111111112, "unit": "proportion", "denominator": 36, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14583333333333334, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5625, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9166666666666667, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7547169811320755, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8679245283018868, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8695652173913043, "unit": "proportion", "denominator": 46, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11320754716981132, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.19811320754717, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.235849056603773, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6274509803921569, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7549019607843137, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8311688311688312, "unit": "proportion", "denominator": 77, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12745098039215685, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.8676470588235294, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.946078431372549, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.59375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.78125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.76, "unit": "proportion", "denominator": 25, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.890625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.609375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.3177570093457944, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5046728971962616, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6296296296296297, "unit": "proportion", "denominator": 54, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.18691588785046728, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.5514018691588785, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.9065420560747666, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.625, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.9, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.15, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6071428571428571, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14473684210526316, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.47368421052631576, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.855263157894737, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.003253471794871795 } }, "gemini-high": { "metrics": { "species_accuracy": { "value": 0.4666666666666667, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4143920595533499, "high": 0.5194830626292767, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6538461538461539, "unit": "proportion", "denominator": 390, "interval": { "low": 0.6036269430051814, "high": 0.7029984854562304, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7137254901960784, "unit": "proportion", "denominator": 255, "interval": { "low": 0.6521739130434783, "high": 0.7706785491396475, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.18717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.14761904761904762, "high": 0.22928258427700063, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.3922962036147562, "unit": "f1", "denominator": null, "interval": { "low": 0.34947100437239237, "high": 0.41544845062666175, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5230616048196749, "unit": "f1", "denominator": null, "interval": { "low": 0.46596133916318977, "high": 0.5539312675022157, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 1.798717948717949, "unit": "points_per_event", "denominator": 390, "interval": { "low": 1.4175098808767035, "high": 2.167989710326679, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.208974358974359, "unit": "points_per_event", "denominator": 390, "interval": { "low": 3.0111641050844433, "high": 3.4043160580658864, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.28205128205128205, "unit": "proportion", "denominator": 390, "interval": { "low": 0.23466340206185568, "high": 0.3317079033771107, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.6190476190476191, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5108695652173914, "high": 0.7191011235955056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.780952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.693069306930693, "high": 0.8623881880733945, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.7926829268292683, "unit": "proportion", "denominator": 82, "interval": { "low": 0.691358024691358, "high": 0.8809523809523809, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1619047619047619, "unit": "proportion", "denominator": 105, "interval": { "low": 0.09259259259259259, "high": 0.24000961538461524, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.5476190476190474, "unit": "points_per_event", "denominator": 105, "interval": { "low": 1.784987068965517, "high": 3.2571957671957663, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.6904761904761907, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.31997619047619, "high": 4.04054054054054, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5107671498194374, "unit": "f1", "denominator": null, "interval": { "low": 0.38266519153619105, "high": 0.540111764994806, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5746130435468672, "unit": "f1", "denominator": null, "interval": { "low": 0.43049834047821506, "high": 0.6076257356191567, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.22857142857142856, "unit": "proportion", "denominator": 105, "interval": { "low": 0.14166482300884956, "high": 0.3218439509059029, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4105263157894737, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3511450381679389, "high": 0.4696497724852357, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6070175438596491, "unit": "proportion", "denominator": 285, "interval": { "low": 0.5480760676156584, "high": 0.6654418377063055, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.6763005780346821, "unit": "proportion", "denominator": 173, "interval": { "low": 0.6012269938650306, "high": 0.7471910112359551, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.19649122807017544, "unit": "proportion", "denominator": 285, "interval": { "low": 0.15034965034965034, "high": 0.2463768115942029, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 1.52280701754386, "unit": "points_per_event", "denominator": 285, "interval": { "low": 1.0907368310360448, "high": 1.9464898800599704, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0315789473684216, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.8032754614689557, "high": 3.2566677852348995, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.32367967828292854, "unit": "f1", "denominator": null, "interval": { "low": 0.26207503429174434, "high": 0.3453264068046811, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.43157290437723805, "unit": "f1", "denominator": null, "interval": { "low": 0.349433379055659, "high": 0.46043520907290814, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.3017543859649123, "unit": "proportion", "denominator": 285, "interval": { "low": 0.24528121159546012, "high": 0.3587812857408835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6580645161290323, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8129032258064516, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8095238095238095, "unit": "proportion", "denominator": 126, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15483870967741936, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.0, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.064516129032258, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4172661870503597, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6618705035971223, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6304347826086957, "unit": "proportion", "denominator": 92, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2446043165467626, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.410071942446043, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.1366906474820144, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.22916666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3854166666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5945945945945946, "unit": "proportion", "denominator": 37, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15625, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.421875, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9322916666666667, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7547169811320755, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9245283018867925, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8163265306122449, "unit": "proportion", "denominator": 49, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.16981132075471697, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.3773584905660377, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.320754716981132, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6078431372549019, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7549019607843137, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8051948051948052, "unit": "proportion", "denominator": 77, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14705882352941177, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.803921568627451, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.9313725490196076, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.625, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7692307692307693, "unit": "proportion", "denominator": 26, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.015625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.734375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.35514018691588783, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.616822429906542, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5757575757575758, "unit": "proportion", "denominator": 66, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2616822429906542, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.2289719626168225, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.957943925233645, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.35, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7142857142857143, "unit": "proportion", "denominator": 7, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.2, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.95, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.2236842105263158, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.39473684210526316, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5666666666666667, "unit": "proportion", "denominator": 30, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.17105263157894737, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.21710526315789475, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.9276315789473684, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "hash_verified_event_usage_ledger", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Saved selected-response usage at retained GBP price/FX buffers; reservations and earlier failed attempts are separate, not free.", "value": 0.004105528205128205 } }, "gemini-medium37": { "metrics": { "species_accuracy": { "value": 0.48717948717948717, "unit": "proportion", "denominator": 390, "interval": { "low": 0.43478260869565216, "high": 0.5398936170212766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5948717948717949, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5424164524421594, "high": 0.6466165413533834, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8189655172413793, "unit": "proportion", "denominator": 232, "interval": { "low": 0.7672413793103449, "high": 0.8674711667181547, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.1076923076923077, "unit": "proportion", "denominator": 390, "interval": { "low": 0.07780899775856549, "high": 0.13994910941475827, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.4834808090277646, "unit": "f1", "denominator": null, "interval": { "low": 0.4369546180370484, "high": 0.5089072946184343, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.557862471955113, "unit": "f1", "denominator": null, "interval": { "low": 0.5041784054273635, "high": 0.5872007245597318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.5166666666666666, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.1967733380480907, "high": 2.825, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.35, "unit": "points_per_event", "denominator": 390, "interval": { "low": 3.162022611005444, "high": 3.536422836851276, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.16666666666666666, "unit": "proportion", "denominator": 390, "interval": { "low": 0.129576370137926, "high": 0.2074468085106383, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.580952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.475, "high": 0.6853932584269663, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6952380952380952, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5929203539823009, "high": 0.7943987817089452, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8356164383561644, "unit": "proportion", "denominator": 73, "interval": { "low": 0.7384615384615385, "high": 0.9210526315789473, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11428571428571428, "unit": "proportion", "denominator": 105, "interval": { "low": 0.053763440860215055, "high": 0.18447393414943003, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.833333333333334, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.1590610047846894, "high": 3.455063703852327, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.6428571428571432, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.29165521978022, "high": 3.9936728639240506, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5553839838322597, "unit": "f1", "denominator": null, "interval": { "low": 0.39146388179096087, "high": 0.5703936970455734, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6710889804639805, "unit": "f1", "denominator": null, "interval": { "low": 0.4730188571640777, "high": 0.689225717263401, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1619047619047619, "unit": "proportion", "denominator": 105, "interval": { "low": 0.0888818565400844, "high": 0.24490296118447372, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.45263157894736844, "unit": "proportion", "denominator": 285, "interval": { "low": 0.3919376332622601, "high": 0.5124577481855636, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5578947368421052, "unit": "proportion", "denominator": 285, "interval": { "low": 0.4966216216216216, "high": 0.6181818181818182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8113207547169812, "unit": "proportion", "denominator": 159, "interval": { "low": 0.7485705931495406, "high": 0.8692819741532977, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.10526315789473684, "unit": "proportion", "denominator": 285, "interval": { "low": 0.0711743772241993, "high": 0.14145524551320818, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.4000000000000004, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.039332877959927, "high": 2.745902965674572, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.2421052631578946, "unit": "points_per_event", "denominator": 285, "interval": { "low": 3.018768465692018, "high": 3.4595083367854182, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.4403752862123052, "unit": "f1", "denominator": null, "interval": { "low": 0.37129940246965343, "high": 0.4632153407410484, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47425030822863634, "unit": "f1", "denominator": null, "interval": { "low": 0.399860894967319, "high": 0.4988472900288214, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.16842105263157894, "unit": "proportion", "denominator": 285, "interval": { "low": 0.12624584717607973, "high": 0.21400814105417318, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6838709677419355, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7806451612903226, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8760330578512396, "unit": "proportion", "denominator": 121, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.0967741935483871, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.4870967741935486, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.132258064516129, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4244604316546763, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.539568345323741, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7866666666666666, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11510791366906475, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.2302158273381294, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.273381294964029, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.2604166666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.375, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6944444444444444, "unit": "proportion", "denominator": 36, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11458333333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.3645833333333333, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.1979166666666665, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7358490566037735, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8679245283018868, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8478260869565217, "unit": "proportion", "denominator": 46, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1320754716981132, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.490566037735849, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.245283018867925, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6568627450980392, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7352941176470589, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8933333333333333, "unit": "proportion", "denominator": 75, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.0784313725490196, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.485294117647059, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.073529411764706, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.59375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8421052631578947, "unit": "proportion", "denominator": 19, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.3125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.40625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.40186915887850466, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5233644859813084, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7678571428571429, "unit": "proportion", "denominator": 56, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.12149532710280374, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.205607476635514, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2336448598130842, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.3, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.75, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.925, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.425, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3684210526315789, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6785714285714286, "unit": "proportion", "denominator": 28, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11842105263157894, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.2171052631578947, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.138157894736842, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "saved_event_telemetry", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Saved provider-token cost estimate for each selected response; not an invoice.", "value": 0.0031772794415384615 } }, "gemini-medium38": { "metrics": { "species_accuracy": { "value": 0.4717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4190205308135899, "high": 0.524632910193255, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.558974358974359, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5064926675391791, "high": 0.6112663220629929, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8440366972477065, "unit": "proportion", "denominator": 218, "interval": { "low": 0.7923655395200537, "high": 0.8909952606635071, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.05970149253731343, "high": 0.11716674472070844, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5056045771004524, "unit": "f1", "denominator": null, "interval": { "low": 0.45544501401993476, "high": 0.5327076861406577, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5444972368774103, "unit": "f1", "denominator": null, "interval": { "low": 0.49047924586762204, "high": 0.5736852004591699, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.588461538461539, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.2754202776440375, "high": 2.8904727910516486, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.3320512820512826, "unit": "points_per_event", "denominator": 390, "interval": { "low": 3.142669685562111, "high": 3.515313079777366, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.14871794871794872, "unit": "proportion", "denominator": 390, "interval": { "low": 0.11232876712328767, "high": 0.18812406439582943, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.580952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.4742268041237113, "high": 0.6857142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6666666666666666, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5634920634920635, "high": 0.7678571428571429, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8714285714285714, "unit": "proportion", "denominator": 70, "interval": { "low": 0.7857142857142857, "high": 0.9444444444444444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08571428571428572, "unit": "proportion", "denominator": 105, "interval": { "low": 0.036036036036036036, "high": 0.14285714285714285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.0285714285714285, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.4081395960446037, "high": 3.588261554621848, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.695238095238096, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.362736103615758, "high": 4.0235962445573294, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5669526975082531, "unit": "f1", "denominator": null, "interval": { "low": 0.4114755691839025, "high": 0.5790927515196521, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6378217846967847, "unit": "f1", "denominator": null, "interval": { "low": 0.4629100153318903, "high": 0.6514793454596086, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.13333333333333333, "unit": "proportion", "denominator": 105, "interval": { "low": 0.06862745098039216, "high": 0.2075565103680178, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.43157894736842106, "unit": "proportion", "denominator": 285, "interval": { "low": 0.37037037037037035, "high": 0.49264705882352944, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.519298245614035, "unit": "proportion", "denominator": 285, "interval": { "low": 0.4603745470448582, "high": 0.5804195804195804, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.831081081081081, "unit": "proportion", "denominator": 148, "interval": { "low": 0.7664233576642335, "high": 0.888199671172817, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08771929824561403, "unit": "proportion", "denominator": 285, "interval": { "low": 0.05647710348993429, "high": 0.12318840579710146, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.4263157894736844, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.072126114466494, "high": 2.7690972222222223, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.1982456140350877, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.9765808473500477, "high": 3.4180670289855075, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.449802005001963, "unit": "f1", "denominator": null, "interval": { "low": 0.37127089388336454, "high": 0.47522819991095683, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.47286877448924314, "unit": "f1", "denominator": null, "interval": { "low": 0.3903104269030242, "high": 0.49959887682946746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1543859649122807, "unit": "proportion", "denominator": 285, "interval": { "low": 0.11301369863013698, "high": 0.19860744494890836, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6838709677419355, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7483870967741936, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9137931034482759, "unit": "proportion", "denominator": 116, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.06451612903225806, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.6548387096774193, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.138709677419355, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4028776978417266, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5179856115107914, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7777777777777778, "unit": "proportion", "denominator": 72, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11510791366906475, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.258992805755396, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2661870503597124, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.22916666666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3125, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7333333333333333, "unit": "proportion", "denominator": 30, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.08333333333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.34375, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.125, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7547169811320755, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8301886792452831, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9090909090909091, "unit": "proportion", "denominator": 44, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.07547169811320754, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.8962264150943398, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.367924528301887, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6470588235294118, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7058823529411765, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9166666666666666, "unit": "proportion", "denominator": 72, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.058823529411764705, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.5294117647058822, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.019607843137255, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.46875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.59375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7894736842105263, "unit": "proportion", "denominator": 19, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.1875, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.4375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.38317757009345793, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4953271028037383, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7735849056603774, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11214953271028037, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.2803738317757007, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2149532710280373, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.3, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.35, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8571428571428571, "unit": "proportion", "denominator": 7, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.075, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.325, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.21052631578947367, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3026315789473684, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6956521739130435, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.1513157894736843, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.0723684210526314, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "estimated", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "verified_selected_result_token_reconstruction", "coverage": { "observed_events": 0, "estimated_events": 390, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Selected-response tokens repriced with the run's retained list prices and GBP buffers; no provider invoice. Earlier failed/uncertain attempts are separate.", "value": 0.007509248717948718 } }, "gemini38-low": { "metrics": { "species_accuracy": { "value": 0.47692307692307695, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4239394719722433, "high": 0.5298512308586936, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.5666666666666667, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5149981343283582, "high": 0.618798955613577, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8416289592760181, "unit": "proportion", "denominator": 221, "interval": { "low": 0.7912621359223301, "high": 0.8879310344827587, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08974358974358974, "unit": "proportion", "denominator": 390, "interval": { "low": 0.0625, "high": 0.11928934010152284, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5087793128538374, "unit": "f1", "denominator": null, "interval": { "low": 0.4570642681703775, "high": 0.536713242085056, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5479161830733634, "unit": "f1", "denominator": null, "interval": { "low": 0.4922230580296374, "high": 0.5779988760915987, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.583333333333334, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.2615898199294486, "high": 2.8913070926657882, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.352564102564103, "unit": "points_per_event", "denominator": 390, "interval": { "low": 3.1653420940625665, "high": 3.5404725443521503, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.15384615384615385, "unit": "proportion", "denominator": 390, "interval": { "low": 0.11764705882352941, "high": 0.1935503636938646, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.6095238095238096, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5, "high": 0.7142857142857143, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6952380952380952, "unit": "proportion", "denominator": 105, "interval": { "low": 0.5899935897435897, "high": 0.7962962962962963, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8767123287671232, "unit": "proportion", "denominator": 73, "interval": { "low": 0.796875, "high": 0.948051948051948, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.08571428571428572, "unit": "proportion", "denominator": 105, "interval": { "low": 0.036036036036036036, "high": 0.14285714285714285, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.157142857142857, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.539600703491402, "high": 3.7255133185349614, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.776190476190476, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.428571428571429, "high": 4.120004807692308, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5725306789382419, "unit": "f1", "denominator": null, "interval": { "low": 0.4168189666063947, "high": 0.5925458024118739, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6679524587612823, "unit": "f1", "denominator": null, "interval": { "low": 0.48628879437412725, "high": 0.6913034361471861, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.12380952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.06, "high": 0.19642857142857142, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4280701754385965, "unit": "proportion", "denominator": 285, "interval": { "low": 0.36899845594200625, "high": 0.48829625884732053, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.519298245614035, "unit": "proportion", "denominator": 285, "interval": { "low": 0.458955223880597, "high": 0.5787545787545788, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8243243243243243, "unit": "proportion", "denominator": 148, "interval": { "low": 0.7597390013087687, "high": 0.8843602825745682, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.0912280701754386, "unit": "proportion", "denominator": 285, "interval": { "low": 0.058823529411764705, "high": 0.1263951202837243, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.3719298245614033, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.0157315340909094, "high": 2.727272727272727, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.196491228070176, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.973851909184727, "high": 3.4177914236989766, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.45035712094922625, "unit": "f1", "denominator": null, "interval": { "low": 0.3745020999083098, "high": 0.4754769636778676, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4619047394351038, "unit": "f1", "denominator": null, "interval": { "low": 0.3841047178546768, "high": 0.48766868069524877, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1649122807017544, "unit": "proportion", "denominator": 285, "interval": { "low": 0.12177121771217712, "high": 0.211222284200251, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.6903225806451613, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7548387096774194, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9145299145299145, "unit": "proportion", "denominator": 117, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.06451612903225806, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.7096774193548385, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.193548387096774, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.4172661870503597, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5251798561151079, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7945205479452054, "unit": "proportion", "denominator": 73, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1079136690647482, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.3633093525179856, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2985611510791366, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.21875, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3229166666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6774193548387096, "unit": "proportion", "denominator": 31, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.10416666666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.0833333333333333, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.0729166666666665, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.7735849056603774, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8113207547169812, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9534883720930233, "unit": "proportion", "denominator": 43, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.03773584905660377, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 4.160377358490566, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.443396226415095, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.6470588235294118, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.7254901960784313, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8918918918918919, "unit": "proportion", "denominator": 74, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.0784313725490196, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.4754901960784315, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.063725490196078, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.5625, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8181818181818182, "unit": "proportion", "denominator": 22, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.546875, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.640625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.37383177570093457, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4766355140186916, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7843137254901961, "unit": "proportion", "denominator": 51, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.102803738317757, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.308411214953271, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.196261682242991, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.25, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.625, "unit": "proportion", "denominator": 8, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.15, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.475, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.225, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.21052631578947367, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.3026315789473684, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6956521739130435, "unit": "proportion", "denominator": 23, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09210526315789473, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.9802631578947368, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.0328947368421053, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.006153087179487179 } }, "gemini38-high": { "metrics": { "species_accuracy": { "value": 0.5794871794871795, "unit": "proportion", "denominator": 390, "interval": { "low": 0.5274151436031331, "high": 0.6308900523560209, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6974358974358974, "unit": "proportion", "denominator": 390, "interval": { "low": 0.6505102040816326, "high": 0.7435233160621761, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8308823529411765, "unit": "proportion", "denominator": 272, "interval": { "low": 0.7829181494661922, "high": 0.8745868644067795, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11794871794871795, "unit": "proportion", "denominator": 390, "interval": { "low": 0.08638743455497382, "high": 0.15151700377861746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.5865229883529476, "unit": "f1", "denominator": null, "interval": { "low": 0.5337536679544457, "high": 0.6122990938092412, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.6466791922865834, "unit": "f1", "denominator": null, "interval": { "low": 0.5884976338984914, "high": 0.6750990008665992, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 2.6961538461538463, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.3501163192858354, "high": 3.0261194029850746, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.632051282051282, "unit": "points_per_event", "denominator": 390, "interval": { "low": 3.444303629587349, "high": 3.814863940181281, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.18717948717948718, "unit": "proportion", "denominator": 390, "interval": { "low": 0.14786751361161524, "high": 0.22894736842105262, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.7142857142857143, "unit": "proportion", "denominator": 105, "interval": { "low": 0.6224489795918368, "high": 0.8, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8285714285714286, "unit": "proportion", "denominator": 105, "interval": { "low": 0.7563025210084033, "high": 0.897196261682243, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8620689655172413, "unit": "proportion", "denominator": 87, "interval": { "low": 0.7821782178217822, "high": 0.9325842696629213, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11428571428571428, "unit": "proportion", "denominator": 105, "interval": { "low": 0.05504587155963303, "high": 0.1810344827586207, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 3.295238095238095, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.6409057811120924, "high": 3.8990624999999994, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 4.057142857142857, "unit": "points_per_event", "denominator": 105, "interval": { "low": 3.736836231203008, "high": 4.357142857142857, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.661547217954781, "unit": "f1", "denominator": null, "interval": { "low": 0.48494878400077984, "high": 0.6728746328076685, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.7718050876139112, "unit": "f1", "denominator": null, "interval": { "low": 0.565773581334243, "high": 0.7850204049422798, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.1523809523809524, "unit": "proportion", "denominator": 105, "interval": { "low": 0.08256880733944955, "high": 0.2298879310344827, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.5298245614035088, "unit": "proportion", "denominator": 285, "interval": { "low": 0.46964398946418556, "high": 0.5907476245845225, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.6491228070175439, "unit": "proportion", "denominator": 285, "interval": { "low": 0.5912162162162162, "high": 0.706081383803158, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.8162162162162162, "unit": "proportion", "denominator": 185, "interval": { "low": 0.7566103297968583, "high": 0.8715083798882681, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.11929824561403508, "unit": "proportion", "denominator": 285, "interval": { "low": 0.08243727598566308, "high": 0.15916955017301038, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 2.4754385964912284, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.0758966934429295, "high": 2.857606329682479, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.4754385964912284, "unit": "points_per_event", "denominator": 285, "interval": { "low": 3.2491525423728818, "high": 3.697115384615385, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.5333835274757305, "unit": "f1", "denominator": null, "interval": { "low": 0.4552077201535471, "high": 0.5603667026108957, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.5607365288847423, "unit": "f1", "denominator": null, "interval": { "low": 0.47855170580244705, "high": 0.5891034565909417, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.2, "unit": "proportion", "denominator": 285, "interval": { "low": 0.15441176470588236, "high": 0.24919019933554787, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.7612903225806451, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.864516129032258, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8805970149253731, "unit": "proportion", "denominator": 134, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1032258064516129, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.6032258064516127, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.312903225806451, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.5323741007194245, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6690647482014388, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7956989247311828, "unit": "proportion", "denominator": 93, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1366906474820144, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.4964028776978417, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.6115107913669067, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3541666666666667, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.46875, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7555555555555555, "unit": "proportion", "denominator": 45, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.11458333333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.5208333333333333, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.5625, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.8113207547169812, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9622641509433962, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.8431372549019608, "unit": "proportion", "denominator": 51, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.1509433962264151, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.5754716981132075, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.4245283018867925, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.7352941176470589, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8137254901960784, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9036144578313253, "unit": "proportion", "denominator": 83, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.0784313725490196, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.6176470588235294, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.254901960784314, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.78125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.88, "unit": "proportion", "denominator": 25, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.09375, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 3.28125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.0625, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.48598130841121495, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6355140186915887, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7647058823529411, "unit": "proportion", "denominator": 68, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.14953271028037382, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.2616822429906542, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.4766355140186915, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.5, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.55, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.9090909090909091, "unit": "proportion", "denominator": 11, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.05, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.575, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.075, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.3157894736842105, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.4473684210526316, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7058823529411765, "unit": "proportion", "denominator": 34, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.13157894736842105, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.243421052631579, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.4276315789473686, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "hash_verified_usage_ledger_plus_selected_canary", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "789 saved reconciled selected-response costs plus one prior canary cost reconstructed from its hash-bound result tokens and retained price/FX buffer; earlier failed or uncertain attempts remain separate. Not an invoice.", "value": 0.013991210256410256 } }, "qwen-repaired": { "metrics": { "species_accuracy": { "value": 0.35384615384615387, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3036649214659686, "high": 0.40601659535579276, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8461538461538461, "unit": "proportion", "denominator": 390, "interval": { "low": 0.8068459657701712, "high": 0.883419689119171, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.41818181818181815, "unit": "proportion", "denominator": 330, "interval": { "low": 0.36079316348973606, "high": 0.47648955418568933, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.49230769230769234, "unit": "proportion", "denominator": 390, "interval": { "low": 0.43902439024390244, "high": 0.5467989648936833, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.1860547887327449, "unit": "f1", "denominator": null, "interval": { "low": 0.15939678648592506, "high": 0.20091256108108368, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.38165084868255367, "unit": "f1", "denominator": null, "interval": { "low": 0.32696776715061543, "high": 0.4121283304227359, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.0576923076923077, "unit": "points_per_event", "denominator": 390, "interval": { "low": -0.34278350515463923, "high": 0.46542663394851297, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.6089743589743595, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.3973327628361862, "high": 2.824001265822785, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.5102564102564102, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4565189520624303, "high": 0.5646446199083737, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.42857142857142855, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3220290278589519, "high": 0.5339805825242718, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8761904761904762, "unit": "proportion", "denominator": 105, "interval": { "low": 0.81, "high": 0.9363636363636364, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4891304347826087, "unit": "proportion", "denominator": 92, "interval": { "low": 0.37142857142857144, "high": 0.6049382716049383, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.44761904761904764, "unit": "proportion", "denominator": 105, "interval": { "low": 0.339995145631068, "high": 0.5575221238938053, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.7619047619047619, "unit": "points_per_event", "denominator": 105, "interval": { "low": -0.05884103641456582, "high": 1.5833333333333333, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.5981232222673714, "high": 3.394743421052631, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.2921770224254696, "unit": "f1", "denominator": null, "interval": { "low": 0.1935114689109603, "high": 0.31441578346465565, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4260914910371432, "unit": "f1", "denominator": null, "interval": { "low": 0.2822042254951504, "high": 0.4585230175526228, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.44761904761904764, "unit": "proportion", "denominator": 105, "interval": { "low": 0.339995145631068, "high": 0.5575221238938053, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.3263157894736842, "unit": "proportion", "denominator": 285, "interval": { "low": 0.2690909090909091, "high": 0.3843810096153846, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8350877192982457, "unit": "proportion", "denominator": 285, "interval": { "low": 0.7873134328358209, "high": 0.8803995738841542, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.3907563025210084, "unit": "proportion", "denominator": 238, "interval": { "low": 0.3252032520325203, "high": 0.45689655172413796, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5087719298245614, "unit": "proportion", "denominator": 285, "interval": { "low": 0.44755244755244755, "high": 0.5700008143322475, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.2017543859649123, "unit": "points_per_event", "denominator": 285, "interval": { "low": -0.6484112507386728, "high": 0.25650633549578444, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.4649122807017543, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.2206392610424954, "high": 2.7101449275362324, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.17004896020114546, "unit": "f1", "denominator": null, "interval": { "low": 0.13144020489747812, "high": 0.1843991466197147, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3357376906535436, "unit": "f1", "denominator": null, "interval": { "low": 0.25951014813091833, "high": 0.3640701099927701, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5333333333333333, "unit": "proportion", "denominator": 285, "interval": { "low": 0.47183098591549294, "high": 0.5947040974258715, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.535483870967742, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9806451612903225, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5460526315789473, "unit": "proportion", "denominator": 152, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.44516129032258067, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.229032258064516, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.5193548387096776, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.2949640287769784, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8848920863309353, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 123, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5899280575539568, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6115107913669064, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.4820143884892087, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.14583333333333334, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5729166666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2545454545454545, "unit": "proportion", "denominator": 55, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4270833333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.8645833333333334, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.3229166666666667, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.6792452830188679, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9811320754716981, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.6923076923076923, "unit": "proportion", "denominator": 52, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.3018867924528302, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.6132075471698113, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.122641509433962, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.46078431372549017, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9803921568627451, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.47, "unit": "proportion", "denominator": 100, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5196078431372549, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5098039215686274, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.2058823529411766, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.28125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.96875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2903225806451613, "unit": "proportion", "denominator": 31, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.859375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.578125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.29906542056074764, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8598130841121495, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.34782608695652173, "unit": "proportion", "denominator": 92, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5607476635514018, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5373831775700935, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.453271028037383, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.0, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.45, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.0, "unit": "proportion", "denominator": 9, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.45, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.55, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.18421052631578946, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6052631578947368, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.30434782608695654, "unit": "proportion", "denominator": 46, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6842105263157895, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.486842105263158, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "Selected evaluated response attempt only: 786 first-pass outputs, four successful repair outputs. Four superseded first-pass length charges are excluded from this comparison and retained in the audit below.", "value": 0.0017576954532692308 } }, "qwen-first-pass": { "metrics": { "species_accuracy": { "value": 0.35128205128205126, "unit": "proportion", "denominator": 390, "interval": { "low": 0.3010471204188482, "high": 0.4036001432626392, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8358974358974359, "unit": "proportion", "denominator": 390, "interval": { "low": 0.7964824120603015, "high": 0.8737373737373737, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.42024539877300615, "unit": "proportion", "denominator": 326, "interval": { "low": 0.3630947802197802, "high": 0.47896440129449835, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4846153846153846, "unit": "proportion", "denominator": 390, "interval": { "low": 0.43116883116883115, "high": 0.5382716049382716, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "macro_f1": { "value": 0.1879247540821163, "unit": "f1", "denominator": null, "interval": { "low": 0.1609243544252662, "high": 0.2028975237292849, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.3806680916022355, "unit": "f1", "denominator": null, "interval": { "low": 0.3259749743486162, "high": 0.41099754806701305, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "adjusted_tis": { "value": 0.07948717948717948, "unit": "points_per_event", "denominator": 390, "interval": { "low": -0.3155234392714081, "high": 0.48195305404178007, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.5923076923076924, "unit": "points_per_event", "denominator": 390, "interval": { "low": 2.3818845044178643, "high": 2.8076923076923075, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "any_rank_conflict_rate": { "value": 0.5025641025641026, "unit": "proportion", "denominator": 390, "interval": { "low": 0.4487798121748531, "high": 0.5566502463054187, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } }, "conditions": { "day": { "n_events": 105, "n_species": 24, "metrics": { "species_accuracy": { "value": 0.42857142857142855, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3220290278589519, "high": 0.5339805825242718, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8666666666666667, "unit": "proportion", "denominator": 105, "interval": { "low": 0.7982456140350878, "high": 0.9285714285714286, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.4945054945054945, "unit": "proportion", "denominator": 91, "interval": { "low": 0.3764705882352941, "high": 0.6104076479076477, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.4380952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3333333333333333, "high": 0.5447339246119731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": 0.8095238095238095, "unit": "points_per_event", "denominator": 105, "interval": { "low": 0.0147953240702222, "high": 1.602176139272913, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 3.0, "unit": "points_per_event", "denominator": 105, "interval": { "low": 2.5981232222673714, "high": 3.394743421052631, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.3007704642615128, "unit": "f1", "denominator": null, "interval": { "low": 0.19920298270245912, "high": 0.32366330650773373, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.4260914910371432, "unit": "f1", "denominator": null, "interval": { "low": 0.2822042254951504, "high": 0.4585230175526228, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.4380952380952381, "unit": "proportion", "denominator": 105, "interval": { "low": 0.3333333333333333, "high": 0.5447339246119731, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "night_ir": { "n_events": 285, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.32280701754385965, "unit": "proportion", "denominator": 285, "interval": { "low": 0.2659923705136471, "high": 0.3811188811188811, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "species_accuracy" }, "answer_coverage": { "value": 0.8245614035087719, "unit": "proportion", "denominator": 285, "interval": { "low": 0.7755867523138554, "high": 0.8709705518714173, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "answer_coverage" }, "accuracy_among_answers": { "value": 0.39148936170212767, "unit": "proportion", "denominator": 235, "interval": { "low": 0.3254897336293008, "high": 0.4581714807436918, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "accuracy_among_answers" }, "wrong_species_rate_all_events": { "value": 0.5017543859649123, "unit": "proportion", "denominator": 285, "interval": { "low": 0.44074074074074077, "high": 0.5632224380767362, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "wrong_species_rate_all_events" }, "adjusted_tis": { "value": -0.18947368421052635, "unit": "points_per_event", "denominator": 285, "interval": { "low": -0.6300536881419234, "high": 0.26531281365005016, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "adjusted_tis" }, "raw_tis": { "value": 2.4421052631578952, "unit": "points_per_event", "denominator": 285, "interval": { "low": 2.1982102272727277, "high": 2.687961132022686, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "raw_tis" }, "macro_f1": { "value": 0.1688338010912495, "unit": "f1", "denominator": null, "interval": { "low": 0.13034962230182134, "high": 0.18289251890430513, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "macro_f1" }, "reference_species_macro_f1": { "value": 0.33333853035964645, "unit": "f1", "denominator": null, "interval": { "low": 0.25735694659590364, "high": 0.3610954860418332, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "reference_species_macro_f1" }, "any_rank_conflict_rate": { "value": 0.5263157894736842, "unit": "proportion", "denominator": 285, "interval": { "low": 0.4641509433962264, "high": 0.5871886120996441, "level": 0.95, "method": "source_cluster", "scope": "pointwise", "valid_resamples": 10000 }, "source_kind": "saved_v1_subgroup_interval", "interval_status": "ok", "metric_key": "any_rank_conflict_rate" } } }, "easy": { "n_events": 155, "n_species": 38, "metrics": { "species_accuracy": { "value": 0.5290322580645161, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9548387096774194, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.5540540540540541, "unit": "proportion", "denominator": 148, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4258064516129032, "unit": "proportion", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 1.2838709677419355, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.47741935483871, "unit": "points_per_event", "denominator": 155, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium": { "n_events": 139, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.2949640287769784, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8848920863309353, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.3333333333333333, "unit": "proportion", "denominator": 123, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5899280575539568, "unit": "proportion", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6115107913669064, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.4820143884892087, "unit": "points_per_event", "denominator": 139, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard": { "n_events": 96, "n_species": 39, "metrics": { "species_accuracy": { "value": 0.14583333333333334, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.5729166666666666, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2545454545454545, "unit": "proportion", "denominator": 55, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.4270833333333333, "unit": "proportion", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.8645833333333334, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.3229166666666667, "unit": "points_per_event", "denominator": 96, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | day": { "n_events": 53, "n_species": 21, "metrics": { "species_accuracy": { "value": 0.6792452830188679, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9622641509433962, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.7058823529411765, "unit": "proportion", "denominator": 51, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.2830188679245283, "unit": "proportion", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 2.707547169811321, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 4.122641509433962, "unit": "points_per_event", "denominator": 53, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "easy | night_ir": { "n_events": 102, "n_species": 33, "metrics": { "species_accuracy": { "value": 0.45098039215686275, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.9509803921568627, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.4742268041237113, "unit": "proportion", "denominator": 97, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5, "unit": "proportion", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": 0.5441176470588235, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 3.142156862745098, "unit": "points_per_event", "denominator": 102, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | day": { "n_events": 32, "n_species": 17, "metrics": { "species_accuracy": { "value": 0.28125, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.96875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.2903225806451613, "unit": "proportion", "denominator": 31, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.6875, "unit": "proportion", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.859375, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.578125, "unit": "points_per_event", "denominator": 32, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "medium | night_ir": { "n_events": 107, "n_species": 35, "metrics": { "species_accuracy": { "value": 0.29906542056074764, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.8598130841121495, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.34782608695652173, "unit": "proportion", "denominator": 92, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.5607476635514018, "unit": "proportion", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.5373831775700935, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 2.453271028037383, "unit": "points_per_event", "denominator": 107, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | day": { "n_events": 20, "n_species": 15, "metrics": { "species_accuracy": { "value": 0.0, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.45, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.0, "unit": "proportion", "denominator": 9, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.45, "unit": "proportion", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -1.55, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 0.7, "unit": "points_per_event", "denominator": 20, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } }, "hard | night_ir": { "n_events": 76, "n_species": 32, "metrics": { "species_accuracy": { "value": 0.18421052631578946, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "answer_coverage": { "value": 0.6052631578947368, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "accuracy_among_answers": { "value": 0.30434782608695654, "unit": "proportion", "denominator": 46, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "wrong_species_rate_all_events": { "value": 0.42105263157894735, "unit": "proportion", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "adjusted_tis": { "value": -0.6842105263157895, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" }, "raw_tis": { "value": 1.486842105263158, "unit": "points_per_event", "denominator": 76, "interval": null, "source_kind": "saved_v1_event_outcomes_descriptive" } } } }, "cost": { "status": "recorded", "unit": "GBP_per_event", "denominator": 390, "basis": "selected_response_attempts", "source_kind": "provider_usage_or_saved_generation_metadata", "coverage": { "observed_events": 390, "estimated_events": 0, "missing_events": 0, "population_events": 390, "population_attempts": 390 }, "note": "First pass includes 786 model outputs and four charged HTTP 200 length nonresponses. 785 charges came from result usage and five exact charges from hash-bound provider generation metadata.", "value": 0.0017951259094230768 } } } } } } }, "note": "V1 saved intervals are pointwise candidate estimates; newly crossed class/difficulty cells and prototype class slices are descriptive only. Cohorts are distinct." }