{"id":"W4404451336","doi":"10.48550/arxiv.2411.09607","title":"Initial Nugget Evaluation Results for the TREC 2024 RAG Track with the AutoNuggetizer Framework","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Geophysics and Gravity Measurements","field":"Earth and Planetary Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; National Institute of Standards and Technology","keywords":"Track (disk drive); Computer science; Operating system","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03090228,0.001846476,0.001418717,0.00308826,0.002029389,0.00323,0.002409738,0.002376107,0.00711855],"category_scores_gemma":[0.03623443,0.0004928156,0.00126094,0.001947332,0.001645813,0.003967871,0.002906201,0.002701598,0.004093486],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003096031,"about_ca_system_score_gemma":0.002472696,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0306821,"about_ca_topic_score_gemma":0.04766133,"domain_scores_codex":[0.971602,0.01487556,0.001129545,0.002540246,0.008215964,0.001636643],"domain_scores_gemma":[0.9709269,0.01271042,0.0005883517,0.003964276,0.01029431,0.001515838],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004816044,0.00390256,0.01769014,0.002863028,0.00104202,0.0007477047,0.003996515,0.04135766,0.05363363,0.006130947,0.4226777,0.4411421],"study_design_scores_gemma":[0.003318095,0.009727813,0.09435992,0.0006665169,0.001047923,0.001406394,0.003619074,0.3562094,0.1536861,0.0123172,0.3626489,0.0009926386],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6238344,0.009220016,0.1151891,0.004506859,0.003520079,0.005559366,0.05735589,0.09647133,0.08434292],"genre_scores_gemma":[0.6384352,0.0009797799,0.1507223,0.001950421,0.0005495648,0.002203267,0.1619095,0.005145431,0.03810452],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03090228,"threshold_uncertainty_score":0.1634288,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1101037632247585,"score_gpt":0.223531311574239,"score_spread":0.1134275483494806,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}