{"id":"W4404451336","doi":"10.48550/arxiv.2411.09607","title":"Initial Nugget Evaluation Results for the TREC 2024 RAG Track with the AutoNuggetizer Framework","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Geophysics and Gravity Measurements","field":"Earth and Planetary Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; National Institute of Standards and Technology","keywords":"Track (disk drive); Computer science; Operating system","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001300342,0.0002883993,0.0002161383,0.00007778804,0.0004101362,0.0002344117,0.0007127966,0.0002242103,0.000352225],"category_scores_gemma":[0.00008849052,0.0001681097,0.0002128486,0.0004162711,0.00018104,0.00008204948,0.0001045729,0.0008889082,0.0002554707],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002770336,"about_ca_system_score_gemma":0.0003511566,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001170525,"about_ca_topic_score_gemma":0.002585891,"domain_scores_codex":[0.9982266,0.0002070219,0.000173995,0.0007517503,0.000316214,0.0003244089],"domain_scores_gemma":[0.9981436,0.0006678066,0.0001760705,0.0006972354,0.0002284823,0.00008676611],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001537541,0.00007958153,0.002640372,0.0001811333,0.0008492994,0.00007081555,0.00168523,0.9401475,0.000002558992,0.007563854,0.02108159,0.02416056],"study_design_scores_gemma":[0.001393237,0.0003190608,0.03870788,0.0003156315,0.00181953,0.000003676512,0.0008728199,0.6399878,0.00002923872,0.2982776,0.01760503,0.00066848],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9337804,0.0012765,0.008919447,0.006629846,0.00651086,0.004938779,0.003121448,0.0001985257,0.03462419],"genre_scores_gemma":[0.9960183,0.00005389929,0.00009977209,0.0001744435,0.0004418887,0.000003404544,0.0003001644,0.00001010767,0.002898001],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3001596,"threshold_uncertainty_score":0.6855316,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1101037632247585,"score_gpt":0.223531311574239,"score_spread":0.1134275483494806,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}