{"id":"W4416037147","doi":"10.18653/v1/2025.emnlp-industry.187","title":"GEAR: A Scalable and Interpretable Evaluation Framework for RAG-Based Car Assistant Systems","year":2025,"lang":"","type":"article","venue":"","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Scalability; Natural language; Natural (archaeology); Empirical research; Scale (ratio); Component (thermodynamics)","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.002870709,0.0003610745,0.000496064,0.000326483,0.0007048476,0.001387364,0.0008455433,0.0003168777,0.00009492743],"category_scores_gemma":[0.001357002,0.0003547881,0.0001357224,0.001100517,0.0001320782,0.000324609,0.0003204727,0.0004671659,0.00003879611],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003693609,"about_ca_system_score_gemma":0.0009805313,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002159423,"about_ca_topic_score_gemma":0.00003646065,"domain_scores_codex":[0.9964135,0.0004768753,0.0007345341,0.001223194,0.0005685149,0.0005833613],"domain_scores_gemma":[0.9956122,0.001721019,0.0002962697,0.001385562,0.0007980631,0.0001868599],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00008345191,0.0003885487,0.008589187,0.001064992,0.0002009562,7.297051e-7,0.0006873576,0.1091573,0.0007462149,0.7717724,0.001643657,0.1056652],"study_design_scores_gemma":[0.001032275,0.0001296912,0.004475712,0.0008168429,0.0001884061,0.000001615768,0.0001098854,0.9774149,0.0002382867,0.01038751,0.004880624,0.0003242278],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01099464,0.002403909,0.9727901,0.005337604,0.001150209,0.003514681,0.00002440922,0.0002112989,0.003573122],"genre_scores_gemma":[0.8162012,0.00001108541,0.1805653,0.0005187176,0.0000657796,0.00128461,0.00001152756,0.00002075862,0.001321114],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8682576,"threshold_uncertainty_score":0.9998904,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02182292723303706,"score_gpt":0.3406130022586686,"score_spread":0.3187900750256316,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}