{"id":"W3052028990","doi":"10.1109/iros45743.2020.9341677","title":"Integrated Benchmarking and Design for Reproducible and Accessible Evaluation of Robotic Agents","year":2020,"lang":"en","type":"article","venue":"","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"National Science Foundation","keywords":"Benchmarking; Robotics; Computer science; Robot; Artificial intelligence; Autonomy; Software engineering; Software; Automation; Variance (accounting); Embedded system; Human–computer interaction; Machine learning; Systems engineering; Operating system; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01253915,0.00006120118,0.0001425606,0.0001206117,0.00009635629,0.0003176408,0.0002572986,0.0000167605,0.000202855],"category_scores_gemma":[0.006691527,0.00004209583,0.00001884302,0.0006379074,0.00004217222,0.0002423978,0.0002403621,0.00002475132,0.00000477637],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000007942855,"about_ca_system_score_gemma":0.00004341806,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002307596,"about_ca_topic_score_gemma":0.000005008729,"domain_scores_codex":[0.9979023,0.0001635407,0.0003480882,0.0007178268,0.0007677703,0.0001004907],"domain_scores_gemma":[0.9984644,0.0005375469,0.0001386909,0.0004015425,0.0003926224,0.00006517195],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005316231,0.00003633645,0.008350408,0.0000303895,0.00002540485,3.948081e-7,0.001059001,0.03350232,0.0009500397,0.0006745735,0.1125423,0.8427756],"study_design_scores_gemma":[0.0003347578,0.00007062452,0.01009193,0.00001261248,0.0000321911,3.827513e-7,0.0005481821,0.9809352,0.0008805699,0.005657122,0.001381539,0.00005493497],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2317952,0.0001601357,0.7643501,0.001331953,0.0003067805,0.0006451246,0.000004017551,0.00002495995,0.00138174],"genre_scores_gemma":[0.9566529,0.000004246811,0.0428577,0.0001428142,0.00002725778,0.000007880018,0.000006681961,0.000002854747,0.0002976623],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9474328,"threshold_uncertainty_score":0.8010867,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.6155147781886495,"score_gpt":0.4672346836766887,"score_spread":0.1482800945119608,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}