{"id":"W2754879180","doi":"10.1613/jair.5699","title":"Revisiting the Arcade Learning Environment: Evaluation Protocols and Open Problems for General Agents","year":2018,"lang":"en","type":"preprint","venue":"Journal of Artificial Intelligence Research","topic":"Artificial Intelligence in Games","field":"Computer Science","cited_by":46,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Alberta Innovates; Alberta Innovates - Technology Futures; Alberta Machine Intelligence Institute; Compute Canada; DeepMind; National Science Foundation","keywords":"Variety (cybernetics); Computer science; Benchmark (surveying); Key (lock); Field (mathematics); Data science; Best practice; State (computer science); Management science; Artificial intelligence; Engineering; Computer security; Political science","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","scholarly_communication","open_science","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.03811995,0.000342907,0.0006055817,0.0005078212,0.001165841,0.003567335,0.006081915,0.0003149891,0.0001345076],"category_scores_gemma":[0.004593722,0.0002526689,0.0002372065,0.0004792521,0.0007592599,0.000890793,0.005633087,0.002619629,0.00006736234],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004838548,"about_ca_system_score_gemma":0.001053792,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001097074,"about_ca_topic_score_gemma":0.00001270473,"domain_scores_codex":[0.9913915,0.002288778,0.001944042,0.0008708995,0.002637474,0.0008673542],"domain_scores_gemma":[0.993113,0.001514704,0.001519913,0.0009487123,0.002649259,0.000254399],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000138126,0.0001104691,0.000214429,0.0002063779,0.000111045,0.00001034977,0.005573101,0.03954709,0.001767489,0.01478282,0.0004670065,0.9370717],"study_design_scores_gemma":[0.00003873285,0.0006111695,0.00008206777,0.0009582578,0.00002930158,0.00003039407,0.0008029807,0.6208856,0.02188654,0.3445988,0.009816111,0.0002601043],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02801606,0.0008989503,0.9283075,0.008870284,0.0007574141,0.03242252,0.000007447307,0.00003238358,0.0006873907],"genre_scores_gemma":[0.721561,0.002449942,0.2374952,0.0002858971,0.01444474,0.0224015,0.00001970679,0.0002255563,0.001116586],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9368116,"threshold_uncertainty_score":0.9999925,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4920271193673282,"score_gpt":0.5281472299857938,"score_spread":0.03612011061846554,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}