{"id":"W3211088471","doi":"10.48550/arxiv.2110.13223","title":"Identifying and Benchmarking Natural Out-of-Context Prediction Problems","year":2021,"lang":"en","type":"article","venue":"arXiv (Cornell University)","topic":"Anomaly Detection Techniques and Applications","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Benchmarking; Suite; Computer science; Context (archaeology); Artificial intelligence; Machine learning; Data mining; Data science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0109007,0.001791046,0.001334043,0.002354919,0.001453753,0.002346672,0.003632091,0.003077312,0.0007872127],"category_scores_gemma":[0.05412087,0.0005311101,0.001047556,0.002060018,0.002901185,0.003504762,0.004505922,0.004525941,0.000556598],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001747112,"about_ca_system_score_gemma":0.00168932,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004106137,"about_ca_topic_score_gemma":0.007451204,"domain_scores_codex":[0.990436,0.004388369,0.0006210693,0.002278074,0.001779175,0.000497297],"domain_scores_gemma":[0.9571288,0.02946121,0.002436844,0.006924823,0.002818893,0.001229486],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001546296,0.001749865,0.1672878,0.001513785,0.0008145623,0.0006924747,0.0009593717,0.5683719,0.008206234,0.01618212,0.044532,0.1881435],"study_design_scores_gemma":[0.00005967048,0.0003473194,0.01111913,0.00007645076,0.00004451651,0.0002838735,0.0002816279,0.9555627,0.008499816,0.02002917,0.003638778,0.00005690024],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.8349592,0.003197368,0.1453868,0.002839007,0.0005408417,0.0003315322,0.00559718,0.003136521,0.004011545],"genre_scores_gemma":[0.9173968,0.0004095498,0.0705792,0.000480836,0.0001887992,0.0002161722,0.009716248,0.0002330848,0.0007791739],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.0109007,"threshold_uncertainty_score":0.05764914,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05324615000415524,"score_gpt":0.1882156488672321,"score_spread":0.1349694988630769,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}