{"id":"W4393636116","doi":"10.5281/zenodo.5130736","title":"SemEval-2021 Task 12: Learning with Disagreements","year":2021,"lang":"en","type":"dataset","venue":"Zenodo (CERN European Organization for Nuclear Research)","topic":"Intelligent Tutoring Systems and Adaptive Learning","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Austrian Science Fund; European Commission","keywords":"SemEval; Task (project management); Computer science; Natural language processing; Artificial intelligence; Engineering","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","sts","scholarly_communication","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0008821564,0.0003485786,0.0003456458,0.0003104837,0.002758757,0.002930629,0.00269304,0.0001494553,0.006064342],"category_scores_gemma":[0.0004961917,0.0003310081,0.00009693969,0.0006724425,0.0000874409,0.0004205772,0.00322255,0.001066034,0.009735885],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002524461,"about_ca_system_score_gemma":0.0000181003,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007917451,"about_ca_topic_score_gemma":0.00000170045,"domain_scores_codex":[0.9961958,0.0008417945,0.0003834647,0.001004829,0.0009604078,0.0006136849],"domain_scores_gemma":[0.9975321,0.00004420027,0.000333561,0.001119199,0.0007508855,0.0002201056],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001266215,0.00009057848,0.000001959924,0.0001357438,0.0001203265,0.0001723549,0.0003676401,0.0003263923,0.0001288438,0.001214,0.9806279,0.01680163],"study_design_scores_gemma":[0.0002604446,0.000330361,0.00002810999,0.0003404855,0.00002355355,0.0001686381,0.0001640857,0.0002576532,0.00004947464,0.000008466857,0.9979699,0.0003987734],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"dataset","genre_gemma":"dataset","genre_scores_codex":[0.0002155055,0.0007373237,0.2435202,0.0008993142,0.001925105,0.001827305,0.6963137,0.002396407,0.05216516],"genre_scores_gemma":[0.0008002024,0.000218383,0.0006833746,0.0001036684,0.0005832626,1.317535e-7,0.9833371,0.001048334,0.01322551],"genre_candidate":"dataset","genre_consensus":"dataset","teacher_disagreement_score":0.2870235,"threshold_uncertainty_score":0.9999142,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02906174980591631,"score_gpt":0.2382847953472288,"score_spread":0.2092230455413125,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}