{"id":"W6966305286","doi":"10.48448/7ms7-7a70","title":"MAGNIFICo: Evaluating the In-Context Learning Ability of Large Language Models to Generalize to Novel Interpretations","year":2023,"lang":"en","type":"other","venue":"Open MIND","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University; Minnow Environmental (Canada)","funders":"","keywords":"Parsing; Suite; Natural language; Natural language understanding; Interpretation (philosophy); Semantics (computer science)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005662075,0.002301037,0.0005973405,0.001238097,0.0005958942,0.001634593,0.002131467,0.002013203,0.00499344],"category_scores_gemma":[0.02904853,0.0004339801,0.0009799644,0.0005652673,0.0009760937,0.003604118,0.002606829,0.002506563,0.001486935],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009562511,"about_ca_system_score_gemma":0.001261634,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006691349,"about_ca_topic_score_gemma":0.01062062,"domain_scores_codex":[0.996908,0.001874807,0.0001510533,0.0005508381,0.00038473,0.0001305767],"domain_scores_gemma":[0.9783386,0.01754214,0.0006134644,0.002194406,0.0008122952,0.0004990142],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003724316,0.003585829,0.03158287,0.002713353,0.001100015,0.0007554019,0.00266702,0.3619986,0.01294864,0.01878943,0.07255711,0.4875775],"study_design_scores_gemma":[0.0002996021,0.0008878175,0.004510968,0.0001201772,0.0001269129,0.0002507097,0.0004722305,0.9492086,0.0107444,0.01924828,0.01403731,0.000092985],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7084795,0.002925348,0.1955693,0.002118613,0.0006551553,0.001157901,0.009920871,0.04539973,0.0337736],"genre_scores_gemma":[0.8068382,0.0004812801,0.168563,0.0006567967,0.0001301373,0.0007162427,0.01528384,0.002289037,0.005041413],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.006691349,"threshold_uncertainty_score":0.02994424,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.114095818128599,"score_gpt":0.4190951312379982,"score_spread":0.3049993131093992,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}