{"id":"W6891719042","doi":"10.48448/tcn5-3z78","title":"ANALOGICAL - A Novel Benchmark for Long Text Analogy Evaluation in Large Language Models","year":2022,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Artificial Intelligence in Medicine (Canada)","funders":"","keywords":"Analogy; Benchmark (surveying); Word (group theory); Taxonomy (biology); Language model; Quality (philosophy)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006475259,0.001692872,0.0007582295,0.004659491,0.001060179,0.002263614,0.00219166,0.002103521,0.004552765],"category_scores_gemma":[0.04052372,0.0002632941,0.001225596,0.003144215,0.001159589,0.005942086,0.003280741,0.001895626,0.001580537],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001355916,"about_ca_system_score_gemma":0.001143499,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00271426,"about_ca_topic_score_gemma":0.0050728,"domain_scores_codex":[0.9912972,0.004731401,0.0008988407,0.001522841,0.001365397,0.0001843699],"domain_scores_gemma":[0.9849492,0.009802599,0.0007838074,0.0025586,0.001523284,0.0003825306],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001976065,0.00173525,0.03716071,0.002739965,0.001245907,0.0009163105,0.0012435,0.1882632,0.01098746,0.02901093,0.05919023,0.6655306],"study_design_scores_gemma":[0.000362972,0.001684687,0.009651609,0.0002003214,0.0001780373,0.0007464176,0.00104911,0.8761669,0.0130267,0.07559707,0.02121776,0.0001184406],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.6429334,0.01213954,0.2749183,0.003266886,0.001270572,0.001019905,0.01867808,0.01528076,0.03049266],"genre_scores_gemma":[0.8132138,0.0008190796,0.1544707,0.0008419587,0.0002818602,0.0006113102,0.02612799,0.0005119953,0.003121298],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.006475259,"threshold_uncertainty_score":0.03424484,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05689247138265904,"score_gpt":0.3601712973901034,"score_spread":0.3032788260074444,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}