{"id":"W6950815263","doi":"10.5683/sp3/mhjrio","title":"Evaluating Large Language Models for Narrative Topic Labeling (NLP4DH 2025)","year":2025,"lang":"en","type":"dataset","venue":"Borealis","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University","funders":"","keywords":"Narrative; Task (project management); Language model; Topic model; Computational linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.007099518,0.005103035,0.001506484,0.004868993,0.002511423,0.00255367,0.005822209,0.004001413,0.01761099],"category_scores_gemma":[0.0191317,0.0008491243,0.002264707,0.003362467,0.001413333,0.00342775,0.00471014,0.003350369,0.0215558],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0040591,"about_ca_system_score_gemma":0.003831493,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.07592069,"about_ca_topic_score_gemma":0.1491272,"domain_scores_codex":[0.9929662,0.003186084,0.0004851089,0.001658386,0.001242078,0.0004622182],"domain_scores_gemma":[0.9885206,0.005014659,0.0005093789,0.002821819,0.002186486,0.0009469242],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.001457379,0.001029408,0.00319504,0.001619948,0.0003716124,0.0003287234,0.0003280698,0.006149048,0.002848488,0.001707072,0.9172686,0.06369653],"study_design_scores_gemma":[0.004284734,0.001669567,0.02504684,0.001103834,0.0009207227,0.001894074,0.002589063,0.1401631,0.02546582,0.01029859,0.7861099,0.0004538164],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"dataset","genre_gemma":"dataset","genre_scores_codex":[0.05941229,0.004239672,0.01857602,0.0023389,0.001523883,0.001466336,0.8477809,0.04116015,0.02350184],"genre_scores_gemma":[0.02321129,0.0002142736,0.02078094,0.0003686848,0.00009965312,0.0005272399,0.9496156,0.0005153721,0.004666899],"genre_candidate":"dataset","genre_consensus":"dataset","teacher_disagreement_score":0.07592069,"threshold_uncertainty_score":0.1509576,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06464726281547914,"score_gpt":0.4050891425063637,"score_spread":0.3404418796908846,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}