{"id":"W4401597819","doi":"10.1371/journal.pone.0307741","title":"GPT-4 as an X data annotator: Unraveling its performance on a stance classification task","year":2024,"lang":"en","type":"article","venue":"PLoS ONE","topic":"Topic Modeling","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University; Lakehead University","funders":"Social Sciences and Humanities Research Council of Canada; Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Artificial intelligence; Natural language processing; Task (project management); Annotation; Machine learning; Context (archaeology); Benchmark (surveying); Generalizability theory; Set (abstract data type); Psychology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003682303,0.0001185397,0.000120356,0.00009805051,0.0001033754,0.0002969799,0.001369873,0.00005042605,0.00001230945],"category_scores_gemma":[0.00006274986,0.000116944,0.00001344358,0.0003068888,0.00001398623,0.001422361,0.0002480846,0.0002237518,0.0005017064],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000654945,"about_ca_system_score_gemma":0.0001196451,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000005611612,"about_ca_topic_score_gemma":0.000003482844,"domain_scores_codex":[0.9982816,0.00003868332,0.0002101476,0.000732306,0.0005031764,0.0002340981],"domain_scores_gemma":[0.9982018,0.00005403696,0.00004366757,0.001554497,0.00007231184,0.00007371736],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00006960632,0.002465181,0.001406518,0.001809755,0.0003671121,0.0001120735,0.008491707,0.002019059,0.2227129,0.3813125,0.00153942,0.3776941],"study_design_scores_gemma":[0.00006945645,0.00008492976,0.0003498642,0.0003731721,0.0000126363,0.000002283969,0.00001615014,0.9904945,0.006712166,0.000482245,0.001254656,0.0001479337],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8392908,0.0007179654,0.1543996,0.001834808,0.0003093671,0.0002462699,0.00002517032,0.0006325917,0.002543403],"genre_scores_gemma":[0.9680405,0.0001497203,0.03077085,0.0003230849,0.000230851,0.00001526325,0.00002809274,0.0000154758,0.0004261861],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9884754,"threshold_uncertainty_score":0.6448589,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1955414476420611,"score_gpt":0.3002820790635207,"score_spread":0.1047406314214596,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}