{"id":"W4362575744","doi":"10.22215/etd/2023-15362","title":"Performance Evaluation of Transformer-based NLP Models on Fake News Detection Datasets","year":2023,"lang":"en","type":"dissertation","venue":"","topic":"Misinformation and Its Impacts","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Carleton University","funders":"","keywords":"Transformer; Computer science; Implementation; Artificial intelligence; Social media; Machine learning; Language model; Fake news; Natural language processing; Data mining; Engineering; World Wide Web; Software engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006532005,0.002118485,0.001212477,0.002392411,0.0008396999,0.001571969,0.001863204,0.001721644,0.001849725],"category_scores_gemma":[0.01289335,0.0004311913,0.001171792,0.001609149,0.0006124399,0.003206765,0.001152236,0.002593098,0.002024821],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002839064,"about_ca_system_score_gemma":0.002051148,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.03789572,"about_ca_topic_score_gemma":0.03263419,"domain_scores_codex":[0.9975225,0.00104894,0.0002732386,0.0005177301,0.0004079232,0.0002296134],"domain_scores_gemma":[0.9929373,0.004752146,0.0002470606,0.0006659789,0.001171519,0.0002261565],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002707211,0.001494991,0.01818907,0.0008401937,0.000746686,0.0004215597,0.0005181577,0.4052203,0.007842075,0.003795678,0.03262965,0.5255946],"study_design_scores_gemma":[0.00004127161,0.0001936976,0.001171183,0.00002411582,0.00005160437,0.00008130106,0.0001503955,0.9919742,0.004329113,0.0008667169,0.001089728,0.00002664563],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8414742,0.01003278,0.09622,0.003675458,0.001052531,0.0005029427,0.008037538,0.02278406,0.01622044],"genre_scores_gemma":[0.9242181,0.001579612,0.05499367,0.0004226504,0.0001129971,0.00016405,0.01332192,0.0002551618,0.004931936],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03789572,"threshold_uncertainty_score":0.07535028,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1184904236862483,"score_gpt":0.3944807047835876,"score_spread":0.2759902810973394,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}