{"id":"W4403090726","doi":"10.1101/2024.10.02.616248","title":"PertEval-scFM: Benchmarking Single-Cell Foundation Models for Perturbation Effect Prediction","year":2024,"lang":"en","type":"preprint","venue":"bioRxiv (Cold Spring Harbor Laboratory)","topic":"Single-cell and spatial transcriptomics","field":"Biochemistry, Genetics and Molecular Biology","cited_by":31,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University","funders":"Wellcome Trust","keywords":"Benchmarking; Perturbation (astronomy); Foundation (evidence); Computer science; Econometrics; Mathematics; Physics; Economics; Law; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006329457,0.0006235971,0.0004358744,0.0002009851,0.0002145965,0.0003514571,0.0003448278,0.0009419125,0.00001087052],"category_scores_gemma":[0.000116539,0.0006589529,0.000367871,0.0001816651,0.00007683691,0.00002090349,0.0002696254,0.0004783657,0.00001599484],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002405526,"about_ca_system_score_gemma":0.000362292,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002589951,"about_ca_topic_score_gemma":0.000004517794,"domain_scores_codex":[0.9971899,0.0001310269,0.0005495921,0.001330581,0.0003104516,0.0004884762],"domain_scores_gemma":[0.9982756,0.00005627096,0.0002667376,0.0008108899,0.0004264476,0.0001640083],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0001661734,0.0001487712,0.0003273576,0.001228493,0.0001573622,0.000002873162,0.00001331378,0.001944145,0.995375,0.0001512522,0.000433299,0.00005192556],"study_design_scores_gemma":[0.0010476,0.0008483253,0.0004407262,0.0004570586,0.0004492053,4.748063e-8,0.000001819701,0.07144391,0.9186411,0.00006113564,0.005739636,0.0008693979],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8971335,0.002405934,0.09315117,0.0001053356,0.004602967,0.001812612,0.0004121478,0.0002807581,0.00009553767],"genre_scores_gemma":[0.9921961,0.0002286804,0.004335103,0.0001178659,0.002323073,0.0004850948,0.00004130141,0.0002175458,0.00005524883],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.09506255,"threshold_uncertainty_score":0.9995862,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01549444564827791,"score_gpt":0.2126725885666566,"score_spread":0.1971781429183787,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}