{"id":"W4391670820","doi":"10.48550/arxiv.2402.04355","title":"PQMass: Probabilistic Assessment of the Quality of Generative Models using Probability Mass Estimation","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Simulation Techniques and Applications","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Samsung; Alliance de recherche numérique du Canada; Genentech; Canada Research Chairs; Canadian Institute for Advanced Research","keywords":"Probabilistic logic; Estimation; Generative grammar; Computer science; Statistical model; Quality (philosophy); Econometrics; Statistics; Artificial intelligence; Mathematics; Engineering; Physics","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002494254,0.0002080392,0.0004780664,0.0001959598,0.0001222511,0.00006980183,0.001092216,0.0002159179,0.00005142946],"category_scores_gemma":[0.0005193168,0.0001566096,0.0003723064,0.001171379,0.0003560302,0.000151854,0.001211653,0.0003910379,0.000002388432],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003677997,"about_ca_system_score_gemma":0.0006379155,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002473411,"about_ca_topic_score_gemma":0.00003824521,"domain_scores_codex":[0.9969738,0.0006014998,0.0009284472,0.0008771578,0.0004813629,0.000137699],"domain_scores_gemma":[0.9953846,0.0007167078,0.001141205,0.001659538,0.001042466,0.00005546782],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005470301,0.00005651436,0.001560939,0.00008174953,0.00002024313,2.449334e-7,0.00008081827,0.6678677,0.0003714813,0.3297921,0.00001713583,0.0001456052],"study_design_scores_gemma":[0.00004819966,0.000007151204,0.00126832,0.00004521559,0.0000444654,8.956864e-8,0.00005274215,0.5050043,0.0003817732,0.4930694,0.000002843585,0.00007547181],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.456272,0.00001104,0.5414208,0.0000802316,0.00009281743,0.0007682436,0.0001190456,0.00003532921,0.001200461],"genre_scores_gemma":[0.9744962,0.000004539591,0.02513906,0.000009736524,0.00001600234,0.000005823741,0.000008554382,0.00001075827,0.0003093408],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5182242,"threshold_uncertainty_score":0.6386356,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4992141740380304,"score_gpt":0.391946298061485,"score_spread":0.1072678759765454,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}