{"id":"W4391670820","doi":"10.48550/arxiv.2402.04355","title":"PQMass: Probabilistic Assessment of the Quality of Generative Models using Probability Mass Estimation","year":2024,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Simulation Techniques and Applications","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Samsung; Alliance de recherche numérique du Canada; Genentech; Canada Research Chairs; Canadian Institute for Advanced Research","keywords":"Probabilistic logic; Estimation; Generative grammar; Computer science; Statistical model; Quality (philosophy); Econometrics; Statistics; Artificial intelligence; Mathematics; Engineering; Physics","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02240973,0.001700009,0.002473782,0.00501507,0.001224915,0.004585213,0.004532586,0.003065495,0.003998054],"category_scores_gemma":[0.09725726,0.001220782,0.002413076,0.002525861,0.004094485,0.004954542,0.005791531,0.004000151,0.001008707],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001534637,"about_ca_system_score_gemma":0.002212628,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003657111,"about_ca_topic_score_gemma":0.003174289,"domain_scores_codex":[0.9911872,0.00443073,0.0004955711,0.001277036,0.002342855,0.0002666346],"domain_scores_gemma":[0.9170424,0.06941453,0.00348115,0.006483063,0.002852051,0.0007267346],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000620719,0.0001913691,0.02604564,0.0006576163,0.0008571987,0.0003946355,0.0006756167,0.6457244,0.006856942,0.104693,0.005342249,0.2079406],"study_design_scores_gemma":[0.00003934858,0.00007602777,0.001942922,0.00007130633,0.00004364147,0.0002358633,0.00005667999,0.9223776,0.00232702,0.07129963,0.001472281,0.00005771183],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.008438533,0.0002344924,0.9895475,0.0002033506,0.00002466829,0.00006790696,0.0001674228,0.0007382018,0.0005779602],"genre_scores_gemma":[0.3910428,0.0005719635,0.6032731,0.0005301049,0.0001782766,0.0004963228,0.001826439,0.0008418711,0.001239152],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02240973,"threshold_uncertainty_score":0.1185154,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4992141740380304,"score_gpt":0.391946298061485,"score_spread":0.1072678759765454,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}