{"id":"W4387076386","doi":"10.48550/arxiv.2309.14181","title":"Q-Bench: A Benchmark for General-Purpose Foundation Models on Low-level Vision","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Institute for Catastrophic Loss Reduction","keywords":"Perception; Computer science; Benchmark (surveying); Construct (python library); Correctness; Artificial intelligence; Softmax function; Pipeline (software); Quality (philosophy); Foundation (evidence); Visual perception; Natural language processing; Machine learning; Psychology; Deep learning; Programming language","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004070057,0.0003887052,0.0003330888,0.0004855861,0.0003775982,0.0002453113,0.002049762,0.0003291973,0.00001766852],"category_scores_gemma":[0.00008019755,0.0004585063,0.0002828483,0.0006977989,0.00006404803,0.0004599845,0.00158185,0.0006048729,0.0003475499],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003355673,"about_ca_system_score_gemma":0.0002162137,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006785103,"about_ca_topic_score_gemma":0.00004796329,"domain_scores_codex":[0.9972823,0.0001381563,0.0002851915,0.001704472,0.0001715039,0.0004184215],"domain_scores_gemma":[0.997021,0.0003593596,0.0003442711,0.001824721,0.0002677816,0.0001829287],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002034911,0.0000892742,0.00007777896,0.00005107359,0.00002935087,0.000009664541,0.0001304336,0.6483781,0.00007246788,0.3471719,0.0004369888,0.003532625],"study_design_scores_gemma":[0.0003762424,0.0000716177,0.002431486,0.00007536744,0.00002679096,6.241773e-7,0.000006441953,0.7387437,0.00004748438,0.2575761,0.000299802,0.0003443239],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1706012,0.000004961969,0.8256488,0.0006066005,0.0005279896,0.0009837555,0.00005317152,0.000546435,0.001027089],"genre_scores_gemma":[0.9614499,0.00004841431,0.03491262,0.0001864146,0.0001464023,0.00003267336,0.0002834694,0.00004793604,0.002892123],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7908487,"threshold_uncertainty_score":0.9997867,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1809078712691692,"score_gpt":0.2562941724986156,"score_spread":0.07538630122944642,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}