{"id":"W4400526284","doi":"10.1145/3626772.3657942","title":"Synthetic Test Collections for Retrieval Evaluation","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"Microsoft (Canada)","funders":"Engineering and Physical Sciences Research Council; Universitas Brawijaya","keywords":"Computer science; Test (biology); Information retrieval; Geology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01156175,0.001751023,0.001257865,0.003084418,0.001311976,0.002006672,0.002884077,0.001770447,0.004256327],"category_scores_gemma":[0.05072599,0.0006874891,0.001389674,0.003669319,0.001443447,0.002706455,0.002841922,0.002003505,0.00261832],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001912071,"about_ca_system_score_gemma":0.001708114,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004177076,"about_ca_topic_score_gemma":0.006684469,"domain_scores_codex":[0.987329,0.007129009,0.001140957,0.001201932,0.002770156,0.000428927],"domain_scores_gemma":[0.9404209,0.02860473,0.002679138,0.01413412,0.01284792,0.001313256],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004029228,0.007769399,0.0384036,0.004282747,0.001099499,0.001402416,0.002952463,0.3247366,0.06541169,0.02391292,0.1076696,0.4183299],"study_design_scores_gemma":[0.001190805,0.007109828,0.02729907,0.0004933363,0.000516702,0.001734743,0.002600341,0.693628,0.1244475,0.0213967,0.1190422,0.0005407894],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5937467,0.003330553,0.3310101,0.001024904,0.0009493591,0.009202268,0.03153118,0.009186827,0.02001813],"genre_scores_gemma":[0.6504381,0.0008750692,0.2620406,0.0007713464,0.0002578259,0.009120662,0.07009078,0.001212351,0.005193341],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01156175,"threshold_uncertainty_score":0.06114513,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0516350703716649,"score_gpt":0.3152633073841311,"score_spread":0.2636282370124662,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}