{"id":"W4410502462","doi":"10.1145/3736402","title":"Query Performance Prediction Using Relevance Judgments Generated by Large Language Models","year":2025,"lang":"en","type":"article","venue":"ACM Transactions on Information Systems","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"HORIZON EUROPE Framework Programme; Ministerie van Economische Zaken en Klimaat; China Scholarship Council; Nederlandse Organisatie voor Wetenschappelijk Onderzoek; Ministerie van Onderwijs, Cultuur en Wetenschap; European Commission","keywords":"Relevance (law); Computer science; Interpretability; Recall; Metric (unit); Measure (data warehouse); Precision and recall; Scalar (mathematics); Information retrieval; Data mining; Machine learning; Artificial intelligence; Cognitive psychology; Mathematics; Psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004519051,0.0001731155,0.0001678356,0.0004461779,0.0005468149,0.0004453083,0.0005874636,0.0001331924,0.00001201339],"category_scores_gemma":[0.00001931723,0.0001640389,0.00006292538,0.0009593453,0.00002029032,0.008021818,0.00002173706,0.000250524,0.0001156478],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002568138,"about_ca_system_score_gemma":0.0001341995,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007522427,"about_ca_topic_score_gemma":0.000001182411,"domain_scores_codex":[0.9982315,0.00006589777,0.0006884839,0.0001604842,0.0005349883,0.0003186534],"domain_scores_gemma":[0.9987402,0.00004757777,0.0001799893,0.0006560065,0.0002967508,0.00007949092],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001729531,0.0004290768,0.0003938575,0.0009313038,0.0002084895,0.000002448907,0.01231027,0.7980534,0.00370515,0.0162006,0.007690731,0.1599017],"study_design_scores_gemma":[0.0007070818,0.00005093105,0.00009470028,0.0001228169,0.00001140463,0.000008232812,0.0005216417,0.9813893,0.006123201,0.00002058366,0.01078741,0.0001626666],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.06238215,0.00006102789,0.9336707,0.0000945575,0.0011203,0.0004938426,0.0001778305,0.0003198308,0.001679754],"genre_scores_gemma":[0.9951566,0.00008632135,0.002806622,0.0004613929,0.0000171895,0.0001062747,0.0001017886,0.000005908267,0.00125787],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9327745,"threshold_uncertainty_score":0.668931,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02121226181174902,"score_gpt":0.2639589587016803,"score_spread":0.2427466968899313,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}