{"id":"W4405761894","doi":"10.1007/978-3-031-73147-1_6","title":"Generative Information Retrieval Evaluation","year":2024,"lang":"en","type":"book-chapter","venue":"˜The œinformation retrieval series","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Generative grammar; Information retrieval; Computer science; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","scholarly_communication","insufficient_payload"],"consensus_categories":["scholarly_communication"],"category_scores_codex":[0.002764063,0.0008277795,0.0005900956,0.0009668677,0.0009122537,0.003450769,0.001750136,0.0007776258,0.0008548853],"category_scores_gemma":[0.0001945461,0.0006221308,0.0004491769,0.0007989409,0.0003295786,0.01883673,0.0007693816,0.001379753,0.0108083],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008437615,"about_ca_system_score_gemma":0.001135328,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001470453,"about_ca_topic_score_gemma":0.00001079442,"domain_scores_codex":[0.9928686,0.00009616636,0.002058686,0.0003846072,0.003927784,0.000664154],"domain_scores_gemma":[0.9942608,0.0001546256,0.001188048,0.001339647,0.002793808,0.0002631009],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0003208579,0.00001044738,0.00000101827,0.000269812,0.0001709856,0.000005175094,0.006810323,0.0003227727,0.00002065392,0.9298235,0.01909082,0.04315364],"study_design_scores_gemma":[0.0006209241,0.000344028,0.00004420525,0.0001490063,0.0001790522,0.0001187407,0.0003065024,0.04254797,0.000730437,0.0428812,0.9111804,0.000897565],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.0001389209,0.0007241198,0.0697245,0.005732427,0.005998142,0.00460261,0.0006635609,0.001396521,0.9110192],"genre_scores_gemma":[0.02313273,0.001755047,0.005390282,0.008778134,0.0020825,0.0002427294,0.009132164,0.0002012662,0.9492851],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.8920895,"threshold_uncertainty_score":0.999623,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02459505392412266,"score_gpt":0.2591735759338347,"score_spread":0.234578522009712,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}