{"id":"W4385688709","doi":"10.1145/3578337.3605115","title":"A is for Adele: An Offline Evaluation Metric for Instant Search","year":2023,"lang":"en","type":"article","venue":"","topic":"Web Data Mining and Analysis","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Instant; Computer science; Metric (unit); Instant messaging; Search engine; Information retrieval; Key (lock); Search problem; Incremental heuristic search; Beam search; Search algorithm; World Wide Web; Computer security; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009113801,0.001528943,0.001557839,0.006610231,0.0009213239,0.003289448,0.001510268,0.001885091,0.002926843],"category_scores_gemma":[0.06500841,0.0002547656,0.0006699526,0.004061837,0.0008442723,0.006691298,0.001714025,0.001388374,0.001657863],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001481583,"about_ca_system_score_gemma":0.001616668,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003143894,"about_ca_topic_score_gemma":0.003454003,"domain_scores_codex":[0.9792188,0.007213732,0.002323265,0.001327795,0.00920873,0.0007076585],"domain_scores_gemma":[0.9220666,0.04768605,0.006034245,0.007072811,0.01483331,0.002306936],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004397969,0.002161203,0.05393954,0.00400606,0.0008184981,0.0004284713,0.001375873,0.05358647,0.03567732,0.02428066,0.03714154,0.7821864],"study_design_scores_gemma":[0.0005555072,0.009791954,0.07137202,0.0008157305,0.0007018326,0.002468703,0.00206381,0.7868542,0.04524051,0.02640064,0.05312046,0.0006146682],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.491695,0.01558192,0.3888416,0.001963595,0.001248344,0.003132322,0.009110703,0.009042278,0.07938433],"genre_scores_gemma":[0.8629107,0.0008498278,0.1263512,0.0002644258,0.0002157473,0.0008490844,0.003782244,0.0005198357,0.004256928],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.009113801,"threshold_uncertainty_score":0.048199,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2237518965002702,"score_gpt":0.4305085783964246,"score_spread":0.2067566818961543,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}