{"id":"W4385688709","doi":"10.1145/3578337.3605115","title":"A is for Adele: An Offline Evaluation Metric for Instant Search","year":2023,"lang":"en","type":"article","venue":"","topic":"Web Data Mining and Analysis","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Instant; Computer science; Metric (unit); Instant messaging; Search engine; Information retrieval; Key (lock); Search problem; Incremental heuristic search; Beam search; Search algorithm; World Wide Web; Computer security; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002047999,0.0000610681,0.0001036249,0.0004268459,0.0001168815,0.0001310473,0.0004768198,0.00002944129,0.00003030374],"category_scores_gemma":[0.0002218234,0.00004968408,0.00006843422,0.002103289,0.000007698733,0.0003718319,0.00009364367,0.00002874373,0.00004129334],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000254663,"about_ca_system_score_gemma":0.0001143127,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004155266,"about_ca_topic_score_gemma":0.00002246942,"domain_scores_codex":[0.9989187,0.00003679472,0.0001496294,0.0003344522,0.0003479128,0.0002125764],"domain_scores_gemma":[0.9990143,0.0001867062,0.00002843363,0.0004229014,0.00028606,0.00006155914],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001476348,0.00009274416,0.0002598301,0.00003959854,0.00008183896,7.402344e-7,0.0008948154,0.001222103,0.0009987876,0.02150336,0.04379235,0.9310991],"study_design_scores_gemma":[0.0003866934,0.0001730502,0.0003822582,0.000003172492,0.00002405139,3.602773e-7,0.0001133839,0.9883683,0.001771369,0.001136084,0.007565486,0.00007584312],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02755789,0.00002657102,0.9690143,0.0023888,0.00006777961,0.0003446375,0.00006856123,0.0001816575,0.0003497439],"genre_scores_gemma":[0.8154963,0.00001310305,0.1805484,0.00049763,0.0001053566,0.0001971575,0.000339638,0.00001118619,0.002791218],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9871461,"threshold_uncertainty_score":0.2026058,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2237518965002702,"score_gpt":0.4305085783964246,"score_spread":0.2067566818961543,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}