{"id":"W3093924713","doi":"10.1145/3340531.3411915","title":"Offline Evaluation by Maximum Similarity to an Ideal Ranking","year":2020,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":42,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Ranking (information retrieval); Relevance (law); Learning to rank; Similarity (geometry); Correctness; Computer science; Rank (graph theory); Ideal (ethics); Information retrieval; Set (abstract data type); Measure (data warehouse); Data mining; Ranking SVM; Similarity measure; Artificial intelligence; Mathematics; Algorithm","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01105145,0.001241178,0.00237667,0.00541257,0.001046072,0.004470822,0.001419305,0.001511502,0.003874717],"category_scores_gemma":[0.04585423,0.0004269626,0.0008787614,0.003228139,0.001951563,0.005851598,0.002538011,0.001280217,0.001793667],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002309776,"about_ca_system_score_gemma":0.002269524,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001476309,"about_ca_topic_score_gemma":0.001654219,"domain_scores_codex":[0.9788303,0.009344744,0.001272147,0.002280738,0.007441528,0.0008306174],"domain_scores_gemma":[0.9725537,0.0139476,0.002132246,0.00508428,0.005341034,0.0009412464],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002464238,0.0009772201,0.01809894,0.001031128,0.0004192555,0.0003968089,0.0009894874,0.1504358,0.02511398,0.1549172,0.01450716,0.6306488],"study_design_scores_gemma":[0.0001242458,0.001759954,0.005836084,0.0001355335,0.0001280184,0.0007743268,0.0004593682,0.8558958,0.02408022,0.09581926,0.01484787,0.0001393195],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.1009945,0.001438828,0.8796562,0.0003538461,0.0001577379,0.0003251036,0.0003846034,0.001879271,0.01480994],"genre_scores_gemma":[0.6580149,0.00040868,0.3361322,0.0001520893,0.0001813548,0.0003321073,0.0009453655,0.0005138853,0.003319422],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.01105145,"threshold_uncertainty_score":0.05844635,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06998699835595117,"score_gpt":0.3268742644032515,"score_spread":0.2568872660473003,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}