{"id":"W3152534132","doi":"10.1145/3404835.3462947","title":"Evaluation Measures Based on Preference Graphs","year":2021,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Ranking (information retrieval); Preference; Relevance (law); Measure (data warehouse); Computer science; Learning to rank; Similarity (geometry); Flexibility (engineering); Rank (graph theory); Set (abstract data type); Information retrieval; Similarity measure; Mathematics; Artificial intelligence; Statistics; Data mining; Combinatorics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008246115,0.00005301858,0.00004802774,0.00007133,0.0000781941,0.0001435157,0.0002415988,0.00002644589,0.0003436987],"category_scores_gemma":[0.0001796137,0.00004125713,0.00003599556,0.000400278,0.00001021434,0.0003194865,0.00003946412,0.00006587424,0.0001650974],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003258313,"about_ca_system_score_gemma":0.0003362412,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000004987689,"about_ca_topic_score_gemma":0.000006479701,"domain_scores_codex":[0.9983801,0.0001237552,0.0001109454,0.0001424397,0.001123815,0.0001189817],"domain_scores_gemma":[0.9988099,0.00004392333,0.00002266447,0.0003217707,0.0007476594,0.00005403165],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000008693727,0.0001667116,0.0007460889,0.000008606357,0.000007517257,0.00000749151,0.0003025091,0.001889354,0.001780872,0.2127795,0.001918382,0.7803843],"study_design_scores_gemma":[0.0004771429,0.00009606245,0.02070473,0.00001306114,0.000007724317,0.000002507615,0.00002078357,0.9103349,0.06069673,0.00501053,0.002499961,0.0001358182],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02667021,0.00002610614,0.8338059,0.001819368,0.0003383772,0.0002645188,0.000002003714,0.0002080114,0.1368656],"genre_scores_gemma":[0.9893017,0.000002200954,0.009192325,0.0009961271,0.000006507117,0.00001731851,0.000006103088,0.000001479838,0.0004762587],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9626315,"threshold_uncertainty_score":0.376326,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1327439096851362,"score_gpt":0.3104968045635845,"score_spread":0.1777528948784483,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}