{"id":"W2087859840","doi":"10.1007/s10844-009-0096-5","title":"Evaluating information retrieval system performance based on user preference","year":2009,"lang":"en","type":"article","venue":"Journal of Intelligent Information Systems","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Regina","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Relevance (law); Information retrieval; Preference; Rank (graph theory); Precision and recall; Measure (data warehouse); Relevance feedback; Document retrieval; Data mining; Artificial intelligence; Image retrieval; Statistics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00439729,0.0004529562,0.0006554505,0.001687248,0.0003880383,0.001324401,0.0002804422,0.0008868991,0.001732357],"category_scores_gemma":[0.02918341,0.000164888,0.0006145542,0.001056226,0.0002206238,0.001458719,0.0003296096,0.0003624563,0.0006676685],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004118551,"about_ca_system_score_gemma":0.0003644148,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002019558,"about_ca_topic_score_gemma":0.002394897,"domain_scores_codex":[0.9968977,0.001508201,0.0003936879,0.0001888235,0.0008006527,0.0002109832],"domain_scores_gemma":[0.9566723,0.0360355,0.001836147,0.001006465,0.003564788,0.0008847522],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.02355006,0.0027289,0.6423898,0.0007342655,0.001621492,0.0003177202,0.000896187,0.01721783,0.06221827,0.0006039595,0.002331186,0.2453902],"study_design_scores_gemma":[0.0006456163,0.01851963,0.6232491,0.00004643995,0.00169492,0.0008141388,0.001039023,0.3064464,0.04573683,0.0006757414,0.0008721117,0.0002599691],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9954938,0.0002315208,0.002980398,0.00004721873,0.00001459521,0.00003192725,0.0000769563,0.00009124496,0.001032348],"genre_scores_gemma":[0.9975552,0.00005438249,0.001977703,0.00001615745,0.00001246623,0.00001236203,0.0001157366,0.00001186234,0.0002441216],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.00439729,"threshold_uncertainty_score":0.02325535,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07183585331902738,"score_gpt":0.3096554805127487,"score_spread":0.2378196271937213,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}