{"id":"W4385688511","doi":"10.1145/3578337.3605136","title":"Perspectives on Large Language Models for Relevance Judgment","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":152,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Science Foundation","keywords":"Relevance (law); Perspective (graphical); Categorization; Computer science; Point (geometry); Compromise; Cognitive psychology; Psychology; Artificial intelligence; Sociology; Political science; Social science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002076616,0.00006672857,0.0000715523,0.00006358158,0.00006901757,0.0000446269,0.0003524951,0.00002324033,0.00000877091],"category_scores_gemma":[0.0000369736,0.00005673361,0.0000438525,0.0001635745,0.000004984339,0.0001932496,0.0001168692,0.00004649635,0.00009104481],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003626457,"about_ca_system_score_gemma":0.00001579102,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008598452,"about_ca_topic_score_gemma":0.000006547196,"domain_scores_codex":[0.9991925,0.00001079404,0.00008481203,0.0003249525,0.0001499431,0.0002370125],"domain_scores_gemma":[0.9994095,0.00009744758,0.00001834548,0.00040958,0.00002942789,0.00003572859],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002481532,0.00002510827,0.000002854513,0.000006565526,0.00000473317,0.000003622633,0.007171798,0.01032939,0.0001914787,0.9731292,0.002251619,0.006881206],"study_design_scores_gemma":[0.0002147785,0.00003678108,0.00001547055,0.000008079703,8.072948e-7,4.388571e-7,0.001665839,0.9749784,0.0007171552,0.02082691,0.001453043,0.00008223868],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01177074,0.00008234624,0.9768758,0.002190397,0.0001508513,0.0001761847,0.000003432387,0.0004852486,0.00826497],"genre_scores_gemma":[0.8866124,0.00002695053,0.1042002,0.0005425478,0.00008867477,0.00005785385,0.000001709557,0.000009175405,0.008460411],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9646491,"threshold_uncertainty_score":0.231353,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03503074496303086,"score_gpt":0.2930385171724492,"score_spread":0.2580077722094183,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}