{"id":"W4381329270","doi":"10.1145/3589322","title":"Maestro: Automatic Generation of Comprehensive Benchmarks for Question Answering Over Knowledge Graphs","year":2023,"lang":"en","type":"article","venue":"Proceedings of the ACM on Management of Data","topic":"Advanced Graph Neural Networks","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Carleton University","funders":"","keywords":"Computer science; Benchmark (surveying); Question answering; Knowledge base; Vocabulary; Natural language; Information retrieval; Set (abstract data type); Natural language understanding; Knowledge graph; Usability; Artificial intelligence; Natural language processing; Programming language; Human–computer interaction; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003286015,0.0001266707,0.0001925608,0.0001814142,0.00006667955,0.00002875395,0.003945779,0.00003291727,0.000001435403],"category_scores_gemma":[0.00007565413,0.0001031526,0.00006636792,0.0008809985,0.00005939749,0.0007582035,0.003621084,0.00006639669,8.554737e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001340415,"about_ca_system_score_gemma":0.000005209152,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002220491,"about_ca_topic_score_gemma":7.947879e-7,"domain_scores_codex":[0.9988128,0.00000867979,0.0003436856,0.0003690156,0.0002909776,0.0001748253],"domain_scores_gemma":[0.9981825,0.00008908675,0.0004037564,0.001139842,0.0001620451,0.00002277489],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005910853,0.0002930654,0.002184746,0.00435893,0.000354902,9.069694e-7,0.0004049941,0.003033863,0.04857857,0.7534453,0.07105367,0.116232],"study_design_scores_gemma":[0.0008526017,0.0003081538,0.02723791,0.001103665,0.0001105794,0.000001083867,0.0001801299,0.8777387,0.02010193,0.07105391,0.001047971,0.0002633825],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9802012,0.0002187125,0.01508426,0.001200575,0.0007021718,0.001702162,0.00008309697,0.0001944579,0.0006133682],"genre_scores_gemma":[0.8480173,0.0002343549,0.1514501,0.00004335195,0.00003987854,0.00004132933,0.00008456816,0.00001594289,0.00007315879],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8747048,"threshold_uncertainty_score":0.7332304,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1019414078019133,"score_gpt":0.3440120252621289,"score_spread":0.2420706174602156,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}