{"id":"W4396832321","doi":"10.1145/3613904.3642068","title":"OmniActions: Predicting Digital Actions in Response to Real-World Multimodal Sensory Inputs with LLMs","year":2024,"lang":"en","type":"article","venue":"","topic":"Tactile and Sensory Interactions","field":"Neuroscience","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Computer science; Human–computer interaction; Pipeline (software); Context (archaeology); Action (physics); Multimodal interaction; Task (project management); Multimodality; Interaction design; Multimedia; World Wide Web; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008601787,0.001402745,0.0004118662,0.0006866251,0.0001708923,0.0006821121,0.001058202,0.000622928,0.003927552],"category_scores_gemma":[0.003904766,0.0004477862,0.0007407961,0.0001827776,0.0003995878,0.0009384101,0.001097215,0.0007609167,0.0007859085],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006117749,"about_ca_system_score_gemma":0.000757599,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003917269,"about_ca_topic_score_gemma":0.00793045,"domain_scores_codex":[0.9996529,0.00008018696,0.00002462123,0.0001292013,0.00008254519,0.00003045746],"domain_scores_gemma":[0.9987715,0.0008288674,0.0001141255,0.00009778573,0.0001204808,0.00006719529],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00191528,0.0009110486,0.02768906,0.001416632,0.00024564,0.000511541,0.002161408,0.2497996,0.09694801,0.004870426,0.006870338,0.606661],"study_design_scores_gemma":[0.00003561099,0.0004987802,0.002998234,0.00003850852,0.00003865397,0.0000859649,0.0001373259,0.9766962,0.01484986,0.001970899,0.002615239,0.00003470237],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2044459,0.0003585922,0.7615342,0.0001630137,0.00006848112,0.0007375095,0.001447813,0.02823795,0.003006585],"genre_scores_gemma":[0.6119208,0.0001554277,0.3817669,0.0001215385,0.00001331306,0.0008337139,0.001755072,0.0003817079,0.003051494],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.003927552,"threshold_uncertainty_score":0.01313901,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04747479545012518,"score_gpt":0.3295433053003257,"score_spread":0.2820685098502005,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}