{"id":"W4396832321","doi":"10.1145/3613904.3642068","title":"OmniActions: Predicting Digital Actions in Response to Real-World Multimodal Sensory Inputs with LLMs","year":2024,"lang":"en","type":"article","venue":"","topic":"Tactile and Sensory Interactions","field":"Neuroscience","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Computer science; Human–computer interaction; Pipeline (software); Context (archaeology); Action (physics); Multimodal interaction; Task (project management); Multimodality; Interaction design; Multimedia; World Wide Web; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00007434838,0.0001906723,0.0001490772,0.0008912913,0.0002183632,0.0004328301,0.0001252843,0.00005398127,0.0002893957],"category_scores_gemma":[0.0005855294,0.00015959,0.00006179469,0.001288518,0.00005882353,0.001252169,0.00004925831,0.000487745,0.0009647215],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002472776,"about_ca_system_score_gemma":0.0001151162,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005466772,"about_ca_topic_score_gemma":0.002486093,"domain_scores_codex":[0.9983665,0.000117756,0.0002916764,0.0005915045,0.0002734391,0.0003590738],"domain_scores_gemma":[0.9978584,0.001588415,0.00003350024,0.0003111107,0.00003761229,0.0001709517],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"not_applicable","study_design_scores_codex":[0.001941917,0.0005316354,0.00410218,0.00002878216,0.00003041677,0.001142863,0.002395064,0.004023736,0.975474,0.001997127,0.003049166,0.005283084],"study_design_scores_gemma":[0.001656202,0.001072999,0.02206263,0.000697289,0.00006415667,0.001860524,0.003915627,0.04095765,0.392007,0.0002314634,0.533995,0.001479453],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.912169,0.000001177349,0.0004230372,0.003673742,0.0007118857,0.0003449203,0.0001066132,0.0007283577,0.08184124],"genre_scores_gemma":[0.9167333,0.000007308647,0.0001959823,0.0004707563,0.0001176059,0.00006709087,0.000003502232,0.00003644785,0.08236796],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.583467,"threshold_uncertainty_score":0.9998131,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04747479545012518,"score_gpt":0.3295433053003257,"score_spread":0.2820685098502005,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}