From 7775bbafd5a3bdb44d4fd3814fd24cafac655b37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dami=C3=A1n=20Silvani?= <munshkr@users.noreply.github.com>
Date: Sun, 15 Sep 2024 10:08:15 -0300
Subject: [PATCH] Initialize audio on Strudel on first user interaction
 (keydown/mouse) (#294)

Handle mouse and keydown events from the `WebTargetIframe` component and
send a message to the iframe when they occur. This allows
`StrudelWrapper` to initialize audio immediately after detecting user
interaction, ensuring the audio worklet is loaded correctly.

Fixes #286
---
 .../web/src/components/web-target-iframe.tsx    | 17 +++++++++++++++++
 packages/web/src/lib/strudel-wrapper.ts         | 11 +++++++++--
 packages/web/src/routes/frames/strudel.tsx      | 16 ++++++++++++++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/packages/web/src/components/web-target-iframe.tsx b/packages/web/src/components/web-target-iframe.tsx
index 2341c25b..1a9c829b 100644
--- a/packages/web/src/components/web-target-iframe.tsx
+++ b/packages/web/src/components/web-target-iframe.tsx
@@ -37,6 +37,23 @@ export const WebTargetIframe = ({ target, session }: WebTargetIframeProps) => {
     };
   }, [session, ref]);
 
+  // Handle user interactions
+  useEffect(() => {
+    const handleUserInteraction = () => {
+      if (!ref.current) return;
+      const interactionMessage = { type: "user-interaction" };
+      ref.current.contentWindow?.postMessage(interactionMessage, "*");
+    };
+
+    window.addEventListener("click", handleUserInteraction);
+    window.addEventListener("keydown", handleUserInteraction);
+
+    return () => {
+      window.removeEventListener("click", handleUserInteraction);
+      window.removeEventListener("keydown", handleUserInteraction);
+    };
+  }, [ref]);
+
   return (
     <iframe
       ref={ref}
diff --git a/packages/web/src/lib/strudel-wrapper.ts b/packages/web/src/lib/strudel-wrapper.ts
index c5128888..acb4e530 100644
--- a/packages/web/src/lib/strudel-wrapper.ts
+++ b/packages/web/src/lib/strudel-wrapper.ts
@@ -13,7 +13,7 @@ import { registerSoundfonts } from "@strudel/soundfonts";
 import { transpiler } from "@strudel/transpiler";
 import {
   getAudioContext,
-  initAudioOnFirstClick,
+  initAudio,
   registerSynthSounds,
   samples,
   webaudioOutput,
@@ -31,6 +31,7 @@ export class StrudelWrapper {
   protected _onWarning: ErrorHandler;
   protected _repl: any;
   protected _docPatterns: any;
+  protected _audioInitialized: boolean;
   protected framer?: any;
 
   constructor({
@@ -43,10 +44,10 @@ export class StrudelWrapper {
     this._docPatterns = {};
     this._onError = onError || (() => {});
     this._onWarning = onWarning || (() => {});
+    this._audioInitialized = false;
   }
 
   async importModules() {
-    initAudioOnFirstClick();
     // import desired modules and add them to the eval scope
     await evalScope(
       import("@strudel/core"),
@@ -70,6 +71,12 @@ export class StrudelWrapper {
     }
   }
 
+  async initAudio() {
+    if (this._audioInitialized) return;
+    await initAudio();
+    this._audioInitialized = true;
+  }
+
   async initialize() {
     if (this.initialized) return;
 
diff --git a/packages/web/src/routes/frames/strudel.tsx b/packages/web/src/routes/frames/strudel.tsx
index 796f82ba..229b6938 100644
--- a/packages/web/src/routes/frames/strudel.tsx
+++ b/packages/web/src/routes/frames/strudel.tsx
@@ -25,6 +25,22 @@ export function Component() {
     })();
   }, []);
 
+  useEffect(() => {
+    if (!instance) return;
+
+    const handleWindowMessage = async (event: MessageEvent) => {
+      if (event.data.type === "user-interaction") {
+        await instance.initAudio();
+      }
+    };
+
+    window.addEventListener("message", handleWindowMessage);
+
+    return () => {
+      window.removeEventListener("message", handleWindowMessage);
+    };
+  }, [instance]);
+
   useEvalHandler(
     useCallback(
       (msg: EvalMessage) => {