Initialize audio on Strudel on first user interaction (keydown/mouse) (…

…#294)

Handle mouse and keydown events from the `WebTargetIframe` component and
send a message to the iframe when they occur. This allows
`StrudelWrapper` to initialize audio immediately after detecting user
interaction, ensuring the audio worklet is loaded correctly.

Fixes #286

packages/web/src/components/web-target-iframe.tsx

@@ -37,6 +37,23 @@ export const WebTargetIframe = ({ target, session }: WebTargetIframeProps) => {
     };
   }, [session, ref]);
 
+  // Handle user interactions
+  useEffect(() => {
+    const handleUserInteraction = () => {
+      if (!ref.current) return;
+      const interactionMessage = { type: "user-interaction" };
+      ref.current.contentWindow?.postMessage(interactionMessage, "*");
+    };
+
+    window.addEventListener("click", handleUserInteraction);
+    window.addEventListener("keydown", handleUserInteraction);
+
+    return () => {
+      window.removeEventListener("click", handleUserInteraction);
+      window.removeEventListener("keydown", handleUserInteraction);
+    };
+  }, [ref]);
+
   return (
     <iframe
       ref={ref}

packages/web/src/lib/strudel-wrapper.ts

@@ -13,7 +13,7 @@ import { registerSoundfonts } from "@strudel/soundfonts";
 import { transpiler } from "@strudel/transpiler";
 import {
   getAudioContext,
-  initAudioOnFirstClick,
+  initAudio,
   registerSynthSounds,
   samples,
   webaudioOutput,
@@ -31,6 +31,7 @@ export class StrudelWrapper {
   protected _onWarning: ErrorHandler;
   protected _repl: any;
   protected _docPatterns: any;
+  protected _audioInitialized: boolean;
   protected framer?: any;
 
   constructor({
@@ -43,10 +44,10 @@ export class StrudelWrapper {
     this._docPatterns = {};
     this._onError = onError || (() => {});
     this._onWarning = onWarning || (() => {});
+    this._audioInitialized = false;
   }
 
   async importModules() {
-    initAudioOnFirstClick();
     // import desired modules and add them to the eval scope
     await evalScope(
       import("@strudel/core"),
@@ -70,6 +71,12 @@ export class StrudelWrapper {
     }
   }
 
+  async initAudio() {
+    if (this._audioInitialized) return;
+    await initAudio();
+    this._audioInitialized = true;
+  }
+
   async initialize() {
     if (this.initialized) return;
 

packages/web/src/routes/frames/strudel.tsx

@@ -25,6 +25,22 @@ export function Component() {
     })();
   }, []);
 
+  useEffect(() => {
+    if (!instance) return;
+
+    const handleWindowMessage = async (event: MessageEvent) => {
+      if (event.data.type === "user-interaction") {
+        await instance.initAudio();
+      }
+    };
+
+    window.addEventListener("message", handleWindowMessage);
+
+    return () => {
+      window.removeEventListener("message", handleWindowMessage);
+    };
+  }, [instance]);
+
   useEvalHandler(
     useCallback(
       (msg: EvalMessage) => {