Main to dev backmerge

examples/prebuilt-react-integration/src/App.jsx

@@ -4,5 +4,7 @@ import { getRoomCodeFromUrl } from './utils';
 export default function App() {
   const roomCode = getRoomCodeFromUrl();
 
-  return <HMSPrebuilt roomCode={roomCode} />;
+  return (
+    <HMSPrebuilt roomCode={roomCode} />
+  );
 }

packages/hms-video-store/src/IHMSActions.ts

@@ -1,3 +1,4 @@
+import { TranscriptionConfig } from './interfaces/transcription-config';
 import {
   HLSConfig,
   HLSTimedMetadata,
@@ -371,6 +372,18 @@ export interface IHMSActions<T extends HMSGenericTypes = { sessionStore: Record<
    */
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
 
+  /**
+   * If you want to start transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription. Currently we have Caption mode only
+   */
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+
+  /**
+   * If you want to stop transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription you want to stop. Currently we have Caption mode only
+   */
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
+
   /**
    * @alpha
    * Used to define date range metadata in a media playlist.

packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts

@@ -56,7 +56,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
       const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats) =>
         trackStats.jitterBufferDelay &&
         trackStats.jitterBufferEmittedCount &&
-        trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount;
+        (trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount) * 1000;
 
       const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats);
 

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -5,11 +5,11 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSAction } from '../error/HMSAction';
 import { EventBus } from '../events/EventBus';
 import { HMSDeviceChangeEvent, HMSTrackUpdate, HMSUpdateListener } from '../interfaces';
+import { isMobile } from '../internal';
 import { HMSRemoteAudioTrack } from '../media/tracks';
 import { HMSRemotePeer } from '../sdk/models/peer';
 import { Store } from '../sdk/store';
 import HMSLogger from '../utils/logger';
-import { isMobile } from '../utils/support';
 import { sleep } from '../utils/timer-utils';
 
 /**
@@ -41,6 +41,7 @@ export class AudioSinkManager {
   private state = { ...INITIAL_STATE };
   private listener?: HMSUpdateListener;
   private timer: ReturnType<typeof setInterval> | null = null;
+  private autoUnpauseTimer: ReturnType<typeof setInterval> | null = null;
   private earpieceSelected = false;
 
   constructor(private store: Store, private deviceManager: DeviceManager, private eventBus: EventBus) {
@@ -49,6 +50,7 @@ export class AudioSinkManager {
     this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate);
     this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange);
     this.startPollingForDevices();
+    this.startPollingToCheckPausedAudio();
   }
 
   setListener(listener?: HMSUpdateListener) {
@@ -100,6 +102,10 @@ export class AudioSinkManager {
       clearInterval(this.timer);
       this.timer = null;
     }
+    if (this.autoUnpauseTimer) {
+      clearInterval(this.autoUnpauseTimer);
+      this.autoUnpauseTimer = null;
+    }
     this.eventBus.audioTrackAdded.unsubscribe(this.handleTrackAdd);
     this.eventBus.audioTrackRemoved.unsubscribe(this.handleTrackRemove);
     this.eventBus.audioTrackUpdate.unsubscribe(this.handleTrackUpdate);
@@ -109,24 +115,11 @@ export class AudioSinkManager {
   }
 
   private handleAudioPaused = async (event: any) => {
-    const audioEl = event.target as HTMLAudioElement;
-    //@ts-ignore
-    const track = audioEl.srcObject?.getAudioTracks()[0];
-    if (!track?.enabled) {
-      // No need to play if already disabled
-      return;
-    }
-    // this means the audio paused because of external factors(headset removal)
+    // this means the audio paused because of external factors(headset removal, incoming phone call)
     HMSLogger.d(this.TAG, 'Audio Paused', event.target.id);
     const audioTrack = this.store.getTrackById(event.target.id);
     if (audioTrack) {
-      if (isMobile()) {
-        // Play after a delay since mobile devices don't call onDevice change event
-        await sleep(500);
-        this.playAudioFor(audioTrack as HMSRemoteAudioTrack);
-      } else {
-        this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack);
-      }
+      this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack);
     }
   };
 
@@ -272,6 +265,14 @@ export class AudioSinkManager {
     }
   };
 
+  private startPollingToCheckPausedAudio = () => {
+    if (isMobile()) {
+      this.autoUnpauseTimer = setInterval(() => {
+        this.unpauseAudioTracks();
+      }, 5000);
+    }
+  };
+
   private startPollingForDevices = () => {
     // device change supported, no polling needed
     if ('ondevicechange' in navigator.mediaDevices) {
@@ -281,6 +282,7 @@ export class AudioSinkManager {
       (async () => {
         await this.deviceManager.init(true, false);
         await this.autoSelectAudioOutput();
+        this.unpauseAudioTracks();
       })();
     }, 5000);
   };

packages/hms-video-store/src/interfaces/hms.ts

@@ -12,6 +12,7 @@ import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room';
 import { RTMPRecordingConfig } from './rtmp-recording-config';
 import { HMSInteractivityCenter, HMSSessionStore } from './session-store';
 import { HMSScreenShareConfig } from './track-settings';
+import { TranscriptionConfig } from './transcription-config';
 import { HMSAudioListener, HMSConnectionQualityListener, HMSUpdateListener } from './update-listener';
 import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel';
 import { IAudioOutputManager } from '../device-manager/AudioOutputManager';
@@ -61,6 +62,8 @@ export interface HMSInterface {
    */
   startHLSStreaming(params?: HLSConfig): Promise<void>;
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
   getRecordingState(): HMSRecording | undefined;
   getRTMPState(): HMSRTMP | undefined;
   getHLSState(): HMSHLS | undefined;

packages/hms-video-store/src/interfaces/index.ts

@@ -19,3 +19,4 @@ export * from './webrtc-stats';
 export * from './framework-info';
 export * from './get-token';
 export * from './session-store';
+export * from './transcription-config';

packages/hms-video-store/src/interfaces/room.ts

@@ -123,6 +123,7 @@ export interface HLSVariant {
 Transcription related details
 */
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,4 +134,9 @@ export enum HMSTranscriptionMode {
 export interface HMSTranscriptionInfo {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: Date;
+  started_at?: Date;
+  updated_at?: Date;
+  stopped_at?: Date;
+  error?: HMSException;
 }

packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts

@@ -20,6 +20,7 @@ export interface HMSInteractivityCenter {
   stopPoll(pollID: string): Promise<void>;
   addResponsesToPoll(pollID: string, responses: HMSPollQuestionResponseCreateParams[]): Promise<void>;
   fetchLeaderboard(pollID: string, offset: number, count: number): Promise<HMSQuizLeaderboardResponse>;
+  getPollResponses(poll: HMSPoll, self: boolean): Promise<void>;
   getPolls(): Promise<HMSPoll[]>;
   /** @alpha */
   whiteboard: HMSWhiteboardInteractivityCenter;

packages/hms-video-store/src/interfaces/transcription-config.ts

@@ -0,0 +1,5 @@
+import { HMSTranscriptionMode } from './room';
+
+export interface TranscriptionConfig {
+  mode: HMSTranscriptionMode;
+}

packages/hms-video-store/src/interfaces/update-listener.ts

@@ -17,6 +17,7 @@ export enum HMSRoomUpdate {
   SERVER_RECORDING_STATE_UPDATED = 'SERVER_RECORDING_STATE_UPDATED',
   RTMP_STREAMING_STATE_UPDATED = 'RTMP_STREAMING_STATE_UPDATED',
   HLS_STREAMING_STATE_UPDATED = 'HLS_STREAMING_STATE_UPDATED',
+  TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED',
   ROOM_PEER_COUNT_UPDATED = 'ROOM_PEER_COUNT_UPDATED',
 }
 
@@ -54,7 +55,6 @@ export enum HMSPollsUpdate {
   POLL_STOPPED,
   POLLS_LIST,
   POLL_STATS_UPDATED,
-  // POLL_LEADERBOARD_SHARED,
 }
 
 export interface HMSAudioListener {

packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts

@@ -493,7 +493,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
       this.replaceSenderTrack(this.nativeTrack);
     } else {
       this.nativeTrack.enabled = this.enabledStateBeforeBackground;
-      this.replaceSenderTrack(this.nativeTrack);
+      this.replaceSenderTrack(this.processedTrack || this.nativeTrack);
     }
     this.eventBus.localVideoEnabled.publish({ enabled: this.nativeTrack.enabled, track: this });
   };

packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts

@@ -69,6 +69,7 @@ export class HMSVideoTrack extends HMSTrack {
       if (existingTrack?.id === track.id) {
         if (!existingTrack.muted && existingTrack.readyState === 'live') {
           // it's already attached, attaching again would just cause flickering
+          this.reTriggerPlay({ videoElement, stream: srcObject });
           return;
         } else {
           this.reduceSinkCount();
@@ -77,10 +78,22 @@ export class HMSVideoTrack extends HMSTrack {
         this.reduceSinkCount();
       }
     }
-    videoElement.srcObject = new MediaStream([track]);
+    const stream = new MediaStream([track]);
+    videoElement.srcObject = stream;
+    this.reTriggerPlay({ videoElement, stream });
     this.sinkCount++;
   }
 
+  private reTriggerPlay = ({ videoElement, stream }: { videoElement: HTMLVideoElement; stream: MediaStream }) => {
+    setTimeout(() => {
+      if (videoElement.paused) {
+        // This is needed for safari and firefox to work properly
+        videoElement.srcObject = stream;
+        videoElement.play().catch(console.error);
+      }
+    }, 0);
+  };
+
   private reduceSinkCount() {
     if (this.sinkCount > 0) {
       this.sinkCount--;

packages/hms-video-store/src/media/tracks/VideoElementManager.ts

@@ -5,7 +5,7 @@ import { HMSLocalVideoTrack, HMSRemoteVideoTrack } from '../../internal';
 import { HMSIntersectionObserver } from '../../utils/intersection-observer';
 import HMSLogger from '../../utils/logger';
 import { HMSResizeObserver } from '../../utils/resize-observer';
-import { isBrowser } from '../../utils/support';
+import { isBrowser, isMobile } from '../../utils/support';
 
 /**
  * This class is to manager video elements for video tracks.
@@ -45,6 +45,7 @@ export class VideoElementManager {
     this.init();
     HMSLogger.d(this.TAG, `Adding video element for ${this.track}`, this.id);
     this.videoElements.add(videoElement);
+    videoElement.addEventListener('pause', this.resumeVideoPlayback);
     if (this.videoElements.size >= 10) {
       HMSLogger.w(
         this.TAG,
@@ -72,6 +73,7 @@ export class VideoElementManager {
   removeVideoElement(videoElement: HTMLVideoElement): void {
     this.track.removeSink(videoElement);
     this.videoElements.delete(videoElement);
+    videoElement.removeEventListener('pause', this.resumeVideoPlayback);
     this.entries.delete(videoElement);
     this.resizeObserver?.unobserve(videoElement);
     this.intersectionObserver?.unobserve(videoElement);
@@ -82,6 +84,14 @@ export class VideoElementManager {
     return Array.from(this.videoElements);
   }
 
+  private resumeVideoPlayback = (e: Event) => {
+    const element = e.target as HTMLVideoElement;
+    if (!document.hidden && isMobile() && element.paused) {
+      // add sink again to play the video specially in safari iOS
+      this.track.addSink(element);
+    }
+  };
+
   private init() {
     if (isBrowser) {
       this.resizeObserver = HMSResizeObserver;
@@ -172,6 +182,7 @@ export class VideoElementManager {
   cleanup = () => {
     this.videoElements.forEach(videoElement => {
       videoElement.srcObject = null;
+      videoElement.removeEventListener('pause', this.resumeVideoPlayback);
       this.resizeObserver?.unobserve(videoElement);
       this.intersectionObserver?.unobserve(videoElement);
     });

packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts

@@ -28,6 +28,7 @@ export enum HMSNotificationMethod {
   RTMP_UPDATE = 'on-rtmp-update',
   RECORDING_UPDATE = 'on-record-update',
   HLS_UPDATE = 'on-hls-update',
+  TRANSCRIPTION_UPDATE = 'on-transcription-update',
   METADATA_CHANGE = 'on-metadata-change',
   POLL_START = 'on-poll-start',
   POLL_STOP = 'on-poll-stop',

packages/hms-video-store/src/notification-manager/HMSNotifications.ts

@@ -62,6 +62,7 @@ export enum HMSStreamingState {
 }
 
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,6 +134,12 @@ export interface PeerNotification {
 export interface TranscriptionNotification {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: number;
+  started_at?: number;
+  updated_at?: number;
+  stopped_at?: number;
+  peer?: PeerNotificationInfo;
+  error?: ServerError;
 }
 
 export interface RoomState {

packages/hms-video-store/src/notification-manager/managers/PollsManager.ts

@@ -88,8 +88,6 @@ export class PollsManager {
       }
 
       this.updatePollResult(savedPoll, updatedPoll);
-      await this.updatePollResponses(savedPoll, true);
-
       updatedPolls.push(savedPoll);
     }