feat(screenshare): Audio only screenshare (#8922)

* audio only screen share implementation

* clean up

* handle stop screen share from chrome window

* update icon

conference.js

@@ -6,6 +6,7 @@ import Logger from 'jitsi-meet-logger';
 
 import { openConnection } from './connection';
 import { ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants';
+import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from './modules/UI/UIErrors';
 import AuthHandler from './modules/UI/authentication/AuthHandler';
 import UIUtil from './modules/UI/util/UIUtil';
 import mediaDeviceHelper from './modules/devices/mediaDeviceHelper';
@@ -126,6 +127,7 @@ import {
     makePrecallTest
 } from './react/features/prejoin';
 import { disableReceiver, stopReceiver } from './react/features/remote-control';
+import { setScreenAudioShareState, isScreenAudioShared } from './react/features/screen-share/';
 import { toggleScreenshotCaptureEffect } from './react/features/screenshot-capture';
 import { setSharedVideoStatus } from './react/features/shared-video/actions';
 import { AudioMixerEffect } from './react/features/stream-effects/audio-mixer/AudioMixerEffect';
@@ -1546,6 +1548,8 @@ export default {
             this._desktopAudioStream = undefined;
         }
 
+        APP.store.dispatch(setScreenAudioShareState(false));
+
         if (didHaveVideo) {
             promise = promise.then(() => createLocalTracksF({ devices: [ 'video' ] }))
                 .then(([ stream ]) => {
@@ -1662,6 +1666,23 @@ export default {
                 = this._turnScreenSharingOff.bind(this, didHaveVideo);
 
             const desktopVideoStream = desktopStreams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+            const dekstopAudioStream = desktopStreams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
+
+            if (dekstopAudioStream) {
+                dekstopAudioStream.on(
+                    JitsiTrackEvents.LOCAL_TRACK_STOPPED,
+                    () => {
+                        logger.debug(`Local screensharing audio track stopped. ${this.isSharingScreen}`);
+
+                        // Handle case where screen share was stopped from  the browsers 'screen share in progress'
+                        // window. If audio screen sharing is stopped via the normal UX flow this point shouldn't
+                        // be reached.
+                        isScreenAudioShared(APP.store.getState())
+                            && this._untoggleScreenSharing
+                            && this._untoggleScreenSharing();
+                    }
+                );
+            }
 
             if (desktopVideoStream) {
                 desktopVideoStream.on(
@@ -1830,14 +1851,28 @@ export default {
 
         return this._createDesktopTrack(options)
             .then(async streams => {
-                const desktopVideoStream = streams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+                let desktopVideoStream = streams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+
+                this._desktopAudioStream = streams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
+
+                const { audioOnly = false } = options;
+
+                // If we're in audio only mode dispose of the video track otherwise the screensharing state will be
+                // inconsistent.
+                if (audioOnly) {
+                    desktopVideoStream.dispose();
+                    desktopVideoStream = undefined;
+
+                    if (!this._desktopAudioStream) {
+                        return Promise.reject(AUDIO_ONLY_SCREEN_SHARE_NO_TRACK);
+                    }
+                }
 
                 if (desktopVideoStream) {
                     logger.debug(`_switchToScreenSharing is using ${desktopVideoStream} for useVideoStream`);
                     await this.useVideoStream(desktopVideoStream);
                 }
 
-                this._desktopAudioStream = streams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
 
                 if (this._desktopAudioStream) {
                     // If there is a localAudio stream, mix in the desktop audio stream captured by the screen sharing
@@ -1850,7 +1885,9 @@ export default {
                         // If no local stream is present ( i.e. no input audio devices) we use the screen share audio
                         // stream as we would use a regular stream.
                         await this.useAudioStream(this._desktopAudioStream);
+
                     }
+                    APP.store.dispatch(setScreenAudioShareState(true));
                 }
             })
             .then(() => {
@@ -1918,6 +1955,9 @@ export default {
         } else if (error.name === JitsiTrackErrors.SCREENSHARING_GENERIC_ERROR) {
             descriptionKey = 'dialog.screenSharingFailed';
             titleKey = 'dialog.screenSharingFailedTitle';
+        } else if (error === AUDIO_ONLY_SCREEN_SHARE_NO_TRACK) {
+            descriptionKey = 'notify.screenShareNoAudio';
+            titleKey = 'notify.screenShareNoAudioTitle';
         }
 
         APP.UI.messageHandler.showError({
@@ -2409,7 +2449,9 @@ export default {
         });
 
         APP.UI.addListener(
-            UIEvents.TOGGLE_SCREENSHARING, this.toggleScreenSharing.bind(this)
+            UIEvents.TOGGLE_SCREENSHARING, audioOnly => {
+                this.toggleScreenSharing(undefined, { audioOnly });
+            }
         );
 
         /* eslint-disable max-params */

config.js

@@ -428,7 +428,7 @@ var config = {
     // toolbarButtons: [
     //    'microphone', 'camera', 'closedcaptions', 'desktop', 'embedmeeting', 'fullscreen',
     //    'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
-    //    'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
+    //    'livestreaming', 'etherpad', 'sharedvideo', 'shareaudio', 'settings', 'raisehand',
     //    'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
     //    'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
     // ],

lang/main.json

@@ -511,6 +511,8 @@
         "passwordRemovedRemotely": "$t(lockRoomPasswordUppercase) removed by another participant",
         "passwordSetRemotely": "$t(lockRoomPasswordUppercase) set by another participant",
         "raisedHand": "{{name}} would like to speak.",
+        "screenShareNoAudio": " Share audio box was not checked in the window selection screen.",
+        "screenShareNoAudioTitle": "Share audio was not chcked",
         "somebody": "Somebody",
         "startSilentTitle": "You joined with no audio output!",
         "startSilentDescription": "Rejoin the meeting to enable audio",
@@ -752,6 +754,7 @@
             "remoteVideoMute": "Disable camera of participant",
             "security": "Security options",
             "Settings": "Toggle settings",
+            "shareaudio": "Share audio",
             "sharedvideo": "Toggle YouTube video sharing",
             "shareRoom": "Invite someone",
             "shareYourScreen": "Toggle screenshare",
@@ -811,6 +814,7 @@
         "raiseYourHand": "Raise your hand",
         "security": "Security options",
         "Settings": "Settings",
+        "shareaudio": "Share audio",
         "sharedvideo": "Share a YouTube video",
         "shareRoom": "Invite someone",
         "shortcuts": "View shortcuts",

modules/UI/UIErrors.js

@@ -8,3 +8,10 @@
  * @type {{FEEDBACK_REQUEST_IN_PROGRESS: string}}
  */
 export const FEEDBACK_REQUEST_IN_PROGRESS = 'FeedbackRequestInProgress';
+
+/**
+ * Indicated an attempted audio only screen share session with no audio track present
+ *
+ * @type {{AUDIO_ONLY_SCREEN_SHARE_NO_TRACK: string}}
+ */
+export const AUDIO_ONLY_SCREEN_SHARE_NO_TRACK = 'AudioOnlyScreenShareNoTrack';

react/features/app/reducers.any.js

@@ -44,6 +44,7 @@ import '../recent-list/reducer';
 import '../recording/reducer';
 import '../settings/reducer';
 import '../subtitles/reducer';
+import '../screen-share/reducer';
 import '../toolbox/reducer';
 import '../transcribing/reducer';
 import '../video-layout/reducer';

react/features/base/conference/middleware.web.js

@@ -32,7 +32,7 @@ MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
     }
     case TOGGLE_SCREENSHARING: {
         if (typeof APP === 'object') {
-            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING);
+            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING, action.audioOnly);
         }
 
         break;

react/features/base/config/constants.js

@@ -16,7 +16,7 @@ export const _CONFIG_STORE_PREFIX = 'config.js';
 export const TOOLBAR_BUTTONS = [
     'microphone', 'camera', 'closedcaptions', 'desktop', 'embedmeeting', 'fullscreen',
     'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
-    'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
+    'livestreaming', 'etherpad', 'sharedvideo', 'shareaudio', 'settings', 'raisehand',
     'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
     'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone',
     'security', 'toggle-camera'

react/features/base/environment/environment.js

@@ -41,6 +41,15 @@ export function isBrowsersOptimal(browserName: string) {
         .includes(browserName);
 }
 
+/**
+ * Returns whether or not the current OS is Mac.
+ *
+ * @returns {boolean}
+ */
+export function isMacOS() {
+    return Platform.OS === 'macos';
+}
+
 /**
  * Returns whether or not the current browser or the list of passed in browsers
  * is considered suboptimal. Suboptimal means it is a supported browser but has

react/features/base/icons/svg/index.js

@@ -99,6 +99,7 @@ export { default as IconSignalLevel0 } from './signal_cellular_0.svg';
 export { default as IconSignalLevel1 } from './signal_cellular_1.svg';
 export { default as IconSignalLevel2 } from './signal_cellular_2.svg';
 export { default as IconShare } from './share.svg';
+export { default as IconShareAudio } from './share-audio.svg';
 export { default as IconShareDesktop } from './share-desktop.svg';
 export { default as IconShareDoc } from './share-doc.svg';
 export { default as IconShareVideo } from './shared-video.svg';

react/features/base/tracks/actions.js

@@ -257,13 +257,16 @@ export function showNoDataFromSourceVideoError(jitsiTrack) {
  * Signals that the local participant is ending screensharing or beginning the
  * screensharing flow.
  *
+ * @param {boolean} audioOnly - Only share system audio.
  * @returns {{
  *     type: TOGGLE_SCREENSHARING,
+ *     audioOnly: boolean
  * }}
  */
-export function toggleScreensharing() {
+export function toggleScreensharing(audioOnly = false) {
     return {
-        type: TOGGLE_SCREENSHARING
+        type: TOGGLE_SCREENSHARING,
+        audioOnly
     };
 }
 

react/features/base/tracks/middleware.js

@@ -135,7 +135,7 @@ MiddlewareRegistry.register(store => next => action => {
 
     case TOGGLE_SCREENSHARING:
         if (typeof APP === 'object') {
-            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING);
+            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING, action.audioOnly);
         }
         break;
 

react/features/screen-share/actionTypes.js

@@ -0,0 +1,12 @@
+// @flow
+
+/**
+ * Type of action which sets the current state of screen audio sharing.
+ *
+ * {
+ *     type: SET_SCREEN_AUDIO_SHARE_STATE,
+ *     isSharingAudio: boolean
+ * }
+ */
+export const SET_SCREEN_AUDIO_SHARE_STATE = 'SET_SCREEN_AUDIO_SHARE_STATE';
+

react/features/screen-share/actions.js

@@ -0,0 +1,19 @@
+// @flow
+
+import { SET_SCREEN_AUDIO_SHARE_STATE } from './actionTypes';
+
+/**
+ * Updates the current known status of the shared video.
+ *
+ * @param {boolean} isSharingAudio - Is audio currently being shared or not.
+ * @returns {{
+ *     type: SET_SCREEN_AUDIO_SHARE_STATE,
+ *     isSharingAudio: boolean
+ * }}
+ */
+export function setScreenAudioShareState(isSharingAudio: boolean) {
+    return {
+        type: SET_SCREEN_AUDIO_SHARE_STATE,
+        isSharingAudio
+    };
+}

react/features/screen-share/functions.js

@@ -0,0 +1,25 @@
+// @flow
+
+import { isMacOS } from '../base/environment';
+import { browser } from '../base/lib-jitsi-meet';
+
+
+/**
+ * State of audio sharing.
+ *
+ * @param {Object} state - The state of the application.
+ * @returns {boolean}
+ */
+export function isScreenAudioShared(state: Object) {
+    return state['features/screen-share'].isSharingAudio;
+}
+
+/**
+ * Returns the visibility of the audio only screen share button. Currently electron on mac os doesn't
+ * have support for this functionality.
+ *
+ * @returns {boolean}
+ */
+export function isScreenAudioSupported() {
+    return !(browser.isElectron() && isMacOS());
+}

react/features/screen-share/index.js

@@ -0,0 +1,4 @@
+export * from './actions';
+export * from './actionTypes';
+export * from './functions';
+

react/features/screen-share/reducer.js

@@ -0,0 +1,22 @@
+
+import { ReducerRegistry } from '../base/redux';
+
+import { SET_SCREEN_AUDIO_SHARE_STATE } from './actionTypes';
+
+/**
+ * Reduces the Redux actions of the feature features/screen-share.
+ */
+ReducerRegistry.register('features/screen-share', (state = {}, action) => {
+    const { isSharingAudio } = action;
+
+    switch (action.type) {
+    case SET_SCREEN_AUDIO_SHARE_STATE:
+        return {
+            ...state,
+            isSharingAudio
+        };
+
+    default:
+        return state;
+    }
+});

react/features/stream-effects/audio-mixer/AudioMixerEffect.js

@@ -14,6 +14,16 @@ export class AudioMixerEffect {
      */
     _mixAudio: Object;
 
+    /**
+     * MediaStream resulted from mixing.
+     */
+    _mixedMediaStream: Object;
+
+    /**
+     * MediaStreamTrack obtained from mixed stream.
+     */
+    _mixedMediaTrack: Object;
+
     /**
      * Original MediaStream from the JitsiLocalTrack that uses this effect.
      */
@@ -68,7 +78,14 @@ export class AudioMixerEffect {
         this._audioMixer.addMediaStream(this._mixAudio.getOriginalStream());
         this._audioMixer.addMediaStream(this._originalStream);
 
-        return this._audioMixer.start();
+        this._mixedMediaStream = this._audioMixer.start();
+        this._mixedMediaTrack = this._mixedMediaStream.getTracks()[0];
+
+        // Sync the resulting mixed track enabled state with that of the track using the effect.
+        this.setMuted(!this._originalTrack.enabled);
+        this._originalTrack.enabled = true;
+
+        return this._mixedMediaStream;
     }
 
     /**
@@ -77,6 +94,9 @@ export class AudioMixerEffect {
      * @returns {void}
      */
     stopEffect() {
+        // Match state of the original track with that of the mixer track, not doing so can
+        // result in an inconsistent state e.g. redux state is muted yet track is enabled.
+        this._originalTrack.enabled = this._mixedMediaTrack.enabled;
         this._audioMixer.reset();
     }
 
@@ -87,7 +107,7 @@ export class AudioMixerEffect {
      * @returns {void}
      */
     setMuted(muted: boolean) {
-        this._originalTrack.enabled = !muted;
+        this._mixedMediaTrack.enabled = !muted;
     }
 
     /**
@@ -96,6 +116,6 @@ export class AudioMixerEffect {
      * @returns {boolean}
      */
     isMuted() {
-        return !this._originalTrack.enabled;
+        return !this._mixedMediaTrack.enabled;
     }
 }