From 4eb850d24c950c280de7f4761caea31c000dc303 Mon Sep 17 00:00:00 2001 From: levanion Date: Mon, 26 Feb 2024 18:43:51 +0400 Subject: [PATCH 1/3] feat: input and response mode on agent form --- .../src/pages/Agents/AgentForm/AgentForm.tsx | 84 +------------------ .../FormSections/VoicePreferences.tsx | 82 +++++++++++++++++- 2 files changed, 84 insertions(+), 82 deletions(-) diff --git a/apps/ui/src/pages/Agents/AgentForm/AgentForm.tsx b/apps/ui/src/pages/Agents/AgentForm/AgentForm.tsx index 3bd9ff49..57c175f6 100644 --- a/apps/ui/src/pages/Agents/AgentForm/AgentForm.tsx +++ b/apps/ui/src/pages/Agents/AgentForm/AgentForm.tsx @@ -28,6 +28,7 @@ import AgentRunners, { StyledRunnerFieldsWrapper } from './components/AgentRunne import { isVoiceAgent } from 'utils/agentUtils' import VoicePreferences from './FormSections/VoicePreferences' +import RadioButton from 'share-ui/components/RadioButton/RadioButton' type AgentFormProps = { formik: any @@ -52,6 +53,8 @@ const AgentForm = ({ formik, isVoice = true }: AgentFormProps) => { agent_integrations, agent_type, agent_sentiment_analyzer, + agent_voice_response, + agent_voice_input_mode, } = values const { @@ -259,87 +262,6 @@ const AgentForm = ({ formik, isVoice = true }: AgentFormProps) => { voiceSynthesizerOptions={voiceSynthesizerOptions} voiceTranscriberOptions={voiceTranscriberOptions} /> - {/* - - - setFieldValue('agent_voice_response', ['Text'])} - checked={ - agent_voice_response?.length === 1 && agent_voice_response?.includes('Text') - } - /> - setFieldValue('agent_voice_response', ['Voice'])} - checked={ - agent_voice_response?.length === 1 && - agent_voice_response?.includes('Voice') - } - /> - setFieldValue('agent_voice_response', ['Text', 'Voice'])} - checked={agent_voice_response?.length === 2} - /> - */} - - {/* - - - { - if (agent_voice_input_mode?.includes('Text')) { - const filteredInput = agent_voice_input_mode?.filter( - (input: string) => input !== 'Text', - ) - setFieldValue('agent_voice_input_mode', filteredInput) - } else { - setFieldValue('agent_voice_input_mode', [ - ...agent_voice_input_mode, - 'Text', - ]) - } - }} - /> - - - { - if (agent_voice_input_mode?.includes('Voice')) { - const filteredInput = agent_voice_input_mode?.filter( - (input: string) => input !== 'Voice', - ) - setFieldValue('agent_voice_input_mode', filteredInput) - } else { - setFieldValue('agent_voice_input_mode', [ - ...agent_voice_input_mode, - 'Voice', - ]) - } - }} - /> - - */} diff --git a/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx b/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx index eb337b11..f10b8811 100644 --- a/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx +++ b/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx @@ -11,6 +11,11 @@ import { useVoiceOptionsService } from 'plugins/contact/services/voice/useVoiceO import { useModal } from 'hooks' import Loader from 'share-ui/components/Loader/Loader' +import Checkbox from 'share-ui/components/Checkbox/Checkbox' +import TypographyPrimary from 'components/Typography/Primary' +import RadioButton from 'share-ui/components/RadioButton/RadioButton' +import Typography from 'share-ui/components/typography/Typography' + const VoicePreferences = ({ formik, voiceSynthesizerOptions, @@ -24,7 +29,14 @@ const VoicePreferences = ({ const { setFieldValue, values } = formik - const { agent_voice_synthesizer, agent_voice_transcriber, agent_voice_id, agent_type } = values + const { + agent_voice_synthesizer, + agent_voice_transcriber, + agent_voice_id, + agent_type, + agent_voice_response, + agent_voice_input_mode, + } = values const { data: voiceOptions, loading } = useVoiceOptionsService({}) @@ -133,6 +145,74 @@ const VoicePreferences = ({ label='Twilio Phone Number SID' /> )} + + {agent_type === 'text' && ( + <> + + + setFieldValue('agent_voice_response', ['Text'])} + checked={agent_voice_response?.length === 1 && agent_voice_response?.includes('Text')} + /> + setFieldValue('agent_voice_response', ['Voice'])} + checked={agent_voice_response?.length === 1 && agent_voice_response?.includes('Voice')} + /> + setFieldValue('agent_voice_response', ['Text', 'Voice'])} + checked={agent_voice_response?.length === 2} + /> + + + { + if (agent_voice_input_mode?.includes('Text')) { + const filteredInput = agent_voice_input_mode?.filter( + (input: string) => input !== 'Text', + ) + setFieldValue('agent_voice_input_mode', filteredInput) + } else { + setFieldValue('agent_voice_input_mode', [...agent_voice_input_mode, 'Text']) + } + }} + /> + + { + if (agent_voice_input_mode?.includes('Voice')) { + const filteredInput = agent_voice_input_mode?.filter( + (input: string) => input !== 'Voice', + ) + setFieldValue('agent_voice_input_mode', filteredInput) + } else { + setFieldValue('agent_voice_input_mode', [...agent_voice_input_mode, 'Voice']) + } + }} + /> + + )} ) } From fd2ac2208c07911ba12ebcd0454321eb4eefb543 Mon Sep 17 00:00:00 2001 From: levanion Date: Wed, 28 Feb 2024 12:44:43 +0400 Subject: [PATCH 2/3] feat: azure & elevenLabs text to speech --- .../agents/conversational/conversational.py | 4 ++ apps/server/services/voice.py | 56 ++++++++++++++++++- .../FormSections/VoicePreferences.tsx | 5 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/apps/server/agents/conversational/conversational.py b/apps/server/agents/conversational/conversational.py index 4c257255..5c063892 100644 --- a/apps/server/agents/conversational/conversational.py +++ b/apps/server/agents/conversational/conversational.py @@ -100,6 +100,8 @@ async def run( }, ) + yield res + try: configs = agent_with_configs.configs voice_url = None @@ -109,6 +111,8 @@ async def run( except Exception as err: res = f"{res}\n\n{handle_agent_error(err)}" + yield res + history.create_ai_message( res, human_message_id, diff --git a/apps/server/services/voice.py b/apps/server/services/voice.py index 4809cdfa..3fdcdd50 100644 --- a/apps/server/services/voice.py +++ b/apps/server/services/voice.py @@ -21,7 +21,8 @@ def text_to_speech( synthesizers = { "142e60f5-2d46-4b1a-9054-0764e553eed6": playht_text_to_speech, - # TODO: add AzureVoice.id: azure_text_to_speech, when available. + "509fd791-578f-40be-971f-c6753957c307": eleven_labs_text_to_speech, + "dc872426-a95c-4c41-83a2-5e5ed43670cd": azure_text_to_speech, } if configs.synthesizer not in synthesizers: @@ -147,3 +148,56 @@ async def deepgram_speech_to_text( return transcribed_text.strip('"') except Exception as err: raise TranscriberException(str(err)) + + +def eleven_labs_text_to_speech( + text: str, configs: ConfigsOutput, settings: AccountVoiceSettings +) -> bytes: + if settings.ELEVEN_LABS_API_KEY is None or not settings.ELEVEN_LABS_API_KEY: + raise SynthesizerException( + "Please set Eleven Labs API Key in [Voice Integrations](/integrations/voice/elevenlabs) in order to synthesize text to speech." + ) + + url = f"https://api.elevenlabs.io/v1/text-to-speech/{configs.voice_id or configs.default_voice}" + + payload = { + "model_id": "eleven_multilingual_v2", + "text": text, + "voice_settings": {"similarity_boost": 1, "stability": 1, "style": 1}, + } + headers = { + "xi-api-key": settings.ELEVEN_LABS_API_KEY, + "Content-Type": "application/json", + } + + response = requests.post(url, json=payload, headers=headers) + return response.content + + +def azure_text_to_speech( + text: str, configs: ConfigsOutput, settings: AccountVoiceSettings +) -> bytes: + if settings.AZURE_SPEECH_KEY is None or not settings.AZURE_SPEECH_KEY: + raise SynthesizerException( + "Please set Azure Speech Key in [Voice Integrations](/integrations/voice/azure) in order to synthesize text to speech." + ) + + url = f"https://{settings.AZURE_SPEECH_REGION}.tts.speech.microsoft.com/cognitiveservices/v1" + + body = f""" + + + {text} + + + """ + + headers = { + "Ocp-Apim-Subscription-Key": settings.AZURE_SPEECH_KEY, + "Content-Type": "application/ssml+xml", + "X-Microsoft-OutputFormat": "riff-24khz-16bit-mono-pcm", + "User-Agent": "Your application name", + } + + response = requests.post(url, headers=headers, data=body) + return response.content diff --git a/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx b/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx index f10b8811..ad4c6d96 100644 --- a/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx +++ b/apps/ui/src/pages/Agents/AgentForm/FormSections/VoicePreferences.tsx @@ -15,6 +15,7 @@ import Checkbox from 'share-ui/components/Checkbox/Checkbox' import TypographyPrimary from 'components/Typography/Primary' import RadioButton from 'share-ui/components/RadioButton/RadioButton' import Typography from 'share-ui/components/typography/Typography' +import { useEffect } from 'react' const VoicePreferences = ({ formik, @@ -61,9 +62,9 @@ const VoicePreferences = ({ return { name: item.name, sample: item.preview_url, - language: item.language, + language: '-', id: item.voice_id, - gender: item.gender, + gender: item.labels.gender, } }) From 94ee7be68c62c8c2e18fb57edf04798ff4e3afb5 Mon Sep 17 00:00:00 2001 From: levanion Date: Wed, 28 Feb 2024 15:25:25 +0400 Subject: [PATCH 3/3] fix: chat scrolling --- .../ChatMessageList/ChatMessageListV2.tsx | 64 ++++++++++++++++++- .../Icon/Icons/components/ArrowDown.tsx | 40 +++++------- .../Icon/Icons/components/Chats.tsx | 9 ++- 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/apps/ui/src/modals/AIChatModal/components/ChatMessageList/ChatMessageListV2.tsx b/apps/ui/src/modals/AIChatModal/components/ChatMessageList/ChatMessageListV2.tsx index 3ff5e7a0..69634a13 100644 --- a/apps/ui/src/modals/AIChatModal/components/ChatMessageList/ChatMessageListV2.tsx +++ b/apps/ui/src/modals/AIChatModal/components/ChatMessageList/ChatMessageListV2.tsx @@ -17,6 +17,8 @@ import HumanReply from './components/HumanReply' import AiReply from './components/AiReply' import { ReplyStateProps } from '../ReplyBox' +import { ArrowDown } from 'share-ui/components/Icon/Icons' + export enum MessageTypeEnum { AI_MANUAL = 'AI_MANUAL', User = 'User', @@ -49,6 +51,7 @@ const ChatMessageListV2 = ({ const [listIsReady, setListIsReady] = useState(false) const virtuoso = useRef(null) + const scrollerRef = useRef(null) const filteredData = data?.map((chat: any) => { const chatDate = moment(chat?.created_on).format('HH:mm') @@ -95,7 +98,7 @@ const ChatMessageListV2 = ({ const scrollToEnd = () => { virtuoso.current?.scrollToIndex({ - index: initialChat.length + 1, + index: initialChat.length + 2, align: 'end', }) } @@ -150,10 +153,44 @@ const ChatMessageListV2 = ({ // eslint-disable-next-line }, [sessionId]) + const [showScrollButton, setShowScrollButton] = useState(false) + + useEffect(() => { + const handleScroll = () => { + if (scrollerRef.current) { + const { scrollTop, clientHeight, scrollHeight } = scrollerRef.current + const bottomScrollPosition = scrollTop + clientHeight + const isScrollable = scrollHeight - bottomScrollPosition > 1 // Using 1 as a threshold + setShowScrollButton(isScrollable) + } + } + + const scrollContainer = scrollerRef.current + if (scrollContainer) { + scrollContainer.addEventListener('scroll', handleScroll) + + // Initial check in case the list is already scrollable + handleScroll() + } + + return () => { + if (scrollContainer) { + scrollContainer.removeEventListener('scroll', handleScroll) + } + } + }, [initialChat]) + + useEffect(() => { + if (!showScrollButton) scrollToEnd() + }, [data]) + return ( { + scrollerRef.current = ref + }} style={{ height: '100%' }} data={initialChat} totalCount={data.length} @@ -253,6 +290,11 @@ const ChatMessageListV2 = ({ )} /> + {showScrollButton && ( + + + + )} ) } @@ -260,6 +302,8 @@ const ChatMessageListV2 = ({ export default memo(ChatMessageListV2) const StyledRoot = styled.div<{ show: boolean }>` + position: relative; + opacity: 0; width: 100%; @@ -331,3 +375,21 @@ const StyledReplyMessageContainer = styled.div` justify-content: center; /* align-items: center; */ ` +const StyledScrollButton = styled.div` + position: absolute; + bottom: 0; + left: 50%; + transform: translateX(-50%); + + cursor: pointer; + + width: 38px; + height: 38px; + border-radius: 100px; + background-color: #fff; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); // Add box shadow here + + display: flex; + align-items: center; + justify-content: center; +` diff --git a/apps/ui/src/share-ui/components/Icon/Icons/components/ArrowDown.tsx b/apps/ui/src/share-ui/components/Icon/Icons/components/ArrowDown.tsx index 72aad611..123ea522 100644 --- a/apps/ui/src/share-ui/components/Icon/Icons/components/ArrowDown.tsx +++ b/apps/ui/src/share-ui/components/Icon/Icons/components/ArrowDown.tsx @@ -1,30 +1,24 @@ /* eslint-disable */ /* tslint:disable */ -import * as React from 'react'; +import * as React from 'react' export interface ArrowDownProps extends React.SVGAttributes { -size?: string | number; + size?: string | number } -const ArrowDown: React.FC = ({size, ...props}) => ( - - - - - - - - - - - - - - - - - +const ArrowDown: React.FC = ({ size, ...props }) => ( + + -); -ArrowDown.displayName = 'ArrowDown'; -export default ArrowDown; +) +ArrowDown.displayName = 'ArrowDown' +export default ArrowDown /* tslint:enable */ /* eslint-enable */ diff --git a/apps/ui/src/share-ui/components/Icon/Icons/components/Chats.tsx b/apps/ui/src/share-ui/components/Icon/Icons/components/Chats.tsx index e22b7ffb..e7be5e27 100644 --- a/apps/ui/src/share-ui/components/Icon/Icons/components/Chats.tsx +++ b/apps/ui/src/share-ui/components/Icon/Icons/components/Chats.tsx @@ -5,7 +5,14 @@ export interface ChatsProps extends React.SVGAttributes { size?: string | number } const Chats: React.FC = ({ size, ...props }) => ( - +