diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index b43218942..48b2b1b92 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -11,8 +11,6 @@ } from '$lib/apis/rag'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS, WEBUI_BASE_URL } from '$lib/constants'; - import { transcribeAudio } from '$lib/apis/audio'; - import Prompts from './MessageInput/PromptCommands.svelte'; import Suggestions from './MessageInput/Suggestions.svelte'; import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte'; @@ -21,7 +19,8 @@ import Tooltip from '../common/Tooltip.svelte'; import XMark from '$lib/components/icons/XMark.svelte'; import InputMenu from './MessageInput/InputMenu.svelte'; - import { t } from 'i18next'; + import Headphone from '../icons/Headphone.svelte'; + import VoiceRecording from './MessageInput/VoiceRecording.svelte'; const i18n = getContext('i18n'); @@ -33,6 +32,8 @@ export let atSelectedModel: Model | undefined; export let selectedModels: ['']; + let recording = false; + let chatTextAreaElement: HTMLTextAreaElement; let filesInputElement; @@ -48,14 +49,11 @@ export let files = []; - export let speechRecognitionEnabled = true; export let webSearchEnabled = false; export let prompt = ''; export let messages = []; - let speechRecognition; - let visionCapableModels = []; $: visionCapableModels = [...(atSelectedModel ? [atSelectedModel] : selectedModels)].filter( (model) => $models.find((m) => m.id === model)?.info?.meta?.capabilities?.vision ?? true @@ -68,176 +66,11 @@ } } - let mediaRecorder; - let audioChunks = []; - let isRecording = false; - const MIN_DECIBELS = -45; - const scrollToBottom = () => { const element = document.getElementById('messages-container'); element.scrollTop = element.scrollHeight; }; - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - mediaRecorder = new MediaRecorder(stream); - mediaRecorder.onstart = () => { - isRecording = true; - console.log('Recording started'); - }; - mediaRecorder.ondataavailable = (event) => audioChunks.push(event.data); - mediaRecorder.onstop = async () => { - isRecording = false; - console.log('Recording stopped'); - - // Create a blob from the audio chunks - const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }); - - const file = blobToFile(audioBlob, 'recording.wav'); - - const res = await transcribeAudio(localStorage.token, file).catch((error) => { - toast.error(error); - return null; - }); - - if (res) { - prompt = res.text; - await tick(); - chatTextAreaElement?.focus(); - - if (prompt !== '' && $settings?.speechAutoSend === true) { - submitPrompt(prompt, user); - } - } - - // saveRecording(audioBlob); - audioChunks = []; - }; - - // Start recording - mediaRecorder.start(); - - // Monitor silence - monitorSilence(stream); - }; - - const monitorSilence = (stream) => { - const audioContext = new AudioContext(); - const audioStreamSource = audioContext.createMediaStreamSource(stream); - const analyser = audioContext.createAnalyser(); - analyser.minDecibels = MIN_DECIBELS; - audioStreamSource.connect(analyser); - - const bufferLength = analyser.frequencyBinCount; - const domainData = new Uint8Array(bufferLength); - - let lastSoundTime = Date.now(); - - const detectSound = () => { - analyser.getByteFrequencyData(domainData); - - if (domainData.some((value) => value > 0)) { - lastSoundTime = Date.now(); - } - - if (isRecording && Date.now() - lastSoundTime > 3000) { - mediaRecorder.stop(); - audioContext.close(); - return; - } - - window.requestAnimationFrame(detectSound); - }; - - window.requestAnimationFrame(detectSound); - }; - - const saveRecording = (blob) => { - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - document.body.appendChild(a); - a.style = 'display: none'; - a.href = url; - a.download = 'recording.wav'; - a.click(); - window.URL.revokeObjectURL(url); - }; - - const speechRecognitionHandler = () => { - // Check if SpeechRecognition is supported - - if (isRecording) { - if (speechRecognition) { - speechRecognition.stop(); - } - - if (mediaRecorder) { - mediaRecorder.stop(); - } - } else { - isRecording = true; - - if ($settings?.audio?.STTEngine ?? '' !== '') { - startRecording(); - } else { - if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) { - // Create a SpeechRecognition object - speechRecognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)(); - - // Set continuous to true for continuous recognition - speechRecognition.continuous = true; - - // Set the timeout for turning off the recognition after inactivity (in milliseconds) - const inactivityTimeout = 3000; // 3 seconds - - let timeoutId; - // Start recognition - speechRecognition.start(); - - // Event triggered when speech is recognized - speechRecognition.onresult = async (event) => { - // Clear the inactivity timeout - clearTimeout(timeoutId); - - // Handle recognized speech - console.log(event); - const transcript = event.results[Object.keys(event.results).length - 1][0].transcript; - - prompt = `${prompt}${transcript}`; - - await tick(); - chatTextAreaElement?.focus(); - - // Restart the inactivity timeout - timeoutId = setTimeout(() => { - console.log('Speech recognition turned off due to inactivity.'); - speechRecognition.stop(); - }, inactivityTimeout); - }; - - // Event triggered when recognition is ended - speechRecognition.onend = function () { - // Restart recognition after it ends - console.log('recognition ended'); - isRecording = false; - if (prompt !== '' && $settings?.speechAutoSend === true) { - submitPrompt(prompt, user); - } - }; - - // Event triggered when an error occurs - speechRecognition.onerror = function (event) { - console.log(event); - toast.error($i18n.t(`Speech recognition error: {{error}}`, { error: event.error })); - isRecording = false; - }; - } else { - toast.error($i18n.t('SpeechRecognition API is not supported in this browser.')); - } - } - } - }; - const uploadDoc = async (file) => { console.log(file); @@ -601,431 +434,418 @@ } }} /> -
{ - // check if selectedModels support image input - submitPrompt(prompt, user); - }} - > - {#if files.length > 0} -
- {#each files as file, fileIdx} -
- {#if file.type === 'image'} -
- input - {#if atSelectedModel ? visionCapableModels.length === 0 : selectedModels.length !== visionCapableModels.length} - !visionCapableModels.includes(id)) - .join(', ') - })} + + {#if recording} + { + recording = false; + + await tick(); + document.getElementById('chat-textarea')?.focus(); + }} + on:confirm={async (e) => { + const response = e.detail; + prompt = `${prompt}${response} `; + + recording = false; + + await tick(); + document.getElementById('chat-textarea')?.focus(); + }} + /> + {:else} + { + // check if selectedModels support image input + submitPrompt(prompt, user); + }} + > +
+ {#if files.length > 0} +
+ {#each files as file, fileIdx} +
+ {#if file.type === 'image'} +
+ input + {#if atSelectedModel ? visionCapableModels.length === 0 : selectedModels.length !== visionCapableModels.length} + !visionCapableModels.includes(id)) + .join(', ') + })} + > + + + + + {/if} +
+ {:else if file.type === 'doc'} +
- - - - +
+ {#if file.upload_status} + + + + + {:else} + + {/if} +
+ +
+
+ {file.name} +
+ +
{$i18n.t('Document')}
+
+
+ {:else if file.type === 'collection'} +
+
+ + + + +
+ +
+
+ {file?.title ?? `#${file.name}`} +
+ +
{$i18n.t('Collection')}
+
+
{/if} -
- {:else if file.type === 'doc'} -
-
- {#if file.upload_status} + +
+
- -
-
- {file.name} -
- -
{$i18n.t('Document')}
+
- {:else if file.type === 'collection'} -
-
- - - - -
+ {/each} +
+ {/if} -
-
- {file?.title ?? `#${file.name}`} -
- -
{$i18n.t('Collection')}
-
-
- {/if} - -
+
+
+ { + filesInputElement.click(); + }} + onClose={async () => { + await tick(); + chatTextAreaElement?.focus(); + }} + > -
+
- {/each} -
- {/if} -
-
- { - filesInputElement.click(); - }} - onClose={async () => { - await tick(); - chatTextAreaElement?.focus(); - }} - > - - -
+