From d17cdc8068d0291da9b9789b4e2ffb0d551e0427 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 7 Jun 2024 14:49:36 -0700 Subject: [PATCH] feat: video devices support --- .../chat/MessageInput/CallOverlay.svelte | 191 +++++++++++------- .../CallOverlay/VideoInputMenu.svelte | 51 +++++ 2 files changed, 169 insertions(+), 73 deletions(-) create mode 100644 src/lib/components/chat/MessageInput/CallOverlay/VideoInputMenu.svelte diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index 91bab8222..997311b9f 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -5,7 +5,9 @@ import { blobToFile, calculateSHA256, extractSentences, findWordIndices } from '$lib/utils'; import { synthesizeOpenAISpeech, transcribeAudio } from '$lib/apis/audio'; import { toast } from 'svelte-sonner'; + import Tooltip from '$lib/components/common/Tooltip.svelte'; + import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte'; const i18n = getContext('i18n'); @@ -25,13 +27,6 @@ let rmsLevel = 0; let hasStartedSpeaking = false; - let audioContext; - let analyser; - let dataArray; - let audioElement; - let animationFrameId; - - let speechRecognition; let currentUtterance = null; let mediaRecorder; @@ -40,28 +35,6 @@ const MIN_DECIBELS = -45; const VISUALIZER_BUFFER_LENGTH = 300; - let visualizerData = Array(VISUALIZER_BUFFER_LENGTH).fill(0); - - const startAudio = () => { - audioContext = new (window.AudioContext || window.webkitAudioContext)(); - analyser = audioContext.createAnalyser(); - const source = audioContext.createMediaElementSource(audioElement); - source.connect(analyser); - analyser.connect(audioContext.destination); - analyser.fftSize = 32; // Adjust the fftSize - dataArray = new Uint8Array(analyser.frequencyBinCount); - visualize(); - }; - - const visualize = () => { - analyser.getByteFrequencyData(dataArray); - div1Height = dataArray[1] / 2; - div2Height = dataArray[3] / 2; - div3Height = dataArray[5] / 2; - div4Height = dataArray[7] / 2; - animationFrameId = requestAnimationFrame(visualize); - }; - // Function to calculate the RMS level from time domain data const calculateRMS = (data: Uint8Array) => { let sumSquares = 0; @@ -333,23 +306,74 @@ mediaRecorder.start(); }; + let videoInputDevices = []; + let selectedVideoInputDeviceId = null; + + const getVideoInputDevices = async () => { + const devices = await navigator.mediaDevices.enumerateDevices(); + videoInputDevices = devices.filter((device) => device.kind === 'videoinput'); + + videoInputDevices = [ + ...videoInputDevices, + { + deviceId: 'screen', + label: 'Screen Share' + } + ]; + + console.log(videoInputDevices); + + if (selectedVideoInputDeviceId === null && videoInputDevices.length > 0) { + selectedVideoInputDeviceId = videoInputDevices[0].deviceId; + } + }; + const startCamera = async () => { + await getVideoInputDevices(); + if (cameraStream === null) { camera = true; await tick(); try { - const video = document.getElementById('camera-feed'); - if (video) { - cameraStream = await navigator.mediaDevices.getUserMedia({ video: true }); - video.srcObject = cameraStream; - await video.play(); - } + await startVideoStream(); } catch (err) { console.error('Error accessing webcam: ', err); } } }; + const startVideoStream = async () => { + const video = document.getElementById('camera-feed'); + if (video) { + if (selectedVideoInputDeviceId === 'screen') { + cameraStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: 'always' + }, + audio: false + }); + } else { + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { + deviceId: selectedVideoInputDeviceId ? { exact: selectedVideoInputDeviceId } : undefined + } + }); + } + + video.srcObject = cameraStream; + await video.play(); + } + }; + + const stopVideoStream = async () => { + if (cameraStream) { + const tracks = cameraStream.getTracks(); + tracks.forEach((track) => track.stop()); + } + + cameraStream = null; + }; + const takeScreenshot = () => { const video = document.getElementById('camera-feed'); const canvas = document.getElementById('camera-canvas'); @@ -359,14 +383,13 @@ } const context = canvas.getContext('2d'); + // Make the canvas match the video dimensions canvas.width = video.videoWidth; canvas.height = video.videoHeight; - // Draw the flipped image from the video onto the canvas - context.save(); - context.scale(-1, 1); // Flip horizontally - context.drawImage(video, 0, 0, video.videoWidth * -1, video.videoHeight); - context.restore(); + + // Draw the image from the video onto the canvas + context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight); // Convert the canvas to a data base64 URL and console log it const dataURL = canvas.toDataURL('image/png'); @@ -375,13 +398,8 @@ return dataURL; }; - const stopCamera = () => { - if (cameraStream) { - const tracks = cameraStream.getTracks(); - tracks.forEach((track) => track.stop()); - } - - cameraStream = null; + const stopCamera = async () => { + await stopVideoStream(); camera = false; }; @@ -539,35 +557,62 @@
- - + + {:else} + + - + + + + + + + {/if}
diff --git a/src/lib/components/chat/MessageInput/CallOverlay/VideoInputMenu.svelte b/src/lib/components/chat/MessageInput/CallOverlay/VideoInputMenu.svelte new file mode 100644 index 000000000..ead2f7d10 --- /dev/null +++ b/src/lib/components/chat/MessageInput/CallOverlay/VideoInputMenu.svelte @@ -0,0 +1,51 @@ + + + { + if (e.detail === false) { + onClose(); + } + }} +> + + +
+ + {#each devices as device} + { + dispatch('change', device.deviceId); + }} + > +
+
+ {device?.label ?? 'Camera'} +
+
+ + {/each} + +
+