OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-05-15 02:56:09 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

feat: emoji call

Browse Source
This commit is contained in:
Timothy J. Baek 2024-06-14 02:30:36 -07:00
parent 53858c9b0e
commit 2e4f060ebb
2 changed files with 112 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/lib/apis/index.ts
View File

@ -242,7 +242,15 @@ export const generateEmoji = async (
throw error; throw error;
} }
return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? null; const response = res?.choices[0]?.message?.content.replace(/["']/g, '') ?? null;
if (response) {
if (/\p{Extended_Pictographic}/u.test(response)) {
return response.match(/\p{Extended_Pictographic}/gu)[0];
}
}
return null;
}; };
export const generateSearchQuery = async ( export const generateSearchQuery = async (

189
src/lib/components/chat/MessageInput/CallOverlay.svelte
View File

@ -338,7 +338,7 @@
speechSynthesis.speak(currentUtterance); speechSynthesis.speak(currentUtterance);
currentUtterance.onend = async (e) => { currentUtterance.onend = async (e) => {
await new Promise((r) => setTimeout(r, 100)); await new Promise((r) => setTimeout(r, 200));
resolve(e); resolve(e);
}; };
} }
@ -402,27 +402,42 @@
// Audio cache map where key is the content and value is the Audio object. // Audio cache map where key is the content and value is the Audio object.
const audioCache = new Map(); const audioCache = new Map();
const emojiCache = new Map();
const fetchAudio = async (content) => { const fetchAudio = async (content) => {
if (!audioCache.has(content)) { if (!audioCache.has(content)) {
try { try {
const res = await synthesizeOpenAISpeech( // Set the emoji for the content if needed
localStorage.token, if ($settings?.showEmojiInCall ?? false) {
$settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice, const emoji = await generateEmoji(localStorage.token, modelId, content, chatId);
content if (emoji) {
).catch((error) => { emojiCache.set(content, emoji);
console.error(error); }
return null; }
});
if (res) { if ($config.audio.tts.engine !== '') {
const blob = await res.blob(); const res = await synthesizeOpenAISpeech(
const blobUrl = URL.createObjectURL(blob); localStorage.token,
audioCache.set(content, new Audio(blobUrl)); $settings?.audio?.tts?.voice ?? $config?.audio?.tts?.voice,
content
).catch((error) => {
console.error(error);
return null;
});
if (res) {
const blob = await res.blob();
const blobUrl = URL.createObjectURL(blob);
audioCache.set(content, new Audio(blobUrl));
}
} else {
audioCache.set(content, true);
} }
} catch (error) { } catch (error) {
console.error('Error synthesizing speech:', error); console.error('Error synthesizing speech:', error);
} }
} }
return audioCache.get(content); return audioCache.get(content);
}; };
@ -436,19 +451,31 @@
if (audioCache.has(content)) { if (audioCache.has(content)) {
// If content is available in the cache, play it // If content is available in the cache, play it
try {
console.log(
'%c%s',
'color: red; font-size: 20px;',
`Playing audio for content: ${content}`
);
const audio = audioCache.get(content); // Set the emoji for the content if available
await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute if (($settings?.showEmojiInCall ?? false) && emojiCache.has(content)) {
console.log(`Played audio for content: ${content}`); emoji = emojiCache.get(content);
await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop } else {
} catch (error) { emoji = null;
console.error('Error playing audio:', error); }
if ($config.audio.tts.engine !== '') {
try {
console.log(
'%c%s',
'color: red; font-size: 20px;',
`Playing audio for content: ${content}`
);
const audio = audioCache.get(content);
await playAudio(audio); // Here ensure that playAudio is indeed correct method to execute
console.log(`Played audio for content: ${content}`);
await new Promise((resolve) => setTimeout(resolve, 200)); // Wait before retrying to reduce tight loop
} catch (error) {
console.error('Error playing audio:', error);
}
} else {
await speakSpeechSynthesisHandler(content);
} }
} else { } else {
// If not available in the cache, push it back to the queue and delay // If not available in the cache, push it back to the queue and delay
@ -475,20 +502,17 @@
chatStreaming = true; chatStreaming = true;
if ($config.audio.tts.engine !== '') { if (currentMessageId !== id) {
// set currentMessageId to id console.log(`Received chat start event for message ID ${id}`);
if (currentMessageId !== id) {
console.log(`Received chat start event for message ID ${id}`);
currentMessageId = id; currentMessageId = id;
if (audioAbortController) { if (audioAbortController) {
audioAbortController.abort(); audioAbortController.abort();
}
audioAbortController = new AbortController();
// Start monitoring and playing audio for the message ID
monitorAndPlayAudio(id, audioAbortController.signal);
} }
audioAbortController = new AbortController();
// Start monitoring and playing audio for the message ID
monitorAndPlayAudio(id, audioAbortController.signal);
} }
}; };
@ -499,23 +523,21 @@
// "content" here is a sentence from the assistant, // "content" here is a sentence from the assistant,
// there will be many sentences for the same "id" // there will be many sentences for the same "id"
if ($config.audio.tts.engine !== '') { if (currentMessageId === id) {
if (currentMessageId === id) { console.log(`Received chat event for message ID ${id}: ${content}`);
console.log(`Received chat event for message ID ${id}: ${content}`);
try { try {
if (messages[id] === undefined) { if (messages[id] === undefined) {
messages[id] = [content]; messages[id] = [content];
} else { } else {
messages[id].push(content); messages[id].push(content);
}
console.log(content);
fetchAudio(content);
} catch (error) {
console.error('Failed to fetch or play audio:', error);
} }
console.log(content);
fetchAudio(content);
} catch (error) {
console.error('Failed to fetch or play audio:', error);
} }
} }
}; };
@ -525,12 +547,7 @@
// "content" here is the entire message from the assistant // "content" here is the entire message from the assistant
chatStreaming = false; chatStreaming = false;
finishedMessages[id] = true;
if ($config.audio.tts.engine !== '') {
finishedMessages[id] = true;
} else {
speakSpeechSynthesisHandler(content);
}
}; };
eventTarget.addEventListener('chat:start', chatStartHandler); eventTarget.addEventListener('chat:start', chatStartHandler);
@ -561,7 +578,20 @@
<div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6"> <div class="max-w-lg w-full h-screen max-h-[100dvh] flex flex-col justify-between p-3 md:p-6">
{#if camera} {#if camera}
<div class="flex justify-center items-center w-full h-20 min-h-20"> <div class="flex justify-center items-center w-full h-20 min-h-20">
{#if loading} {#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '4.5'
: rmsLevel * 100 > 2
? '4.25'
: rmsLevel * 100 > 1
? '3.75'
: '3.5'}rem;width: 100%; text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg <svg
class="size-12 text-gray-900 dark:text-gray-400" class="size-12 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24" viewBox="0 0 24 24"
@ -598,19 +628,6 @@
r="3" r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg /><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
> >
{:else if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '4.5'
: rmsLevel * 100 > 2
? '4.25'
: rmsLevel * 100 > 1
? '3.75'
: '3.5'}rem;width: 100%; text-align:center;"
>
{emoji}
</div>
{:else} {:else}
<div <div
class=" {rmsLevel * 100 > 4 class=" {rmsLevel * 100 > 4
@ -628,7 +645,20 @@
<div class="flex justify-center items-center flex-1 h-full w-full max-h-full"> <div class="flex justify-center items-center flex-1 h-full w-full max-h-full">
{#if !camera} {#if !camera}
{#if loading} {#if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '13'
: rmsLevel * 100 > 2
? '12'
: rmsLevel * 100 > 1
? '11.5'
: '11'}rem;width:100%;text-align:center;"
>
{emoji}
</div>
{:else if loading}
<svg <svg
class="size-44 text-gray-900 dark:text-gray-400" class="size-44 text-gray-900 dark:text-gray-400"
viewBox="0 0 24 24" viewBox="0 0 24 24"
@ -665,19 +695,6 @@
r="3" r="3"
/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg /><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="3" /></svg
> >
{:else if emoji}
<div
class=" transition-all rounded-full"
style="font-size:{rmsLevel * 100 > 4
? '13'
: rmsLevel * 100 > 2
? '12'
: rmsLevel * 100 > 1
? '11.5'
: '11'}rem;width:100%;text-align:center;"
>
{emoji}
</div>
{:else} {:else}
<div <div
class=" {rmsLevel * 100 > 4 class=" {rmsLevel * 100 > 4