import { v4 as uuidv4 } from 'uuid';
import sha256 from 'js-sha256';
import { getOllamaModels } from '$lib/apis/ollama';
import { getOpenAIModels } from '$lib/apis/openai';
import { getLiteLLMModels } from '$lib/apis/litellm';
export const getModels = async (token: string) => {
let models = await Promise.all([
await getOllamaModels(token).catch((error) => {
console.log(error);
return null;
}),
await getOpenAIModels(token).catch((error) => {
console.log(error);
return null;
}),
await getLiteLLMModels(token).catch((error) => {
console.log(error);
return null;
})
]);
models = models
.filter((models) => models)
.reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []);
return models;
};
//////////////////////////
// Helper functions
//////////////////////////
export const sanitizeResponseContent = (content: string) => {
return content
.replace(/<\|[a-z]*$/, '')
.replace(/<\|[a-z]+\|$/, '')
.replace(/<$/, '')
.replaceAll(/<\|[a-z]+\|>/g, ' ')
.replaceAll(/
/gi, '\n')
.replaceAll('<', '<')
.trim();
};
export const revertSanitizedResponseContent = (content: string) => {
return content.replaceAll('<', '<');
};
export const capitalizeFirstLetter = (string) => {
return string.charAt(0).toUpperCase() + string.slice(1);
};
export const splitStream = (splitOn) => {
let buffer = '';
return new TransformStream({
transform(chunk, controller) {
buffer += chunk;
const parts = buffer.split(splitOn);
parts.slice(0, -1).forEach((part) => controller.enqueue(part));
buffer = parts[parts.length - 1];
},
flush(controller) {
if (buffer) controller.enqueue(buffer);
}
});
};
export const convertMessagesToHistory = (messages) => {
const history = {
messages: {},
currentId: null
};
let parentMessageId = null;
let messageId = null;
for (const message of messages) {
messageId = uuidv4();
if (parentMessageId !== null) {
history.messages[parentMessageId].childrenIds = [
...history.messages[parentMessageId].childrenIds,
messageId
];
}
history.messages[messageId] = {
...message,
id: messageId,
parentId: parentMessageId,
childrenIds: []
};
parentMessageId = messageId;
}
history.currentId = messageId;
return history;
};
export const getGravatarURL = (email) => {
// Trim leading and trailing whitespace from
// an email address and force all characters
// to lower case
const address = String(email).trim().toLowerCase();
// Create a SHA256 hash of the final string
const hash = sha256(address);
// Grab the actual image URL
return `https://www.gravatar.com/avatar/${hash}`;
};
export const canvasPixelTest = () => {
// Test a 1x1 pixel to potentially identify browser/plugin fingerprint blocking or spoofing
// Inspiration: https://github.com/kkapsner/CanvasBlocker/blob/master/test/detectionTest.js
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
canvas.height = 1;
canvas.width = 1;
const imageData = new ImageData(canvas.width, canvas.height);
const pixelValues = imageData.data;
// Generate RGB test data
for (let i = 0; i < imageData.data.length; i += 1) {
if (i % 4 !== 3) {
pixelValues[i] = Math.floor(256 * Math.random());
} else {
pixelValues[i] = 255;
}
}
ctx.putImageData(imageData, 0, 0);
const p = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
// Read RGB data and fail if unmatched
for (let i = 0; i < p.length; i += 1) {
if (p[i] !== pixelValues[i]) {
console.log(
'canvasPixelTest: Wrong canvas pixel RGB value detected:',
p[i],
'at:',
i,
'expected:',
pixelValues[i]
);
console.log('canvasPixelTest: Canvas blocking or spoofing is likely');
return false;
}
}
return true;
};
export const generateInitialsImage = (name) => {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
canvas.width = 100;
canvas.height = 100;
if (!canvasPixelTest()) {
console.log(
'generateInitialsImage: failed pixel test, fingerprint evasion is likely. Using default image.'
);
return '/user.png';
}
ctx.fillStyle = '#F39C12';
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = '#FFFFFF';
ctx.font = '40px Helvetica';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
const sanitizedName = name.trim();
const initials =
sanitizedName.length > 0
? sanitizedName[0] +
(sanitizedName.split(' ').length > 1
? sanitizedName[sanitizedName.lastIndexOf(' ') + 1]
: '')
: '';
ctx.fillText(initials.toUpperCase(), canvas.width / 2, canvas.height / 2);
return canvas.toDataURL();
};
export const copyToClipboard = (text) => {
if (!navigator.clipboard) {
const textArea = document.createElement('textarea');
textArea.value = text;
// Avoid scrolling to bottom
textArea.style.top = '0';
textArea.style.left = '0';
textArea.style.position = 'fixed';
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
try {
const successful = document.execCommand('copy');
const msg = successful ? 'successful' : 'unsuccessful';
console.log('Fallback: Copying text command was ' + msg);
} catch (err) {
console.error('Fallback: Oops, unable to copy', err);
}
document.body.removeChild(textArea);
return;
}
navigator.clipboard.writeText(text).then(
function () {
console.log('Async: Copying to clipboard was successful!');
},
function (err) {
console.error('Async: Could not copy text: ', err);
}
);
};
export const compareVersion = (latest, current) => {
return current === '0.0.0'
? false
: current.localeCompare(latest, undefined, {
numeric: true,
sensitivity: 'case',
caseFirst: 'upper'
}) < 0;
};
export const findWordIndices = (text) => {
const regex = /\[([^\]]+)\]/g;
const matches = [];
let match;
while ((match = regex.exec(text)) !== null) {
matches.push({
word: match[1],
startIndex: match.index,
endIndex: regex.lastIndex - 1
});
}
return matches;
};
export const removeFirstHashWord = (inputString) => {
// Split the string into an array of words
const words = inputString.split(' ');
// Find the index of the first word that starts with #
const index = words.findIndex((word) => word.startsWith('#'));
// Remove the first word with #
if (index !== -1) {
words.splice(index, 1);
}
// Join the remaining words back into a string
const resultString = words.join(' ');
return resultString;
};
export const transformFileName = (fileName) => {
// Convert to lowercase
const lowerCaseFileName = fileName.toLowerCase();
// Remove special characters using regular expression
const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');
// Replace spaces with dashes
const finalFileName = sanitizedFileName.replace(/\s+/g, '-');
return finalFileName;
};
export const calculateSHA256 = async (file) => {
// Create a FileReader to read the file asynchronously
const reader = new FileReader();
// Define a promise to handle the file reading
const readFile = new Promise((resolve, reject) => {
reader.onload = () => resolve(reader.result);
reader.onerror = reject;
});
// Read the file as an ArrayBuffer
reader.readAsArrayBuffer(file);
try {
// Wait for the FileReader to finish reading the file
const buffer = await readFile;
// Convert the ArrayBuffer to a Uint8Array
const uint8Array = new Uint8Array(buffer);
// Calculate the SHA-256 hash using Web Crypto API
const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
// Convert the hash to a hexadecimal string
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
return `${hashHex}`;
} catch (error) {
console.error('Error calculating SHA-256 hash:', error);
throw error;
}
};
export const getImportOrigin = (_chats) => {
// Check what external service chat imports are from
if ('mapping' in _chats[0]) {
return 'openai';
}
return 'webui';
};
const convertOpenAIMessages = (convo) => {
// Parse OpenAI chat messages and create chat dictionary for creating new chats
const mapping = convo['mapping'];
const messages = [];
let currentId = '';
let lastId = null;
for (let message_id in mapping) {
const message = mapping[message_id];
currentId = message_id;
try {
if (
messages.length == 0 &&
(message['message'] == null ||
(message['message']['content']['parts']?.[0] == '' &&
message['message']['content']['text'] == null))
) {
// Skip chat messages with no content
continue;
} else {
const new_chat = {
id: message_id,
parentId: lastId,
childrenIds: message['children'] || [],
role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
content:
message['message']?.['content']?.['parts']?.[0] ||
message['message']?.['content']?.['text'] ||
'',
model: 'gpt-3.5-turbo',
done: true,
context: null
};
messages.push(new_chat);
lastId = currentId;
}
} catch (error) {
console.log('Error with', message, '\nError:', error);
}
}
let history = {};
messages.forEach((obj) => (history[obj.id] = obj));
const chat = {
history: {
currentId: currentId,
messages: history // Need to convert this to not a list and instead a json object
},
models: ['gpt-3.5-turbo'],
messages: messages,
options: {},
timestamp: convo['create_time'],
title: convo['title'] ?? 'New Chat'
};
return chat;
};
const validateChat = (chat) => {
// Because ChatGPT sometimes has features we can't use like DALL-E or migh have corrupted messages, need to validate
const messages = chat.messages;
// Check if messages array is empty
if (messages.length === 0) {
return false;
}
// Last message's children should be an empty array
const lastMessage = messages[messages.length - 1];
if (lastMessage.childrenIds.length !== 0) {
return false;
}
// First message's parent should be null
const firstMessage = messages[0];
if (firstMessage.parentId !== null) {
return false;
}
// Every message's content should be a string
for (let message of messages) {
if (typeof message.content !== 'string') {
return false;
}
}
return true;
};
export const convertOpenAIChats = (_chats) => {
// Create a list of dictionaries with each conversation from import
const chats = [];
let failed = 0;
for (let convo of _chats) {
const chat = convertOpenAIMessages(convo);
if (validateChat(chat)) {
chats.push({
id: convo['id'],
user_id: '',
title: convo['title'],
chat: chat,
timestamp: convo['timestamp']
});
} else {
failed++;
}
}
console.log(failed, 'Conversations could not be imported');
return chats;
};
export const isValidHttpUrl = (string) => {
let url;
try {
url = new URL(string);
} catch (_) {
return false;
}
return url.protocol === 'http:' || url.protocol === 'https:';
};
export const removeEmojis = (str) => {
// Regular expression to match emojis
const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;
// Replace emojis with an empty string
return str.replace(emojiRegex, '');
};
export const extractSentences = (text) => {
// Split the paragraph into sentences based on common punctuation marks
const sentences = text.split(/(?<=[.!?])/);
return sentences
.map((sentence) => removeEmojis(sentence.trim()))
.filter((sentence) => sentence !== '');
};
export const blobToFile = (blob, fileName) => {
// Create a new File object from the Blob
const file = new File([blob], fileName, { type: blob.type });
return file;
};
// templatePrompt replaces any occurrences of the following in the template with the prompt
// {{prompt}} will be replaced with the prompt
// {{prompt:start:}} will be replaced with the first characters of the prompt
// {{prompt:end:}} will be replaced with the last characters of the prompt
// Character length is used as we don't have the ability to tokenize the prompt
export const templatePrompt = (template: string, prompt: string) => {
template = template.replace(/{{prompt}}/g, prompt);
// Replace all instances of {{prompt:start:}} with the first characters of the prompt
const startRegex = /{{prompt:start:(\d+)}}/g;
let startMatch: RegExpMatchArray | null;
while ((startMatch = startRegex.exec(template)) !== null) {
const length = parseInt(startMatch[1]);
template = template.replace(startMatch[0], prompt.substring(0, length));
}
// Replace all instances of {{prompt:end:}} with the last characters of the prompt
const endRegex = /{{prompt:end:(\d+)}}/g;
let endMatch: RegExpMatchArray | null;
while ((endMatch = endRegex.exec(template)) !== null) {
const length = parseInt(endMatch[1]);
template = template.replace(endMatch[0], prompt.substring(prompt.length - length));
}
return template;
export const approximateToHumanReadable = (nanoseconds: number) => {
const seconds = Math.floor((nanoseconds / 1e9) % 60);
const minutes = Math.floor((nanoseconds / 6e10) % 60);
const hours = Math.floor((nanoseconds / 3.6e12) % 24);
const results: string[] = [];
if (seconds >= 0) {
results.push(`${seconds}s`);
}
if (minutes > 0) {
results.push(`${minutes}m`);
}
if (hours > 0) {
results.push(`${hours}h`);
}
return results.reverse().join(' ');
};