diff --git a/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte index b987b3d89..36d71469a 100644 --- a/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte @@ -42,6 +42,10 @@ let OpenAIUrl = ''; let OpenAIKey = ''; + let AzureOpenAIUrl = ''; + let AzureOpenAIKey = ''; + let AzureOpenAIVersion = ''; + let OllamaUrl = ''; let OllamaKey = ''; @@ -85,7 +89,14 @@ return; } - if ((embeddingEngine === 'openai' && OpenAIKey === '') || OpenAIUrl === '') { + if (embeddingEngine === 'openai' && (OpenAIKey === '' || OpenAIUrl === '')) { + toast.error($i18n.t('OpenAI URL/Key required.')); + return; + } + if ( + embeddingEngine === 'azure_openai' && + (AzureOpenAIKey === '' || AzureOpenAIUrl === '' || AzureOpenAIVersion === '') + ) { toast.error($i18n.t('OpenAI URL/Key required.')); return; } @@ -105,6 +116,11 @@ key: OpenAIKey, url: OpenAIUrl }, + azure_openai_config: { + key: AzureOpenAIKey, + url: AzureOpenAIUrl, + version: AzureOpenAIVersion + }, knowledge_id: knowledgeId }).catch(async (error) => { toast.error(`${error}`); @@ -135,6 +151,10 @@ OpenAIKey = embeddingConfig.openai_config.key; OpenAIUrl = embeddingConfig.openai_config.url; + AzureOpenAIKey = embeddingConfig.azure_openai_config.key; + AzureOpenAIUrl = embeddingConfig.azure_openai_config.url; + AzureOpenAIVersion = embeddingConfig.azure_openai_config.version; + OllamaKey = embeddingConfig.ollama_config.key; OllamaUrl = embeddingConfig.ollama_config.url; } @@ -166,7 +186,24 @@ toast.error($i18n.t('Docling Server URL required.')); return; } + if ( + RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling' && + ((RAGConfig.DOCLING_OCR_ENGINE === '' && RAGConfig.DOCLING_OCR_LANG !== '') || + (RAGConfig.DOCLING_OCR_ENGINE !== '' && RAGConfig.DOCLING_OCR_LANG === '')) + ) { + toast.error( + $i18n.t('Both Docling OCR Engine and Language(s) must be provided or both left empty.') + ); + return; + } + if ( + RAGConfig.CONTENT_EXTRACTION_ENGINE === 'datalab_marker' && + !RAGConfig.DATALAB_MARKER_API_KEY + ) { + toast.error($i18n.t('Datalab Marker API Key required.')); + return; + } if ( RAGConfig.CONTENT_EXTRACTION_ENGINE === 'document_intelligence' && (RAGConfig.DOCUMENT_INTELLIGENCE_ENDPOINT === '' || @@ -336,6 +373,7 @@ + @@ -353,6 +391,138 @@ + {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'datalab_marker'} +
+ +
+ +
+
+ {$i18n.t('Languages')} +
+ + +
+ +
+
+ + {$i18n.t('Use LLM')} + +
+
+ +
+
+
+
+ + {$i18n.t('Skip Cache')} + +
+
+ +
+
+
+
+ + {$i18n.t('Force OCR')} + +
+
+ +
+
+
+
+ + {$i18n.t('Paginate')} + +
+
+ +
+
+
+
+ + {$i18n.t('Strip Existing OCR')} + +
+
+ +
+
+
+
+ + {$i18n.t('Disable Image Extraction')} + +
+
+ +
+
+
+
+ + {$i18n.t('Output Format')} + +
+
+ +
+
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
{$i18n.t('Default (SentenceTransformers)')} +
@@ -538,6 +711,26 @@ required={false} /> + {:else if embeddingEngine === 'azure_openai'} +
+
+ + +
+
+ +
+
{/if} @@ -609,7 +802,7 @@ - {#if embeddingEngine === 'ollama' || embeddingEngine === 'openai'} + {#if embeddingEngine === 'ollama' || embeddingEngine === 'openai' || embeddingEngine === 'azure_openai'}
{$i18n.t('Embedding Batch Size')} @@ -819,6 +1012,25 @@
{/if} + {#if RAGConfig.ENABLE_RAG_HYBRID_SEARCH === true} +
+
+ {$i18n.t('Weight of BM25 Retrieval')} +
+
+ +
+
+ {/if} {/if}
diff --git a/src/lib/components/workspace/common/RagConfigModal.svelte b/src/lib/components/workspace/common/RagConfigModal.svelte index 1fe225884..006ffc557 100644 --- a/src/lib/components/workspace/common/RagConfigModal.svelte +++ b/src/lib/components/workspace/common/RagConfigModal.svelte @@ -45,6 +45,10 @@ let OpenAIUrl = RAGConfig.openai_config?.url || ""; let OpenAIKey = RAGConfig.openai_config?.key || ""; + let AzureOpenAIUrl = RAGConfig.azure_openai_config?.url || ""; + let AzureOpenAIKey = RAGConfig.azure_openai_config?.key || ""; + let AzureOpenAIVersion = RAGConfig.azure_openai_config?.version || ""; + let OllamaUrl = RAGConfig.ollama_config?.url || ""; let OllamaKey = RAGConfig.ollama_config?.key || ""; @@ -61,6 +65,10 @@ OpenAIUrl = RAGConfig.openai_config?.url || ""; OpenAIKey = RAGConfig.openai_config?.key || ""; + AzureOpenAIUrl = RAGConfig.azure_openai_config?.url || ""; + AzureOpenAIKey = RAGConfig.azure_openai_config?.key || ""; + AzureOpenAIVersion = RAGConfig.azure_openai_config?.version || ""; + OllamaUrl = RAGConfig.ollama_config?.url || ""; OllamaKey = RAGConfig.ollama_config?.key || ""; needsReindex = false; @@ -101,10 +109,17 @@ return; } - if ((embeddingEngine === 'openai' && OpenAIKey === '') || OpenAIUrl === '') { - toast.error($i18n.t('OpenAI URL/Key required.')); - return; - } + if (embeddingEngine === 'openai' && (OpenAIKey === '' || OpenAIUrl === '')) { + toast.error($i18n.t('OpenAI URL/Key required.')); + return; + } + if ( + embeddingEngine === 'azure_openai' && + (AzureOpenAIKey === '' || AzureOpenAIUrl === '' || AzureOpenAIVersion === '') + ) { + toast.error($i18n.t('OpenAI URL/Key required.')); + return; + } console.log('Update embedding model attempt:', embeddingModel); @@ -121,6 +136,11 @@ key: OpenAIKey, url: OpenAIUrl }, + azure_openai_config: { + key: AzureOpenAIKey, + url: AzureOpenAIUrl, + version: AzureOpenAIVersion + }, knowledge_id: knowledgeId, }).catch(async (error) => { toast.error(`${error}`); @@ -151,6 +171,10 @@ OpenAIKey = embeddingConfig.openai_config.key; OpenAIUrl = embeddingConfig.openai_config.url; + AzureOpenAIKey = embeddingConfig.azure_openai_config.key; + AzureOpenAIUrl = embeddingConfig.azure_openai_config.url; + AzureOpenAIVersion = embeddingConfig.azure_openai_config.version; + OllamaKey = embeddingConfig.ollama_config.key; OllamaUrl = embeddingConfig.ollama_config.url; } @@ -174,7 +198,7 @@ toast.error($i18n.t('Docling Server URL required.')); return; } - if ( + if ( localRAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling' && ((localRAGConfig.DOCLING_OCR_ENGINE === '' && localRAGConfig.DOCLING_OCR_LANG !== '') || (localRAGConfig.DOCLING_OCR_ENGINE !== '' && localRAGConfig.DOCLING_OCR_LANG === '')) @@ -184,6 +208,13 @@ ); return; } + if ( + localRAGConfig.CONTENT_EXTRACTION_ENGINE === 'datalab_marker' && + !localRAGConfig.DATALAB_MARKER_API_KEY + ) { + toast.error($i18n.t('Datalab Marker API Key required.')); + return; + } if ( localRAGConfig.CONTENT_EXTRACTION_ENGINE === 'document_intelligence' && @@ -207,7 +238,8 @@ embedding_model, embedding_batch_size, openai_config, - ollama_config, + ollama_config, + azure_openai_config, LOADED_EMBEDDING_MODELS, DOWNLOADED_EMBEDDING_MODELS, LOADED_RERANKING_MODELS, @@ -229,16 +261,15 @@ localRAGConfig.embedding_batch_size = embeddingBatchSize localRAGConfig.openai_config = {"key": OpenAIKey, "url": OpenAIUrl} localRAGConfig.ollama_config = {"key": OllamaKey, "url": OllamaUrl} - + localRAGConfig.azure_openai_config = {"key": AzureOpenAIKey, "url": AzureOpenAIUrl, "version": AzureOpenAIVersion} + if (needsReindex) { // Reindex knowledge files if reranking model changed const reindexResponse = await reindexSpecificKnowledgeFiles(localStorage.token, knowledgeId, ); - if (reindexResponse.status === true) { + if (reindexResponse) { toast.success($i18n.t('Knowledge files reindexed successfully.')); - } else { - toast.error($i18n.t('Failed to reindex knowledge files.')); } } @@ -290,6 +321,7 @@ + @@ -307,7 +339,139 @@
- {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'} + {:else if localRAGConfig.CONTENT_EXTRACTION_ENGINE === 'datalab_marker'} +
+ +
+ +
+
+ {$i18n.t('Languages')} +
+ + +
+ +
+
+ + {$i18n.t('Use LLM')} + +
+
+ +
+
+
+
+ + {$i18n.t('Skip Cache')} + +
+
+ +
+
+
+
+ + {$i18n.t('Force OCR')} + +
+
+ +
+
+
+
+ + {$i18n.t('Paginate')} + +
+
+ +
+
+
+
+ + {$i18n.t('Strip Existing OCR')} + +
+
+ +
+
+
+
+ + {$i18n.t('Disable Image Extraction')} + +
+
+ +
+
+
+
+ + {$i18n.t('Output Format')} + +
+
+ +
+
+ {:else if localRAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
{$i18n.t('Default (SentenceTransformers)')} +
@@ -492,6 +659,26 @@ required={false} /> + {:else if embeddingEngine === 'azure_openai'} +
+
+ + +
+
+ +
+
{/if} @@ -643,7 +830,7 @@ - {#if embeddingEngine === 'ollama' || embeddingEngine === 'openai'} + {#if embeddingEngine === 'ollama' || embeddingEngine === 'openai'|| embeddingEngine === 'azure_openai'}
{$i18n.t('Embedding Batch Size')} @@ -854,6 +1041,25 @@
{/if} + {#if localRAGConfig.ENABLE_RAG_HYBRID_SEARCH === true} +
+
+ {$i18n.t('Weight of BM25 Retrieval')} +
+
+ +
+
+ {/if} {/if}