From f2c78ac0fb172fc13ea618b9d339942057ccacd1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 25 Oct 2024 22:23:21 -0700 Subject: [PATCH] refac --- backend/open_webui/apps/retrieval/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index 7c7a49eca..e67d1df23 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -667,7 +667,11 @@ def save_docs_to_vector_db( add_start_index=True, ) elif app.state.config.TEXT_SPLITTER == "token": - tiktoken.get_encoding(app.state.config.TIKTOKEN_ENCODING_NAME) + log.info( + f"Using token text splitter: {app.state.config.TIKTOKEN_ENCODING_NAME}" + ) + + tiktoken.get_encoding(str(app.state.config.TIKTOKEN_ENCODING_NAME)) text_splitter = TokenTextSplitter( encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME), chunk_size=app.state.config.CHUNK_SIZE,