From 0ea9e19d793c3b3aecd9532b220932f647ee0f31 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Tue, 23 Apr 2024 19:14:01 +0100 Subject: [PATCH 1/4] feat: add LITELLM_PROXY_PORT to configure internal proxy port --- backend/apps/litellm/main.py | 16 +++++++++------- backend/config.py | 8 ++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/apps/litellm/main.py b/backend/apps/litellm/main.py index 52e0c7002..ad5c2afd5 100644 --- a/backend/apps/litellm/main.py +++ b/backend/apps/litellm/main.py @@ -23,7 +23,12 @@ log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["LITELLM"]) -from config import MODEL_FILTER_ENABLED, MODEL_FILTER_LIST, DATA_DIR +from config import ( + MODEL_FILTER_ENABLED, + MODEL_FILTER_LIST, + DATA_DIR, + LITELLM_PROXY_PORT, +) from litellm.utils import get_llm_provider @@ -90,9 +95,7 @@ async def run_background_process(command): async def start_litellm_background(): log.info("start_litellm_background") # Command to run in the background - command = ( - "litellm --port 14365 --telemetry False --config ./data/litellm/config.yaml" - ) + command = f"litellm --port {LITELLM_PROXY_PORT} --telemetry False --config ./data/litellm/config.yaml" await run_background_process(command) @@ -109,7 +112,6 @@ async def shutdown_litellm_background(): @app.on_event("startup") async def startup_event(): - log.info("startup_event") # TODO: Check config.yaml file and create one asyncio.create_task(start_litellm_background()) @@ -186,7 +188,7 @@ async def get_models(user=Depends(get_current_user)): while not background_process: await asyncio.sleep(0.1) - url = "http://localhost:14365/v1" + url = f"http://localhost:{LITELLM_PROXY_PORT}/v1" r = None try: r = requests.request(method="GET", url=f"{url}/models") @@ -289,7 +291,7 @@ async def delete_model_from_config( async def proxy(path: str, request: Request, user=Depends(get_verified_user)): body = await request.body() - url = "http://localhost:14365" + url = f"http://localhost:{LITELLM_PROXY_PORT}" target_url = f"{url}/{path}" diff --git a/backend/config.py b/backend/config.py index 489e80647..3f67c00d2 100644 --- a/backend/config.py +++ b/backend/config.py @@ -499,3 +499,11 @@ IMAGES_OPENAI_API_KEY = os.getenv("IMAGES_OPENAI_API_KEY", OPENAI_API_KEY) AUDIO_OPENAI_API_BASE_URL = os.getenv("AUDIO_OPENAI_API_BASE_URL", OPENAI_API_BASE_URL) AUDIO_OPENAI_API_KEY = os.getenv("AUDIO_OPENAI_API_KEY", OPENAI_API_KEY) + +#################################### +# LiteLLM +#################################### + +LITELLM_PROXY_PORT = int(os.getenv("LITELLM_PROXY_PORT", "14365")) +if LITELLM_PROXY_PORT < 0 or LITELLM_PROXY_PORT > 65535: + raise ValueError("Invalid port number for LITELLM_PROXY_PORT") From 9e9306fd2b3e2398966c4884f4e0ede370ed5efc Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Tue, 23 Apr 2024 19:19:16 +0100 Subject: [PATCH 2/4] feat: add LITELLM_PROXY_HOST to configure address litellm listens on --- backend/apps/litellm/main.py | 3 ++- backend/config.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/apps/litellm/main.py b/backend/apps/litellm/main.py index ad5c2afd5..b1752f8c6 100644 --- a/backend/apps/litellm/main.py +++ b/backend/apps/litellm/main.py @@ -28,6 +28,7 @@ from config import ( MODEL_FILTER_LIST, DATA_DIR, LITELLM_PROXY_PORT, + LITELLM_PROXY_HOST, ) from litellm.utils import get_llm_provider @@ -95,7 +96,7 @@ async def run_background_process(command): async def start_litellm_background(): log.info("start_litellm_background") # Command to run in the background - command = f"litellm --port {LITELLM_PROXY_PORT} --telemetry False --config ./data/litellm/config.yaml" + command = f"litellm --port {LITELLM_PROXY_PORT} --host {LITELLM_PROXY_HOST} --telemetry False --config ./data/litellm/config.yaml" await run_background_process(command) diff --git a/backend/config.py b/backend/config.py index 3f67c00d2..f421c8aea 100644 --- a/backend/config.py +++ b/backend/config.py @@ -507,3 +507,4 @@ AUDIO_OPENAI_API_KEY = os.getenv("AUDIO_OPENAI_API_KEY", OPENAI_API_KEY) LITELLM_PROXY_PORT = int(os.getenv("LITELLM_PROXY_PORT", "14365")) if LITELLM_PROXY_PORT < 0 or LITELLM_PROXY_PORT > 65535: raise ValueError("Invalid port number for LITELLM_PROXY_PORT") +LITELLM_PROXY_HOST = os.getenv("LITELLM_PROXY_HOST", "127.0.0.1") From 58bead039892136ac16e601d37e0dd87a3a75bf3 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Tue, 23 Apr 2024 19:22:41 +0100 Subject: [PATCH 3/4] fix: DATA_DIR was not respected when loading litellm configs --- backend/apps/litellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/litellm/main.py b/backend/apps/litellm/main.py index b1752f8c6..bdf70615c 100644 --- a/backend/apps/litellm/main.py +++ b/backend/apps/litellm/main.py @@ -96,7 +96,7 @@ async def run_background_process(command): async def start_litellm_background(): log.info("start_litellm_background") # Command to run in the background - command = f"litellm --port {LITELLM_PROXY_PORT} --host {LITELLM_PROXY_HOST} --telemetry False --config ./data/litellm/config.yaml" + command = f"litellm --port {LITELLM_PROXY_PORT} --host {LITELLM_PROXY_HOST} --telemetry False --config {LITELLM_CONFIG_DIR}" await run_background_process(command) From 5245d037aca3dd3e885be0bd723767cce69b087d Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Tue, 23 Apr 2024 19:25:43 +0100 Subject: [PATCH 4/4] fix: harden litellm exec command to prevent unintended commands logic was previously to split on space for arguments, but if any of the user controlled variables LITELLM_PROXY_HOST or DATA_DIR had spaces in them, this would not behave correctly. --- backend/apps/litellm/main.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/apps/litellm/main.py b/backend/apps/litellm/main.py index bdf70615c..119e9107e 100644 --- a/backend/apps/litellm/main.py +++ b/backend/apps/litellm/main.py @@ -1,3 +1,5 @@ +import sys + from fastapi import FastAPI, Depends, HTTPException from fastapi.routing import APIRoute from fastapi.middleware.cors import CORSMiddleware @@ -70,7 +72,7 @@ async def run_background_process(command): log.info(f"Executing command: {command}") # Execute the command and create a subprocess process = await asyncio.create_subprocess_exec( - *command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + *command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) background_process = process log.info("Subprocess started successfully.") @@ -96,7 +98,17 @@ async def run_background_process(command): async def start_litellm_background(): log.info("start_litellm_background") # Command to run in the background - command = f"litellm --port {LITELLM_PROXY_PORT} --host {LITELLM_PROXY_HOST} --telemetry False --config {LITELLM_CONFIG_DIR}" + command = [ + "litellm", + "--port", + str(LITELLM_PROXY_PORT), + "--host", + LITELLM_PROXY_HOST, + "--telemetry", + "False", + "--config", + LITELLM_CONFIG_DIR, + ] await run_background_process(command)