From 95851a78fac944aa2d09c63a5c1b21f9c8d896ed Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Thu, 25 Jul 2024 11:21:43 -0400 Subject: [PATCH 1/7] feat: add RouteLLM Pipeline --- .../pipelines/providers/routellm_pipeline.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 examples/pipelines/providers/routellm_pipeline.py diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py new file mode 100644 index 0000000..a6d3ce6 --- /dev/null +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -0,0 +1,110 @@ +""" +title: RouteLLM Pipeline +author: justinh-rahb +date: 2024-07-25 +version: 0.1.0 +license: MIT +description: A pipeline for routing LLM requests using RouteLLM framework, compatible with OpenAI API. +requirements: routellm, pydantic, requests +""" + +from typing import List, Union, Generator, Iterator +from pydantic import BaseModel +import os +import logging +from routellm.controller import Controller + +class Pipeline: + class Valves(BaseModel): + ROUTELLM_ROUTER: str = "mf" + ROUTELLM_STRONG_MODEL: str = "gpt-4o" + ROUTELLM_WEAK_MODEL: str = "gpt-4o-mini" + ROUTELLM_STRONG_API_KEY: str = "sk-your-api-key" + ROUTELLM_WEAK_API_KEY: str = "sk-your-api-key" + ROUTELLM_STRONG_BASE_URL: str = "https://api.openai.com/v1" + ROUTELLM_WEAK_BASE_URL: str = "https://api.openai.com/v1" + ROUTELLM_THRESHOLD: float = 0.11593 + + def __init__(self): + self.id = "routellm" + self.name = "RouteLLM" + self.valves = self.Valves() + self.controller = None + + # Set the environment variables for API keys and base URLs + self._set_environment_variables() + + self._initialize_controller() + + def _set_environment_variables(self): + os.environ["OPENAI_API_KEY"] = self.valves.ROUTELLM_STRONG_API_KEY + logging.info(f"Setting OPENAI_API_KEY to: {os.environ['OPENAI_API_KEY']}") + + os.environ["WEAK_MODEL_API_KEY"] = self.valves.ROUTELLM_WEAK_API_KEY + logging.info(f"Setting WEAK_MODEL_API_KEY to: {os.environ['WEAK_MODEL_API_KEY']}") + + if self.valves.ROUTELLM_STRONG_BASE_URL: + os.environ['OPENAI_BASE_URL'] = self.valves.ROUTELLM_STRONG_BASE_URL + logging.info(f"Setting OPENAI_BASE_URL to: {os.environ['OPENAI_BASE_URL']}") + + if self.valves.ROUTELLM_WEAK_BASE_URL: + os.environ['WEAK_MODEL_BASE_URL'] = self.valves.ROUTELLM_WEAK_BASE_URL + logging.info(f"Setting WEAK_MODEL_BASE_URL to: {os.environ['WEAK_MODEL_BASE_URL']}") + + def pipelines(self) -> List[dict]: + return [{"id": f"routellm.{self.valves.ROUTELLM_ROUTER}", "name": f"RouteLLM/{self.valves.ROUTELLM_ROUTER}"}] + + async def on_startup(self): + logging.info(f"on_startup:{__name__}") + + async def on_shutdown(self): + logging.info(f"on_shutdown:{__name__}") + + async def on_valves_updated(self): + logging.info(f"on_valves_updated:{__name__}") + self._set_environment_variables() + self._initialize_controller() + + def _initialize_controller(self): + try: + strong_model = self.valves.ROUTELLM_STRONG_MODEL + weak_model = self.valves.ROUTELLM_WEAK_MODEL + + # Adjust model names if base URLs are provided + if self.valves.ROUTELLM_STRONG_BASE_URL: + strong_model = f"openai/{strong_model}" + if self.valves.ROUTELLM_WEAK_BASE_URL: + weak_model = f"openai/{weak_model}" + + self.controller = Controller( + routers=[self.valves.ROUTELLM_ROUTER], + strong_model=strong_model, + weak_model=weak_model + ) + logging.info("RouteLLM controller initialized successfully") + except Exception as e: + logging.error(f"Error initializing RouteLLM controller: {e}") + self.controller = None + + def pipe( + self, user_message: str, model_id: str, messages: List[dict], body: dict + ) -> Union[str, Generator, Iterator]: + if not self.controller: + return "Error: RouteLLM controller not initialized. Please update valves with valid API keys and configuration." + + try: + response = self.controller.chat.completions.create( + model=f"router-{self.valves.ROUTELLM_ROUTER}-{self.valves.ROUTELLM_THRESHOLD}", + messages=messages, + max_tokens=body.get("max_tokens", 4096), + temperature=body.get("temperature", 0.8), + stream=body.get("stream", False), + ) + + if body.get("stream", False): + return (chunk for chunk in response) + else: + return response + except Exception as e: + logging.error(f"Error in pipe: {e}") + return f"Error: {e}" From e49419ccd3aab3c33ebca83d5f93da1e8ef0816f Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Fri, 26 Jul 2024 09:31:27 -0400 Subject: [PATCH 2/7] enh: provider-agnostic --- .../pipelines/providers/routellm_pipeline.py | 55 +++++++------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index a6d3ce6..134ba03 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -2,7 +2,7 @@ title: RouteLLM Pipeline author: justinh-rahb date: 2024-07-25 -version: 0.1.0 +version: 0.2.0 license: MIT description: A pipeline for routing LLM requests using RouteLLM framework, compatible with OpenAI API. requirements: routellm, pydantic, requests @@ -10,7 +10,6 @@ requirements: routellm, pydantic, requests from typing import List, Union, Generator, Iterator from pydantic import BaseModel -import os import logging from routellm.controller import Controller @@ -31,38 +30,19 @@ class Pipeline: self.valves = self.Valves() self.controller = None - # Set the environment variables for API keys and base URLs - self._set_environment_variables() - self._initialize_controller() - def _set_environment_variables(self): - os.environ["OPENAI_API_KEY"] = self.valves.ROUTELLM_STRONG_API_KEY - logging.info(f"Setting OPENAI_API_KEY to: {os.environ['OPENAI_API_KEY']}") - - os.environ["WEAK_MODEL_API_KEY"] = self.valves.ROUTELLM_WEAK_API_KEY - logging.info(f"Setting WEAK_MODEL_API_KEY to: {os.environ['WEAK_MODEL_API_KEY']}") - - if self.valves.ROUTELLM_STRONG_BASE_URL: - os.environ['OPENAI_BASE_URL'] = self.valves.ROUTELLM_STRONG_BASE_URL - logging.info(f"Setting OPENAI_BASE_URL to: {os.environ['OPENAI_BASE_URL']}") - - if self.valves.ROUTELLM_WEAK_BASE_URL: - os.environ['WEAK_MODEL_BASE_URL'] = self.valves.ROUTELLM_WEAK_BASE_URL - logging.info(f"Setting WEAK_MODEL_BASE_URL to: {os.environ['WEAK_MODEL_BASE_URL']}") - def pipelines(self) -> List[dict]: return [{"id": f"routellm.{self.valves.ROUTELLM_ROUTER}", "name": f"RouteLLM/{self.valves.ROUTELLM_ROUTER}"}] async def on_startup(self): - logging.info(f"on_startup:{__name__}") + logging.info(f"on_startup: {__name__}") async def on_shutdown(self): - logging.info(f"on_shutdown:{__name__}") + logging.info(f"on_shutdown: {__name__}") async def on_valves_updated(self): - logging.info(f"on_valves_updated:{__name__}") - self._set_environment_variables() + logging.info(f"on_valves_updated: {__name__}") self._initialize_controller() def _initialize_controller(self): @@ -70,12 +50,10 @@ class Pipeline: strong_model = self.valves.ROUTELLM_STRONG_MODEL weak_model = self.valves.ROUTELLM_WEAK_MODEL - # Adjust model names if base URLs are provided - if self.valves.ROUTELLM_STRONG_BASE_URL: - strong_model = f"openai/{strong_model}" - if self.valves.ROUTELLM_WEAK_BASE_URL: - weak_model = f"openai/{weak_model}" - + # Set the API keys as environment variables + import os + os.environ["OPENAI_API_KEY"] = self.valves.ROUTELLM_STRONG_API_KEY + self.controller = Controller( routers=[self.valves.ROUTELLM_ROUTER], strong_model=strong_model, @@ -93,12 +71,19 @@ class Pipeline: return "Error: RouteLLM controller not initialized. Please update valves with valid API keys and configuration." try: - response = self.controller.chat.completions.create( - model=f"router-{self.valves.ROUTELLM_ROUTER}-{self.valves.ROUTELLM_THRESHOLD}", + model_name = f"router-{self.valves.ROUTELLM_ROUTER}-{self.valves.ROUTELLM_THRESHOLD}" + + # Prepare parameters, excluding 'model' and 'messages' if they're in body + params = {k: v for k, v in body.items() if k not in ['model', 'messages'] and v is not None} + + # Ensure 'user' is a string if present + if 'user' in params and not isinstance(params['user'], str): + params['user'] = str(params['user']) + + response = self.controller.completion( + model=model_name, messages=messages, - max_tokens=body.get("max_tokens", 4096), - temperature=body.get("temperature", 0.8), - stream=body.get("stream", False), + **params ) if body.get("stream", False): From 984d5b9f58f65282fe0c69ecf27c1b3ffe3b0b83 Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:32:05 -0400 Subject: [PATCH 3/7] fix: suffix --- .../pipelines/providers/routellm_pipeline.py | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index 134ba03..87296dc 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -9,25 +9,50 @@ requirements: routellm, pydantic, requests """ from typing import List, Union, Generator, Iterator -from pydantic import BaseModel +from pydantic import BaseModel, Field import logging from routellm.controller import Controller class Pipeline: class Valves(BaseModel): - ROUTELLM_ROUTER: str = "mf" - ROUTELLM_STRONG_MODEL: str = "gpt-4o" - ROUTELLM_WEAK_MODEL: str = "gpt-4o-mini" - ROUTELLM_STRONG_API_KEY: str = "sk-your-api-key" - ROUTELLM_WEAK_API_KEY: str = "sk-your-api-key" - ROUTELLM_STRONG_BASE_URL: str = "https://api.openai.com/v1" - ROUTELLM_WEAK_BASE_URL: str = "https://api.openai.com/v1" - ROUTELLM_THRESHOLD: float = 0.11593 + ROUTELLM_ROUTER: str = Field( + default="mf", description="Identifier for the RouteLLM router." + ) + ROUTELLM_STRONG_MODEL: str = Field( + default="gpt-4o", description="Identifier for the strong model." + ) + ROUTELLM_WEAK_MODEL: str = Field( + default="gpt-4o-mini", description="Identifier for the weak model." + ) + ROUTELLM_STRONG_API_KEY: str = Field( + default="sk-your-api-key", + description="API key for accessing the strong model." + ) + ROUTELLM_WEAK_API_KEY: str = Field( + default="sk-your-api-key", + description="API key for accessing the weak model." + ) + ROUTELLM_STRONG_BASE_URL: str = Field( + default="https://api.openai.com/v1", + description="Base URL for the strong model's API." + ) + ROUTELLM_WEAK_BASE_URL: str = Field( + default="https://api.openai.com/v1", + description="Base URL for the weak model's API." + ) + ROUTELLM_THRESHOLD: float = Field( + default=0.11593, + description="Threshold value for determining when to use the strong model." + ) + ROUTELLM_SUFFIX: str = Field( + default="OpenAI", + description="Suffix to use for model identifier and name." + ) def __init__(self): - self.id = "routellm" - self.name = "RouteLLM" self.valves = self.Valves() + self.id = f"routellm-{self.valves.ROUTELLM_SUFFIX.lower()}" + self.name = f"RouteLLM/{self.valves.ROUTELLM_SUFFIX}" self.controller = None self._initialize_controller() @@ -75,10 +100,6 @@ class Pipeline: # Prepare parameters, excluding 'model' and 'messages' if they're in body params = {k: v for k, v in body.items() if k not in ['model', 'messages'] and v is not None} - - # Ensure 'user' is a string if present - if 'user' in params and not isinstance(params['user'], str): - params['user'] = str(params['user']) response = self.controller.completion( model=model_name, From a26527679b6e06d32c68689c06d401d4d693cedf Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:54:43 -0400 Subject: [PATCH 4/7] fix: one-pipe manifold --- examples/pipelines/providers/routellm_pipeline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index 87296dc..e61ddc0 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -50,15 +50,16 @@ class Pipeline: ) def __init__(self): + self.type = "manifold" self.valves = self.Valves() - self.id = f"routellm-{self.valves.ROUTELLM_SUFFIX.lower()}" - self.name = f"RouteLLM/{self.valves.ROUTELLM_SUFFIX}" + self.id = "routellm" + self.name = f"RouteLLM/" self.controller = None self._initialize_controller() def pipelines(self) -> List[dict]: - return [{"id": f"routellm.{self.valves.ROUTELLM_ROUTER}", "name": f"RouteLLM/{self.valves.ROUTELLM_ROUTER}"}] + return [{"id": f"{self.valves.ROUTELLM_SUFFIX.lower()}", "name": f"{self.valves.ROUTELLM_SUFFIX}"}] async def on_startup(self): logging.info(f"on_startup: {__name__}") From 316446e7a8fe357b39ae87c3a6109d0cf70852e2 Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:55:42 -0400 Subject: [PATCH 5/7] version bump --- examples/pipelines/providers/routellm_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index e61ddc0..032c979 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -2,7 +2,7 @@ title: RouteLLM Pipeline author: justinh-rahb date: 2024-07-25 -version: 0.2.0 +version: 0.2.1 license: MIT description: A pipeline for routing LLM requests using RouteLLM framework, compatible with OpenAI API. requirements: routellm, pydantic, requests From 0afe4aa8498de1bcd2a9f421f6a34a83497dc71a Mon Sep 17 00:00:00 2001 From: Justin Hayes <52832301+justinh-rahb@users.noreply.github.com> Date: Fri, 26 Jul 2024 11:07:14 -0400 Subject: [PATCH 6/7] fix: cast `user` as string --- examples/pipelines/providers/routellm_pipeline.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index 032c979..f7203c9 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -2,7 +2,7 @@ title: RouteLLM Pipeline author: justinh-rahb date: 2024-07-25 -version: 0.2.1 +version: 0.2.2 license: MIT description: A pipeline for routing LLM requests using RouteLLM framework, compatible with OpenAI API. requirements: routellm, pydantic, requests @@ -55,7 +55,7 @@ class Pipeline: self.id = "routellm" self.name = f"RouteLLM/" self.controller = None - + self._initialize_controller() def pipelines(self) -> List[dict]: @@ -102,12 +102,16 @@ class Pipeline: # Prepare parameters, excluding 'model' and 'messages' if they're in body params = {k: v for k, v in body.items() if k not in ['model', 'messages'] and v is not None} + # Ensure 'user' is a string if present + if 'user' in params and not isinstance(params['user'], str): + params['user'] = str(params['user']) + response = self.controller.completion( model=model_name, messages=messages, **params ) - + if body.get("stream", False): return (chunk for chunk in response) else: From aafa4c0837be9598104f232188a48e3b2638fac6 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 2 Aug 2024 09:09:08 -0400 Subject: [PATCH 7/7] Fixes --- .../pipelines/providers/routellm_pipeline.py | 77 +++++++++---------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/examples/pipelines/providers/routellm_pipeline.py b/examples/pipelines/providers/routellm_pipeline.py index f7203c9..bfc0cea 100644 --- a/examples/pipelines/providers/routellm_pipeline.py +++ b/examples/pipelines/providers/routellm_pipeline.py @@ -2,12 +2,13 @@ title: RouteLLM Pipeline author: justinh-rahb date: 2024-07-25 -version: 0.2.2 +version: 0.2.3 license: MIT description: A pipeline for routing LLM requests using RouteLLM framework, compatible with OpenAI API. requirements: routellm, pydantic, requests """ +import os from typing import List, Union, Generator, Iterator from pydantic import BaseModel, Field import logging @@ -15,8 +16,9 @@ from routellm.controller import Controller class Pipeline: class Valves(BaseModel): - ROUTELLM_ROUTER: str = Field( - default="mf", description="Identifier for the RouteLLM router." + ROUTELLM_SUFFIX: str = Field( + default="OpenAI", + description="Suffix to use for model identifier and name." ) ROUTELLM_STRONG_MODEL: str = Field( default="gpt-4o", description="Identifier for the strong model." @@ -24,51 +26,33 @@ class Pipeline: ROUTELLM_WEAK_MODEL: str = Field( default="gpt-4o-mini", description="Identifier for the weak model." ) - ROUTELLM_STRONG_API_KEY: str = Field( - default="sk-your-api-key", - description="API key for accessing the strong model." - ) - ROUTELLM_WEAK_API_KEY: str = Field( - default="sk-your-api-key", - description="API key for accessing the weak model." - ) - ROUTELLM_STRONG_BASE_URL: str = Field( + ROUTELLM_BASE_URL: str = Field( default="https://api.openai.com/v1", - description="Base URL for the strong model's API." + description="Base URL for the API." ) - ROUTELLM_WEAK_BASE_URL: str = Field( - default="https://api.openai.com/v1", - description="Base URL for the weak model's API." + ROUTELLM_API_KEY: str = Field( + default="sk-your-api-key", + description="API key for accessing models." + ) + ROUTELLM_ROUTER: str = Field( + default="mf", description="Identifier for the RouteLLM routing model." ) ROUTELLM_THRESHOLD: float = Field( default=0.11593, description="Threshold value for determining when to use the strong model." ) - ROUTELLM_SUFFIX: str = Field( - default="OpenAI", - description="Suffix to use for model identifier and name." - ) def __init__(self): self.type = "manifold" - self.valves = self.Valves() self.id = "routellm" - self.name = f"RouteLLM/" + self.name = "RouteLLM/" + + # Initialize valves with environment variables if available + self.valves = self.Valves( + ROUTELLM_API_KEY=os.getenv("OPENAI_API_KEY", "") + ) + self.controller = None - - self._initialize_controller() - - def pipelines(self) -> List[dict]: - return [{"id": f"{self.valves.ROUTELLM_SUFFIX.lower()}", "name": f"{self.valves.ROUTELLM_SUFFIX}"}] - - async def on_startup(self): - logging.info(f"on_startup: {__name__}") - - async def on_shutdown(self): - logging.info(f"on_shutdown: {__name__}") - - async def on_valves_updated(self): - logging.info(f"on_valves_updated: {__name__}") self._initialize_controller() def _initialize_controller(self): @@ -76,9 +60,8 @@ class Pipeline: strong_model = self.valves.ROUTELLM_STRONG_MODEL weak_model = self.valves.ROUTELLM_WEAK_MODEL - # Set the API keys as environment variables - import os - os.environ["OPENAI_API_KEY"] = self.valves.ROUTELLM_STRONG_API_KEY + # Set the API key as an environment variable + os.environ["OPENAI_API_KEY"] = self.valves.ROUTELLM_API_KEY self.controller = Controller( routers=[self.valves.ROUTELLM_ROUTER], @@ -90,11 +73,25 @@ class Pipeline: logging.error(f"Error initializing RouteLLM controller: {e}") self.controller = None + def pipelines(self) -> List[dict]: + return [{"id": f"{self.valves.ROUTELLM_SUFFIX.lower()}", "name": f"{self.valves.ROUTELLM_SUFFIX}"}] + + async def on_startup(self): + logging.info(f"on_startup: {__name__}") + self._initialize_controller() + + async def on_shutdown(self): + logging.info(f"on_shutdown: {__name__}") + + async def on_valves_updated(self): + logging.info(f"on_valves_updated: {__name__}") + self._initialize_controller() + def pipe( self, user_message: str, model_id: str, messages: List[dict], body: dict ) -> Union[str, Generator, Iterator]: if not self.controller: - return "Error: RouteLLM controller not initialized. Please update valves with valid API keys and configuration." + return "Error: RouteLLM controller not initialized. Please update valves with valid API key and configuration." try: model_name = f"router-{self.valves.ROUTELLM_ROUTER}-{self.valves.ROUTELLM_THRESHOLD}"