chore

examples/providers/anthropic_manifold_pipeline.py

@@ -0,0 +1,123 @@
+"""
+title: Anthropic Manifold Pipeline
+author: justinh-rahb
+date: 2024-05-27
+version: 1.0
+license: MIT
+description: A pipeline for generating text using the Anthropic API.
+requirements: requests, anthropic
+environment_variables: ANTHROPIC_API_KEY
+"""
+
+import os
+from anthropic import Anthropic, RateLimitError, APIStatusError, APIConnectionError
+
+from schemas import OpenAIChatMessage
+from typing import List, Union, Generator, Iterator
+from pydantic import BaseModel
+import requests
+
+
+class Pipeline:
+    class Valves(BaseModel):
+        ANTHROPIC_API_KEY: str = ""
+
+    def __init__(self):
+        self.type = "manifold"
+        self.id = "anthropic"
+        self.name = "anthropic/"
+
+        self.valves = self.Valves(
+            **{"ANTHROPIC_API_KEY": os.getenv("ANTHROPIC_API_KEY")}
+        )
+        self.client = Anthropic(api_key=self.valves.ANTHROPIC_API_KEY)
+
+    def get_anthropic_models(self):
+        # In the future, this could fetch models dynamically from Anthropic
+        return [
+            {"id": "claude-3-haiku-20240307", "name": "claude-3-haiku"},
+            {"id": "claude-3-opus-20240229", "name": "claude-3-opus"},
+            {"id": "claude-3-sonnet-20240229", "name": "claude-3-sonnet"},
+            # Add other Anthropic models here as they become available
+        ]
+
+    async def on_startup(self):
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    async def on_valves_updated(self):
+        # This function is called when the valves are updated.
+        self.client = Anthropic(api_key=self.valves.ANTHROPIC_API_KEY)
+        pass
+
+    # Pipelines are the models that are available in the manifold.
+    # It can be a list or a function that returns a list.
+    def pipelines(self) -> List[dict]:
+        return self.get_anthropic_models()
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        try:
+            if body.get("stream", False):
+                return self.stream_response(model_id, messages, body)
+            else:
+                return self.get_completion(model_id, messages, body)
+        except (RateLimitError, APIStatusError, APIConnectionError) as e:
+            return f"Error: {e}"
+
+    def stream_response(
+        self, model_id: str, messages: List[dict], body: dict
+    ) -> Generator:
+        max_tokens = (
+            body.get("max_tokens") if body.get("max_tokens") is not None else 4096
+        )
+        temperature = (
+            body.get("temperature") if body.get("temperature") is not None else 0.8
+        )
+        top_k = body.get("top_k") if body.get("top_k") is not None else 40
+        top_p = body.get("top_p") if body.get("top_p") is not None else 0.9
+        stop_sequences = body.get("stop") if body.get("stop") is not None else []
+
+        stream = self.client.messages.create(
+            model=model_id,
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            stop_sequences=stop_sequences,
+            stream=True,
+        )
+
+        for chunk in stream:
+            if chunk.type == "content_block_start":
+                yield chunk.content_block.text
+            elif chunk.type == "content_block_delta":
+                yield chunk.delta.text
+
+    def get_completion(self, model_id: str, messages: List[dict], body: dict) -> str:
+        max_tokens = (
+            body.get("max_tokens") if body.get("max_tokens") is not None else 4096
+        )
+        temperature = (
+            body.get("temperature") if body.get("temperature") is not None else 0.8
+        )
+        top_k = body.get("top_k") if body.get("top_k") is not None else 40
+        top_p = body.get("top_p") if body.get("top_p") is not None else 0.9
+        stop_sequences = body.get("stop") if body.get("stop") is not None else []
+
+        response = self.client.messages.create(
+            model=model_id,
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            stop_sequences=stop_sequences,
+        )
+        return response.content[0].text

examples/providers/azure_openai_pipeline.py

@@ -0,0 +1,59 @@
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+import requests
+
+
+class Pipeline:
+    def __init__(self):
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "azure_openai_pipeline"
+        self.name = "Azure OpenAI Pipeline"
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        # This is where you can add your custom pipelines like RAG.
+        print(f"pipe:{__name__}")
+
+        print(messages)
+        print(user_message)
+
+        AZURE_OPENAI_API_KEY = "your-azure-openai-api-key-here"
+        AZURE_OPENAI_ENDPOINT = "your-azure-openai-endpoint-here"
+        DEPLOYMENT_NAME = "your-deployment-name-here"
+        MODEL = "gpt-3.5-turbo"
+
+        headers = {"api-key": AZURE_OPENAI_API_KEY, "Content-Type": "application/json"}
+
+        url = f"{AZURE_OPENAI_ENDPOINT}/openai/deployments/{DEPLOYMENT_NAME}/chat/completions?api-version=2023-10-01-preview"
+
+        try:
+            r = requests.post(
+                url=url,
+                json={**body, "model": MODEL},
+                headers=headers,
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/cohere_manifold_pipeline.py

@@ -0,0 +1,154 @@
+"""
+title: Cohere Manifold Pipeline
+author: justinh-rahb
+date: 2024-05-28
+version: 1.0
+license: MIT
+description: A pipeline for generating text using the Anthropic API.
+requirements: requests
+environment_variables: COHERE_API_KEY
+"""
+
+import os
+import json
+from schemas import OpenAIChatMessage
+from typing import List, Union, Generator, Iterator
+from pydantic import BaseModel
+import requests
+
+
+class Pipeline:
+    class Valves(BaseModel):
+        COHERE_API_BASE_URL: str = "https://api.cohere.com/v1"
+        COHERE_API_KEY: str = ""
+
+    def __init__(self):
+        self.type = "manifold"
+        self.id = "cohere"
+        self.name = "cohere/"
+
+        self.valves = self.Valves(**{"COHERE_API_KEY": os.getenv("COHERE_API_KEY")})
+
+        self.pipelines = self.get_cohere_models()
+
+    async def on_startup(self):
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    async def on_valves_updated(self):
+        # This function is called when the valves are updated.
+
+        self.pipelines = self.get_cohere_models()
+
+        pass
+
+    def get_cohere_models(self):
+        if self.valves.COHERE_API_KEY:
+            try:
+                headers = {}
+                headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}"
+                headers["Content-Type"] = "application/json"
+
+                r = requests.get(
+                    f"{self.valves.COHERE_API_BASE_URL}/models", headers=headers
+                )
+
+                models = r.json()
+                return [
+                    {
+                        "id": model["name"],
+                        "name": model["name"] if "name" in model else model["name"],
+                    }
+                    for model in models["models"]
+                ]
+            except Exception as e:
+
+                print(f"Error: {e}")
+                return [
+                    {
+                        "id": self.id,
+                        "name": "Could not fetch models from Cohere, please update the API Key in the valves.",
+                    },
+                ]
+        else:
+            return []
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        try:
+            if body.get("stream", False):
+                return self.stream_response(user_message, model_id, messages, body)
+            else:
+                return self.get_completion(user_message, model_id, messages, body)
+        except Exception as e:
+            return f"Error: {e}"
+
+    def stream_response(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Generator:
+
+        headers = {}
+        headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}"
+        headers["Content-Type"] = "application/json"
+
+        r = requests.post(
+            url=f"{self.valves.COHERE_API_BASE_URL}/chat",
+            json={
+                "model": model_id,
+                "chat_history": [
+                    {
+                        "role": "USER" if message["role"] == "user" else "CHATBOT",
+                        "message": message["content"],
+                    }
+                    for message in messages[:-1]
+                ],
+                "message": user_message,
+                "stream": True,
+            },
+            headers=headers,
+            stream=True,
+        )
+
+        r.raise_for_status()
+
+        for line in r.iter_lines():
+            if line:
+                try:
+                    line = json.loads(line)
+                    if line["event_type"] == "text-generation":
+                        yield line["text"]
+                except:
+                    pass
+
+    def get_completion(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> str:
+        headers = {}
+        headers["Authorization"] = f"Bearer {self.valves.COHERE_API_KEY}"
+        headers["Content-Type"] = "application/json"
+
+        r = requests.post(
+            url=f"{self.valves.COHERE_API_BASE_URL}/chat",
+            json={
+                "model": model_id,
+                "chat_history": [
+                    {
+                        "role": "USER" if message["role"] == "user" else "CHATBOT",
+                        "message": message["content"],
+                    }
+                    for message in messages[:-1]
+                ],
+                "message": user_message,
+            },
+            headers=headers,
+        )
+
+        r.raise_for_status()
+        data = r.json()
+
+        return data["text"] if "text" in data else "No response from Cohere."

examples/providers/litellm_manifold_pipeline.py

@@ -0,0 +1,103 @@
+"""
+title: LiteLLM Manifold Pipeline
+author: open-webui
+date: 2024-05-30
+version: 1.0
+license: MIT
+description: A manifold pipeline that uses LiteLLM.
+"""
+
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+from pydantic import BaseModel
+import requests
+
+
+class Pipeline:
+
+    class Valves(BaseModel):
+        LITELLM_BASE_URL: str
+
+    def __init__(self):
+        # You can also set the pipelines that are available in this pipeline.
+        # Set manifold to True if you want to use this pipeline as a manifold.
+        # Manifold pipelines can have multiple pipelines.
+        self.type = "manifold"
+
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "litellm_manifold"
+
+        # Optionally, you can set the name of the manifold pipeline.
+        self.name = "LiteLLM: "
+
+        # Initialize rate limits
+        self.valves = self.Valves(**{"LITELLM_BASE_URL": "http://localhost:4001"})
+        self.pipelines = []
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    async def on_valves_updated(self):
+        # This function is called when the valves are updated.
+
+        self.pipelines = self.get_litellm_models()
+        pass
+
+    def get_litellm_models(self):
+        if self.valves.LITELLM_BASE_URL:
+            try:
+                r = requests.get(f"{self.valves.LITELLM_BASE_URL}/v1/models")
+                models = r.json()
+                return [
+                    {
+                        "id": model["id"],
+                        "name": model["name"] if "name" in model else model["id"],
+                    }
+                    for model in models["data"]
+                ]
+            except Exception as e:
+                print(f"Error: {e}")
+                return [
+                    {
+                        "id": self.id,
+                        "name": "Could not fetch models from LiteLLM, please update the URL in the valves.",
+                    },
+                ]
+        else:
+            return []
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        if "user" in body:
+            print("######################################")
+            print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})')
+            print(f"# Message: {user_message}")
+            print("######################################")
+
+        try:
+            r = requests.post(
+                url=f"{self.valves.LITELLM_BASE_URL}/v1/chat/completions",
+                json={**body, "model": model_id, "user_id": body["user"]["id"]},
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/litellm_subprocess_manifold_pipeline.py

@@ -0,0 +1,211 @@
+"""
+title: LiteLLM Subprocess Manifold Pipeline
+author: open-webui
+date: 2024-05-30
+version: 1.0
+license: MIT
+description: A manifold pipeline that uses LiteLLM as a subprocess.
+requirements: yaml, litellm[proxy]
+"""
+
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+from pydantic import BaseModel
+import requests
+
+
+import os
+import asyncio
+import subprocess
+import yaml
+
+
+class Pipeline:
+    class Valves(BaseModel):
+        LITELLM_CONFIG_DIR: str = "./litellm/config.yaml"
+        LITELLM_PROXY_PORT: int = 4001
+        LITELLM_PROXY_HOST: str = "127.0.0.1"
+        litellm_config: dict = {}
+
+    def __init__(self):
+        # You can also set the pipelines that are available in this pipeline.
+        # Set manifold to True if you want to use this pipeline as a manifold.
+        # Manifold pipelines can have multiple pipelines.
+        self.type = "manifold"
+
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "litellm_subprocess_manifold"
+
+        # Optionally, you can set the name of the manifold pipeline.
+        self.name = "LiteLLM: "
+
+        # Initialize Valves
+        self.valves = self.Valves(**{"LITELLM_CONFIG_DIR": f"./litellm/config.yaml"})
+        self.background_process = None
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+
+        # Check if the config file exists
+        if not os.path.exists(self.valves.LITELLM_CONFIG_DIR):
+            with open(self.valves.LITELLM_CONFIG_DIR, "w") as file:
+                yaml.dump(
+                    {
+                        "general_settings": {},
+                        "litellm_settings": {},
+                        "model_list": [],
+                        "router_settings": {},
+                    },
+                    file,
+                )
+
+            print(
+                f"Config file not found. Created a default config file at {self.valves.LITELLM_CONFIG_DIR}"
+            )
+
+        with open(self.valves.LITELLM_CONFIG_DIR, "r") as file:
+            litellm_config = yaml.safe_load(file)
+
+        self.valves.litellm_config = litellm_config
+
+        asyncio.create_task(self.start_litellm_background())
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        await self.shutdown_litellm_background()
+        pass
+
+    async def on_valves_updated(self):
+        # This function is called when the valves are updated.
+
+        print(f"on_valves_updated:{__name__}")
+
+        with open(self.valves.LITELLM_CONFIG_DIR, "r") as file:
+            litellm_config = yaml.safe_load(file)
+
+        self.valves.litellm_config = litellm_config
+
+        await self.shutdown_litellm_background()
+        await self.start_litellm_background()
+        pass
+
+    async def run_background_process(self, command):
+        print("run_background_process")
+
+        try:
+            # Log the command to be executed
+            print(f"Executing command: {command}")
+
+            # Execute the command and create a subprocess
+            process = await asyncio.create_subprocess_exec(
+                *command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            self.background_process = process
+            print("Subprocess started successfully.")
+
+            # Capture STDERR for debugging purposes
+            stderr_output = await process.stderr.read()
+            stderr_text = stderr_output.decode().strip()
+            if stderr_text:
+                print(f"Subprocess STDERR: {stderr_text}")
+
+            # log.info output line by line
+            async for line in process.stdout:
+                print(line.decode().strip())
+
+            # Wait for the process to finish
+            returncode = await process.wait()
+            print(f"Subprocess exited with return code {returncode}")
+        except Exception as e:
+            print(f"Failed to start subprocess: {e}")
+            raise  # Optionally re-raise the exception if you want it to propagate
+
+    async def start_litellm_background(self):
+        print("start_litellm_background")
+        # Command to run in the background
+        command = [
+            "litellm",
+            "--port",
+            str(self.valves.LITELLM_PROXY_PORT),
+            "--host",
+            self.valves.LITELLM_PROXY_HOST,
+            "--telemetry",
+            "False",
+            "--config",
+            self.valves.LITELLM_CONFIG_DIR,
+        ]
+
+        await self.run_background_process(command)
+
+    async def shutdown_litellm_background(self):
+        print("shutdown_litellm_background")
+
+        if self.background_process:
+            self.background_process.terminate()
+            await self.background_process.wait()  # Ensure the process has terminated
+            print("Subprocess terminated")
+            self.background_process = None
+
+    def get_litellm_models(self):
+        if self.background_process:
+            try:
+                r = requests.get(
+                    f"http://{self.valves.LITELLM_PROXY_HOST}:{self.valves.LITELLM_PROXY_PORT}/v1/models"
+                )
+                models = r.json()
+                return [
+                    {
+                        "id": model["id"],
+                        "name": model["name"] if "name" in model else model["id"],
+                    }
+                    for model in models["data"]
+                ]
+            except Exception as e:
+                print(f"Error: {e}")
+                return [
+                    {
+                        "id": self.id,
+                        "name": "Could not fetch models from LiteLLM, please update the URL in the valves.",
+                    },
+                ]
+        else:
+            return []
+
+    # Pipelines are the models that are available in the manifold.
+    # It can be a list or a function that returns a list.
+    def pipelines(self) -> List[dict]:
+        return self.get_litellm_models()
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        if "user" in body:
+            print("######################################")
+            print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})')
+            print(f"# Message: {user_message}")
+            print("######################################")
+
+        try:
+            r = requests.post(
+                url=f"http://{self.valves.LITELLM_PROXY_HOST}:{self.valves.LITELLM_PROXY_PORT}/v1/chat/completions",
+                json={**body, "model": model_id, "user_id": body["user"]["id"]},
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/llama_cpp_pipeline.py

@@ -0,0 +1,61 @@
+"""
+title: Llama C++ Pipeline
+author: open-webui
+date: 2024-05-30
+version: 1.0
+license: MIT
+description: A pipeline for generating responses using the Llama C++ library.
+requirements: llama-cpp-python
+"""
+
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+
+
+class Pipeline:
+    def __init__(self):
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "llama_cpp_pipeline"
+
+        self.name = "Llama C++ Pipeline"
+        self.llm = None
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        from llama_cpp import Llama
+
+        self.llm = Llama(
+            model_path="./models/llama3.gguf",
+            # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+            # seed=1337, # Uncomment to set a specific seed
+            # n_ctx=2048, # Uncomment to increase the context window
+        )
+
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        # This is where you can add your custom pipelines like RAG.
+        print(f"pipe:{__name__}")
+
+        print(messages)
+        print(user_message)
+        print(body)
+
+        response = self.llm.create_chat_completion_openai_v1(
+            messages=messages,
+            stream=body["stream"],
+        )
+
+        return response

examples/providers/mlx_pipeline.py

@@ -0,0 +1,109 @@
+"""
+title: MLX Pipeline
+author: justinh-rahb
+date: 2024-05-27
+version: 1.1
+license: MIT
+description: A pipeline for generating text using Apple MLX Framework.
+requirements: requests, mlx-lm, huggingface-hub
+environment_variables: MLX_HOST, MLX_PORT, MLX_MODEL, MLX_STOP, MLX_SUBPROCESS, HUGGINGFACE_TOKEN
+"""
+
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+import requests
+import os
+import subprocess
+import logging
+from huggingface_hub import login
+
+
+class Pipeline:
+    def __init__(self):
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "mlx_pipeline"
+        self.name = "MLX Pipeline"
+        self.host = os.getenv("MLX_HOST", "localhost")
+        self.port = os.getenv("MLX_PORT", "8080")
+        self.model = os.getenv("MLX_MODEL", "mistralai/Mistral-7B-Instruct-v0.2")
+        self.stop_sequence = os.getenv("MLX_STOP", "[INST]").split(
+            ","
+        )  # Default stop sequence is [INST]
+        self.subprocess = os.getenv("MLX_SUBPROCESS", "true").lower() == "true"
+        self.huggingface_token = os.getenv("HUGGINGFACE_TOKEN", None)
+
+        if self.huggingface_token:
+            login(self.huggingface_token)
+
+        if self.subprocess:
+            self.start_mlx_server()
+
+    def start_mlx_server(self):
+        if not os.getenv("MLX_PORT"):
+            self.port = self.find_free_port()
+            command = f"mlx_lm.server --model {self.model} --port {self.port}"
+            self.server_process = subprocess.Popen(command, shell=True)
+            logging.info(f"Started MLX server on port {self.port}")
+
+    def find_free_port(self):
+        import socket
+
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.bind(("", 0))
+        port = s.getsockname()[1]
+        s.close()
+        return port
+
+    async def on_startup(self):
+        logging.info(f"on_startup:{__name__}")
+
+    async def on_shutdown(self):
+        if self.subprocess and hasattr(self, "server_process"):
+            self.server_process.terminate()
+            logging.info(f"Terminated MLX server on port {self.port}")
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        logging.info(f"pipe:{__name__}")
+
+        url = f"http://{self.host}:{self.port}/v1/chat/completions"
+        headers = {"Content-Type": "application/json"}
+
+        # Extract and validate parameters from the request body
+        max_tokens = body.get("max_tokens", 4096)
+        if not isinstance(max_tokens, int) or max_tokens < 0:
+            max_tokens = 4096  # Default to 4096 if invalid
+
+        temperature = body.get("temperature", 0.8)
+        if not isinstance(temperature, (int, float)) or temperature < 0:
+            temperature = 0.8  # Default to 0.8 if invalid
+
+        repeat_penalty = body.get("repeat_penalty", 1.0)
+        if not isinstance(repeat_penalty, (int, float)) or repeat_penalty < 0:
+            repeat_penalty = 1.0  # Default to 1.0 if invalid
+
+        payload = {
+            "messages": messages,
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+            "repetition_penalty": repeat_penalty,
+            "stop": self.stop_sequence,
+            "stream": body.get("stream", False),
+        }
+
+        try:
+            r = requests.post(
+                url, headers=headers, json=payload, stream=body.get("stream", False)
+            )
+            r.raise_for_status()
+
+            if body.get("stream", False):
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/ollama_manifold_pipeline.py

@@ -0,0 +1,93 @@
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+from pydantic import BaseModel
+import requests
+
+
+class Pipeline:
+
+    class Valves(BaseModel):
+        OLLAMA_BASE_URL: str
+
+    def __init__(self):
+        # You can also set the pipelines that are available in this pipeline.
+        # Set manifold to True if you want to use this pipeline as a manifold.
+        # Manifold pipelines can have multiple pipelines.
+        self.type = "manifold"
+
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "ollama_manifold"
+
+        # Optionally, you can set the name of the manifold pipeline.
+        self.name = "Ollama: "
+
+        self.valves = self.Valves(**{"OLLAMA_BASE_URL": "http://localhost:11435"})
+        self.pipelines = []
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        self.pipelines = self.get_ollama_models()
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    async def on_valves_updated(self):
+        # This function is called when the valves are updated.
+        print(f"on_valves_updated:{__name__}")
+        self.pipelines = self.get_ollama_models()
+        pass
+
+    def get_ollama_models(self):
+        if self.valves.OLLAMA_BASE_URL:
+            try:
+                r = requests.get(f"{self.valves.OLLAMA_BASE_URL}/api/tags")
+                models = r.json()
+                return [
+                    {"id": model["model"], "name": model["name"]}
+                    for model in models["models"]
+                ]
+            except Exception as e:
+                print(f"Error: {e}")
+                return [
+                    {
+                        "id": self.id,
+                        "name": "Could not fetch models from Ollama, please update the URL in the valves.",
+                    },
+                ]
+        else:
+            return []
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        # This is where you can add your custom pipelines like RAG.
+
+        if "user" in body:
+            print("######################################")
+            print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})')
+            print(f"# Message: {user_message}")
+            print("######################################")
+
+        try:
+            r = requests.post(
+                url=f"{self.valves.OLLAMA_BASE_URL}/v1/chat/completions",
+                json={**body, "model": model_id},
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/ollama_pipeline.py

@@ -0,0 +1,55 @@
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+import requests
+
+
+class Pipeline:
+    def __init__(self):
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "ollama_pipeline"
+        self.name = "Ollama Pipeline"
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        # This is where you can add your custom pipelines like RAG.
+        print(f"pipe:{__name__}")
+
+        OLLAMA_BASE_URL = "http://localhost:11434"
+        MODEL = "llama3"
+
+        if "user" in body:
+            print("######################################")
+            print(f'# User: {body["user"]["name"]} ({body["user"]["id"]})')
+            print(f"# Message: {user_message}")
+            print("######################################")
+
+        try:
+            r = requests.post(
+                url=f"{OLLAMA_BASE_URL}/v1/chat/completions",
+                json={**body, "model": MODEL},
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"

examples/providers/openai_pipeline.py

@@ -0,0 +1,57 @@
+from typing import List, Union, Generator, Iterator
+from schemas import OpenAIChatMessage
+import requests
+
+
+class Pipeline:
+    def __init__(self):
+        # Optionally, you can set the id and name of the pipeline.
+        # Assign a unique identifier to the pipeline.
+        # The identifier must be unique across all pipelines.
+        # The identifier must be an alphanumeric string that can include underscores or hyphens. It cannot contain spaces, special characters, slashes, or backslashes.
+        self.id = "openai_pipeline"
+        self.name = "OpenAI Pipeline"
+        pass
+
+    async def on_startup(self):
+        # This function is called when the server is started.
+        print(f"on_startup:{__name__}")
+        pass
+
+    async def on_shutdown(self):
+        # This function is called when the server is stopped.
+        print(f"on_shutdown:{__name__}")
+        pass
+
+    def pipe(
+        self, user_message: str, model_id: str, messages: List[dict], body: dict
+    ) -> Union[str, Generator, Iterator]:
+        # This is where you can add your custom pipelines like RAG.
+        print(f"pipe:{__name__}")
+
+        print(messages)
+        print(user_message)
+
+        OPENAI_API_KEY = "your-openai-api-key-here"
+        MODEL = "gpt-3.5-turbo"
+
+        headers = {}
+        headers["Authorization"] = f"Bearer {OPENAI_API_KEY}"
+        headers["Content-Type"] = "application/json"
+
+        try:
+            r = requests.post(
+                url="https://api.openai.com/v1/chat/completions",
+                json={**body, "model": MODEL},
+                headers=headers,
+                stream=True,
+            )
+
+            r.raise_for_status()
+
+            if body["stream"]:
+                return r.iter_lines()
+            else:
+                return r.json()
+        except Exception as e:
+            return f"Error: {e}"