Add LLMs to liteLLM config 4

backend/beyond_the_loop/routers/openai.py

@@ -557,15 +557,12 @@ async def generate_chat_completion(
     if BYPASS_MODEL_ACCESS_CONTROL:
         bypass_filter = True
 
-    idx = 0
-
     payload = {**form_data}
     metadata = payload.pop("metadata", None)
 
     model_id = form_data.get("model")
     model_info = Models.get_model_by_id(model_id)
 
-    chat_id = metadata.get("chat_id", "no_chat_id")
     has_chat_id = "chat_id" in metadata and metadata["chat_id"] is not None
 
     # Initialize the credit cost variable

litellm-config.yaml

@@ -26,31 +26,12 @@ model_list:
       arena_elo: 1116
       knowledge_cutoff: "September 2021"
       context_window: 16000
-  - model_name: 'Claude 3 Haiku'
-    litellm_params:
-      model: 'vertex_ai/claude-3-haiku@20240307'
-      vertex_project: 'testllm-94944'
-      vertex_location: 'europe-west4'
-    model_info:
-      description: 'Lightning-fast responses for simple queries.'
-      arena_elo: 1178
-      knowledge_cutoff: 'August 2023'
-      context_window: 200_000
-  - model_name: 'Claude 3.5 Sonnet'
-    litellm_params:
-      model: 'vertex_ai/claude-3-5-sonnet@20240620'
-      vertex_project: 'testllm-94944'
-      vertex_location: 'europe-west1'
-    model_info:
-      description: "Anthropic's latest model."
-      arena_elo: 1271
-      knowledge_cutoff: 'August 2023'
-      context_window: 200_000
   - model_name: 'Gemini 1.5 Pro'
     litellm_params:
       model: 'gemini-1.5-pro'
-      vertex_project: 'testllm-94944'
+      vertex_project: 'beyond-chat-1111'
       vertex_location: 'europe-west3'
+      max_tokens: 800
     model_info:
       description: "Google's leading model with a very large context window."
       arena_elo: 1265
@@ -58,14 +39,24 @@ model_list:
       context_window: 2_000_000
   - model_name: 'Gemini 1.5 Flash'
     litellm_params:
+      max_tokens: 800
       model: 'gemini-1.5-flash-001'
-      vertex_project: 'testllm-94944'
+      vertex_project: 'beyond-chat-1111'
       vertex_location: 'europe-west3'
     model_info:
       description: 'Optimized for speed and efficiency.'
       arena_elo: 1231
       knowledge_cutoff: 'November 2023'
       context_window: 1_000_000
-
+  - model_name: 'Claude 3.5 Sonnet v2'
+    litellm_params:
+      model: 'claude-3-sonnet-20240229'
+      api_key: os.environ/ANTHROPIC_API_KEY
+      max_tokens: 800
+    model_info:
+      description: 'Lightning-fast responses for simple queries.'
+      arena_elo: 1178
+      knowledge_cutoff: 'February 2024'
+      context_window: 200_000
 
 general_settings: {}