Introduce /models command to change model mid-conversation

Signed-off-by: Sid Sun <sid@sidsun.com>

.env.example

@@ -1,3 +0,0 @@
-API_TOKEN=1234567890:efwjbfjbew_ewbfjkbwjkbfkjbwrjkvbkjr
-OPENAI_ENDPOINT=http://localhost:3000/ollama/v1/
-OPENAI_API_KEY=sk-12345667890abcdefghijkl

.gitignore

@@ -2,6 +2,7 @@ out
 .idea/
 .vscode
 *.env
+data/
 *.iml
 *.DS_Store
 vendor/

README.md

@@ -4,30 +4,12 @@
 
 ## Configuration
 
-The configuration is loaded from environment variables, here are the environment variables you can set:
+The configuration is loaded a yaml file, en example is provided in `example.yaml` and should be self-explanatory, it has to be stored in `config.yaml` under either:
 
-### OpenAI Endpoint Options
-
-- `OPENAI_ENDPOINT`: The base OpenAI API compatible endpoint. example: `http://localhost:3000/ollama/v1/`. http & https both work.
-- `OPENAI_API_KEY`: The `Bearer` token API Key, example: `sk-12345667890abcdefghijkl`
-
-### Model Options
-
-- `MODEL`: The model to use. Default is `llama3:instruct`.
-- `MODEL_TWEAK_LEVEL`: The level of tweaking to apply to the model. Set to `advanced` to make Penalty tweaks take effect, else - none are provided to API. Default is `minimal`, i.e. no Penalty parameters.
-
-### Model Tweaks
-
-- `MAX_TOKENS`: The maximum number of tokens that the model can generate. Default is `1024`.
-- `TEMPERATURE`: Controls the randomness of the model's output. Higher values make the output more random. Default is `0.8`.
-- `REPEAT_PENALTY`: Penalty for repeating the same token. Default is `1.2`.
-- `CONTEXT_LENGTH`: The maximum number of tokens in the context. Default is `8192`.
-- `PRESENCE_PENALTY`: Penalty for using tokens that are not in the context. Default is `1.5`.
-- `FREQUENCY_PENALTY`: Penalty for using tokens that are used frequently. Default is `1.0`.
-
-### Bot Configuration
-
-- `API_TOKEN`: The Telegram Bot API token.
+- current directory
+- `data` directory
+- `config` directory
+- `data/config` directory
 
 ## Usage
 
@@ -39,19 +21,24 @@ If you send messages without a reply, bot treats that as starting a new chat / t
 
 ### Commands
 
-The bot supports two commands:
+The bot supports three commands:
 
 1. To set system prompt, use `/reset <system prompt>`.
    - The default system prompt is `You are a friendly assistant`.
    - Once you set a custom system prompt, it will remain set until you either change it or bot is restarted.
-2. To regenerate a response, reply to the message you want to regenerate from and send `/resend`
+   - This takes effect immediately after you set it, even in earlier conversations.
+2. To regenerate a response, reply to the message you want to regenerate from and send `/resend`.
    - Once the bot is restarted, the conversation history is lost and thread can't be continued.
+3. To change the model being used, use `/models`.
+   - It will present you with the available model, friendly names and basic config.
+   - Select the model using the inline keyboard.
+   - This takes effect immediately after you set it, even in earlier conversations.
 
 ## How to run
 
 ### Docker Compose
 
-Copy the docker-compose.yml in this repo, create your env file in `dev.env` and run:
+Copy the docker-compose.yml in this repo, create your config file in `data/config/config.yaml` and run:
 
 ```bash
 docker compose up
@@ -59,19 +46,12 @@ docker compose up
 
 ### Shell
 
-In the fish shell, you can just do:
-
-```fish
-env (cat dev.env | xargs -L 1) make serve
-```
-
-bash:
+In the shell, you can just do:
 
 ```bash
-env $(cat dev.env | xargs -L 1) make serve
+make serve
 ```
 
-
 ## Contributing
 
 Contributions are welcome. Please submit a pull request or create an issue if you have any improvements or suggestions.

cmd/config/config.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-	"github.com/sid-sun/openwebui-bot/pkg/bot/contract"
 	"github.com/spf13/viper"
 )
 
@@ -9,47 +8,64 @@ var GlobalConfig Config
 
 // Config contains all the neccessary configurations
 type Config struct {
-	ModelTweaks contract.ModelTweaks
-	ModelOpts   ModelOptions
-	OpenAIAPI   OpenAI
-	Bot         BotConfig
+	Models     map[string]Model
+	ModelNames []string
+	OpenAIAPI  OpenAI
+	Bot        BotConfig
 }
 
 // Load reads all config from env to config
 func Load() Config {
 	viper.AutomaticEnv()
 
-	// Model Tweaks
-	viper.SetDefault("MAX_TOKENS", 1024)
-	viper.SetDefault("TEMPERATURE", 0.8)
-	viper.SetDefault("REPEAT_PENALTY", 1.2)
-	viper.SetDefault("CONTEXT_LENGTH", 8192)
-	viper.SetDefault("PRESENCE_PENALTY", 1.5)
-	viper.SetDefault("FREQUENCY_PENALTY", 1.0)
-	// Model Options
-	viper.SetDefault("MODEL", "llama3:instruct")
-	viper.SetDefault("MODEL_TWEAK_LEVEL", "minimal")
+	viper.SetConfigName("config")      // name of config file (without extension)
+	viper.SetConfigType("yaml")        // REQUIRED if the config file does not have the extension in the name
+	viper.AddConfigPath(".")           // optionally look for config in the working directory
+	viper.AddConfigPath("config")      // optionally look for config in the working directory
+	viper.AddConfigPath("data")        // optionally look for config in the working directory
+	viper.AddConfigPath("data/config") // optionally look for config in the working directory
+	viper.ReadInConfig()               // Find and read the config file
+
+	// Set default models
+	viper.SetDefault("models", []Model{
+		defaultModel, // set in model.go
+	})
+
+	modelList := make([]Model, 1)
+	err := viper.UnmarshalKey("models", &modelList)
+	if err != nil {
+		panic(err)
+	}
+
+	// Initialize modelNames and models from modelList
+	modelNames := make([]string, len(modelList))
+	models := make(map[string]Model)
+	for i, model := range modelList {
+		modelNames[i] = model.Name
+		if _, ok := models[model.Name]; ok {
+			panic("duplicate model name")
+		}
+		models[model.Name] = model
+	}
+	if _, ok := models["default"]; !ok {
+		panic("default model not found")
+	}
+
+	// print models
+	// for name, model := range models {
+	// 	fmt.Printf("%s: %+v\n", name, model)
+	// }
 
 	GlobalConfig = Config{
 		Bot: BotConfig{
-			tkn: viper.GetString("API_TOKEN"),
+			tkn: viper.GetString("api_token"),
 		},
 		OpenAIAPI: OpenAI{
-			Endpoint: viper.GetString("OPENAI_ENDPOINT"),
-			APIKey:   viper.GetString("OPENAI_API_KEY"),
-		},
-		ModelOpts: ModelOptions{
-			Model:           viper.GetString("MODEL"),
-			modelTweakLevel: viper.GetString("MODEL_TWEAK_LEVEL"),
-		},
-		ModelTweaks: contract.ModelTweaks{
-			ContextLength:    viper.GetInt("CONTEXT_LENGTH"),
-			MaxTokens:        viper.GetInt("MAX_TOKENS"),
-			FrequencyPenalty: viper.GetFloat64("FREQUENCY_PENALTY"),
-			PresencePenalty:  viper.GetFloat64("PRESENCE_PENALTY"),
-			Temperature:      viper.GetFloat64("TEMPERATURE"),
-			RepeatPenalty:    viper.GetFloat64("REPEAT_PENALTY"),
+			Endpoint: viper.GetString("openai.endpoint"),
+			APIKey:   viper.GetString("openai.api_key"),
 		},
+		Models:     models,
+		ModelNames: modelNames,
 	}
 	return GlobalConfig
 }

cmd/config/model.go

@@ -0,0 +1,40 @@
+package config
+
+import "github.com/sid-sun/openwebui-bot/pkg/bot/contract"
+
+var defaultModel = Model{
+	Name:            "default",
+	Model:           "llama3:instruct",
+	modelTweakLevel: "basic",
+	Tweaks: contract.ModelTweaks{
+		ContextLength:    8192,
+		MaxTokens:        1024,
+		Temperature:      0.8,
+		RepeatPenalty:    1.2,
+		PresencePenalty:  1.5,
+		FrequencyPenalty: 1.0,
+	},
+}
+
+type Model struct {
+	Name            string               `mapstructure:"name"`
+	Model           string               `mapstructure:"model"`
+	modelTweakLevel string               `mapstructure:"tweak_level"`
+	Tweaks          contract.ModelTweaks `mapstructure:"tweaks"`
+}
+
+func (m Model) UseMinimalTweaks() bool {
+	return m.modelTweakLevel != "advanced"
+}
+
+func (m Model) GetAdvancedTweaks() contract.ModelTweaks {
+	return m.Tweaks
+}
+
+func (m Model) GetBasicTweaks() contract.BasicModelTweaks {
+	return contract.BasicModelTweaks{
+		ContextLength: m.Tweaks.ContextLength,
+		MaxTokens:     m.Tweaks.MaxTokens,
+		Temperature:   m.Tweaks.Temperature,
+	}
+}

cmd/config/model_options.go

@@ -1,10 +0,0 @@
-package config
-
-type ModelOptions struct {
-	Model           string
-	modelTweakLevel string
-}
-
-func (m ModelOptions) UseMinimalTweaks() bool {
-	return m.modelTweakLevel != "advanced"
-}

docker-compose.yml

@@ -6,7 +6,8 @@ services:
     #   context: .
     #   dockerfile: Dockerfile
     image: realsidsun/openwebui-telegram:latest
-    env_file:
-      - dev.env
+    volumes:
+      - ./data/store:/app/store
+      - ./data/config:/app/config
     network_mode: host
     restart: unless-stopped

example.yaml

@@ -0,0 +1,12 @@
+api_token: "1234567890:efwjbfjbew_ewbfjkbwjkbfkjbwrjkvbkjr"
+openai:
+  api_key: "sk-12345667890abcdefghijkl"
+  endpoint: "http://localhost:3000/ollama/v1/"
+models:
+  - name: "default"
+    model: "llama3:instruct"
+    tweak_level: "basic" # any value but "advanced" is basic.
+    tweaks:
+      context_length: 8192
+      max_tokens: 1024
+      temperature: 0.8

pkg/bot/contract/openai.go

@@ -24,12 +24,12 @@ type ModelOptions struct {
 }
 
 type ModelTweaks struct {
-	ContextLength    int     `json:"context_length"`
-	MaxTokens        int     `json:"max_tokens"`
-	Temperature      float64 `json:"temperature"`
-	FrequencyPenalty float64 `json:"frequency_penalty"`
-	PresencePenalty  float64 `json:"presence_penalty"`
-	RepeatPenalty    float64 `json:"repeat_penalty"`
+	ContextLength    int     `json:"context_length" mapstructure:"context_length"`
+	MaxTokens        int     `json:"max_tokens" mapstructure:"max_tokens"`
+	Temperature      float64 `json:"temperature" mapstructure:"temperature"`
+	FrequencyPenalty float64 `json:"frequency_penalty" mapstructure:"frequency_penalty"`
+	PresencePenalty  float64 `json:"presence_penalty" mapstructure:"presence_penalty"`
+	RepeatPenalty    float64 `json:"repeat_penalty" mapstructure:"repeat_penalty"`
 }
 
 type ChatMessage struct {

pkg/bot/handlers/models/models.go

@@ -0,0 +1,54 @@
+package models
+
+import (
+	"log/slog"
+	"strings"
+
+	"github.com/sid-sun/openwebui-bot/pkg/bot/store"
+	tele "gopkg.in/telebot.v3"
+)
+
+var logger = slog.Default().With(slog.String("package", "Models"))
+
+func GetModelsHandler(b *tele.Bot) tele.HandlerFunc {
+	return func(c tele.Context) error {
+		logger.Info("[Models] [GetModels] [Attempt]")
+
+		modelInfoMessage, modelOptions := getInlineKeyboardMarkup(store.ModelStore[c.Chat().ID])
+
+		mkp := b.NewMarkup()
+		mkp.InlineKeyboard = [][]tele.InlineButton{
+			modelOptions,
+		}
+
+		b.Send(c.Chat(), modelInfoMessage, mkp)
+
+		logger.Info("[Models] [GetModels] [Success]")
+		return nil
+	}
+}
+
+func CallbackHandler(b *tele.Bot) tele.HandlerFunc {
+	return func(c tele.Context) error {
+		logger.Info("[Models] [Callback] [Success]", slog.String("scope", "callback"))
+
+		model, found := strings.CutPrefix(c.Callback().Data, "model_")
+		if !found {
+			c.Send("requested model size not available")
+		}
+
+		store.ModelStore[c.Chat().ID] = model
+		modelInfoMessage, modelOptions := getInlineKeyboardMarkup(store.ModelStore[c.Chat().ID])
+		mkp := b.NewMarkup()
+		mkp.InlineKeyboard = [][]tele.InlineButton{
+			modelOptions,
+		}
+		_, err := b.Edit(c.Callback().Message, modelInfoMessage, mkp)
+		if err != nil {
+			logger.Error("[Models] [Callback] [Error]", slog.String("error", err.Error()))
+		}
+
+		logger.Info("[Models] [Callback] [Success]", slog.String("scope", "callback"))
+		return nil
+	}
+}

pkg/bot/handlers/models/utils.go

@@ -0,0 +1,29 @@
+package models
+
+import (
+	"fmt"
+
+	"github.com/sid-sun/openwebui-bot/cmd/config"
+	tele "gopkg.in/telebot.v3"
+)
+
+func getInlineKeyboardMarkup(currentModel string) (string, []tele.InlineButton) {
+	var modelOptions []tele.InlineButton
+	modelInfoMessage := "Here are the available models: \n\n"
+	if currentModel == "" {
+		currentModel = "default"
+	}
+	for _, modelName := range config.GlobalConfig.ModelNames {
+		options := config.GlobalConfig.Models[modelName]
+		text := modelName
+		if currentModel == modelName {
+			text = fmt.Sprintf("*%s*", modelName)
+		}
+		modelInfoMessage += fmt.Sprintf("%s (%s) - %d\n", modelName, options.Model, options.Tweaks.ContextLength)
+		modelOptions = append(modelOptions, tele.InlineButton{
+			Data: "model_" + modelName,
+			Text: text,
+		})
+	}
+	return modelInfoMessage, modelOptions
+}

pkg/bot/router/router.go

@@ -6,9 +6,11 @@ import (
 
 	"github.com/sid-sun/openwebui-bot/cmd/config"
 	"github.com/sid-sun/openwebui-bot/pkg/bot/handlers/completion"
+	"github.com/sid-sun/openwebui-bot/pkg/bot/handlers/models"
 	"github.com/sid-sun/openwebui-bot/pkg/bot/handlers/reset"
 	"github.com/sid-sun/openwebui-bot/pkg/bot/store"
 	tele "gopkg.in/telebot.v3"
+	"gopkg.in/telebot.v3/middleware"
 )
 
 type Bot struct {
@@ -19,9 +21,16 @@ type Bot struct {
 func (b Bot) Start() {
 	store.BotUsername = b.bot.Me.Username
 	slog.Info("[StartBot] Started Bot", slog.String("bot_name", b.bot.Me.FirstName))
-	r := b.bot.Group()
-	r.Use(StripCommand("/reset"))
-	r.Handle("/reset", reset.Handler)
+	// Register Special Commands
+	resetGroup := b.bot.Group()
+	resetGroup.Use(StripCommand("/reset"))
+	resetGroup.Handle("/reset", reset.Handler)
+	// Callbacks
+	callbackGroup := b.bot.Group()
+	callbackGroup.Use(middleware.AutoRespond())
+	callbackGroup.Handle("/models", models.GetModelsHandler(b.bot))
+	callbackGroup.Handle(tele.OnCallback, models.CallbackHandler(b.bot))
+	// Add all other handlers
 	b.bot.Handle("/resend", completion.Handler(b.bot, true))
 	b.bot.Handle(tele.OnText, completion.Handler(b.bot, false))
 	b.bot.Start()

pkg/bot/service/openai.go

@@ -27,49 +27,41 @@ func generateMessages(chatID int64, promptID int, messages []contract.ChatMessag
 }
 
 func generateAPIPayloadMinimal(chatID int64, promptID int) contract.ChatCompletionPayloadMinimal {
+	model := getModel(chatID)
 	x := contract.ChatCompletionPayloadMinimal{
 		ModelOptions: contract.ModelOptions{
-			Model:  config.GlobalConfig.ModelOpts.Model,
+			Model:  model.Model,
 			Stream: true,
 		},
 		Messages: generateMessages(chatID, promptID, []contract.ChatMessage{{
 			Role:    "system",
 			Content: getSystemPrompt(chatID),
 		}}),
-		BasicModelTweaks: contract.BasicModelTweaks{
-			Temperature:   config.GlobalConfig.ModelTweaks.Temperature,
-			MaxTokens:     config.GlobalConfig.ModelTweaks.MaxTokens,
-			ContextLength: config.GlobalConfig.ModelTweaks.ContextLength,
-		},
+		BasicModelTweaks: model.GetBasicTweaks(),
 	}
 	return x
 }
 
 func generateAPIPayload(chatID int64, promptID int) contract.ChatCompletionPayload {
+	model := getModel(chatID)
 	x := contract.ChatCompletionPayload{
 		ModelOptions: contract.ModelOptions{
-			Model:  config.GlobalConfig.ModelOpts.Model,
+			Model:  model.Model,
 			Stream: true,
 		},
 		Messages: generateMessages(chatID, promptID, []contract.ChatMessage{{
 			Role:    "system",
 			Content: getSystemPrompt(chatID),
 		}}),
-		ModelTweaks: contract.ModelTweaks{
-			MaxTokens:        config.GlobalConfig.ModelTweaks.MaxTokens,
-			Temperature:      config.GlobalConfig.ModelTweaks.Temperature,
-			RepeatPenalty:    config.GlobalConfig.ModelTweaks.RepeatPenalty,
-			ContextLength:    config.GlobalConfig.ModelTweaks.ContextLength,
-			PresencePenalty:  config.GlobalConfig.ModelTweaks.PresencePenalty,
-			FrequencyPenalty: config.GlobalConfig.ModelTweaks.FrequencyPenalty,
-		},
+		ModelTweaks: model.GetAdvancedTweaks(),
 	}
 	return x
 }
 
 func GetChatResponseStream(chatID int64, promptID int, uc chan contract.CompletionUpdate) error {
 	var payload any
-	if config.GlobalConfig.ModelOpts.UseMinimalTweaks() {
+	model := getModel(chatID)
+	if model.UseMinimalTweaks() {
 		payload = generateAPIPayloadMinimal(chatID, promptID)
 	} else {
 		payload = generateAPIPayload(chatID, promptID)

pkg/bot/service/utils.go

@@ -1,6 +1,9 @@
 package service
 
-import "github.com/sid-sun/openwebui-bot/pkg/bot/store"
+import (
+	"github.com/sid-sun/openwebui-bot/cmd/config"
+	"github.com/sid-sun/openwebui-bot/pkg/bot/store"
+)
 
 func getRole(from string) string {
 	if from == store.BotUsername {
@@ -15,3 +18,10 @@ func getSystemPrompt(chatID int64) string {
 	}
 	return store.SystemPromptStore[chatID]
 }
+
+func getModel(chatID int64) config.Model {
+	if store.ModelStore[chatID] == "" {
+		return config.GlobalConfig.Models["default"]
+	}
+	return config.GlobalConfig.Models[store.ModelStore[chatID]]
+}

pkg/bot/store/store.go

@@ -7,8 +7,10 @@ import (
 var ChatStore map[int64]map[int]*contract.MessageLink
 var SystemPromptStore map[int64]string
 var BotUsername string
+var ModelStore map[int64]string
 
 func NewStore() {
 	ChatStore = make(map[int64]map[int]*contract.MessageLink)
 	SystemPromptStore = make(map[int64]string)
+	ModelStore = make(map[int64]string)
 }

store/chat_store.json

@@ -0,0 +1 @@
+{}