revert

backend/apps/webui/internal/migrations/004_add_archived.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 009_add_models.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
 
 Some examples (model - class or model name)::
 
@@ -37,25 +37,10 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    @migrator.create_model
+    migrator.add_fields("chat", archived=pw.BooleanField(default=False))
-    class Model(pw.Model):
-        id = pw.TextField(unique=True)
-        user_id = pw.TextField()
-        base_model_id = pw.TextField(null=True)
-
-        name = pw.TextField()
-
-        meta = pw.TextField()
-        params = pw.TextField()
-
-        created_at = pw.BigIntegerField(null=False)
-        updated_at = pw.BigIntegerField(null=False)
-
-        class Meta:
-            table_name = "model"
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_model("model")
+    migrator.remove_fields("chat", "archived")

backend/apps/webui/internal/migrations/005_add_updated_at.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 009_add_models.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
 
 Some examples (model - class or model name)::
 
@@ -37,25 +37,94 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    @migrator.create_model
+    if isinstance(database, pw.SqliteDatabase):
-    class Model(pw.Model):
+        migrate_sqlite(migrator, database, fake=fake)
-        id = pw.TextField(unique=True)
+    else:
-        user_id = pw.TextField()
+        migrate_external(migrator, database, fake=fake)
-        base_model_id = pw.TextField(null=True)
 
-        name = pw.TextField()
 
-        meta = pw.TextField()
+def migrate_sqlite(migrator: Migrator, database: pw.Database, *, fake=False):
-        params = pw.TextField()
+    # Adding fields created_at and updated_at to the 'chat' table
+    migrator.add_fields(
+        "chat",
+        created_at=pw.DateTimeField(null=True),  # Allow null for transition
+        updated_at=pw.DateTimeField(null=True),  # Allow null for transition
+    )
 
-        created_at = pw.BigIntegerField(null=False)
+    # Populate the new fields from an existing 'timestamp' field
-        updated_at = pw.BigIntegerField(null=False)
+    migrator.sql(
+        "UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL"
+    )
 
-        class Meta:
+    # Now that the data has been copied, remove the original 'timestamp' field
-            table_name = "model"
+    migrator.remove_fields("chat", "timestamp")
+
+    # Update the fields to be not null now that they are populated
+    migrator.change_fields(
+        "chat",
+        created_at=pw.DateTimeField(null=False),
+        updated_at=pw.DateTimeField(null=False),
+    )
+
+
+def migrate_external(migrator: Migrator, database: pw.Database, *, fake=False):
+    # Adding fields created_at and updated_at to the 'chat' table
+    migrator.add_fields(
+        "chat",
+        created_at=pw.BigIntegerField(null=True),  # Allow null for transition
+        updated_at=pw.BigIntegerField(null=True),  # Allow null for transition
+    )
+
+    # Populate the new fields from an existing 'timestamp' field
+    migrator.sql(
+        "UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL"
+    )
+
+    # Now that the data has been copied, remove the original 'timestamp' field
+    migrator.remove_fields("chat", "timestamp")
+
+    # Update the fields to be not null now that they are populated
+    migrator.change_fields(
+        "chat",
+        created_at=pw.BigIntegerField(null=False),
+        updated_at=pw.BigIntegerField(null=False),
+    )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_model("model")
+    if isinstance(database, pw.SqliteDatabase):
+        rollback_sqlite(migrator, database, fake=fake)
+    else:
+        rollback_external(migrator, database, fake=fake)
+
+
+def rollback_sqlite(migrator: Migrator, database: pw.Database, *, fake=False):
+    # Recreate the timestamp field initially allowing null values for safe transition
+    migrator.add_fields("chat", timestamp=pw.DateTimeField(null=True))
+
+    # Copy the earliest created_at date back into the new timestamp field
+    # This assumes created_at was originally a copy of timestamp
+    migrator.sql("UPDATE chat SET timestamp = created_at")
+
+    # Remove the created_at and updated_at fields
+    migrator.remove_fields("chat", "created_at", "updated_at")
+
+    # Finally, alter the timestamp field to not allow nulls if that was the original setting
+    migrator.change_fields("chat", timestamp=pw.DateTimeField(null=False))
+
+
+def rollback_external(migrator: Migrator, database: pw.Database, *, fake=False):
+    # Recreate the timestamp field initially allowing null values for safe transition
+    migrator.add_fields("chat", timestamp=pw.BigIntegerField(null=True))
+
+    # Copy the earliest created_at date back into the new timestamp field
+    # This assumes created_at was originally a copy of timestamp
+    migrator.sql("UPDATE chat SET timestamp = created_at")
+
+    # Remove the created_at and updated_at fields
+    migrator.remove_fields("chat", "created_at", "updated_at")
+
+    # Finally, alter the timestamp field to not allow nulls if that was the original setting
+    migrator.change_fields("chat", timestamp=pw.BigIntegerField(null=False))

backend/apps/webui/internal/migrations/006_migrate_timestamps_and_charfields.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 009_add_models.py.
+"""Peewee migrations -- 006_migrate_timestamps_and_charfields.py.
 
 Some examples (model - class or model name)::
 
@@ -37,25 +37,94 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    @migrator.create_model
+    # Alter the tables with timestamps
-    class Model(pw.Model):
+    migrator.change_fields(
-        id = pw.TextField(unique=True)
+        "chatidtag",
-        user_id = pw.TextField()
+        timestamp=pw.BigIntegerField(),
-        base_model_id = pw.TextField(null=True)
+    )
-
+    migrator.change_fields(
-        name = pw.TextField()
+        "document",
-
+        timestamp=pw.BigIntegerField(),
-        meta = pw.TextField()
+    )
-        params = pw.TextField()
+    migrator.change_fields(
-
+        "modelfile",
-        created_at = pw.BigIntegerField(null=False)
+        timestamp=pw.BigIntegerField(),
-        updated_at = pw.BigIntegerField(null=False)
+    )
-
+    migrator.change_fields(
-        class Meta:
+        "prompt",
-            table_name = "model"
+        timestamp=pw.BigIntegerField(),
+    )
+    migrator.change_fields(
+        "user",
+        timestamp=pw.BigIntegerField(),
+    )
+    # Alter the tables with varchar to text where necessary
+    migrator.change_fields(
+        "auth",
+        password=pw.TextField(),
+    )
+    migrator.change_fields(
+        "chat",
+        title=pw.TextField(),
+    )
+    migrator.change_fields(
+        "document",
+        title=pw.TextField(),
+        filename=pw.TextField(),
+    )
+    migrator.change_fields(
+        "prompt",
+        title=pw.TextField(),
+    )
+    migrator.change_fields(
+        "user",
+        profile_image_url=pw.TextField(),
+    )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_model("model")
+    if isinstance(database, pw.SqliteDatabase):
+        # Alter the tables with timestamps
+        migrator.change_fields(
+            "chatidtag",
+            timestamp=pw.DateField(),
+        )
+        migrator.change_fields(
+            "document",
+            timestamp=pw.DateField(),
+        )
+        migrator.change_fields(
+            "modelfile",
+            timestamp=pw.DateField(),
+        )
+        migrator.change_fields(
+            "prompt",
+            timestamp=pw.DateField(),
+        )
+        migrator.change_fields(
+            "user",
+            timestamp=pw.DateField(),
+        )
+    migrator.change_fields(
+        "auth",
+        password=pw.CharField(max_length=255),
+    )
+    migrator.change_fields(
+        "chat",
+        title=pw.CharField(),
+    )
+    migrator.change_fields(
+        "document",
+        title=pw.CharField(),
+        filename=pw.CharField(),
+    )
+    migrator.change_fields(
+        "prompt",
+        title=pw.CharField(),
+    )
+    migrator.change_fields(
+        "user",
+        profile_image_url=pw.CharField(),
+    )

backend/apps/webui/internal/migrations/007_add_user_last_active_at.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 009_add_models.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
 
 Some examples (model - class or model name)::
 
@@ -37,25 +37,43 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    @migrator.create_model
+    # Adding fields created_at and updated_at to the 'user' table
-    class Model(pw.Model):
+    migrator.add_fields(
-        id = pw.TextField(unique=True)
+        "user",
-        user_id = pw.TextField()
+        created_at=pw.BigIntegerField(null=True),  # Allow null for transition
-        base_model_id = pw.TextField(null=True)
+        updated_at=pw.BigIntegerField(null=True),  # Allow null for transition
+        last_active_at=pw.BigIntegerField(null=True),  # Allow null for transition
+    )
 
-        name = pw.TextField()
+    # Populate the new fields from an existing 'timestamp' field
+    migrator.sql(
+        'UPDATE "user" SET created_at = timestamp, updated_at = timestamp, last_active_at = timestamp WHERE timestamp IS NOT NULL'
+    )
 
-        meta = pw.TextField()
+    # Now that the data has been copied, remove the original 'timestamp' field
-        params = pw.TextField()
+    migrator.remove_fields("user", "timestamp")
 
-        created_at = pw.BigIntegerField(null=False)
+    # Update the fields to be not null now that they are populated
-        updated_at = pw.BigIntegerField(null=False)
+    migrator.change_fields(
-
+        "user",
-        class Meta:
+        created_at=pw.BigIntegerField(null=False),
-            table_name = "model"
+        updated_at=pw.BigIntegerField(null=False),
+        last_active_at=pw.BigIntegerField(null=False),
+    )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_model("model")
+    # Recreate the timestamp field initially allowing null values for safe transition
+    migrator.add_fields("user", timestamp=pw.BigIntegerField(null=True))
+
+    # Copy the earliest created_at date back into the new timestamp field
+    # This assumes created_at was originally a copy of timestamp
+    migrator.sql('UPDATE "user" SET timestamp = created_at')
+
+    # Remove the created_at and updated_at fields
+    migrator.remove_fields("user", "created_at", "updated_at", "last_active_at")
+
+    # Finally, alter the timestamp field to not allow nulls if that was the original setting
+    migrator.change_fields("user", timestamp=pw.BigIntegerField(null=False))

backend/apps/webui/internal/migrations/008_add_memory.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 009_add_models.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
 
 Some examples (model - class or model name)::
 
@@ -35,27 +35,19 @@ with suppress(ImportError):
 
 
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
-    """Write your migrations here."""
-
     @migrator.create_model
-    class Model(pw.Model):
+    class Memory(pw.Model):
-        id = pw.TextField(unique=True)
+        id = pw.CharField(max_length=255, unique=True)
-        user_id = pw.TextField()
+        user_id = pw.CharField(max_length=255)
-        base_model_id = pw.TextField(null=True)
+        content = pw.TextField(null=False)
-
-        name = pw.TextField()
-
-        meta = pw.TextField()
-        params = pw.TextField()
-
-        created_at = pw.BigIntegerField(null=False)
         updated_at = pw.BigIntegerField(null=False)
+        created_at = pw.BigIntegerField(null=False)
 
         class Meta:
-            table_name = "model"
+            table_name = "memory"
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_model("model")
+    migrator.remove_model("memory")

backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py

@@ -1,7 +1,10 @@
-"""Peewee migrations -- 017_add_user_oauth_sub.py.
+"""Peewee migrations -- 009_add_models.py.
+
 Some examples (model - class or model name)::
+
     > Model = migrator.orm['table_name']            # Return model in current state by name
     > Model = migrator.ModelClass                   # Return model in current state by name
+
     > migrator.sql(sql)                             # Run custom SQL
     > migrator.run(func, *args, **kwargs)           # Run python function with the given args
     > migrator.create_model(Model)                  # Create a model (could be used as decorator)
@@ -18,13 +21,16 @@ Some examples (model - class or model name)::
     > migrator.drop_index(model, *col_names)
     > migrator.drop_not_null(model, *field_names)
     > migrator.drop_constraints(model, *constraints)
+
 """
 
 from contextlib import suppress
 
 import peewee as pw
 from peewee_migrate import Migrator
+import json
 
+from utils.misc import parse_ollama_modelfile
 
 with suppress(ImportError):
     import playhouse.postgres_ext as pw_pext
@@ -33,13 +39,92 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    migrator.add_fields(
+    # Fetch data from 'modelfile' table and insert into 'model' table
-        "user",
+    migrate_modelfile_to_model(migrator, database)
-        oauth_sub=pw.TextField(null=True, unique=True),
+    # Drop the 'modelfile' table
+    migrator.remove_model("modelfile")
+
+
+def migrate_modelfile_to_model(migrator: Migrator, database: pw.Database):
+    ModelFile = migrator.orm["modelfile"]
+    Model = migrator.orm["model"]
+
+    modelfiles = ModelFile.select()
+
+    for modelfile in modelfiles:
+        # Extract and transform data in Python
+
+        modelfile.modelfile = json.loads(modelfile.modelfile)
+        meta = json.dumps(
+            {
+                "description": modelfile.modelfile.get("desc"),
+                "profile_image_url": modelfile.modelfile.get("imageUrl"),
+                "ollama": {"modelfile": modelfile.modelfile.get("content")},
+                "suggestion_prompts": modelfile.modelfile.get("suggestionPrompts"),
+                "categories": modelfile.modelfile.get("categories"),
+                "user": {**modelfile.modelfile.get("user", {}), "community": True},
+            }
+        )
+
+        info = parse_ollama_modelfile(modelfile.modelfile.get("content"))
+
+        # Insert the processed data into the 'model' table
+        Model.create(
+            id=f"ollama-{modelfile.tag_name}",
+            user_id=modelfile.user_id,
+            base_model_id=info.get("base_model_id"),
+            name=modelfile.modelfile.get("title"),
+            meta=meta,
+            params=json.dumps(info.get("params", {})),
+            created_at=modelfile.timestamp,
+            updated_at=modelfile.timestamp,
         )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_fields("user", "oauth_sub")
+    recreate_modelfile_table(migrator, database)
+    move_data_back_to_modelfile(migrator, database)
+    migrator.remove_model("model")
+
+
+def recreate_modelfile_table(migrator: Migrator, database: pw.Database):
+    query = """
+    CREATE TABLE IF NOT EXISTS modelfile (
+        user_id TEXT,
+        tag_name TEXT,
+        modelfile JSON,
+        timestamp BIGINT
+    )
+    """
+    migrator.sql(query)
+
+
+def move_data_back_to_modelfile(migrator: Migrator, database: pw.Database):
+    Model = migrator.orm["model"]
+    Modelfile = migrator.orm["modelfile"]
+
+    models = Model.select()
+
+    for model in models:
+        # Extract and transform data in Python
+        meta = json.loads(model.meta)
+
+        modelfile_data = {
+            "title": model.name,
+            "desc": meta.get("description"),
+            "imageUrl": meta.get("profile_image_url"),
+            "content": meta.get("ollama", {}).get("modelfile"),
+            "suggestionPrompts": meta.get("suggestion_prompts"),
+            "categories": meta.get("categories"),
+            "user": {k: v for k, v in meta.get("user", {}).items() if k != "community"},
+        }
+
+        # Insert the processed data back into the 'modelfile' table
+        Modelfile.create(
+            user_id=model.user_id,
+            tag_name=model.id,
+            modelfile=modelfile_data,
+            timestamp=model.created_at,
+        )

backend/apps/webui/internal/migrations/011_add_user_settings.py

@@ -1,7 +1,10 @@
-"""Peewee migrations -- 017_add_user_oauth_sub.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
+
 Some examples (model - class or model name)::
+
     > Model = migrator.orm['table_name']            # Return model in current state by name
     > Model = migrator.ModelClass                   # Return model in current state by name
+
     > migrator.sql(sql)                             # Run custom SQL
     > migrator.run(func, *args, **kwargs)           # Run python function with the given args
     > migrator.create_model(Model)                  # Create a model (could be used as decorator)
@@ -18,6 +21,7 @@ Some examples (model - class or model name)::
     > migrator.drop_index(model, *col_names)
     > migrator.drop_not_null(model, *field_names)
     > migrator.drop_constraints(model, *constraints)
+
 """
 
 from contextlib import suppress
@@ -33,13 +37,12 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    migrator.add_fields(
+    # Adding fields settings to the 'user' table
-        "user",
+    migrator.add_fields("user", settings=pw.TextField(null=True))
-        oauth_sub=pw.TextField(null=True, unique=True),
-    )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_fields("user", "oauth_sub")
+    # Remove the settings field
+    migrator.remove_fields("user", "settings")

backend/apps/webui/internal/migrations/012_add_tools.py

@@ -1,7 +1,10 @@
-"""Peewee migrations -- 017_add_user_oauth_sub.py.
+"""Peewee migrations -- 009_add_models.py.
+
 Some examples (model - class or model name)::
+
     > Model = migrator.orm['table_name']            # Return model in current state by name
     > Model = migrator.ModelClass                   # Return model in current state by name
+
     > migrator.sql(sql)                             # Run custom SQL
     > migrator.run(func, *args, **kwargs)           # Run python function with the given args
     > migrator.create_model(Model)                  # Create a model (could be used as decorator)
@@ -18,6 +21,7 @@ Some examples (model - class or model name)::
     > migrator.drop_index(model, *col_names)
     > migrator.drop_not_null(model, *field_names)
     > migrator.drop_constraints(model, *constraints)
+
 """
 
 from contextlib import suppress
@@ -33,13 +37,25 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    migrator.add_fields(
+    @migrator.create_model
-        "user",
+    class Tool(pw.Model):
-        oauth_sub=pw.TextField(null=True, unique=True),
+        id = pw.TextField(unique=True)
-    )
+        user_id = pw.TextField()
+
+        name = pw.TextField()
+        content = pw.TextField()
+        specs = pw.TextField()
+
+        meta = pw.TextField()
+
+        created_at = pw.BigIntegerField(null=False)
+        updated_at = pw.BigIntegerField(null=False)
+
+        class Meta:
+            table_name = "tool"
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_fields("user", "oauth_sub")
+    migrator.remove_model("tool")

backend/apps/webui/internal/migrations/013_add_user_info.py

@@ -1,7 +1,10 @@
-"""Peewee migrations -- 017_add_user_oauth_sub.py.
+"""Peewee migrations -- 002_add_local_sharing.py.
+
 Some examples (model - class or model name)::
+
     > Model = migrator.orm['table_name']            # Return model in current state by name
     > Model = migrator.ModelClass                   # Return model in current state by name
+
     > migrator.sql(sql)                             # Run custom SQL
     > migrator.run(func, *args, **kwargs)           # Run python function with the given args
     > migrator.create_model(Model)                  # Create a model (could be used as decorator)
@@ -18,6 +21,7 @@ Some examples (model - class or model name)::
     > migrator.drop_index(model, *col_names)
     > migrator.drop_not_null(model, *field_names)
     > migrator.drop_constraints(model, *constraints)
+
 """
 
 from contextlib import suppress
@@ -33,13 +37,12 @@ with suppress(ImportError):
 def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your migrations here."""
 
-    migrator.add_fields(
+    # Adding fields info to the 'user' table
-        "user",
+    migrator.add_fields("user", info=pw.TextField(null=True))
-        oauth_sub=pw.TextField(null=True, unique=True),
-    )
 
 
 def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
     """Write your rollback migrations here."""
 
-    migrator.remove_fields("user", "oauth_sub")
+    # Remove the settings field
+    migrator.remove_fields("user", "info")