diff --git a/backend/apps/webui/internal/migrations/004_add_archived.py b/backend/apps/webui/internal/migrations/004_add_archived.py index 548ec7cdc..d01c06b4e 100644 --- a/backend/apps/webui/internal/migrations/004_add_archived.py +++ b/backend/apps/webui/internal/migrations/004_add_archived.py @@ -1,4 +1,4 @@ -"""Peewee migrations -- 009_add_models.py. +"""Peewee migrations -- 002_add_local_sharing.py. Some examples (model - class or model name):: @@ -37,25 +37,10 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - @migrator.create_model - class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) - - name = pw.TextField() - - meta = pw.TextField() - params = pw.TextField() - - created_at = pw.BigIntegerField(null=False) - updated_at = pw.BigIntegerField(null=False) - - class Meta: - table_name = "model" + migrator.add_fields("chat", archived=pw.BooleanField(default=False)) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_model("model") + migrator.remove_fields("chat", "archived") diff --git a/backend/apps/webui/internal/migrations/005_add_updated_at.py b/backend/apps/webui/internal/migrations/005_add_updated_at.py index 548ec7cdc..950866ef0 100644 --- a/backend/apps/webui/internal/migrations/005_add_updated_at.py +++ b/backend/apps/webui/internal/migrations/005_add_updated_at.py @@ -1,4 +1,4 @@ -"""Peewee migrations -- 009_add_models.py. +"""Peewee migrations -- 002_add_local_sharing.py. Some examples (model - class or model name):: @@ -37,25 +37,94 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - @migrator.create_model - class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) + if isinstance(database, pw.SqliteDatabase): + migrate_sqlite(migrator, database, fake=fake) + else: + migrate_external(migrator, database, fake=fake) - name = pw.TextField() - meta = pw.TextField() - params = pw.TextField() +def migrate_sqlite(migrator: Migrator, database: pw.Database, *, fake=False): + # Adding fields created_at and updated_at to the 'chat' table + migrator.add_fields( + "chat", + created_at=pw.DateTimeField(null=True), # Allow null for transition + updated_at=pw.DateTimeField(null=True), # Allow null for transition + ) - created_at = pw.BigIntegerField(null=False) - updated_at = pw.BigIntegerField(null=False) + # Populate the new fields from an existing 'timestamp' field + migrator.sql( + "UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL" + ) - class Meta: - table_name = "model" + # Now that the data has been copied, remove the original 'timestamp' field + migrator.remove_fields("chat", "timestamp") + + # Update the fields to be not null now that they are populated + migrator.change_fields( + "chat", + created_at=pw.DateTimeField(null=False), + updated_at=pw.DateTimeField(null=False), + ) + + +def migrate_external(migrator: Migrator, database: pw.Database, *, fake=False): + # Adding fields created_at and updated_at to the 'chat' table + migrator.add_fields( + "chat", + created_at=pw.BigIntegerField(null=True), # Allow null for transition + updated_at=pw.BigIntegerField(null=True), # Allow null for transition + ) + + # Populate the new fields from an existing 'timestamp' field + migrator.sql( + "UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL" + ) + + # Now that the data has been copied, remove the original 'timestamp' field + migrator.remove_fields("chat", "timestamp") + + # Update the fields to be not null now that they are populated + migrator.change_fields( + "chat", + created_at=pw.BigIntegerField(null=False), + updated_at=pw.BigIntegerField(null=False), + ) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_model("model") + if isinstance(database, pw.SqliteDatabase): + rollback_sqlite(migrator, database, fake=fake) + else: + rollback_external(migrator, database, fake=fake) + + +def rollback_sqlite(migrator: Migrator, database: pw.Database, *, fake=False): + # Recreate the timestamp field initially allowing null values for safe transition + migrator.add_fields("chat", timestamp=pw.DateTimeField(null=True)) + + # Copy the earliest created_at date back into the new timestamp field + # This assumes created_at was originally a copy of timestamp + migrator.sql("UPDATE chat SET timestamp = created_at") + + # Remove the created_at and updated_at fields + migrator.remove_fields("chat", "created_at", "updated_at") + + # Finally, alter the timestamp field to not allow nulls if that was the original setting + migrator.change_fields("chat", timestamp=pw.DateTimeField(null=False)) + + +def rollback_external(migrator: Migrator, database: pw.Database, *, fake=False): + # Recreate the timestamp field initially allowing null values for safe transition + migrator.add_fields("chat", timestamp=pw.BigIntegerField(null=True)) + + # Copy the earliest created_at date back into the new timestamp field + # This assumes created_at was originally a copy of timestamp + migrator.sql("UPDATE chat SET timestamp = created_at") + + # Remove the created_at and updated_at fields + migrator.remove_fields("chat", "created_at", "updated_at") + + # Finally, alter the timestamp field to not allow nulls if that was the original setting + migrator.change_fields("chat", timestamp=pw.BigIntegerField(null=False)) diff --git a/backend/apps/webui/internal/migrations/006_migrate_timestamps_and_charfields.py b/backend/apps/webui/internal/migrations/006_migrate_timestamps_and_charfields.py index 548ec7cdc..caca14d32 100644 --- a/backend/apps/webui/internal/migrations/006_migrate_timestamps_and_charfields.py +++ b/backend/apps/webui/internal/migrations/006_migrate_timestamps_and_charfields.py @@ -1,4 +1,4 @@ -"""Peewee migrations -- 009_add_models.py. +"""Peewee migrations -- 006_migrate_timestamps_and_charfields.py. Some examples (model - class or model name):: @@ -37,25 +37,94 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - @migrator.create_model - class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) - - name = pw.TextField() - - meta = pw.TextField() - params = pw.TextField() - - created_at = pw.BigIntegerField(null=False) - updated_at = pw.BigIntegerField(null=False) - - class Meta: - table_name = "model" + # Alter the tables with timestamps + migrator.change_fields( + "chatidtag", + timestamp=pw.BigIntegerField(), + ) + migrator.change_fields( + "document", + timestamp=pw.BigIntegerField(), + ) + migrator.change_fields( + "modelfile", + timestamp=pw.BigIntegerField(), + ) + migrator.change_fields( + "prompt", + timestamp=pw.BigIntegerField(), + ) + migrator.change_fields( + "user", + timestamp=pw.BigIntegerField(), + ) + # Alter the tables with varchar to text where necessary + migrator.change_fields( + "auth", + password=pw.TextField(), + ) + migrator.change_fields( + "chat", + title=pw.TextField(), + ) + migrator.change_fields( + "document", + title=pw.TextField(), + filename=pw.TextField(), + ) + migrator.change_fields( + "prompt", + title=pw.TextField(), + ) + migrator.change_fields( + "user", + profile_image_url=pw.TextField(), + ) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_model("model") + if isinstance(database, pw.SqliteDatabase): + # Alter the tables with timestamps + migrator.change_fields( + "chatidtag", + timestamp=pw.DateField(), + ) + migrator.change_fields( + "document", + timestamp=pw.DateField(), + ) + migrator.change_fields( + "modelfile", + timestamp=pw.DateField(), + ) + migrator.change_fields( + "prompt", + timestamp=pw.DateField(), + ) + migrator.change_fields( + "user", + timestamp=pw.DateField(), + ) + migrator.change_fields( + "auth", + password=pw.CharField(max_length=255), + ) + migrator.change_fields( + "chat", + title=pw.CharField(), + ) + migrator.change_fields( + "document", + title=pw.CharField(), + filename=pw.CharField(), + ) + migrator.change_fields( + "prompt", + title=pw.CharField(), + ) + migrator.change_fields( + "user", + profile_image_url=pw.CharField(), + ) diff --git a/backend/apps/webui/internal/migrations/007_add_user_last_active_at.py b/backend/apps/webui/internal/migrations/007_add_user_last_active_at.py index 548ec7cdc..dd176ba73 100644 --- a/backend/apps/webui/internal/migrations/007_add_user_last_active_at.py +++ b/backend/apps/webui/internal/migrations/007_add_user_last_active_at.py @@ -1,4 +1,4 @@ -"""Peewee migrations -- 009_add_models.py. +"""Peewee migrations -- 002_add_local_sharing.py. Some examples (model - class or model name):: @@ -37,25 +37,43 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - @migrator.create_model - class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) + # Adding fields created_at and updated_at to the 'user' table + migrator.add_fields( + "user", + created_at=pw.BigIntegerField(null=True), # Allow null for transition + updated_at=pw.BigIntegerField(null=True), # Allow null for transition + last_active_at=pw.BigIntegerField(null=True), # Allow null for transition + ) - name = pw.TextField() + # Populate the new fields from an existing 'timestamp' field + migrator.sql( + 'UPDATE "user" SET created_at = timestamp, updated_at = timestamp, last_active_at = timestamp WHERE timestamp IS NOT NULL' + ) - meta = pw.TextField() - params = pw.TextField() + # Now that the data has been copied, remove the original 'timestamp' field + migrator.remove_fields("user", "timestamp") - created_at = pw.BigIntegerField(null=False) - updated_at = pw.BigIntegerField(null=False) - - class Meta: - table_name = "model" + # Update the fields to be not null now that they are populated + migrator.change_fields( + "user", + created_at=pw.BigIntegerField(null=False), + updated_at=pw.BigIntegerField(null=False), + last_active_at=pw.BigIntegerField(null=False), + ) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_model("model") + # Recreate the timestamp field initially allowing null values for safe transition + migrator.add_fields("user", timestamp=pw.BigIntegerField(null=True)) + + # Copy the earliest created_at date back into the new timestamp field + # This assumes created_at was originally a copy of timestamp + migrator.sql('UPDATE "user" SET timestamp = created_at') + + # Remove the created_at and updated_at fields + migrator.remove_fields("user", "created_at", "updated_at", "last_active_at") + + # Finally, alter the timestamp field to not allow nulls if that was the original setting + migrator.change_fields("user", timestamp=pw.BigIntegerField(null=False)) diff --git a/backend/apps/webui/internal/migrations/008_add_memory.py b/backend/apps/webui/internal/migrations/008_add_memory.py index 548ec7cdc..9307aa4d5 100644 --- a/backend/apps/webui/internal/migrations/008_add_memory.py +++ b/backend/apps/webui/internal/migrations/008_add_memory.py @@ -1,4 +1,4 @@ -"""Peewee migrations -- 009_add_models.py. +"""Peewee migrations -- 002_add_local_sharing.py. Some examples (model - class or model name):: @@ -35,27 +35,19 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): - """Write your migrations here.""" - @migrator.create_model - class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) - - name = pw.TextField() - - meta = pw.TextField() - params = pw.TextField() - - created_at = pw.BigIntegerField(null=False) + class Memory(pw.Model): + id = pw.CharField(max_length=255, unique=True) + user_id = pw.CharField(max_length=255) + content = pw.TextField(null=False) updated_at = pw.BigIntegerField(null=False) + created_at = pw.BigIntegerField(null=False) class Meta: - table_name = "model" + table_name = "memory" def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_model("model") + migrator.remove_model("memory") diff --git a/backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py b/backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py index eaa3fa5fe..2ef814c06 100644 --- a/backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py +++ b/backend/apps/webui/internal/migrations/010_migrate_modelfiles_to_models.py @@ -1,7 +1,10 @@ -"""Peewee migrations -- 017_add_user_oauth_sub.py. +"""Peewee migrations -- 009_add_models.py. + Some examples (model - class or model name):: + > Model = migrator.orm['table_name'] # Return model in current state by name > Model = migrator.ModelClass # Return model in current state by name + > migrator.sql(sql) # Run custom SQL > migrator.run(func, *args, **kwargs) # Run python function with the given args > migrator.create_model(Model) # Create a model (could be used as decorator) @@ -18,13 +21,16 @@ Some examples (model - class or model name):: > migrator.drop_index(model, *col_names) > migrator.drop_not_null(model, *field_names) > migrator.drop_constraints(model, *constraints) + """ from contextlib import suppress import peewee as pw from peewee_migrate import Migrator +import json +from utils.misc import parse_ollama_modelfile with suppress(ImportError): import playhouse.postgres_ext as pw_pext @@ -33,13 +39,92 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - migrator.add_fields( - "user", - oauth_sub=pw.TextField(null=True, unique=True), - ) + # Fetch data from 'modelfile' table and insert into 'model' table + migrate_modelfile_to_model(migrator, database) + # Drop the 'modelfile' table + migrator.remove_model("modelfile") + + +def migrate_modelfile_to_model(migrator: Migrator, database: pw.Database): + ModelFile = migrator.orm["modelfile"] + Model = migrator.orm["model"] + + modelfiles = ModelFile.select() + + for modelfile in modelfiles: + # Extract and transform data in Python + + modelfile.modelfile = json.loads(modelfile.modelfile) + meta = json.dumps( + { + "description": modelfile.modelfile.get("desc"), + "profile_image_url": modelfile.modelfile.get("imageUrl"), + "ollama": {"modelfile": modelfile.modelfile.get("content")}, + "suggestion_prompts": modelfile.modelfile.get("suggestionPrompts"), + "categories": modelfile.modelfile.get("categories"), + "user": {**modelfile.modelfile.get("user", {}), "community": True}, + } + ) + + info = parse_ollama_modelfile(modelfile.modelfile.get("content")) + + # Insert the processed data into the 'model' table + Model.create( + id=f"ollama-{modelfile.tag_name}", + user_id=modelfile.user_id, + base_model_id=info.get("base_model_id"), + name=modelfile.modelfile.get("title"), + meta=meta, + params=json.dumps(info.get("params", {})), + created_at=modelfile.timestamp, + updated_at=modelfile.timestamp, + ) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_fields("user", "oauth_sub") + recreate_modelfile_table(migrator, database) + move_data_back_to_modelfile(migrator, database) + migrator.remove_model("model") + + +def recreate_modelfile_table(migrator: Migrator, database: pw.Database): + query = """ + CREATE TABLE IF NOT EXISTS modelfile ( + user_id TEXT, + tag_name TEXT, + modelfile JSON, + timestamp BIGINT + ) + """ + migrator.sql(query) + + +def move_data_back_to_modelfile(migrator: Migrator, database: pw.Database): + Model = migrator.orm["model"] + Modelfile = migrator.orm["modelfile"] + + models = Model.select() + + for model in models: + # Extract and transform data in Python + meta = json.loads(model.meta) + + modelfile_data = { + "title": model.name, + "desc": meta.get("description"), + "imageUrl": meta.get("profile_image_url"), + "content": meta.get("ollama", {}).get("modelfile"), + "suggestionPrompts": meta.get("suggestion_prompts"), + "categories": meta.get("categories"), + "user": {k: v for k, v in meta.get("user", {}).items() if k != "community"}, + } + + # Insert the processed data back into the 'modelfile' table + Modelfile.create( + user_id=model.user_id, + tag_name=model.id, + modelfile=modelfile_data, + timestamp=model.created_at, + ) diff --git a/backend/apps/webui/internal/migrations/011_add_user_settings.py b/backend/apps/webui/internal/migrations/011_add_user_settings.py index eaa3fa5fe..a1620dcad 100644 --- a/backend/apps/webui/internal/migrations/011_add_user_settings.py +++ b/backend/apps/webui/internal/migrations/011_add_user_settings.py @@ -1,7 +1,10 @@ -"""Peewee migrations -- 017_add_user_oauth_sub.py. +"""Peewee migrations -- 002_add_local_sharing.py. + Some examples (model - class or model name):: + > Model = migrator.orm['table_name'] # Return model in current state by name > Model = migrator.ModelClass # Return model in current state by name + > migrator.sql(sql) # Run custom SQL > migrator.run(func, *args, **kwargs) # Run python function with the given args > migrator.create_model(Model) # Create a model (could be used as decorator) @@ -18,6 +21,7 @@ Some examples (model - class or model name):: > migrator.drop_index(model, *col_names) > migrator.drop_not_null(model, *field_names) > migrator.drop_constraints(model, *constraints) + """ from contextlib import suppress @@ -33,13 +37,12 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - migrator.add_fields( - "user", - oauth_sub=pw.TextField(null=True, unique=True), - ) + # Adding fields settings to the 'user' table + migrator.add_fields("user", settings=pw.TextField(null=True)) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_fields("user", "oauth_sub") + # Remove the settings field + migrator.remove_fields("user", "settings") diff --git a/backend/apps/webui/internal/migrations/012_add_tools.py b/backend/apps/webui/internal/migrations/012_add_tools.py index eaa3fa5fe..4a68eea55 100644 --- a/backend/apps/webui/internal/migrations/012_add_tools.py +++ b/backend/apps/webui/internal/migrations/012_add_tools.py @@ -1,7 +1,10 @@ -"""Peewee migrations -- 017_add_user_oauth_sub.py. +"""Peewee migrations -- 009_add_models.py. + Some examples (model - class or model name):: + > Model = migrator.orm['table_name'] # Return model in current state by name > Model = migrator.ModelClass # Return model in current state by name + > migrator.sql(sql) # Run custom SQL > migrator.run(func, *args, **kwargs) # Run python function with the given args > migrator.create_model(Model) # Create a model (could be used as decorator) @@ -18,6 +21,7 @@ Some examples (model - class or model name):: > migrator.drop_index(model, *col_names) > migrator.drop_not_null(model, *field_names) > migrator.drop_constraints(model, *constraints) + """ from contextlib import suppress @@ -33,13 +37,25 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - migrator.add_fields( - "user", - oauth_sub=pw.TextField(null=True, unique=True), - ) + @migrator.create_model + class Tool(pw.Model): + id = pw.TextField(unique=True) + user_id = pw.TextField() + + name = pw.TextField() + content = pw.TextField() + specs = pw.TextField() + + meta = pw.TextField() + + created_at = pw.BigIntegerField(null=False) + updated_at = pw.BigIntegerField(null=False) + + class Meta: + table_name = "tool" def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_fields("user", "oauth_sub") + migrator.remove_model("tool") diff --git a/backend/apps/webui/internal/migrations/013_add_user_info.py b/backend/apps/webui/internal/migrations/013_add_user_info.py index eaa3fa5fe..0f68669cc 100644 --- a/backend/apps/webui/internal/migrations/013_add_user_info.py +++ b/backend/apps/webui/internal/migrations/013_add_user_info.py @@ -1,7 +1,10 @@ -"""Peewee migrations -- 017_add_user_oauth_sub.py. +"""Peewee migrations -- 002_add_local_sharing.py. + Some examples (model - class or model name):: + > Model = migrator.orm['table_name'] # Return model in current state by name > Model = migrator.ModelClass # Return model in current state by name + > migrator.sql(sql) # Run custom SQL > migrator.run(func, *args, **kwargs) # Run python function with the given args > migrator.create_model(Model) # Create a model (could be used as decorator) @@ -18,6 +21,7 @@ Some examples (model - class or model name):: > migrator.drop_index(model, *col_names) > migrator.drop_not_null(model, *field_names) > migrator.drop_constraints(model, *constraints) + """ from contextlib import suppress @@ -33,13 +37,12 @@ with suppress(ImportError): def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" - migrator.add_fields( - "user", - oauth_sub=pw.TextField(null=True, unique=True), - ) + # Adding fields info to the 'user' table + migrator.add_fields("user", info=pw.TextField(null=True)) def rollback(migrator: Migrator, database: pw.Database, *, fake=False): """Write your rollback migrations here.""" - migrator.remove_fields("user", "oauth_sub") + # Remove the settings field + migrator.remove_fields("user", "info")