Improved sql migration to script migration tool #428

bot/cpl-workspace.json

@@ -21,7 +21,8 @@
       "checks": "tools/checks/checks.json",
       "get-version": "tools/get_version/get-version.json",
       "post-build": "tools/post_build/post-build.json",
-      "set-version": "tools/set_version/set-version.json"
+      "set-version": "tools/set_version/set-version.json",
+      "migration-to-sql": "tools/migration_to_sql/tools/migration-to-sql.json"
     },
     "Scripts": {
       "format": "black ./",

bot/tools/migration_to_sql/__init__.py

@@ -0,0 +1 @@
+# imports

bot/tools/migration_to_sql/application.py

@@ -0,0 +1,15 @@
+from cpl_core.application import ApplicationABC
+from cpl_core.configuration import ConfigurationABC
+from cpl_core.console import Console
+from cpl_core.dependency_injection import ServiceProviderABC
+
+
+class Application(ApplicationABC):
+    def __init__(self, config: ConfigurationABC, services: ServiceProviderABC):
+        ApplicationABC.__init__(self, config, services)
+
+    async def configure(self):
+        pass
+
+    async def main(self):
+        Console.write_line("Finished")

bot/tools/migration_to_sql/appsettings.json

@@ -0,0 +1,22 @@
+{
+  "TimeFormatSettings": {
+    "DateFormat": "%Y-%m-%d",
+    "TimeFormat": "%H:%M:%S",
+    "DateTimeFormat": "%Y-%m-%d %H:%M:%S.%f",
+    "DateTimeLogFormat": "%Y-%m-%d_%H-%M-%S"
+  },
+  "LoggingSettings": {
+    "Path": "logs/",
+    "Filename": "log_$start_time.log",
+    "ConsoleLogLevel": "ERROR",
+    "FileLogLevel": "WARN"
+  },
+  "BotLoggingSettings": {
+    "Database": {
+      "Path": "logs/$date_now/",
+      "Filename": "database.log",
+      "ConsoleLogLevel": "ERROR",
+      "FileLogLevel": "INFO"
+    }
+  }
+}

bot/tools/migration_to_sql/main.py

@@ -0,0 +1,26 @@
+import asyncio
+
+from cpl_core.application import ApplicationBuilder
+
+from bot_data.startup_migration_extension import StartupMigrationExtension
+from migration_to_sql.application import Application
+from migration_to_sql.mock.database_extension import DatabaseExtension
+from migration_to_sql.startup import Startup
+from migration_to_sql.startup_mock_extension import StartupMockExtension
+
+
+async def main():
+    app_builder = (
+        ApplicationBuilder(Application)
+        .use_extension(StartupMockExtension)
+        .use_extension(StartupMigrationExtension)
+        .use_extension(DatabaseExtension)
+        .use_startup(Startup)
+    )
+
+    app: Application = await app_builder.build_async()
+    await app.run_async()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

bot/tools/migration_to_sql/migration-to-sql.json

@@ -0,0 +1,46 @@
+{
+  "ProjectSettings": {
+    "Name": "migration-to-sql",
+    "Version": {
+      "Major": "0",
+      "Minor": "0",
+      "Micro": "0"
+    },
+    "Author": "",
+    "AuthorEmail": "",
+    "Description": "",
+    "LongDescription": "",
+    "URL": "",
+    "CopyrightDate": "",
+    "CopyrightName": "",
+    "LicenseName": "",
+    "LicenseDescription": "",
+    "Dependencies": [
+      "cpl-core>=2023.10.0"
+    ],
+    "DevDependencies": [
+      "cpl-cli>=2023.4.0.post3"
+    ],
+    "PythonVersion": ">=3.10.4",
+    "PythonPath": {
+      "linux": ""
+    },
+    "Classifiers": []
+  },
+  "BuildSettings": {
+    "ProjectType": "console",
+    "SourcePath": "",
+    "OutputPath": "../../dist",
+    "Main": "migration_to_sql.main",
+    "EntryPoint": "migration-to-sql",
+    "IncludePackageData": false,
+    "Included": [],
+    "Excluded": [
+      "*/__pycache__",
+      "*/logs",
+      "*/tests"
+    ],
+    "PackageData": {},
+    "ProjectReferences": []
+  }
+}

bot/tools/migration_to_sql/mock/database_extension.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+
+from cpl_core.application.application_extension_abc import ApplicationExtensionABC
+from cpl_core.configuration import ConfigurationABC
+from cpl_core.dependency_injection import ServiceProviderABC
+
+from migration_to_sql.mock.migration_service import MigrationService
+
+
+class DatabaseExtension(ApplicationExtensionABC):
+    def __init__(self):
+        pass
+
+    async def run(self, config: ConfigurationABC, services: ServiceProviderABC):
+        config.add_configuration("Database_StartTime", str(datetime.now()))
+        migrations: MigrationService = services.get_service(MigrationService)
+        migrations.migrate()

bot/tools/migration_to_sql/mock/migration_service.py

@@ -0,0 +1,36 @@
+from cpl_core.console import Console
+from cpl_core.dependency_injection import ServiceProviderABC
+from cpl_query.extension import List
+
+from bot_data.abc.migration_abc import MigrationABC
+from bot_data.model.migration_history import MigrationHistory
+from migration_to_sql.mock.mock_db_context import MockDBContext
+
+
+class MigrationService:
+    def __init__(
+        self,
+        services: ServiceProviderABC,
+        db: MockDBContext,
+    ):
+        self._services = services
+
+        self._db = db
+        self._cursor = db.cursor
+
+        self._migrations: List[MigrationABC] = (
+            List(type, MigrationABC.__subclasses__()).order_by(lambda x: x.name.split("_")[0]).then_by(lambda x: x.prio)
+        )
+
+    def migrate(self):
+        for migration in self._migrations:
+            migration_id = migration.__name__
+            try:
+                migration_as_service: MigrationABC = self._services.get_service(migration)
+                self._db.set_migration(migration_as_service.name)
+                migration_as_service.upgrade()
+                self._cursor.execute(MigrationHistory(migration_id).insert_string)
+                self._db.save_changes()
+
+            except Exception as e:
+                Console.error(e)

bot/tools/migration_to_sql/mock/mock_cursor.py

@@ -0,0 +1,9 @@
+from mysql.connector.cursor import MySQLCursorBuffered
+
+
+class MockCursor(MySQLCursorBuffered):
+    def __init__(self):
+        MySQLCursorBuffered.__init__(self)
+
+    def fetchone(self, *args, **kwargs):
+        pass

bot/tools/migration_to_sql/mock/mock_db_context.py

@@ -0,0 +1,53 @@
+import os
+import textwrap
+from typing import Optional
+
+from cpl_core.database import DatabaseSettings
+from cpl_core.database.context import DatabaseContextABC
+
+from migration_to_sql.mock.mock_cursor import MockCursor
+
+
+class MockDBContext(DatabaseContextABC):
+    def __init__(self):
+        DatabaseContextABC.__init__(self)
+        self._migration_version: Optional[str] = None
+        self._migration_name: Optional[str] = None
+        self._migration_script: Optional[str] = None
+
+    @property
+    def cursor(self) -> MockCursor:
+        cursor = MockCursor()
+        cursor.execute = self.execute
+        return cursor
+
+    def set_migration(self, name: str):
+        self._migration_name = name.split("_")[1].replace("Migration", "")
+        self._migration_version = name.split("_")[0]
+        self._migration_script = ""
+
+    def connect(self, database_settings: DatabaseSettings):
+        pass
+
+    def select(self, statement: str) -> list[tuple]:
+        pass
+
+    def execute(self, query: str, *args, **kwargs):
+        if "MigrationHistory" in query:
+            return None
+        else:
+            self._migration_script += f"{textwrap.dedent(query)}\n\n"
+
+    def save_changes(self):
+        path = f"../../src/bot_data/scripts/{self._migration_version}"
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        script = textwrap.dedent(self._migration_script)
+        with open(f"{path}/{self._migration_name}.sql", "w+") as f:
+            f.write(script)
+            f.close()
+
+        self._migration_name = None
+        self._migration_version = None
+        self._migration_script = None

bot/tools/migration_to_sql/startup.py

@@ -0,0 +1,47 @@
+from typing import Optional, Type, Callable
+
+from cpl_core.application import StartupABC
+from cpl_core.configuration import ConfigurationABC
+from cpl_core.dependency_injection import ServiceCollectionABC
+from cpl_core.dependency_injection import ServiceProviderABC
+from cpl_core.environment import ApplicationEnvironment
+
+from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC
+from bot_core.configuration.bot_logging_settings import BotLoggingSettings
+from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings
+from bot_core.logging.database_logger import DatabaseLogger
+from migration_to_sql.mock.migration_service import MigrationService
+
+
+class Startup(StartupABC):
+    def __init__(self):
+        StartupABC.__init__(self)
+
+    def configure_configuration(
+        self, configuration: ConfigurationABC, environment: ApplicationEnvironment
+    ) -> ConfigurationABC:
+        configuration.add_json_file("appsettings.json")
+        configuration.add_configuration(FeatureFlagsSettings, FeatureFlagsSettings())
+        self._configure_settings_with_sub_settings(
+            configuration, BotLoggingSettings, lambda x: x.files, lambda x: x.key
+        )
+        return configuration
+
+    def configure_services(
+        self, services: ServiceCollectionABC, environment: ApplicationEnvironment
+    ) -> ServiceProviderABC:
+        services.add_logging()
+        services.add_singleton(CustomFileLoggerABC, DatabaseLogger)
+        services.add_transient(MigrationService)
+        return services.build_service_provider()
+
+    @staticmethod
+    def _configure_settings_with_sub_settings(
+        config: ConfigurationABC, settings_type: Type, list_atr: Callable, atr: Callable
+    ):
+        settings: Optional[settings_type] = config.get_configuration(settings_type)
+        if settings is None:
+            return
+
+        for sub_settings in list_atr(settings):
+            config.add_configuration(f"{type(sub_settings).__name__}_{atr(sub_settings)}", sub_settings)

bot/tools/migration_to_sql/startup_mock_extension.py

@@ -0,0 +1,20 @@
+from cpl_core.application import StartupExtensionABC
+from cpl_core.configuration import ConfigurationABC
+from cpl_core.database import DatabaseSettings
+from cpl_core.database.context import DatabaseContextABC
+from cpl_core.dependency_injection import ServiceCollectionABC
+from cpl_core.environment import ApplicationEnvironmentABC
+
+from migration_to_sql.mock.mock_db_context import MockDBContext
+
+
+class StartupMockExtension(StartupExtensionABC):
+    def __init__(self):
+        pass
+
+    def configure_configuration(self, config: ConfigurationABC, env: ApplicationEnvironmentABC):
+        pass
+
+    def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC):
+        services.add_db_context(MockDBContext, DatabaseSettings())
+        services.add_transient(DatabaseContextABC, MockDBContext)