Added version logic & fixed downgrade scripts #428

bot/src/bot_data/scripts/0.2.2/1_AutoRole_down.sql

@@ -1,4 +1,4 @@
-DROP TABLE `AutoRole`;
-
 DROP TABLE `AutoRoleRules`;
 
+
+DROP TABLE `AutoRoles`;

bot/src/bot_data/scripts/0.3.0/1_Api_down.sql

@@ -1,4 +1,4 @@
-DROP TABLE `AuthUsers`;
-
 DROP TABLE `AuthUserUsersRelations`;
 
+
+DROP TABLE `AuthUsers`;

bot/src/bot_data/scripts/1.0.0/2_UserJoinedGameServer_down.sql

@@ -1,6 +1,8 @@
-DROP TABLE `GameServers`;
-
 DROP TABLE `UserJoinedGameServer`;
 
+
 DROP TABLE `UserGameIdents`;
 
+
+DROP TABLE `GameServers`;
+

bot/src/bot_data/scripts/1.1.0/1_Achievements_down.sql

@@ -1,6 +1,10 @@
+DROP TABLE `UserGotAchievements`;
+
 DROP TABLE `Achievements`;
 
 ALTER TABLE Users DROP COLUMN MessageCount;
 
 ALTER TABLE Users DROP COLUMN ReactionCount;
 
+DROP TABLE `AchievementsHistory`;
+

bot/src/bot_data/scripts/1.1.0/2_Config_down.sql

@@ -1,8 +1,24 @@
+DROP TABLE `CFG_ServerTeamRoleIds`;
+
+DROP TABLE `CFG_ServerTeamRoleIdsHistory`;
+
+DROP TABLE `CFG_ServerAFKChannelIds`;
+
+DROP TABLE `CFG_ServerAFKChannelIdsHistory`;
+
 DROP TABLE `CFG_Server`;
 
-DROP TABLE `CFG_Technician`;
+DROP TABLE `CFG_ServerHistory`;
 
 DROP TABLE `CFG_TechnicianPingUrls`;
 
+DROP TABLE `CFG_TechnicianPingUrlsHistory`;
+
 DROP TABLE `CFG_TechnicianIds`;
 
+DROP TABLE `CFG_TechnicianIdsHistory`;
+
+DROP TABLE `CFG_Technician`;
+
+DROP TABLE `CFG_TechnicianHistory`;
+

bot/src/bot_data/scripts/1.2.0/3_SteamSpecialOffer_down.sql

@@ -2,11 +2,10 @@ DROP TABLE `SteamSpecialOffers`;
 
 
 ALTER TABLE CFG_Server
-    DROP COLUMN ShortRoleNameSetOnlyHighest;
+    DROP COLUMN GameOfferNotificationChatId;
-
 
 
 ALTER TABLE CFG_ServerHistory
-    DROP COLUMN ShortRoleNameSetOnlyHighest;
+    DROP COLUMN GameOfferNotificationChatId;
 
 

bot/src/bot_data/service/migration_service.py

@@ -4,7 +4,9 @@ import os
 from cpl_core.database.context import DatabaseContextABC
 from cpl_core.dependency_injection import ServiceProviderABC
 from cpl_query.extension import List
+from packaging import version
 
+import bot
 from bot_core.logging.database_logger import DatabaseLogger
 from bot_data.model.migration import Migration
 from bot_data.model.migration_history import MigrationHistory
@@ -23,7 +25,7 @@ class MigrationService:
         self._db = db
         self._cursor = db.cursor
 
-    def _migrate_from_old_to_new(self):
+    def _get_migration_history(self) -> List[MigrationHistory]:
         results = self._db.select(
             """
             SELECT * FROM `MigrationHistory`
@@ -33,7 +35,15 @@ class MigrationService:
         for result in results:
             applied_migrations.add(MigrationHistory(result[0], result[1]))
 
-        for migration in applied_migrations:
+        return applied_migrations
+
+    def _migrate_from_old_to_new(self):
+        self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'")
+        result = self._cursor.fetchone()
+        if not result:
+            return
+
+        for migration in self._get_migration_history():
             if not migration.migration_id.endswith("Migration"):
                 continue
 
@@ -41,20 +51,37 @@ class MigrationService:
             self._cursor.execute(migration.change_id_string(migration.migration_id.replace("Migration", "")))
             self._db.save_changes()
 
-    def _load_up_scripts(self) -> List[Migration]:
+    def _load_scripts(self, upgrade: bool = True) -> List[Migration]:
         migrations = List(Migration)
         path = "../../src/bot_data/scripts"
 
         if not os.path.exists(path):
             raise Exception("Migration path not found")
 
-        folders = List(str, glob.glob(f"{path}/*")).order_by()
+        folders = List(str, glob.glob(f"{path}/*"))
+        if upgrade:
+            folders = folders.order_by()
+        else:
+            folders = folders.order_by_descending()
 
         for folder in folders:
             splitted = folder.split("/")
-            version = splitted[len(splitted) - 1]
+            version_str = splitted[len(splitted) - 1]
 
-            for file in List(str, os.listdir(folder)).where(lambda x: x.endswith("_up.sql")).order_by().to_list():
+            # upgrade do not run migrations from higher versions
+            if upgrade and version.Version(version_str) > version.Version(bot.__version__):
+                break
+            # downgrade run migrations from higher versions
+            if not upgrade and version.Version(version_str) <= version.Version(bot.__version__):
+                continue
+
+            files = List(str, os.listdir(folder)).where(lambda x: x.endswith(f"_{'up' if upgrade else 'down'}.sql"))
+            if upgrade:
+                files = files.order_by()
+            else:
+                files = files.order_by_descending()
+
+            for file in files.to_list():
                 if not file.endswith(".sql"):
                     continue
 
@@ -70,11 +97,11 @@ class MigrationService:
                     script = f.read()
                     f.close()
 
-                migrations.add(Migration(name, version, script))
+                migrations.add(Migration(name, version_str, script))
 
         return migrations
 
-    def _execute(self, migrations: List[Migration]):
+    def _execute(self, migrations: List[Migration], upgrade: bool = True):
         for migration in migrations:
             active_statement = ""
             try:
@@ -88,10 +115,14 @@ class MigrationService:
                         f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration.name)}",
                     )
                     migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration.name))
-                    if migration_from_db is not None and len(migration_from_db) > 0:
+                    if upgrade and migration_from_db is not None and len(migration_from_db) > 0:
+                        continue
+                    elif not upgrade and (migration_from_db is None or len(migration_from_db) == 0):
                         continue
 
-                self._logger.debug(__name__, f"Running migration: {migration.name}")
+                self._logger.debug(
+                    __name__, f"Running {'upgrade' if upgrade else 'downgrade'} migration: {migration.name}"
+                )
 
                 for statement in migration.script.split("\n\n"):
                     if statement in ["", "\n"]:
@@ -99,11 +130,18 @@ class MigrationService:
                     active_statement = statement
                     self._cursor.execute(statement + ";")
 
-                self._cursor.execute(MigrationHistory(migration.name).insert_string)
+                self._cursor.execute(
+                    MigrationHistory(migration.name).insert_string
+                    if upgrade
+                    else MigrationHistory(migration.name).delete_string
+                )
                 self._db.save_changes()
             except Exception as e:
-                self._logger.fatal(__name__, f"Migration failed: {migration.name}\n{active_statement}", e)
+                self._logger.fatal(
+                    __name__, f"Migration failed: {migration.version}-{migration.name}\n{active_statement}", e
+                )
 
     def migrate(self):
         self._migrate_from_old_to_new()
-        self._execute(self._load_up_scripts())
+        self._execute(self._load_scripts())
+        self._execute(self._load_scripts(False), False)