From fe5b0207c0ed9827ebdcd9448262e7a312e0b26b Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Mon, 13 Nov 2023 23:38:04 +0100 Subject: [PATCH] Added version logic & fixed downgrade scripts #428 --- .../scripts/0.2.2/1_AutoRole_down.sql | 4 +- bot/src/bot_data/scripts/0.3.0/1_Api_down.sql | 4 +- .../1.0.0/2_UserJoinedGameServer_down.sql | 6 +- .../scripts/1.1.0/1_Achievements_down.sql | 4 ++ .../bot_data/scripts/1.1.0/2_Config_down.sql | 18 +++++- .../1.2.0/3_SteamSpecialOffer_down.sql | 5 +- bot/src/bot_data/service/migration_service.py | 64 +++++++++++++++---- 7 files changed, 82 insertions(+), 23 deletions(-) diff --git a/bot/src/bot_data/scripts/0.2.2/1_AutoRole_down.sql b/bot/src/bot_data/scripts/0.2.2/1_AutoRole_down.sql index c6caf074..6e92bfe4 100644 --- a/bot/src/bot_data/scripts/0.2.2/1_AutoRole_down.sql +++ b/bot/src/bot_data/scripts/0.2.2/1_AutoRole_down.sql @@ -1,4 +1,4 @@ -DROP TABLE `AutoRole`; - DROP TABLE `AutoRoleRules`; + +DROP TABLE `AutoRoles`; diff --git a/bot/src/bot_data/scripts/0.3.0/1_Api_down.sql b/bot/src/bot_data/scripts/0.3.0/1_Api_down.sql index 49473912..46021597 100644 --- a/bot/src/bot_data/scripts/0.3.0/1_Api_down.sql +++ b/bot/src/bot_data/scripts/0.3.0/1_Api_down.sql @@ -1,4 +1,4 @@ -DROP TABLE `AuthUsers`; - DROP TABLE `AuthUserUsersRelations`; + +DROP TABLE `AuthUsers`; \ No newline at end of file diff --git a/bot/src/bot_data/scripts/1.0.0/2_UserJoinedGameServer_down.sql b/bot/src/bot_data/scripts/1.0.0/2_UserJoinedGameServer_down.sql index 2b885152..e630662a 100644 --- a/bot/src/bot_data/scripts/1.0.0/2_UserJoinedGameServer_down.sql +++ b/bot/src/bot_data/scripts/1.0.0/2_UserJoinedGameServer_down.sql @@ -1,6 +1,8 @@ -DROP TABLE `GameServers`; - DROP TABLE `UserJoinedGameServer`; + DROP TABLE `UserGameIdents`; + +DROP TABLE `GameServers`; + diff --git a/bot/src/bot_data/scripts/1.1.0/1_Achievements_down.sql b/bot/src/bot_data/scripts/1.1.0/1_Achievements_down.sql index 186492f9..ffdb52a3 100644 --- a/bot/src/bot_data/scripts/1.1.0/1_Achievements_down.sql +++ b/bot/src/bot_data/scripts/1.1.0/1_Achievements_down.sql @@ -1,6 +1,10 @@ +DROP TABLE `UserGotAchievements`; + DROP TABLE `Achievements`; ALTER TABLE Users DROP COLUMN MessageCount; ALTER TABLE Users DROP COLUMN ReactionCount; +DROP TABLE `AchievementsHistory`; + diff --git a/bot/src/bot_data/scripts/1.1.0/2_Config_down.sql b/bot/src/bot_data/scripts/1.1.0/2_Config_down.sql index 54ec30b8..d774b897 100644 --- a/bot/src/bot_data/scripts/1.1.0/2_Config_down.sql +++ b/bot/src/bot_data/scripts/1.1.0/2_Config_down.sql @@ -1,8 +1,24 @@ +DROP TABLE `CFG_ServerTeamRoleIds`; + +DROP TABLE `CFG_ServerTeamRoleIdsHistory`; + +DROP TABLE `CFG_ServerAFKChannelIds`; + +DROP TABLE `CFG_ServerAFKChannelIdsHistory`; + DROP TABLE `CFG_Server`; -DROP TABLE `CFG_Technician`; +DROP TABLE `CFG_ServerHistory`; DROP TABLE `CFG_TechnicianPingUrls`; +DROP TABLE `CFG_TechnicianPingUrlsHistory`; + DROP TABLE `CFG_TechnicianIds`; +DROP TABLE `CFG_TechnicianIdsHistory`; + +DROP TABLE `CFG_Technician`; + +DROP TABLE `CFG_TechnicianHistory`; + diff --git a/bot/src/bot_data/scripts/1.2.0/3_SteamSpecialOffer_down.sql b/bot/src/bot_data/scripts/1.2.0/3_SteamSpecialOffer_down.sql index e2e1ae6c..2cca3310 100644 --- a/bot/src/bot_data/scripts/1.2.0/3_SteamSpecialOffer_down.sql +++ b/bot/src/bot_data/scripts/1.2.0/3_SteamSpecialOffer_down.sql @@ -2,11 +2,10 @@ DROP TABLE `SteamSpecialOffers`; ALTER TABLE CFG_Server - DROP COLUMN ShortRoleNameSetOnlyHighest; - + DROP COLUMN GameOfferNotificationChatId; ALTER TABLE CFG_ServerHistory - DROP COLUMN ShortRoleNameSetOnlyHighest; + DROP COLUMN GameOfferNotificationChatId; diff --git a/bot/src/bot_data/service/migration_service.py b/bot/src/bot_data/service/migration_service.py index 4754e2cb..e9d2e327 100644 --- a/bot/src/bot_data/service/migration_service.py +++ b/bot/src/bot_data/service/migration_service.py @@ -4,7 +4,9 @@ import os from cpl_core.database.context import DatabaseContextABC from cpl_core.dependency_injection import ServiceProviderABC from cpl_query.extension import List +from packaging import version +import bot from bot_core.logging.database_logger import DatabaseLogger from bot_data.model.migration import Migration from bot_data.model.migration_history import MigrationHistory @@ -23,7 +25,7 @@ class MigrationService: self._db = db self._cursor = db.cursor - def _migrate_from_old_to_new(self): + def _get_migration_history(self) -> List[MigrationHistory]: results = self._db.select( """ SELECT * FROM `MigrationHistory` @@ -33,7 +35,15 @@ class MigrationService: for result in results: applied_migrations.add(MigrationHistory(result[0], result[1])) - for migration in applied_migrations: + return applied_migrations + + def _migrate_from_old_to_new(self): + self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'") + result = self._cursor.fetchone() + if not result: + return + + for migration in self._get_migration_history(): if not migration.migration_id.endswith("Migration"): continue @@ -41,20 +51,37 @@ class MigrationService: self._cursor.execute(migration.change_id_string(migration.migration_id.replace("Migration", ""))) self._db.save_changes() - def _load_up_scripts(self) -> List[Migration]: + def _load_scripts(self, upgrade: bool = True) -> List[Migration]: migrations = List(Migration) path = "../../src/bot_data/scripts" if not os.path.exists(path): raise Exception("Migration path not found") - folders = List(str, glob.glob(f"{path}/*")).order_by() + folders = List(str, glob.glob(f"{path}/*")) + if upgrade: + folders = folders.order_by() + else: + folders = folders.order_by_descending() for folder in folders: splitted = folder.split("/") - version = splitted[len(splitted) - 1] + version_str = splitted[len(splitted) - 1] - for file in List(str, os.listdir(folder)).where(lambda x: x.endswith("_up.sql")).order_by().to_list(): + # upgrade do not run migrations from higher versions + if upgrade and version.Version(version_str) > version.Version(bot.__version__): + break + # downgrade run migrations from higher versions + if not upgrade and version.Version(version_str) <= version.Version(bot.__version__): + continue + + files = List(str, os.listdir(folder)).where(lambda x: x.endswith(f"_{'up' if upgrade else 'down'}.sql")) + if upgrade: + files = files.order_by() + else: + files = files.order_by_descending() + + for file in files.to_list(): if not file.endswith(".sql"): continue @@ -70,11 +97,11 @@ class MigrationService: script = f.read() f.close() - migrations.add(Migration(name, version, script)) + migrations.add(Migration(name, version_str, script)) return migrations - def _execute(self, migrations: List[Migration]): + def _execute(self, migrations: List[Migration], upgrade: bool = True): for migration in migrations: active_statement = "" try: @@ -88,10 +115,14 @@ class MigrationService: f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration.name)}", ) migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration.name)) - if migration_from_db is not None and len(migration_from_db) > 0: + if upgrade and migration_from_db is not None and len(migration_from_db) > 0: + continue + elif not upgrade and (migration_from_db is None or len(migration_from_db) == 0): continue - self._logger.debug(__name__, f"Running migration: {migration.name}") + self._logger.debug( + __name__, f"Running {'upgrade' if upgrade else 'downgrade'} migration: {migration.name}" + ) for statement in migration.script.split("\n\n"): if statement in ["", "\n"]: @@ -99,11 +130,18 @@ class MigrationService: active_statement = statement self._cursor.execute(statement + ";") - self._cursor.execute(MigrationHistory(migration.name).insert_string) + self._cursor.execute( + MigrationHistory(migration.name).insert_string + if upgrade + else MigrationHistory(migration.name).delete_string + ) self._db.save_changes() except Exception as e: - self._logger.fatal(__name__, f"Migration failed: {migration.name}\n{active_statement}", e) + self._logger.fatal( + __name__, f"Migration failed: {migration.version}-{migration.name}\n{active_statement}", e + ) def migrate(self): self._migrate_from_old_to_new() - self._execute(self._load_up_scripts()) + self._execute(self._load_scripts()) + self._execute(self._load_scripts(False), False)