sh-edraft.de/sh_discord_bot
4
1
Fork 1
Code Issues 21 Pull Requests Actions Packages Projects 3 Releases 27 Wiki Activity

Finished script migration #428

Browse Source
This commit is contained in:
Sven Heidemann 2023-11-13 22:54:47 +01:00
parent 4e12ba5ffe
commit dd3bfa68c6
14 changed files with 160 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bot/src/bot_core/service/data_integrity_service.py
View File

@ -92,7 +92,7 @@ class DataIntegrityService:
except Exception as e:
self._logger.error(__name__, f"Cannot get user", e)
def _check_servers(self):
def check_servers(self):
self._logger.debug(__name__, f"Start checking Servers table")
for g in self._bot.guilds:
g: discord.Guild = g
@ -411,7 +411,7 @@ class DataIntegrityService:
await self._check_default_role()
self._check_known_users()
self._check_servers()
self.check_servers()
self._check_clients()
self._check_users()
self._check_user_joins()

17
bot/src/bot_data/model/migration.py Normal file
View File

@ -0,0 +1,17 @@
class Migration:
def __init__(self, name: str, version: str, script: str):
self._name = name
self._version = version
self._script = script
@property
def name(self) -> str:
return self._name
@property
def version(self) -> str:
return self._version
@property
def script(self) -> str:
return self._script

16
bot/src/bot_data/model/migration_history.py
View File

@ -2,10 +2,12 @@ from cpl_core.database import TableABC
class MigrationHistory(TableABC):
def __init__(self, id: str):
def __init__(self, id: str, created_at=None, modified_at=None):
self._id = id
TableABC.__init__(self)
self._created_at = created_at if created_at is not None else self._created_at
self._modified_at = modified_at if modified_at is not None else self._modified_at
@property
def migration_id(self) -> str:
@ -39,7 +41,17 @@ class MigrationHistory(TableABC):
return str(
f"""
UPDATE `MigrationHistory`
SET LastModifiedAt` = '{self._modified_at}'
SET `LastModifiedAt` = '{self._modified_at}'
WHERE `MigrationId` = '{self._id}';
"""
)
def change_id_string(self, new_name: str) -> str:
return str(
f"""
UPDATE `MigrationHistory`
SET `LastModifiedAt` = '{self._modified_at}',
`MigrationId` = '{new_name}'
WHERE `MigrationId` = '{self._id}';
"""
)

8
bot/src/bot_data/scripts/0.1.0/1_Initial_up.sql
View File

@ -1,3 +1,11 @@
CREATE TABLE IF NOT EXISTS `MigrationHistory`
(
`MigrationId` VARCHAR(255),
`CreatedAt` DATETIME(6),
`LastModifiedAt` DATETIME(6),
PRIMARY KEY (`MigrationId`)
);
CREATE TABLE IF NOT EXISTS `Servers`
(
`ServerId` BIGINT NOT NULL AUTO_INCREMENT,

0
bot/src/bot_data/scripts/0.3.0/4_AutoRoleFix_down.sql → bot/src/bot_data/scripts/0.3.0/4_AutoRoleFix1_down.sql
View File

0
bot/src/bot_data/scripts/0.3.0/4_AutoRoleFix_up.sql → bot/src/bot_data/scripts/0.3.0/4_AutoRoleFix1_up.sql
View File

11
bot/src/bot_data/scripts/1.0.0/5_DBHistory_up.sql
View File

@ -638,9 +638,6 @@ CREATE TABLE IF NOT EXISTS `UsersHistory`
`Id` BIGINT(20) NOT NULL,
`DiscordId` BIGINT(20) NOT NULL,
`XP` BIGINT(20) NOT NULL DEFAULT 0,
`ReactionCount` BIGINT(20) NOT NULL DEFAULT 0,
`MessageCount` BIGINT(20) NOT NULL DEFAULT 0,
`Birthday` DATE NULL,
`ServerId` BIGINT(20) DEFAULT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -654,9 +651,9 @@ CREATE TRIGGER `TR_UsersUpdate`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ServerId`,
`DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ServerId,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;
@ -667,9 +664,9 @@ CREATE TRIGGER `TR_UsersDelete`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ServerId`,
`Deleted`, `DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId, TRUE,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ServerId, TRUE,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;

30
bot/src/bot_data/scripts/1.1.0/2_Config_up.sql
View File

@ -101,9 +101,6 @@ CREATE TABLE IF NOT EXISTS `CFG_ServerHistory`
`HelpVoiceChannelId` BIGINT NOT NULL,
`TeamChannelId` BIGINT NOT NULL,
`LoginMessageChannelId` BIGINT NOT NULL,
`DefaultRoleId` BIGINT NULL,
`ShortRoleNameSetOnlyHighest` BOOLEAN NOT NULL DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`ServerId` BIGINT NOT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -131,9 +128,6 @@ BEGIN
`HelpVoiceChannelId`,
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`FeatureFlags`,
`ServerId`,
`DateFrom`,
`DateTo`)
@ -151,9 +145,6 @@ BEGIN
OLD.HelpVoiceChannelId,
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -180,10 +171,7 @@ BEGIN
`HelpVoiceChannelId`,
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`ServerId`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
`DateTo`)
@ -201,9 +189,6 @@ BEGIN
OLD.HelpVoiceChannelId,
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
TRUE,
OLD.LastModifiedAt,
@ -322,9 +307,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForRestart` BIGINT NOT NULL DEFAULT 8,
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
`DateTo` DATETIME(6) NOT NULL
@ -342,9 +324,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
VALUES (OLD.Id,
@ -352,9 +331,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
END;;
@ -371,9 +347,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
`DateTo`)
@ -382,9 +355,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));

15
bot/src/bot_data/scripts/1.1.7/2_FixUpdates_up.sql
View File

@ -25,7 +25,6 @@ CREATE TABLE IF NOT EXISTS `CFG_ServerHistory`
`TeamChannelId` BIGINT NOT NULL,
`LoginMessageChannelId` BIGINT NOT NULL,
`DefaultRoleId` BIGINT NULL,
`ShortRoleNameSetOnlyHighest` BOOLEAN NOT NULL DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`ServerId` BIGINT NOT NULL,
`Deleted` BOOL DEFAULT FALSE,
@ -55,7 +54,6 @@ BEGIN
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`FeatureFlags`,
`ServerId`,
`DateFrom`,
@ -75,7 +73,6 @@ BEGIN
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
OLD.LastModifiedAt,
@ -104,7 +101,6 @@ BEGIN
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`ServerId`,
`FeatureFlags`,
`Deleted`,
@ -125,7 +121,6 @@ BEGIN
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
TRUE,
@ -140,8 +135,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForRestart` BIGINT NOT NULL DEFAULT 8,
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -160,8 +153,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
@ -170,8 +161,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -189,8 +178,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
@ -200,8 +187,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,

15
bot/src/bot_data/scripts/1.2.0/1_FixUserHistory_up.sql
View File

@ -1,9 +1,3 @@
ALTER TABLE UsersHistory
ADD COLUMN ReactionCount BIGINT NOT NULL DEFAULT 0 AFTER XP;
ALTER TABLE UsersHistory
ADD COLUMN MessageCount BIGINT NOT NULL DEFAULT 0 AFTER ReactionCount;
ALTER TABLE `Users`
CHANGE `CreatedAt` `CreatedAt` DATETIME(6) NULL DEFAULT CURRENT_TIMESTAMP(6);;
@ -17,7 +11,6 @@ CREATE TABLE IF NOT EXISTS `UsersHistory`
`XP` BIGINT(20) NOT NULL DEFAULT 0,
`ReactionCount` BIGINT(20) NOT NULL DEFAULT 0,
`MessageCount` BIGINT(20) NOT NULL DEFAULT 0,
`Birthday` DATE NULL,
`ServerId` BIGINT(20) DEFAULT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -31,9 +24,9 @@ CREATE TRIGGER `TR_UsersUpdate`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `ServerId`,
`DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.ServerId,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;
@ -44,9 +37,9 @@ CREATE TRIGGER `TR_UsersDelete`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `ServerId`,
`Deleted`, `DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId, TRUE,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.ServerId, TRUE,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;

5
bot/src/bot_data/scripts/1.2.0/4_MaxSteamOfferCount_up.sql
View File

@ -15,7 +15,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -35,7 +34,6 @@ BEGIN
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
@ -45,7 +43,6 @@ BEGIN
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -64,7 +61,6 @@ BEGIN
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
@ -75,7 +71,6 @@ BEGIN
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,

120
bot/src/bot_data/service/migration_service.py
View File

@ -1,9 +1,12 @@
import glob
import os
from cpl_core.database.context import DatabaseContextABC
from cpl_core.dependency_injection import ServiceProviderABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.migration_abc import MigrationABC
from bot_data.model.migration import Migration
from bot_data.model.migration_history import MigrationHistory
@ -20,14 +23,64 @@ class MigrationService:
self._db = db
self._cursor = db.cursor
self._migrations: List[MigrationABC] = (
List(type, MigrationABC.__subclasses__()).order_by(lambda x: x.name.split("_")[0]).then_by(lambda x: x.prio)
)
# self._migrations: List[MigrationABC] = (
# List(type, MigrationABC.__subclasses__()).order_by(lambda x: x.name.split("_")[0]).then_by(lambda x: x.prio)
# )
def migrate(self):
self._logger.info(__name__, f"Running Migrations")
for migration in self._migrations:
migration_id = migration.__name__
def _migrate_from_old_to_new(self):
results = self._db.select(
"""
SELECT * FROM `MigrationHistory`
"""
)
applied_migrations = List(MigrationHistory)
for result in results:
applied_migrations.add(MigrationHistory(result[0], result[1]))
for migration in applied_migrations:
if not migration.migration_id.endswith("Migration"):
continue
self._logger.debug(__name__, f"Migrate old migration {migration.migration_id} to new method")
self._cursor.execute(migration.change_id_string(migration.migration_id.replace("Migration", "")))
self._db.save_changes()
def _load_up_scripts(self) -> List[Migration]:
migrations = List(Migration)
path = "../../src/bot_data/scripts"
if not os.path.exists(path):
raise Exception("Migration path not found")
folders = List(str, glob.glob(f"{path}/*")).order_by()
for folder in folders:
splitted = folder.split("/")
version = splitted[len(splitted) - 1]
for file in List(str, os.listdir(folder)).where(lambda x: x.endswith("_up.sql")).order_by().to_list():
if not file.endswith(".sql"):
continue
name = str(file.split(".sql")[0])
if name.endswith("_up"):
name = name.replace("_up", "")
elif name.endswith("_down"):
name = name.replace("_down", "")
name = name.split("_")[1]
with open(f"{folder}/{file}", "r") as f:
script = f.read()
f.close()
migrations.add(Migration(name, version, script))
return migrations
def _execute(self, migrations: List[Migration]):
for migration in migrations:
active_statement = ""
try:
# check if table exists
self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'")
@ -36,17 +89,52 @@ class MigrationService:
# there is a table named "tableName"
self._logger.trace(
__name__,
f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration_id)}",
f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration.name)}",
)
migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration_id))
migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration.name))
if migration_from_db is not None and len(migration_from_db) > 0:
continue
self._logger.debug(__name__, f"Running Migration {migration_id}")
migration_as_service: MigrationABC = self._services.get_service(migration)
migration_as_service.upgrade()
self._cursor.execute(MigrationHistory(migration_id).insert_string)
self._db.save_changes()
self._logger.debug(__name__, f"Running migration: {migration.name}")
for statement in migration.script.split("\n\n"):
if statement in ["", "\n"]:
continue
active_statement = statement
self._cursor.execute(statement + ";")
self._cursor.execute(MigrationHistory(migration.name).insert_string)
self._db.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get migration with id {migration}", e)
self._logger.fatal(__name__, f"Migration failed: {migration.name}\n{active_statement}", e)
def migrate(self):
self._migrate_from_old_to_new()
self._execute(self._load_up_scripts())
# def migrate(self):
# self._logger.info(__name__, f"Running Migrations")
# for migration in self._migrations:
# migration_id = migration.__name__
# try:
# # check if table exists
# self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'")
# result = self._cursor.fetchone()
# if result:
# # there is a table named "tableName"
# self._logger.trace(
# __name__,
# f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration_id)}",
# )
# migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration_id))
# if migration_from_db is not None and len(migration_from_db) > 0:
# continue
#
# self._logger.debug(__name__, f"Running Migration {migration_id}")
# migration_as_service: MigrationABC = self._services.get_service(migration)
# migration_as_service.upgrade()
# self._cursor.execute(MigrationHistory(migration_id).insert_string)
# self._db.save_changes()
#
# except Exception as e:
# self._logger.error(__name__, f"Cannot get migration with id {migration}", e)

2
bot/src/bot_data/service/technician_config_seeder.py
View File

@ -32,6 +32,8 @@ class TechnicianConfigSeeder(DataSeederABC):
8,
8,
1000000,
100,
False,
{},
List(int, [240160344557879316]),
List(

5
bot/src/modules/config/events/config_on_ready_event.py
View File

@ -4,6 +4,7 @@ from cpl_discord.events import OnReadyABC
from cpl_discord.service import DiscordBotServiceABC
from bot_core.service.config_service import ConfigService
from bot_core.service.data_integrity_service import DataIntegrityService
from bot_data.abc.server_repository_abc import ServerRepositoryABC
@ -15,6 +16,7 @@ class ConfigOnReadyEvent(OnReadyABC):
bot: DiscordBotServiceABC,
servers: ServerRepositoryABC,
config_service: ConfigService,
data_integrity_service: DataIntegrityService,
):
OnReadyABC.__init__(self)
@ -23,7 +25,10 @@ class ConfigOnReadyEvent(OnReadyABC):
self._bot = bot
self._servers = servers
self._config_service = config_service
self._data_integrity_service = data_integrity_service
async def on_ready(self):
self._data_integrity_service.check_servers()
for guild in self._bot.guilds:
await self._config_service.reload_server_config(self._servers.get_server_by_discord_id(guild.id))