#428 #436

Merged
edraft merged 19 commits from #428 into dev 2023-11-15 16:56:32 +01:00
14 changed files with 160 additions and 88 deletions
Showing only changes of commit dd3bfa68c6 - Show all commits

View File

@ -92,7 +92,7 @@ class DataIntegrityService:
except Exception as e:
self._logger.error(__name__, f"Cannot get user", e)
def _check_servers(self):
def check_servers(self):
self._logger.debug(__name__, f"Start checking Servers table")
for g in self._bot.guilds:
g: discord.Guild = g
@ -411,7 +411,7 @@ class DataIntegrityService:
await self._check_default_role()
self._check_known_users()
self._check_servers()
self.check_servers()
self._check_clients()
self._check_users()
self._check_user_joins()

View File

@ -0,0 +1,17 @@
class Migration:
def __init__(self, name: str, version: str, script: str):
self._name = name
self._version = version
self._script = script
@property
def name(self) -> str:
return self._name
@property
def version(self) -> str:
return self._version
@property
def script(self) -> str:
return self._script

View File

@ -2,10 +2,12 @@ from cpl_core.database import TableABC
class MigrationHistory(TableABC):
def __init__(self, id: str):
def __init__(self, id: str, created_at=None, modified_at=None):
self._id = id
TableABC.__init__(self)
self._created_at = created_at if created_at is not None else self._created_at
self._modified_at = modified_at if modified_at is not None else self._modified_at
@property
def migration_id(self) -> str:
@ -39,7 +41,17 @@ class MigrationHistory(TableABC):
return str(
f"""
UPDATE `MigrationHistory`
SET LastModifiedAt` = '{self._modified_at}'
SET `LastModifiedAt` = '{self._modified_at}'
WHERE `MigrationId` = '{self._id}';
"""
)
def change_id_string(self, new_name: str) -> str:
return str(
f"""
UPDATE `MigrationHistory`
SET `LastModifiedAt` = '{self._modified_at}',
`MigrationId` = '{new_name}'
WHERE `MigrationId` = '{self._id}';
"""
)

View File

@ -1,3 +1,11 @@
CREATE TABLE IF NOT EXISTS `MigrationHistory`
(
`MigrationId` VARCHAR(255),
`CreatedAt` DATETIME(6),
`LastModifiedAt` DATETIME(6),
PRIMARY KEY (`MigrationId`)
);
CREATE TABLE IF NOT EXISTS `Servers`
(
`ServerId` BIGINT NOT NULL AUTO_INCREMENT,

View File

@ -638,9 +638,6 @@ CREATE TABLE IF NOT EXISTS `UsersHistory`
`Id` BIGINT(20) NOT NULL,
`DiscordId` BIGINT(20) NOT NULL,
`XP` BIGINT(20) NOT NULL DEFAULT 0,
`ReactionCount` BIGINT(20) NOT NULL DEFAULT 0,
`MessageCount` BIGINT(20) NOT NULL DEFAULT 0,
`Birthday` DATE NULL,
`ServerId` BIGINT(20) DEFAULT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -654,9 +651,9 @@ CREATE TRIGGER `TR_UsersUpdate`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ServerId`,
`DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ServerId,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;
@ -667,9 +664,9 @@ CREATE TRIGGER `TR_UsersDelete`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ServerId`,
`Deleted`, `DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId, TRUE,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ServerId, TRUE,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;

View File

@ -101,9 +101,6 @@ CREATE TABLE IF NOT EXISTS `CFG_ServerHistory`
`HelpVoiceChannelId` BIGINT NOT NULL,
`TeamChannelId` BIGINT NOT NULL,
`LoginMessageChannelId` BIGINT NOT NULL,
`DefaultRoleId` BIGINT NULL,
`ShortRoleNameSetOnlyHighest` BOOLEAN NOT NULL DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`ServerId` BIGINT NOT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -131,9 +128,6 @@ BEGIN
`HelpVoiceChannelId`,
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`FeatureFlags`,
`ServerId`,
`DateFrom`,
`DateTo`)
@ -151,9 +145,6 @@ BEGIN
OLD.HelpVoiceChannelId,
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -180,10 +171,7 @@ BEGIN
`HelpVoiceChannelId`,
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`ServerId`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
`DateTo`)
@ -201,9 +189,6 @@ BEGIN
OLD.HelpVoiceChannelId,
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
TRUE,
OLD.LastModifiedAt,
@ -322,9 +307,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForRestart` BIGINT NOT NULL DEFAULT 8,
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
`DateTo` DATETIME(6) NOT NULL
@ -342,9 +324,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
VALUES (OLD.Id,
@ -352,9 +331,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
END;;
@ -371,9 +347,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
`DateTo`)
@ -382,9 +355,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));

View File

@ -25,7 +25,6 @@ CREATE TABLE IF NOT EXISTS `CFG_ServerHistory`
`TeamChannelId` BIGINT NOT NULL,
`LoginMessageChannelId` BIGINT NOT NULL,
`DefaultRoleId` BIGINT NULL,
`ShortRoleNameSetOnlyHighest` BOOLEAN NOT NULL DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`ServerId` BIGINT NOT NULL,
`Deleted` BOOL DEFAULT FALSE,
@ -55,7 +54,6 @@ BEGIN
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`FeatureFlags`,
`ServerId`,
`DateFrom`,
@ -75,7 +73,6 @@ BEGIN
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
OLD.LastModifiedAt,
@ -104,7 +101,6 @@ BEGIN
`TeamChannelId`,
`LoginMessageChannelId`,
`DefaultRoleId`,
`ShortRoleNameSetOnlyHighest`,
`ServerId`,
`FeatureFlags`,
`Deleted`,
@ -125,7 +121,6 @@ BEGIN
OLD.TeamChannelId,
OLD.LoginMessageChannelId,
OLD.DefaultRoleId,
OLD.ShortRoleNameSetOnlyHighest,
OLD.FeatureFlags,
OLD.ServerId,
TRUE,
@ -140,8 +135,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForRestart` BIGINT NOT NULL DEFAULT 8,
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -160,8 +153,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
@ -170,8 +161,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -189,8 +178,6 @@ BEGIN
`WaitForRestart`,
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
@ -200,8 +187,6 @@ BEGIN
OLD.WaitForRestart,
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,

View File

@ -1,9 +1,3 @@
ALTER TABLE UsersHistory
ADD COLUMN ReactionCount BIGINT NOT NULL DEFAULT 0 AFTER XP;
ALTER TABLE UsersHistory
ADD COLUMN MessageCount BIGINT NOT NULL DEFAULT 0 AFTER ReactionCount;
ALTER TABLE `Users`
Review

Hier werden Tabellen angelegt, welche im Downgrade nicht gedroppt werden

Hier werden Tabellen angelegt, welche im Downgrade nicht gedroppt werden
CHANGE `CreatedAt` `CreatedAt` DATETIME(6) NULL DEFAULT CURRENT_TIMESTAMP(6);;
@ -17,7 +11,6 @@ CREATE TABLE IF NOT EXISTS `UsersHistory`
`XP` BIGINT(20) NOT NULL DEFAULT 0,
`ReactionCount` BIGINT(20) NOT NULL DEFAULT 0,
`MessageCount` BIGINT(20) NOT NULL DEFAULT 0,
`Birthday` DATE NULL,
`ServerId` BIGINT(20) DEFAULT NULL,
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -31,9 +24,9 @@ CREATE TRIGGER `TR_UsersUpdate`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `ServerId`,
`DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.ServerId,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;
@ -44,9 +37,9 @@ CREATE TRIGGER `TR_UsersDelete`
ON `Users`
FOR EACH ROW
BEGIN
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `Birthday`, `ServerId`,
INSERT INTO `UsersHistory` (`Id`, `DiscordId`, `XP`, `ReactionCount`, `MessageCount`, `ServerId`,
`Deleted`, `DateFrom`, `DateTo`)
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.Birthday, OLD.ServerId, TRUE,
VALUES (OLD.UserId, OLD.DiscordId, OLD.XP, OLD.ReactionCount, OLD.MessageCount, OLD.ServerId, TRUE,
OLD.LastModifiedAt, CURRENT_TIMESTAMP(6));
END;;

View File

@ -15,7 +15,6 @@ CREATE TABLE IF NOT EXISTS `CFG_TechnicianHistory`
`WaitForShutdown` BIGINT NOT NULL DEFAULT 8,
`CacheMaxMessages` BIGINT NOT NULL DEFAULT 1000000,
`MaxSteamOfferCount` BIGINT NOT NULL DEFAULT 250,
`Maintenance` BOOLEAN DEFAULT FALSE,
`FeatureFlags` JSON NULL DEFAULT ('{}'),
`Deleted` BOOL DEFAULT FALSE,
`DateFrom` DATETIME(6) NOT NULL,
@ -35,7 +34,6 @@ BEGIN
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`DateFrom`,
`DateTo`)
@ -45,7 +43,6 @@ BEGIN
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
OLD.LastModifiedAt,
CURRENT_TIMESTAMP(6));
@ -64,7 +61,6 @@ BEGIN
`WaitForShutdown`,
`CacheMaxMessages`,
`MaxSteamOfferCount`,
`Maintenance`,
`FeatureFlags`,
`Deleted`,
`DateFrom`,
@ -75,7 +71,6 @@ BEGIN
OLD.WaitForShutdown,
OLD.CacheMaxMessages,
OLD.MaxSteamOfferCount,
OLD.Maintenance,
OLD.FeatureFlags,
TRUE,
OLD.LastModifiedAt,

View File

@ -1,9 +1,12 @@
import glob
import os
from cpl_core.database.context import DatabaseContextABC
from cpl_core.dependency_injection import ServiceProviderABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.migration_abc import MigrationABC
from bot_data.model.migration import Migration
from bot_data.model.migration_history import MigrationHistory
@ -20,14 +23,64 @@ class MigrationService:
self._db = db
self._cursor = db.cursor
self._migrations: List[MigrationABC] = (
List(type, MigrationABC.__subclasses__()).order_by(lambda x: x.name.split("_")[0]).then_by(lambda x: x.prio)
)
# self._migrations: List[MigrationABC] = (
# List(type, MigrationABC.__subclasses__()).order_by(lambda x: x.name.split("_")[0]).then_by(lambda x: x.prio)
# )
def migrate(self):
self._logger.info(__name__, f"Running Migrations")
for migration in self._migrations:
migration_id = migration.__name__
def _migrate_from_old_to_new(self):
results = self._db.select(
"""
SELECT * FROM `MigrationHistory`
"""
)
applied_migrations = List(MigrationHistory)
for result in results:
applied_migrations.add(MigrationHistory(result[0], result[1]))
for migration in applied_migrations:
if not migration.migration_id.endswith("Migration"):
continue
self._logger.debug(__name__, f"Migrate old migration {migration.migration_id} to new method")

Sollte hier nicht vielleicht die Nachricht "Migrate new migration {migration.migration_id} to old method" geloggt werden?

Jedenfalls steht in Zeile 52 die selbe Nachricht drinne.

Sollte hier nicht vielleicht die Nachricht `"Migrate new migration {migration.migration_id} to old method"` geloggt werden? Jedenfalls steht in Zeile 52 die selbe Nachricht drinne.
self._cursor.execute(migration.change_id_string(migration.migration_id.replace("Migration", "")))
self._db.save_changes()
def _load_up_scripts(self) -> List[Migration]:
migrations = List(Migration)
path = "../../src/bot_data/scripts"
if not os.path.exists(path):
raise Exception("Migration path not found")
folders = List(str, glob.glob(f"{path}/*")).order_by()
for folder in folders:
splitted = folder.split("/")
version = splitted[len(splitted) - 1]
for file in List(str, os.listdir(folder)).where(lambda x: x.endswith("_up.sql")).order_by().to_list():
if not file.endswith(".sql"):
continue
name = str(file.split(".sql")[0])
if name.endswith("_up"):
name = name.replace("_up", "")
elif name.endswith("_down"):
name = name.replace("_down", "")
name = name.split("_")[1]
with open(f"{folder}/{file}", "r") as f:
script = f.read()
f.close()
migrations.add(Migration(name, version, script))
return migrations
def _execute(self, migrations: List[Migration]):
for migration in migrations:
active_statement = ""
try:
# check if table exists
self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'")
@ -36,17 +89,52 @@ class MigrationService:
# there is a table named "tableName"
Review

Wie ich es verstehe, sammelst du Downgrades von der höchsten Version bis zum Botversion und führst sie an einer anderen Stelle aus. Sollte es da nicht zu einem Fehler führen, da du im Downgrade Tabellen Droppst, die evtl. nicht existieren?

Wie ich es verstehe, sammelst du Downgrades von der höchsten Version bis zum Botversion und führst sie an einer anderen Stelle aus. Sollte es da nicht zu einem Fehler führen, da du im Downgrade Tabellen Droppst, die evtl. nicht existieren?
self._logger.trace(
Review

Bei einem Downgrade macht du ein continue wenn die Bedingung zutrifft, während du beim Upgrade ein break abrufst. Gibt es hier einen Fall, in dem die Ordner evtl. nicht in richtiger Reihenfolge stehen und due damit versuchst relevante Ordner abzufangen?

Bei einem Downgrade macht du ein `continue` wenn die Bedingung zutrifft, während du beim Upgrade ein `break` abrufst. Gibt es hier einen Fall, in dem die Ordner evtl. nicht in richtiger Reihenfolge stehen und due damit versuchst relevante Ordner abzufangen?
__name__,
f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration_id)}",
f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration.name)}",
)
migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration_id))
migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration.name))
if migration_from_db is not None and len(migration_from_db) > 0:
continue
self._logger.debug(__name__, f"Running Migration {migration_id}")
migration_as_service: MigrationABC = self._services.get_service(migration)
migration_as_service.upgrade()
self._cursor.execute(MigrationHistory(migration_id).insert_string)
self._db.save_changes()
self._logger.debug(__name__, f"Running migration: {migration.name}")
for statement in migration.script.split("\n\n"):
if statement in ["", "\n"]:
continue
active_statement = statement
self._cursor.execute(statement + ";")
self._cursor.execute(MigrationHistory(migration.name).insert_string)
self._db.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get migration with id {migration}", e)
self._logger.fatal(__name__, f"Migration failed: {migration.name}\n{active_statement}", e)
def migrate(self):
self._migrate_from_old_to_new()
self._execute(self._load_up_scripts())
# def migrate(self):
# self._logger.info(__name__, f"Running Migrations")
# for migration in self._migrations:
# migration_id = migration.__name__
# try:
# # check if table exists
# self._cursor.execute("SHOW TABLES LIKE 'MigrationHistory'")
# result = self._cursor.fetchone()
# if result:
# # there is a table named "tableName"
# self._logger.trace(
# __name__,
# f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration_id)}",
# )
# migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration_id))
# if migration_from_db is not None and len(migration_from_db) > 0:
# continue
#
# self._logger.debug(__name__, f"Running Migration {migration_id}")
# migration_as_service: MigrationABC = self._services.get_service(migration)
# migration_as_service.upgrade()
# self._cursor.execute(MigrationHistory(migration_id).insert_string)
# self._db.save_changes()
#
# except Exception as e:
# self._logger.error(__name__, f"Cannot get migration with id {migration}", e)

View File

@ -32,6 +32,8 @@ class TechnicianConfigSeeder(DataSeederABC):
8,
8,
1000000,
100,
False,
{},
List(int, [240160344557879316]),
List(

View File

@ -4,6 +4,7 @@ from cpl_discord.events import OnReadyABC
from cpl_discord.service import DiscordBotServiceABC
from bot_core.service.config_service import ConfigService
from bot_core.service.data_integrity_service import DataIntegrityService
from bot_data.abc.server_repository_abc import ServerRepositoryABC
@ -15,6 +16,7 @@ class ConfigOnReadyEvent(OnReadyABC):
bot: DiscordBotServiceABC,
servers: ServerRepositoryABC,
config_service: ConfigService,
data_integrity_service: DataIntegrityService,
):
OnReadyABC.__init__(self)
@ -23,7 +25,10 @@ class ConfigOnReadyEvent(OnReadyABC):
self._bot = bot
self._servers = servers
self._config_service = config_service
self._data_integrity_service = data_integrity_service
async def on_ready(self):
self._data_integrity_service.check_servers()
for guild in self._bot.guilds:
await self._config_service.reload_server_config(self._servers.get_server_by_discord_id(guild.id))