From ac238b360a2e3b62346bbb4db99fa0dd6e3237b4 Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 02:34:27 +0200 Subject: [PATCH 1/8] Added feature-flags --- src/bot/config/appsettings.development.json | 8 +++ .../config/appsettings.edrafts-pc-ubuntu.json | 8 +++ src/bot/config/appsettings.production.json | 8 +++ src/bot/config/appsettings.staging.json | 8 +++ src/bot/startup.py | 9 ++- src/bot/startup_discord_extension.py | 53 +++++++++------ .../configuration/feature_flags_settings.py | 65 +++++++++++++++++++ 7 files changed, 138 insertions(+), 21 deletions(-) create mode 100644 src/bot_core/configuration/feature_flags_settings.py diff --git a/src/bot/config/appsettings.development.json b/src/bot/config/appsettings.development.json index 53ed5d70ec..d75fdfea29 100644 --- a/src/bot/config/appsettings.development.json +++ b/src/bot/config/appsettings.development.json @@ -17,6 +17,14 @@ "de" ] }, + "FeatureFlag": { + "AdminModule": true, + "BaseModule": true, + "BootLogModule": true, + "DatabaseModule": true, + "ModeratorModule": true, + "PermissionModule": true + }, "DiscordBot": { "Token": "OTk4MTU5NjczODkzMDYwNzM4.GN3QyA.yvWO6L7Eu36gXQ7ARDs0Jg2J1VqIDnHLou5lT4", "Prefix": "!kd " diff --git a/src/bot/config/appsettings.edrafts-pc-ubuntu.json b/src/bot/config/appsettings.edrafts-pc-ubuntu.json index 8c26c3ea80..6523b2d9f4 100644 --- a/src/bot/config/appsettings.edrafts-pc-ubuntu.json +++ b/src/bot/config/appsettings.edrafts-pc-ubuntu.json @@ -19,6 +19,14 @@ "Token": "OTk4MTYwNDI3Njg5MTgxMjM3.GI7h67.BqD6Lu1Tz0MuG8iktYrcLnHi1pNozyMiWFGTKI", "Prefix": "!ke " }, + "FeatureFlag": { + "AdminModule": true, + "BaseModule": true, + "BootLogModule": true, + "DatabaseModule": true, + "ModeratorModule": true, + "PermissionModule": true + }, "Bot": { "910199451145076828": { "MessageDeleteTimer": 2 diff --git a/src/bot/config/appsettings.production.json b/src/bot/config/appsettings.production.json index 918722728f..1f2dd507fe 100644 --- a/src/bot/config/appsettings.production.json +++ b/src/bot/config/appsettings.production.json @@ -28,6 +28,14 @@ "Buffered": "true", "AuthPlugin": "mysql_native_password" }, + "FeatureFlag": { + "AdminModule": true, + "BaseModule": true, + "BootLogModule": true, + "DatabaseModule": true, + "ModeratorModule": true, + "PermissionModule": true + }, "Bot": { "650366049023295514": { "MessageDeleteTimer": 2 diff --git a/src/bot/config/appsettings.staging.json b/src/bot/config/appsettings.staging.json index b69da3ddc1..3ddd1c3f4e 100644 --- a/src/bot/config/appsettings.staging.json +++ b/src/bot/config/appsettings.staging.json @@ -17,6 +17,14 @@ "de" ] }, + "FeatureFlag": { + "AdminModule": true, + "BaseModule": true, + "BootLogModule": true, + "DatabaseModule": true, + "ModeratorModule": true, + "PermissionModule": true + }, "Bot": { "910199451145076828": { "MessageDeleteTimer": 4 diff --git a/src/bot/startup.py b/src/bot/startup.py index db5fd50892..4f9363407a 100644 --- a/src/bot/startup.py +++ b/src/bot/startup.py @@ -10,6 +10,7 @@ from cpl_core.environment import ApplicationEnvironment from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe from bot_core.service.client_utils_service import ClientUtilsService from bot_core.service.message_service import MessageService @@ -39,6 +40,7 @@ class Startup(StartupABC): self._start_time = datetime.now() self._config: Optional[ConfigurationABC] = None + self._feature_flags: Optional[FeatureFlagsSettings] = None def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironment) -> ConfigurationABC: environment.set_working_directory(os.path.dirname(os.path.realpath(__file__))) @@ -52,6 +54,7 @@ class Startup(StartupABC): configuration.add_configuration('Startup_StartTime', str(self._start_time)) self._config = configuration + self._feature_flags = configuration.get_configuration(FeatureFlagsSettings) return configuration def configure_services(self, services: ServiceCollectionABC, environment: ApplicationEnvironment) -> ServiceProviderABC: @@ -61,7 +64,8 @@ class Startup(StartupABC): services.add_db_context(DBContext, self._config.get_configuration(DatabaseSettings)) # general services - services.add_transient(BaseHelperABC, BaseHelperService) + if self._feature_flags.base_module: + services.add_transient(BaseHelperABC, BaseHelperService) services.add_transient(MessageServiceABC, MessageService) services.add_transient(ClientUtilsServiceABC, ClientUtilsService) @@ -69,7 +73,8 @@ class Startup(StartupABC): services.add_transient(DateTimeOffsetPipe) # module services - services.add_singleton(PermissionServiceABC, PermissionService) + if self._feature_flags.permission_module: + services.add_singleton(PermissionServiceABC, PermissionService) # data services services.add_transient(ServerRepositoryABC, ServerRepositoryService) diff --git a/src/bot/startup_discord_extension.py b/src/bot/startup_discord_extension.py index 4d20e684d5..8e7b93ecb7 100644 --- a/src/bot/startup_discord_extension.py +++ b/src/bot/startup_discord_extension.py @@ -1,3 +1,5 @@ +from typing import Optional + from cpl_core.application import StartupExtensionABC from cpl_core.configuration import ConfigurationABC from cpl_core.dependency_injection import ServiceCollectionABC @@ -5,6 +7,7 @@ from cpl_core.environment import ApplicationEnvironmentABC from cpl_discord import get_discord_collection from cpl_discord.discord_event_types_enum import DiscordEventTypesEnum +from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings from modules.admin.command.restart_command import RestartCommand from modules.admin.command.shutdown_command import ShutdownCommand from modules.base.command.afk_command import AFKCommand @@ -27,40 +30,52 @@ from modules.permission.events.permission_on_ready_event import PermissionOnRead class StartupDiscordExtension(StartupExtensionABC): def __init__(self): - pass + self._feature_flags: Optional[FeatureFlagsSettings] = None def configure_configuration(self, config: ConfigurationABC, env: ApplicationEnvironmentABC): - pass + self._feature_flags = config.get_configuration(FeatureFlagsSettings) def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC): services.add_discord() dc = get_discord_collection(services) """ commands """ # admin - dc.add_command(RestartCommand) - dc.add_command(ShutdownCommand) + if self._feature_flags.admin_module: + dc.add_command(RestartCommand) + dc.add_command(ShutdownCommand) # moderator - dc.add_command(PurgeCommand) + if self._feature_flags.moderator_module: + dc.add_command(PurgeCommand) # simple - dc.add_command(AFKCommand) - dc.add_command(HelpCommand) - dc.add_command(InfoCommand) - dc.add_command(PingCommand) - dc.add_command(UserInfoCommand) + if self._feature_flags.base_module: + dc.add_command(AFKCommand) + dc.add_command(HelpCommand) + dc.add_command(InfoCommand) + dc.add_command(PingCommand) + dc.add_command(UserInfoCommand) """ events """ # on_command_error - dc.add_event(DiscordEventTypesEnum.on_command_error.value, BaseOnCommandErrorEvent) + if self._feature_flags.base_module: + dc.add_event(DiscordEventTypesEnum.on_command_error.value, BaseOnCommandErrorEvent) # on_member_join - dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberJoinEvent) + if self._feature_flags.base_module: + dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberJoinEvent) # on_member_remove - dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberRemoveEvent) + if self._feature_flags.base_module: + dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberRemoveEvent) # on_member_update - dc.add_event(DiscordEventTypesEnum.on_member_update.value, PermissionOnMemberUpdateEvent) + if self._feature_flags.permission_module: + dc.add_event(DiscordEventTypesEnum.on_member_update.value, PermissionOnMemberUpdateEvent) # on_message - dc.add_event(DiscordEventTypesEnum.on_message.value, BaseOnMessageEvent) + if self._feature_flags.base_module: + dc.add_event(DiscordEventTypesEnum.on_message.value, BaseOnMessageEvent) # on_voice_state_update - dc.add_event(DiscordEventTypesEnum.on_voice_state_update.value, BaseOnVoiceStateUpdateEvent) + if self._feature_flags.base_module: + dc.add_event(DiscordEventTypesEnum.on_voice_state_update.value, BaseOnVoiceStateUpdateEvent) # on_ready - dc.add_event(DiscordEventTypesEnum.on_ready.value, DatabaseOnReadyEvent) - dc.add_event(DiscordEventTypesEnum.on_ready.value, PermissionOnReadyEvent) - dc.add_event(DiscordEventTypesEnum.on_ready.value, BootLogOnReadyEvent) # has to be last + if self._feature_flags.database_module: + dc.add_event(DiscordEventTypesEnum.on_ready.value, DatabaseOnReadyEvent) + if self._feature_flags.permission_module: + dc.add_event(DiscordEventTypesEnum.on_ready.value, PermissionOnReadyEvent) + if self._feature_flags.boot_log_module: + dc.add_event(DiscordEventTypesEnum.on_ready.value, BootLogOnReadyEvent) # has to be last diff --git a/src/bot_core/configuration/feature_flags_settings.py b/src/bot_core/configuration/feature_flags_settings.py new file mode 100644 index 0000000000..8d2807445d --- /dev/null +++ b/src/bot_core/configuration/feature_flags_settings.py @@ -0,0 +1,65 @@ +import traceback +from typing import Optional + +from cpl_core.configuration.configuration_model_abc import ConfigurationModelABC +from cpl_core.console import Console + + +class FeatureFlagsSettings(ConfigurationModelABC): + + def __init__(self): + ConfigurationModelABC.__init__(self) + + self._admin_module = False # 02.10.2022 #48 + self._base_module = True # 02.10.2022 #48 + self._boot_log_module = True # 02.10.2022 #48 + self._database_module = True # 02.10.2022 #48 + self._moderator_module = False # 02.10.2022 #48 + self._permission_module = True # 02.10.2022 #48 + + @property + def admin_module(self) -> bool: + return self._admin_module + + @property + def base_module(self) -> bool: + return self._base_module + + @property + def boot_log_module(self) -> bool: + return self._boot_log_module + + @property + def database_module(self) -> bool: + return self._database_module + + @property + def moderator_module(self) -> bool: + return self._moderator_module + + @property + def permission_module(self) -> bool: + return self._permission_module + + def from_dict(self, settings: dict): + try: + if 'AdminModule' in settings: + self._admin_module = settings['AdminModule'] + + if 'BaseModule' in settings: + self._base_module = settings['BaseModule'] + + if 'BootLogModule' in settings: + self._boot_log_module = settings['BootLogModule'] + + if 'DatabaseModule' in settings: + self._database_module = settings['DatabaseModule'] + + if 'ModeratorModule' in settings: + self._moderator_module = settings['ModeratorModule'] + + if 'PermissionModule' in settings: + self._permission_module = settings['PermissionModule'] + except Exception as e: + Console.error(f'[ ERROR ] [ {__name__} ]: Reading error in {type(self).__name__} settings') + Console.error(f'[ EXCEPTION ] [ {__name__} ]: {e} -> {traceback.format_exc()}') From 890cb6f6b5487b2d0786365131a957a3c10b62c7 Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 02:54:08 +0200 Subject: [PATCH 2/8] Added feature-flags #48 --- src/bot/config/appsettings.development.json | 2 +- .../config/appsettings.edrafts-pc-ubuntu.json | 2 +- src/bot/config/appsettings.production.json | 2 +- src/bot/config/appsettings.staging.json | 2 +- src/bot/main.py | 14 ++++++---- src/bot/startup.py | 14 ---------- src/bot/startup_settings_extension.py | 28 +++++++++++++++++++ src/modules/boot_log/boot_log_extension.py | 5 ++++ src/modules/database/database_extension.py | 4 +++ 9 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 src/bot/startup_settings_extension.py diff --git a/src/bot/config/appsettings.development.json b/src/bot/config/appsettings.development.json index d75fdfea29..44dbc1664a 100644 --- a/src/bot/config/appsettings.development.json +++ b/src/bot/config/appsettings.development.json @@ -17,7 +17,7 @@ "de" ] }, - "FeatureFlag": { + "FeatureFlags": { "AdminModule": true, "BaseModule": true, "BootLogModule": true, diff --git a/src/bot/config/appsettings.edrafts-pc-ubuntu.json b/src/bot/config/appsettings.edrafts-pc-ubuntu.json index 6523b2d9f4..20585cb528 100644 --- a/src/bot/config/appsettings.edrafts-pc-ubuntu.json +++ b/src/bot/config/appsettings.edrafts-pc-ubuntu.json @@ -19,7 +19,7 @@ "Token": "OTk4MTYwNDI3Njg5MTgxMjM3.GI7h67.BqD6Lu1Tz0MuG8iktYrcLnHi1pNozyMiWFGTKI", "Prefix": "!ke " }, - "FeatureFlag": { + "FeatureFlags": { "AdminModule": true, "BaseModule": true, "BootLogModule": true, diff --git a/src/bot/config/appsettings.production.json b/src/bot/config/appsettings.production.json index 1f2dd507fe..515c60afab 100644 --- a/src/bot/config/appsettings.production.json +++ b/src/bot/config/appsettings.production.json @@ -28,7 +28,7 @@ "Buffered": "true", "AuthPlugin": "mysql_native_password" }, - "FeatureFlag": { + "FeatureFlags": { "AdminModule": true, "BaseModule": true, "BootLogModule": true, diff --git a/src/bot/config/appsettings.staging.json b/src/bot/config/appsettings.staging.json index 3ddd1c3f4e..605fe7fc3e 100644 --- a/src/bot/config/appsettings.staging.json +++ b/src/bot/config/appsettings.staging.json @@ -17,7 +17,7 @@ "de" ] }, - "FeatureFlag": { + "FeatureFlags": { "AdminModule": true, "BaseModule": true, "BootLogModule": true, diff --git a/src/bot/main.py b/src/bot/main.py index 0bfb5a82c8..29c81fc113 100644 --- a/src/bot/main.py +++ b/src/bot/main.py @@ -8,6 +8,7 @@ from bot.application import Application from bot.startup import Startup from bot.startup_discord_extension import StartupDiscordExtension from bot.startup_migration_extension import StartupMigrationExtension +from bot.startup_settings_extension import StartupSettingsExtension from modules.boot_log.boot_log_extension import BootLogExtension from modules.database.database_extension import DatabaseExtension @@ -18,12 +19,13 @@ class Program: self.app: Optional[Application] = None async def start(self): - app_builder = ApplicationBuilder(Application) - app_builder.use_extension(StartupDiscordExtension) - app_builder.use_extension(StartupMigrationExtension) - app_builder.use_extension(BootLogExtension) - app_builder.use_extension(DatabaseExtension) - app_builder.use_startup(Startup) + app_builder = ApplicationBuilder(Application) \ + .use_extension(StartupSettingsExtension) \ + .use_extension(StartupDiscordExtension) \ + .use_extension(StartupMigrationExtension) \ + .use_extension(BootLogExtension) \ + .use_extension(DatabaseExtension) \ + .use_startup(Startup) self.app: Application = await app_builder.build_async() await self.app.run_async() diff --git a/src/bot/startup.py b/src/bot/startup.py index 4f9363407a..689c981925 100644 --- a/src/bot/startup.py +++ b/src/bot/startup.py @@ -1,5 +1,3 @@ -import os -from datetime import datetime from typing import Optional from cpl_core.application import StartupABC @@ -37,22 +35,10 @@ class Startup(StartupABC): def __init__(self): StartupABC.__init__(self) - self._start_time = datetime.now() - self._config: Optional[ConfigurationABC] = None self._feature_flags: Optional[FeatureFlagsSettings] = None def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironment) -> ConfigurationABC: - environment.set_working_directory(os.path.dirname(os.path.realpath(__file__))) - configuration.add_environment_variables('KDB_') - configuration.add_environment_variables('DISCORD_') - - configuration.add_json_file(f'config/appsettings.json', optional=False) - configuration.add_json_file(f'config/appsettings.{environment.environment_name}.json', optional=True) - configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True) - - configuration.add_configuration('Startup_StartTime', str(self._start_time)) - self._config = configuration self._feature_flags = configuration.get_configuration(FeatureFlagsSettings) return configuration diff --git a/src/bot/startup_settings_extension.py b/src/bot/startup_settings_extension.py new file mode 100644 index 0000000000..8c27e69b63 --- /dev/null +++ b/src/bot/startup_settings_extension.py @@ -0,0 +1,28 @@ +import os +from datetime import datetime + +from cpl_core.application import StartupExtensionABC +from cpl_core.configuration import ConfigurationABC +from cpl_core.dependency_injection import ServiceCollectionABC +from cpl_core.environment import ApplicationEnvironmentABC + + +class StartupSettingsExtension(StartupExtensionABC): + + def __init__(self): + self._start_time = datetime.now() + + def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironmentABC): + # this shit has to be done here because we need settings in subsequent startup extensions + environment.set_working_directory(os.path.dirname(os.path.realpath(__file__))) + configuration.add_environment_variables('KDB_') + configuration.add_environment_variables('DISCORD_') + + configuration.add_json_file(f'config/appsettings.json', optional=False) + configuration.add_json_file(f'config/appsettings.{environment.environment_name}.json', optional=True) + configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True) + + configuration.add_configuration('Startup_StartTime', str(self._start_time)) + + def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC): + pass diff --git a/src/modules/boot_log/boot_log_extension.py b/src/modules/boot_log/boot_log_extension.py index 8bb36c7ef8..42db4b8eba 100644 --- a/src/modules/boot_log/boot_log_extension.py +++ b/src/modules/boot_log/boot_log_extension.py @@ -5,6 +5,8 @@ from cpl_core.configuration import ConfigurationABC from cpl_core.dependency_injection import ServiceProviderABC from cpl_core.logging import LoggerABC +from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings + class BootLogExtension(ApplicationExtensionABC): @@ -12,6 +14,9 @@ class BootLogExtension(ApplicationExtensionABC): pass async def run(self, config: ConfigurationABC, services: ServiceProviderABC): + feature_flags: FeatureFlagsSettings = config.get_configuration(FeatureFlagsSettings) + if not feature_flags.boot_log_module: + return logger: LoggerABC = services.get_service(LoggerABC) logger.debug(__name__, 'BootLog extension started') config.add_configuration('Bot_StartTime', str(datetime.now())) diff --git a/src/modules/database/database_extension.py b/src/modules/database/database_extension.py index edcca9b9b0..883752bb0b 100644 --- a/src/modules/database/database_extension.py +++ b/src/modules/database/database_extension.py @@ -5,6 +5,7 @@ from cpl_core.configuration import ConfigurationABC from cpl_core.dependency_injection import ServiceProviderABC from cpl_core.logging import LoggerABC +from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings from bot_data.service.migration_service import MigrationService @@ -14,6 +15,9 @@ class DatabaseExtension(ApplicationExtensionABC): pass async def run(self, config: ConfigurationABC, services: ServiceProviderABC): + feature_flags: FeatureFlagsSettings = config.get_configuration(FeatureFlagsSettings) + if not feature_flags.database_module: + return logger: LoggerABC = services.get_service(LoggerABC) logger.debug(__name__, 'Database extension started') config.add_configuration('Database_StartTime', str(datetime.now())) From 12cd5c904562234903cac38dfb85eba11f7b1b97 Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 12:14:44 +0200 Subject: [PATCH 3/8] Added custom logging support #45 --- src/bot/application.py | 30 +++++++--- .../config/appsettings.edrafts-pc-ubuntu.json | 16 ++++- src/bot/startup.py | 29 +++++++-- src/bot/startup_discord_extension.py | 47 ++++++++------- src/bot_core/abc/custom_file_logger_abc.py | 60 +++++++++++++++++++ .../configuration/bot_logging_settings.py | 33 ++++++++++ .../configuration/file_logging_settings.py | 24 ++++++++ src/bot_core/logging/__init__.py | 1 + src/bot_core/logging/command_logger.py | 11 ++++ src/bot_core/logging/event_logger.py | 11 ++++ src/modules/admin/command/restart_command.py | 4 +- src/modules/admin/command/shutdown_command.py | 4 +- src/modules/base/command/afk_command.py | 4 +- src/modules/base/command/help_command.py | 4 +- src/modules/base/command/info_command.py | 4 +- src/modules/base/command/ping_command.py | 4 +- src/modules/base/command/user_info_command.py | 4 +- .../configuration/base_server_settings.py | 7 ++- .../base/events/base_on_command_event.py | 41 +++++++++++++ .../moderator/command/purge_command.py | 4 +- 20 files changed, 286 insertions(+), 56 deletions(-) create mode 100644 src/bot_core/abc/custom_file_logger_abc.py create mode 100644 src/bot_core/configuration/bot_logging_settings.py create mode 100644 src/bot_core/configuration/file_logging_settings.py create mode 100644 src/bot_core/logging/__init__.py create mode 100644 src/bot_core/logging/command_logger.py create mode 100644 src/bot_core/logging/event_logger.py create mode 100644 src/modules/base/events/base_on_command_event.py diff --git a/src/bot/application.py b/src/bot/application.py index f2088027f3..4b80eb9b6f 100644 --- a/src/bot/application.py +++ b/src/bot/application.py @@ -1,4 +1,4 @@ -from typing import Optional, Type +from typing import Optional, Type, Callable from cpl_core.configuration import ConfigurationABC, ConfigurationModelABC from cpl_core.console import Console @@ -9,8 +9,11 @@ from cpl_discord.configuration import DiscordBotSettings from cpl_discord.service import DiscordBotServiceABC, DiscordBotService from cpl_translation import TranslatePipe, TranslationServiceABC, TranslationSettings +from bot_core.configuration.bot_logging_settings import BotLoggingSettings from bot_core.configuration.bot_settings import BotSettings from bot_core.configuration.server_settings import ServerSettings +from bot_core.logging.command_logger import CommandLogger +from bot_core.logging.event_logger import EventLogger from modules.base.configuration.base_server_settings import BaseServerSettings from modules.base.configuration.base_settings import BaseSettings from modules.boot_log.configuration.boot_log_server_settings import BootLogServerSettings @@ -24,6 +27,10 @@ class Application(DiscordBotApplicationABC): def __init__(self, config: ConfigurationABC, services: ServiceProviderABC): DiscordBotApplicationABC.__init__(self, config, services) + self._services = services + self._cmd_logger: Optional[LoggerABC] = None + self._event_logger: Optional[LoggerABC] = None + # cpl-core self._logger: LoggerABC = services.get_service(LoggerABC) # cpl-discord @@ -33,27 +40,32 @@ class Application(DiscordBotApplicationABC): self._translation: TranslationServiceABC = services.get_service(TranslationServiceABC) self._translate: TranslatePipe = services.get_service(TranslatePipe) - def _configure_settings_with_servers(self, settings: Type, server_settings: Type): + def _configure_settings_with_sub_settings(self, settings: Type, list_atr: Callable, atr: Callable): settings: Optional[settings] = self._configuration.get_configuration(settings) if settings is None: return - for server in settings.servers: - self._logger.trace(__name__, f'Saved config: {type(server).__name__}_{server.id}') - self._configuration.add_configuration(f'{type(server).__name__}_{server.id}', server) + for sub_settings in list_atr(settings): + self._logger.trace(__name__, f'Saved config: {type(sub_settings).__name__}_{atr(sub_settings)}') + self._configuration.add_configuration(f'{type(sub_settings).__name__}_{atr(sub_settings)}', sub_settings) async def configure(self): self._translation.load_by_settings(self._configuration.get_configuration(TranslationSettings)) - self._configure_settings_with_servers(BotSettings, ServerSettings) - self._configure_settings_with_servers(BaseSettings, BaseServerSettings) - self._configure_settings_with_servers(BootLogSettings, BootLogServerSettings) - self._configure_settings_with_servers(PermissionSettings, PermissionServerSettings) + self._configure_settings_with_sub_settings(BotSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(BaseSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(BootLogSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(PermissionSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(BotLoggingSettings, lambda x: x.files, lambda x: x.key) + # custom loggers need to be loaded AFTER config is loaded correctly + self._cmd_logger: LoggerABC = self._services.get_service(CommandLogger) + self._event_logger: LoggerABC = self._services.get_service(EventLogger) async def main(self): try: self._logger.debug(__name__, f'Starting...\n') self._logger.trace(__name__, f'Try to start {DiscordBotService.__name__}') await self._bot.start_async() + await self._bot.stop_async() except Exception as e: self._logger.error(__name__, 'Start failed', e) diff --git a/src/bot/config/appsettings.edrafts-pc-ubuntu.json b/src/bot/config/appsettings.edrafts-pc-ubuntu.json index 20585cb528..d4138b2d3c 100644 --- a/src/bot/config/appsettings.edrafts-pc-ubuntu.json +++ b/src/bot/config/appsettings.edrafts-pc-ubuntu.json @@ -1,10 +1,24 @@ { "LoggingSettings": { "Path": "logs/", - "Filename": "log_dev.log", + "Filename": "dev.log", "ConsoleLogLevel": "DEBUG", "FileLogLevel": "TRACE" }, + "BotLoggingSettings": { + "Command": { + "Path": "logs/", + "Filename": "commands.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "TRACE" + }, + "Event": { + "Path": "logs/", + "Filename": "event.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "TRACE" + } + }, "DatabaseSettings": { "Host": "localhost", "User": "kd_kdb", diff --git a/src/bot/startup.py b/src/bot/startup.py index 689c981925..3063c7b3b3 100644 --- a/src/bot/startup.py +++ b/src/bot/startup.py @@ -1,3 +1,5 @@ +import os +from datetime import datetime from typing import Optional from cpl_core.application import StartupABC @@ -5,10 +7,13 @@ from cpl_core.configuration import ConfigurationABC from cpl_core.database import DatabaseSettings from cpl_core.dependency_injection import ServiceProviderABC, ServiceCollectionABC from cpl_core.environment import ApplicationEnvironment +from cpl_core.logging import LoggerABC from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings +from bot_core.logging.command_logger import CommandLogger +from bot_core.logging.event_logger import EventLogger from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe from bot_core.service.client_utils_service import ClientUtilsService from bot_core.service.message_service import MessageService @@ -35,23 +40,36 @@ class Startup(StartupABC): def __init__(self): StartupABC.__init__(self) + self._start_time = datetime.now() + self._config: Optional[ConfigurationABC] = None - self._feature_flags: Optional[FeatureFlagsSettings] = None def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironment) -> ConfigurationABC: + environment.set_working_directory(os.path.dirname(os.path.realpath(__file__))) + configuration.add_environment_variables('KDB_') + configuration.add_environment_variables('DISCORD_') + + configuration.add_json_file(f'config/appsettings.json', optional=False) + configuration.add_json_file(f'config/appsettings.{environment.environment_name}.json', optional=True) + configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True) + + configuration.add_configuration('Startup_StartTime', str(self._start_time)) + self._config = configuration - self._feature_flags = configuration.get_configuration(FeatureFlagsSettings) return configuration def configure_services(self, services: ServiceCollectionABC, environment: ApplicationEnvironment) -> ServiceProviderABC: services.add_logging() + # custom logging + services.add_singleton(LoggerABC, CommandLogger) + services.add_singleton(LoggerABC, EventLogger) + services.add_translation() services.add_db_context(DBContext, self._config.get_configuration(DatabaseSettings)) # general services - if self._feature_flags.base_module: - services.add_transient(BaseHelperABC, BaseHelperService) + services.add_transient(BaseHelperABC, BaseHelperService) services.add_transient(MessageServiceABC, MessageService) services.add_transient(ClientUtilsServiceABC, ClientUtilsService) @@ -59,8 +77,7 @@ class Startup(StartupABC): services.add_transient(DateTimeOffsetPipe) # module services - if self._feature_flags.permission_module: - services.add_singleton(PermissionServiceABC, PermissionService) + services.add_singleton(PermissionServiceABC, PermissionService) # data services services.add_transient(ServerRepositoryABC, ServerRepositoryService) diff --git a/src/bot/startup_discord_extension.py b/src/bot/startup_discord_extension.py index 8e7b93ecb7..dbf3281d58 100644 --- a/src/bot/startup_discord_extension.py +++ b/src/bot/startup_discord_extension.py @@ -15,6 +15,7 @@ from modules.base.command.help_command import HelpCommand from modules.base.command.info_command import InfoCommand from modules.base.command.ping_command import PingCommand from modules.base.events.base_on_command_error_event import BaseOnCommandErrorEvent +from modules.base.events.base_on_command_event import BaseOnCommandEvent from modules.moderator.command.purge_command import PurgeCommand from modules.base.command.user_info_command import UserInfoCommand from modules.base.events.base_on_member_join_event import BaseOnMemberJoinEvent @@ -38,44 +39,46 @@ class StartupDiscordExtension(StartupExtensionABC): def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC): services.add_discord() dc = get_discord_collection(services) - """ commands """ - # admin + """modules""" if self._feature_flags.admin_module: + """ commands """ dc.add_command(RestartCommand) dc.add_command(ShutdownCommand) - # moderator - if self._feature_flags.moderator_module: - dc.add_command(PurgeCommand) - # simple + """ events """ + if self._feature_flags.base_module: + """ commands """ dc.add_command(AFKCommand) dc.add_command(HelpCommand) dc.add_command(InfoCommand) dc.add_command(PingCommand) dc.add_command(UserInfoCommand) - """ events """ - # on_command_error - if self._feature_flags.base_module: + """ events """ + dc.add_event(DiscordEventTypesEnum.on_command.value, BaseOnCommandEvent) dc.add_event(DiscordEventTypesEnum.on_command_error.value, BaseOnCommandErrorEvent) - # on_member_join - if self._feature_flags.base_module: dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberJoinEvent) - # on_member_remove - if self._feature_flags.base_module: dc.add_event(DiscordEventTypesEnum.on_member_join.value, BaseOnMemberRemoveEvent) - # on_member_update - if self._feature_flags.permission_module: - dc.add_event(DiscordEventTypesEnum.on_member_update.value, PermissionOnMemberUpdateEvent) - # on_message - if self._feature_flags.base_module: dc.add_event(DiscordEventTypesEnum.on_message.value, BaseOnMessageEvent) - # on_voice_state_update - if self._feature_flags.base_module: dc.add_event(DiscordEventTypesEnum.on_voice_state_update.value, BaseOnVoiceStateUpdateEvent) - # on_ready + if self._feature_flags.database_module: + """ commands """ + """ events """ dc.add_event(DiscordEventTypesEnum.on_ready.value, DatabaseOnReadyEvent) + + if self._feature_flags.moderator_module: + """ commands """ + dc.add_command(PurgeCommand) + """ events """ + if self._feature_flags.permission_module: + """ commands """ + """ events """ dc.add_event(DiscordEventTypesEnum.on_ready.value, PermissionOnReadyEvent) + dc.add_event(DiscordEventTypesEnum.on_member_update.value, PermissionOnMemberUpdateEvent) + + # has to be last! if self._feature_flags.boot_log_module: - dc.add_event(DiscordEventTypesEnum.on_ready.value, BootLogOnReadyEvent) # has to be last + """ commands """ + """ events """ + dc.add_event(DiscordEventTypesEnum.on_ready.value, BootLogOnReadyEvent) diff --git a/src/bot_core/abc/custom_file_logger_abc.py b/src/bot_core/abc/custom_file_logger_abc.py new file mode 100644 index 0000000000..f5e9ca5ebd --- /dev/null +++ b/src/bot_core/abc/custom_file_logger_abc.py @@ -0,0 +1,60 @@ +from abc import ABC, abstractmethod + +from cpl_core.configuration import ConfigurationABC +from cpl_core.environment import ApplicationEnvironmentABC +from cpl_core.logging import LoggingSettings, Logger, LoggingLevelEnum, LoggerABC +from cpl_core.time import TimeFormatSettings + +from bot_core.configuration.file_logging_settings import FileLoggingSettings + + +class CustomFileLoggerABC(Logger, ABC): + + @abstractmethod + def __init__(self, key: str, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC): + self._key = key + settings: LoggingSettings = config.get_configuration(f'{FileLoggingSettings.__name__}_{key}') + Logger.__init__(self, settings, time_format, env) + self._begin_log() + + def _begin_log(self): + console_level = self._console.value + self._console = LoggingLevelEnum.OFF + self.info(__name__, f'Starting...\n') + self._console = LoggingLevelEnum(console_level) + + def _get_string(self, name_list_as_str: str, level: LoggingLevelEnum, message: str) -> str: + names = name_list_as_str.split(' ') + log_level = level.name + string = f'<{self._get_datetime_now()}> [ {log_level} ] ' + for name in names: + string += f'[ {name} ] ' + string += f': {message}' + return string + + def header(self, string: str): + super().header(string) + + def trace(self, name: str, message: str): + name = f'{name} {self._key}' + super().trace(name, message) + + def debug(self, name: str, message: str): + name = f'{name} {self._key}' + super().debug(name, message) + + def info(self, name: str, message: str): + name = f'{name} {self._key}' + super().info(name, message) + + def warn(self, name: str, message: str): + name = f'{name} {self._key}' + super().warn(name, message) + + def error(self, name: str, message: str, ex: Exception = None): + name = f'{name} {self._key}' + super().error(name, message, ex) + + def fatal(self, name: str, message: str, ex: Exception = None): + name = f'{name} {self._key}' + super().fatal(name, message, ex) diff --git a/src/bot_core/configuration/bot_logging_settings.py b/src/bot_core/configuration/bot_logging_settings.py new file mode 100644 index 0000000000..37ecd6ed8b --- /dev/null +++ b/src/bot_core/configuration/bot_logging_settings.py @@ -0,0 +1,33 @@ +import traceback + +from cpl_core.configuration.configuration_model_abc import ConfigurationModelABC +from cpl_core.console import Console, ForegroundColorEnum +from cpl_query.extension import List + +from bot_core.configuration.file_logging_settings import FileLoggingSettings + + +class BotLoggingSettings(ConfigurationModelABC): + + def __init__(self): + ConfigurationModelABC.__init__(self) + self._files: List[FileLoggingSettings] = List(FileLoggingSettings) + + @property + def files(self) -> List[FileLoggingSettings]: + return self._files + + def from_dict(self, settings: dict): + try: + files = List(FileLoggingSettings) + for s in settings: + st = FileLoggingSettings() + settings[s]['Key'] = s + st.from_dict(settings[s]) + files.append(st) + self._files = files + except Exception as e: + Console.set_foreground_color(ForegroundColorEnum.red) + Console.write_line(f'[ ERROR ] [ {__name__} ]: Reading error in {type(self).__name__} settings') + Console.write_line(f'[ EXCEPTION ] [ {__name__} ]: {e} -> {traceback.format_exc()}') + Console.set_foreground_color(ForegroundColorEnum.default) diff --git a/src/bot_core/configuration/file_logging_settings.py b/src/bot_core/configuration/file_logging_settings.py new file mode 100644 index 0000000000..50f3ddb001 --- /dev/null +++ b/src/bot_core/configuration/file_logging_settings.py @@ -0,0 +1,24 @@ +import traceback + +from cpl_core.console import Console +from cpl_core.logging import LoggingSettings + + +class FileLoggingSettings(LoggingSettings): + + def __init__(self): + LoggingSettings.__init__(self) + + self._key = '' + + @property + def key(self) -> str: + return self._key + + def from_dict(self, settings: dict): + try: + self._key = settings['Key'] + super().from_dict(settings) + except Exception as e: + Console.error(f'[ ERROR ] [ {__name__} ]: Reading error in {type(self).__name__} settings') + Console.error(f'[ EXCEPTION ] [ {__name__} ]: {e} -> {traceback.format_exc()}') diff --git a/src/bot_core/logging/__init__.py b/src/bot_core/logging/__init__.py new file mode 100644 index 0000000000..425ab6c146 --- /dev/null +++ b/src/bot_core/logging/__init__.py @@ -0,0 +1 @@ +# imports diff --git a/src/bot_core/logging/command_logger.py b/src/bot_core/logging/command_logger.py new file mode 100644 index 0000000000..16d6f15b85 --- /dev/null +++ b/src/bot_core/logging/command_logger.py @@ -0,0 +1,11 @@ +from cpl_core.configuration import ConfigurationABC +from cpl_core.environment import ApplicationEnvironmentABC +from cpl_core.time import TimeFormatSettings + +from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC + + +class CommandLogger(CustomFileLoggerABC): + + def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC): + CustomFileLoggerABC.__init__(self, 'Command', config, time_format, env) diff --git a/src/bot_core/logging/event_logger.py b/src/bot_core/logging/event_logger.py new file mode 100644 index 0000000000..7c7ff9316b --- /dev/null +++ b/src/bot_core/logging/event_logger.py @@ -0,0 +1,11 @@ +from cpl_core.configuration import ConfigurationABC +from cpl_core.environment import ApplicationEnvironmentABC +from cpl_core.time import TimeFormatSettings + +from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC + + +class EventLogger(CustomFileLoggerABC): + + def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC): + CustomFileLoggerABC.__init__(self, 'Event', config, time_format, env) diff --git a/src/modules/admin/command/restart_command.py b/src/modules/admin/command/restart_command.py index e8de4f5b27..62a1b5176d 100644 --- a/src/modules/admin/command/restart_command.py +++ b/src/modules/admin/command/restart_command.py @@ -1,5 +1,4 @@ from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -8,6 +7,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger from modules.permission.abc.permission_service_abc import PermissionServiceABC @@ -15,7 +15,7 @@ class RestartCommand(DiscordCommandABC): def __init__( self, - logger: LoggerABC, + logger: CommandLogger, config: ConfigurationABC, message_service: MessageServiceABC, bot: DiscordBotServiceABC, diff --git a/src/modules/admin/command/shutdown_command.py b/src/modules/admin/command/shutdown_command.py index f4e6849913..1a44e83632 100644 --- a/src/modules/admin/command/shutdown_command.py +++ b/src/modules/admin/command/shutdown_command.py @@ -1,5 +1,4 @@ from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -8,6 +7,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger from modules.permission.abc.permission_service_abc import PermissionServiceABC @@ -15,7 +15,7 @@ class ShutdownCommand(DiscordCommandABC): def __init__( self, - logger: LoggerABC, + logger: CommandLogger, config: ConfigurationABC, message_service: MessageServiceABC, bot: DiscordBotServiceABC, diff --git a/src/modules/base/command/afk_command.py b/src/modules/base/command/afk_command.py index 10eb6af104..42edc5e99e 100644 --- a/src/modules/base/command/afk_command.py +++ b/src/modules/base/command/afk_command.py @@ -1,5 +1,4 @@ from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -9,6 +8,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger from modules.base.configuration.base_server_settings import BaseServerSettings @@ -16,7 +16,7 @@ class AFKCommand(DiscordCommandABC): def __init__( self, - logger: LoggerABC, + logger: CommandLogger, config: ConfigurationABC, message_service: MessageServiceABC, bot: DiscordBotServiceABC, diff --git a/src/modules/base/command/help_command.py b/src/modules/base/command/help_command.py index 4bf9747178..c1f69d3ad7 100644 --- a/src/modules/base/command/help_command.py +++ b/src/modules/base/command/help_command.py @@ -1,5 +1,4 @@ from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from discord.ext import commands @@ -7,6 +6,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger from modules.base.configuration.base_server_settings import BaseServerSettings @@ -15,7 +15,7 @@ class HelpCommand(DiscordCommandABC): def __init__( self, config: ConfigurationABC, - logger: LoggerABC, + logger: CommandLogger, message_service: MessageServiceABC, bot: DiscordBotServiceABC, client_utils: ClientUtilsServiceABC diff --git a/src/modules/base/command/info_command.py b/src/modules/base/command/info_command.py index d3c7de6575..55d2b9500a 100644 --- a/src/modules/base/command/info_command.py +++ b/src/modules/base/command/info_command.py @@ -2,7 +2,6 @@ from datetime import datetime import discord from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -12,6 +11,7 @@ from discord.ext.commands import Context import bot from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger class InfoCommand(DiscordCommandABC): @@ -19,7 +19,7 @@ class InfoCommand(DiscordCommandABC): def __init__( self, config: ConfigurationABC, - logger: LoggerABC, + logger: CommandLogger, message_service: MessageServiceABC, bot: DiscordBotServiceABC, client_utils: ClientUtilsServiceABC, diff --git a/src/modules/base/command/ping_command.py b/src/modules/base/command/ping_command.py index f8a796ef28..9402bcd4e8 100644 --- a/src/modules/base/command/ping_command.py +++ b/src/modules/base/command/ping_command.py @@ -1,4 +1,3 @@ -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -7,13 +6,14 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger class PingCommand(DiscordCommandABC): def __init__( self, - logger: LoggerABC, + logger: CommandLogger, message_service: MessageServiceABC, bot: DiscordBotServiceABC, client_utils: ClientUtilsServiceABC, diff --git a/src/modules/base/command/user_info_command.py b/src/modules/base/command/user_info_command.py index d9d646ba2c..139c818021 100644 --- a/src/modules/base/command/user_info_command.py +++ b/src/modules/base/command/user_info_command.py @@ -2,7 +2,6 @@ from typing import Optional import discord from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe @@ -11,6 +10,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.logging.command_logger import CommandLogger from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.abc.user_joined_server_repository_abc import UserJoinedServerRepositoryABC @@ -23,7 +23,7 @@ class UserInfoCommand(DiscordCommandABC): def __init__( self, config: ConfigurationABC, - logger: LoggerABC, + logger: CommandLogger, message_service: MessageServiceABC, bot: DiscordBotServiceABC, client_utils: ClientUtilsServiceABC, diff --git a/src/modules/base/configuration/base_server_settings.py b/src/modules/base/configuration/base_server_settings.py index 14bba3eb59..d2875f547c 100644 --- a/src/modules/base/configuration/base_server_settings.py +++ b/src/modules/base/configuration/base_server_settings.py @@ -2,6 +2,7 @@ import traceback from cpl_core.configuration.configuration_model_abc import ConfigurationModelABC from cpl_core.console import Console +from cpl_query.extension import List class BaseServerSettings(ConfigurationModelABC): @@ -13,7 +14,9 @@ class BaseServerSettings(ConfigurationModelABC): self._max_voice_state_hours: int = 0 self._xp_per_message: int = 0 self._xp_per_ontime_hour: int = 0 - self._afk_channel_ids: list[int] = [] + self._afk_channel_ids: List[int] = List(int) + self._afk_command_channel_id: int = 0 + self._help_command_reference_url: str = '' @property def id(self) -> int: @@ -32,7 +35,7 @@ class BaseServerSettings(ConfigurationModelABC): return self._xp_per_ontime_hour @property - def afk_channel_ids(self) -> list[int]: + def afk_channel_ids(self) -> List[int]: return self._afk_channel_ids @property diff --git a/src/modules/base/events/base_on_command_event.py b/src/modules/base/events/base_on_command_event.py new file mode 100644 index 0000000000..7b6d6bfdc5 --- /dev/null +++ b/src/modules/base/events/base_on_command_event.py @@ -0,0 +1,41 @@ +import datetime +import traceback +import uuid + +from cpl_core.time import TimeFormatSettings +from cpl_discord.events import OnCommandABC +from cpl_discord.service import DiscordBotServiceABC +from cpl_translation import TranslatePipe +from discord.ext import commands +from discord.ext.commands import Context, CommandError + +from cpl_core.logging import LoggerABC +from cpl_discord.events.on_command_error_abc import OnCommandErrorABC + +from bot_core.abc.message_service_abc import MessageServiceABC +from bot_core.configuration.bot_settings import BotSettings +from bot_core.logging.command_logger import CommandLogger + + +class BaseOnCommandEvent(OnCommandABC): + + def __init__( + self, + logger: CommandLogger, + bot: DiscordBotServiceABC, + messenger: MessageServiceABC, + bot_settings: BotSettings, + time_format_settings: TimeFormatSettings, + translate: TranslatePipe + ): + OnCommandABC.__init__(self) + self._logger = logger + self._bot = bot + self._messenger = messenger + self._bot_settings = bot_settings + self._time_format_settings = time_format_settings + self._t = translate + + async def on_command(self, ctx: Context): + self._logger.warn(__name__, f'Received command afk {ctx}') + pass diff --git a/src/modules/moderator/command/purge_command.py b/src/modules/moderator/command/purge_command.py index 07ecd4bf92..9e9510523a 100644 --- a/src/modules/moderator/command/purge_command.py +++ b/src/modules/moderator/command/purge_command.py @@ -1,7 +1,6 @@ import asyncio from cpl_core.configuration import ConfigurationABC -from cpl_core.logging import LoggerABC from cpl_discord.command import DiscordCommandABC from cpl_translation import TranslatePipe from discord.ext import commands @@ -10,6 +9,7 @@ from discord.ext.commands import Context from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC from bot_core.abc.message_service_abc import MessageServiceABC from bot_core.configuration.server_settings import ServerSettings +from bot_core.logging.command_logger import CommandLogger from modules.permission.abc.permission_service_abc import PermissionServiceABC @@ -17,7 +17,7 @@ class PurgeCommand(DiscordCommandABC): def __init__( self, - logger: LoggerABC, + logger: CommandLogger, config: ConfigurationABC, message_service: MessageServiceABC, permissions: PermissionServiceABC, From e2749f6a816cb57c79d686d2358c60713372929d Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 13:13:23 +0200 Subject: [PATCH 4/8] Improved logging --- src/bot/application.py | 27 ++----------------- .../config/appsettings.edrafts-pc-ubuntu.json | 12 ++++++--- src/bot/startup.py | 25 ++++++++--------- src/bot/startup_settings_extension.py | 24 +++++++++++++++++ src/bot_core/abc/custom_file_logger_abc.py | 6 ++--- src/bot_core/helper/__init__.py | 0 src/bot_core/helper/log_message_helper.py | 9 +++++++ src/bot_core/logging/database_logger.py | 11 ++++++++ .../{event_logger.py => message_logger.py} | 4 +-- src/bot_core/service/message_service.py | 7 ++--- src/bot_data/db_context.py | 5 ++-- src/bot_data/migration/initial_migration.py | 5 ++-- .../service/client_repository_service.py | 5 ++-- .../service/known_user_repository_service.py | 11 ++++---- src/bot_data/service/migration_service.py | 4 +-- .../service/server_repository_service.py | 5 ++-- .../user_joined_server_repository_service.py | 6 ++--- .../user_joined_voice_channel_service.py | 8 +++--- .../service/user_repository_service.py | 5 ++-- .../base/events/base_on_command_event.py | 12 ++------- .../base/events/base_on_message_event.py | 6 +++-- .../boot_log/boot_log_on_ready_event.py | 4 +-- src/modules/database/database_extension.py | 3 ++- .../database/database_on_ready_event.py | 8 +++--- 24 files changed, 118 insertions(+), 94 deletions(-) create mode 100644 src/bot_core/helper/__init__.py create mode 100644 src/bot_core/helper/log_message_helper.py create mode 100644 src/bot_core/logging/database_logger.py rename src/bot_core/logging/{event_logger.py => message_logger.py} (73%) diff --git a/src/bot/application.py b/src/bot/application.py index 4b80eb9b6f..7af5db7912 100644 --- a/src/bot/application.py +++ b/src/bot/application.py @@ -1,6 +1,6 @@ from typing import Optional, Type, Callable -from cpl_core.configuration import ConfigurationABC, ConfigurationModelABC +from cpl_core.configuration import ConfigurationABC from cpl_core.console import Console from cpl_core.dependency_injection import ServiceProviderABC from cpl_core.logging import LoggerABC @@ -11,14 +11,10 @@ from cpl_translation import TranslatePipe, TranslationServiceABC, TranslationSet from bot_core.configuration.bot_logging_settings import BotLoggingSettings from bot_core.configuration.bot_settings import BotSettings -from bot_core.configuration.server_settings import ServerSettings from bot_core.logging.command_logger import CommandLogger -from bot_core.logging.event_logger import EventLogger -from modules.base.configuration.base_server_settings import BaseServerSettings +from bot_core.logging.database_logger import DatabaseLogger from modules.base.configuration.base_settings import BaseSettings -from modules.boot_log.configuration.boot_log_server_settings import BootLogServerSettings from modules.boot_log.configuration.boot_log_settings import BootLogSettings -from modules.permission.configuration.permission_server_settings import PermissionServerSettings from modules.permission.configuration.permission_settings import PermissionSettings @@ -28,8 +24,6 @@ class Application(DiscordBotApplicationABC): DiscordBotApplicationABC.__init__(self, config, services) self._services = services - self._cmd_logger: Optional[LoggerABC] = None - self._event_logger: Optional[LoggerABC] = None # cpl-core self._logger: LoggerABC = services.get_service(LoggerABC) @@ -40,25 +34,8 @@ class Application(DiscordBotApplicationABC): self._translation: TranslationServiceABC = services.get_service(TranslationServiceABC) self._translate: TranslatePipe = services.get_service(TranslatePipe) - def _configure_settings_with_sub_settings(self, settings: Type, list_atr: Callable, atr: Callable): - settings: Optional[settings] = self._configuration.get_configuration(settings) - if settings is None: - return - - for sub_settings in list_atr(settings): - self._logger.trace(__name__, f'Saved config: {type(sub_settings).__name__}_{atr(sub_settings)}') - self._configuration.add_configuration(f'{type(sub_settings).__name__}_{atr(sub_settings)}', sub_settings) - async def configure(self): self._translation.load_by_settings(self._configuration.get_configuration(TranslationSettings)) - self._configure_settings_with_sub_settings(BotSettings, lambda x: x.servers, lambda x: x.id) - self._configure_settings_with_sub_settings(BaseSettings, lambda x: x.servers, lambda x: x.id) - self._configure_settings_with_sub_settings(BootLogSettings, lambda x: x.servers, lambda x: x.id) - self._configure_settings_with_sub_settings(PermissionSettings, lambda x: x.servers, lambda x: x.id) - self._configure_settings_with_sub_settings(BotLoggingSettings, lambda x: x.files, lambda x: x.key) - # custom loggers need to be loaded AFTER config is loaded correctly - self._cmd_logger: LoggerABC = self._services.get_service(CommandLogger) - self._event_logger: LoggerABC = self._services.get_service(EventLogger) async def main(self): try: diff --git a/src/bot/config/appsettings.edrafts-pc-ubuntu.json b/src/bot/config/appsettings.edrafts-pc-ubuntu.json index d4138b2d3c..7afbf20dc2 100644 --- a/src/bot/config/appsettings.edrafts-pc-ubuntu.json +++ b/src/bot/config/appsettings.edrafts-pc-ubuntu.json @@ -1,7 +1,7 @@ { "LoggingSettings": { "Path": "logs/", - "Filename": "dev.log", + "Filename": "bot.log", "ConsoleLogLevel": "DEBUG", "FileLogLevel": "TRACE" }, @@ -12,9 +12,15 @@ "ConsoleLogLevel": "DEBUG", "FileLogLevel": "TRACE" }, - "Event": { + "Database": { "Path": "logs/", - "Filename": "event.log", + "Filename": "database.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "TRACE" + }, + "Message": { + "Path": "logs/", + "Filename": "message.log", "ConsoleLogLevel": "DEBUG", "FileLogLevel": "TRACE" } diff --git a/src/bot/startup.py b/src/bot/startup.py index 3063c7b3b3..177d3a9ebc 100644 --- a/src/bot/startup.py +++ b/src/bot/startup.py @@ -10,10 +10,11 @@ from cpl_core.environment import ApplicationEnvironment from cpl_core.logging import LoggerABC from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC +from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC from bot_core.abc.message_service_abc import MessageServiceABC -from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings from bot_core.logging.command_logger import CommandLogger -from bot_core.logging.event_logger import EventLogger +from bot_core.logging.database_logger import DatabaseLogger +from bot_core.logging.message_logger import MessageLogger from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe from bot_core.service.client_utils_service import ClientUtilsService from bot_core.service.message_service import MessageService @@ -45,16 +46,6 @@ class Startup(StartupABC): self._config: Optional[ConfigurationABC] = None def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironment) -> ConfigurationABC: - environment.set_working_directory(os.path.dirname(os.path.realpath(__file__))) - configuration.add_environment_variables('KDB_') - configuration.add_environment_variables('DISCORD_') - - configuration.add_json_file(f'config/appsettings.json', optional=False) - configuration.add_json_file(f'config/appsettings.{environment.environment_name}.json', optional=True) - configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True) - - configuration.add_configuration('Startup_StartTime', str(self._start_time)) - self._config = configuration return configuration @@ -62,7 +53,8 @@ class Startup(StartupABC): services.add_logging() # custom logging services.add_singleton(LoggerABC, CommandLogger) - services.add_singleton(LoggerABC, EventLogger) + services.add_singleton(LoggerABC, DatabaseLogger) + services.add_singleton(LoggerABC, MessageLogger) services.add_translation() @@ -87,4 +79,9 @@ class Startup(StartupABC): services.add_transient(UserJoinedServerRepositoryABC, UserJoinedServerRepositoryService) services.add_transient(UserJoinedVoiceChannelRepositoryABC, UserJoinedVoiceChannelRepositoryService) - return services.build_service_provider() + provider = services.build_service_provider() + # instantiate custom logger + for c in CustomFileLoggerABC.__subclasses__(): + i: LoggerABC = provider.get_service(c) + + return provider diff --git a/src/bot/startup_settings_extension.py b/src/bot/startup_settings_extension.py index 8c27e69b63..0a9b80911d 100644 --- a/src/bot/startup_settings_extension.py +++ b/src/bot/startup_settings_extension.py @@ -1,10 +1,20 @@ import os from datetime import datetime +from typing import Callable, Type, Optional from cpl_core.application import StartupExtensionABC from cpl_core.configuration import ConfigurationABC from cpl_core.dependency_injection import ServiceCollectionABC from cpl_core.environment import ApplicationEnvironmentABC +from cpl_core.logging import LoggerABC + +from bot_core.configuration.bot_logging_settings import BotLoggingSettings +from bot_core.configuration.bot_settings import BotSettings +from bot_core.logging.command_logger import CommandLogger +from bot_core.logging.database_logger import DatabaseLogger +from modules.base.configuration.base_settings import BaseSettings +from modules.boot_log.configuration.boot_log_settings import BootLogSettings +from modules.permission.configuration.permission_settings import PermissionSettings class StartupSettingsExtension(StartupExtensionABC): @@ -23,6 +33,20 @@ class StartupSettingsExtension(StartupExtensionABC): configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True) configuration.add_configuration('Startup_StartTime', str(self._start_time)) + self._configure_settings_with_sub_settings(configuration, BotSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(configuration, BaseSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(configuration, BootLogSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(configuration, PermissionSettings, lambda x: x.servers, lambda x: x.id) + self._configure_settings_with_sub_settings(configuration, BotLoggingSettings, lambda x: x.files, lambda x: x.key) def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC): pass + + @staticmethod + def _configure_settings_with_sub_settings(config: ConfigurationABC, settings: Type, list_atr: Callable, atr: Callable): + settings: Optional[settings] = config.get_configuration(settings) + if settings is None: + return + + for sub_settings in list_atr(settings): + config.add_configuration(f'{type(sub_settings).__name__}_{atr(sub_settings)}', sub_settings) diff --git a/src/bot_core/abc/custom_file_logger_abc.py b/src/bot_core/abc/custom_file_logger_abc.py index f5e9ca5ebd..c7354f65e2 100644 --- a/src/bot_core/abc/custom_file_logger_abc.py +++ b/src/bot_core/abc/custom_file_logger_abc.py @@ -20,15 +20,15 @@ class CustomFileLoggerABC(Logger, ABC): def _begin_log(self): console_level = self._console.value self._console = LoggingLevelEnum.OFF - self.info(__name__, f'Starting...\n') + self.info(__name__, f'Starting...') self._console = LoggingLevelEnum(console_level) def _get_string(self, name_list_as_str: str, level: LoggingLevelEnum, message: str) -> str: names = name_list_as_str.split(' ') log_level = level.name - string = f'<{self._get_datetime_now()}> [ {log_level} ] ' + string = f'<{self._get_datetime_now()}> [ {log_level} ]' for name in names: - string += f'[ {name} ] ' + string += f' [ {name} ]' string += f': {message}' return string diff --git a/src/bot_core/helper/__init__.py b/src/bot_core/helper/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/bot_core/helper/log_message_helper.py b/src/bot_core/helper/log_message_helper.py new file mode 100644 index 0000000000..403bf68855 --- /dev/null +++ b/src/bot_core/helper/log_message_helper.py @@ -0,0 +1,9 @@ +import discord + + +class LogMessageHelper: + + @staticmethod + def get_log_string(message: discord.Message): + content = message.content.replace("\n", "\n\t") + return f'{message.author} @ {message.channel} -> \n\t{content}' diff --git a/src/bot_core/logging/database_logger.py b/src/bot_core/logging/database_logger.py new file mode 100644 index 0000000000..dfc1f637e8 --- /dev/null +++ b/src/bot_core/logging/database_logger.py @@ -0,0 +1,11 @@ +from cpl_core.configuration import ConfigurationABC +from cpl_core.environment import ApplicationEnvironmentABC +from cpl_core.time import TimeFormatSettings + +from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC + + +class DatabaseLogger(CustomFileLoggerABC): + + def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC): + CustomFileLoggerABC.__init__(self, 'Database', config, time_format, env) diff --git a/src/bot_core/logging/event_logger.py b/src/bot_core/logging/message_logger.py similarity index 73% rename from src/bot_core/logging/event_logger.py rename to src/bot_core/logging/message_logger.py index 7c7ff9316b..78ce56e11e 100644 --- a/src/bot_core/logging/event_logger.py +++ b/src/bot_core/logging/message_logger.py @@ -5,7 +5,7 @@ from cpl_core.time import TimeFormatSettings from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC -class EventLogger(CustomFileLoggerABC): +class MessageLogger(CustomFileLoggerABC): def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC): - CustomFileLoggerABC.__init__(self, 'Event', config, time_format, env) + CustomFileLoggerABC.__init__(self, 'Message', config, time_format, env) diff --git a/src/bot_core/service/message_service.py b/src/bot_core/service/message_service.py index f7696c6356..d1b8f5f04c 100644 --- a/src/bot_core/service/message_service.py +++ b/src/bot_core/service/message_service.py @@ -4,19 +4,20 @@ from typing import Union import discord from cpl_core.configuration.configuration_abc import ConfigurationABC from cpl_core.database.context.database_context_abc import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_discord.service import DiscordBotServiceABC from cpl_query.extension import List from discord.ext.commands import Context from bot_core.abc.message_service_abc import MessageServiceABC from bot_core.configuration.server_settings import ServerSettings +from bot_core.helper.log_message_helper import LogMessageHelper +from bot_core.logging.message_logger import MessageLogger from bot_data.abc.client_repository_abc import ClientRepositoryABC class MessageService(MessageServiceABC): - def __init__(self, config: ConfigurationABC, logger: LoggerABC, bot: DiscordBotServiceABC, clients: ClientRepositoryABC, db: DatabaseContextABC): + def __init__(self, config: ConfigurationABC, logger: MessageLogger, bot: DiscordBotServiceABC, clients: ClientRepositoryABC, db: DatabaseContextABC): self._config = config self._logger = logger self._bot = bot @@ -35,7 +36,7 @@ class MessageService(MessageServiceABC): server_st: ServerSettings = self._config.get_configuration(f'ServerSettings_{message.guild.id}') if not mass_delete: await asyncio.sleep(server_st.message_delete_timer) - self._logger.debug(__name__, f'Try to delete message:\n\t{message}\n\t{message.content}') + self._logger.debug(__name__, f'Try to delete message: {LogMessageHelper.get_log_string(message)}') guild_id = message.guild.id try: await message.delete() diff --git a/src/bot_data/db_context.py b/src/bot_data/db_context.py index 8ad5414c37..fb0df29027 100644 --- a/src/bot_data/db_context.py +++ b/src/bot_data/db_context.py @@ -1,11 +1,12 @@ from cpl_core.database import DatabaseSettings from cpl_core.database.context import DatabaseContext -from cpl_core.logging import LoggerABC + +from bot_core.logging.database_logger import DatabaseLogger class DBContext(DatabaseContext): - def __init__(self, logger: LoggerABC): + def __init__(self, logger: DatabaseLogger): self._logger = logger diff --git a/src/bot_data/migration/initial_migration.py b/src/bot_data/migration/initial_migration.py index fa6a251d43..76dd481d8d 100644 --- a/src/bot_data/migration/initial_migration.py +++ b/src/bot_data/migration/initial_migration.py @@ -1,12 +1,11 @@ -from cpl_core.logging import LoggerABC - +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.migration_abc import MigrationABC from bot_data.db_context import DBContext class InitialMigration(MigrationABC): - def __init__(self, logger: LoggerABC, db: DBContext): + def __init__(self, logger: DatabaseLogger, db: DBContext): self._logger = logger self._db = db self._cursor = db.cursor diff --git a/src/bot_data/service/client_repository_service.py b/src/bot_data/service/client_repository_service.py index 7756f13ccf..ba875a256f 100644 --- a/src/bot_data/service/client_repository_service.py +++ b/src/bot_data/service/client_repository_service.py @@ -1,8 +1,9 @@ from typing import Optional + from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_query.extension import List +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.client_repository_abc import ClientRepositoryABC from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.model.client import Client @@ -10,7 +11,7 @@ from bot_data.model.client import Client class ClientRepositoryService(ClientRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC): self._logger = logger self._context = db_context diff --git a/src/bot_data/service/known_user_repository_service.py b/src/bot_data/service/known_user_repository_service.py index 80bdf134a5..fc1121792f 100644 --- a/src/bot_data/service/known_user_repository_service.py +++ b/src/bot_data/service/known_user_repository_service.py @@ -1,16 +1,17 @@ from typing import Optional -from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC -from cpl_query.extension import List -from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC +from cpl_core.database.context import DatabaseContextABC +from cpl_query.extension import List + +from bot_core.logging.database_logger import DatabaseLogger +from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.model.known_user import KnownUser class KnownUserRepositoryService(KnownUserRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC): self._logger = logger self._context = db_context diff --git a/src/bot_data/service/migration_service.py b/src/bot_data/service/migration_service.py index a0e8ed151c..fef42f2165 100644 --- a/src/bot_data/service/migration_service.py +++ b/src/bot_data/service/migration_service.py @@ -2,15 +2,15 @@ from typing import Type from cpl_core.database.context import DatabaseContextABC from cpl_core.dependency_injection import ServiceProviderABC -from cpl_core.logging import LoggerABC +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.migration_abc import MigrationABC from bot_data.model.migration_history import MigrationHistory class MigrationService: - def __init__(self, logger: LoggerABC, services: ServiceProviderABC, db: DatabaseContextABC): + def __init__(self, logger: DatabaseLogger, services: ServiceProviderABC, db: DatabaseContextABC): self._logger = logger self._services = services diff --git a/src/bot_data/service/server_repository_service.py b/src/bot_data/service/server_repository_service.py index ff585f49ec..728d2bc679 100644 --- a/src/bot_data/service/server_repository_service.py +++ b/src/bot_data/service/server_repository_service.py @@ -1,15 +1,16 @@ from typing import Optional + from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_query.extension import List +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.model.server import Server class ServerRepositoryService(ServerRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC): self._logger = logger self._context = db_context diff --git a/src/bot_data/service/user_joined_server_repository_service.py b/src/bot_data/service/user_joined_server_repository_service.py index c42bf76b1e..29f4e50b00 100644 --- a/src/bot_data/service/user_joined_server_repository_service.py +++ b/src/bot_data/service/user_joined_server_repository_service.py @@ -1,18 +1,18 @@ from typing import Optional from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_query.extension import List + +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.user_joined_server_repository_abc import \ UserJoinedServerRepositoryABC from bot_data.abc.user_repository_abc import UserRepositoryABC -from bot_data.model.user import User from bot_data.model.user_joined_server import UserJoinedServer class UserJoinedServerRepositoryService(UserJoinedServerRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, users: UserRepositoryABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, users: UserRepositoryABC): self._logger = logger self._context = db_context diff --git a/src/bot_data/service/user_joined_voice_channel_service.py b/src/bot_data/service/user_joined_voice_channel_service.py index 05e9824400..d6b1b5968c 100644 --- a/src/bot_data/service/user_joined_voice_channel_service.py +++ b/src/bot_data/service/user_joined_voice_channel_service.py @@ -1,17 +1,17 @@ from typing import Optional from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_query.extension import List + +from bot_core.logging.database_logger import DatabaseLogger +from bot_data.abc.user_joined_voice_channel_abc import UserJoinedVoiceChannelRepositoryABC from bot_data.abc.user_repository_abc import UserRepositoryABC from bot_data.model.user_joined_voice_channel import UserJoinedVoiceChannel -from bot_data.abc.user_joined_voice_channel_abc import UserJoinedVoiceChannelRepositoryABC - class UserJoinedVoiceChannelRepositoryService(UserJoinedVoiceChannelRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, users: UserRepositoryABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, users: UserRepositoryABC): self._logger = logger self._context = db_context diff --git a/src/bot_data/service/user_repository_service.py b/src/bot_data/service/user_repository_service.py index f95db48a67..f2543e5dd7 100644 --- a/src/bot_data/service/user_repository_service.py +++ b/src/bot_data/service/user_repository_service.py @@ -1,8 +1,9 @@ from typing import Optional + from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_query.extension import List +from bot_core.logging.database_logger import DatabaseLogger from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.abc.user_repository_abc import UserRepositoryABC from bot_data.model.user import User @@ -10,7 +11,7 @@ from bot_data.model.user import User class UserRepositoryService(UserRepositoryABC): - def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC): + def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC): self._logger = logger self._context = db_context diff --git a/src/modules/base/events/base_on_command_event.py b/src/modules/base/events/base_on_command_event.py index 7b6d6bfdc5..bbe548e74f 100644 --- a/src/modules/base/events/base_on_command_event.py +++ b/src/modules/base/events/base_on_command_event.py @@ -1,16 +1,8 @@ -import datetime -import traceback -import uuid - from cpl_core.time import TimeFormatSettings from cpl_discord.events import OnCommandABC from cpl_discord.service import DiscordBotServiceABC from cpl_translation import TranslatePipe -from discord.ext import commands -from discord.ext.commands import Context, CommandError - -from cpl_core.logging import LoggerABC -from cpl_discord.events.on_command_error_abc import OnCommandErrorABC +from discord.ext.commands import Context from bot_core.abc.message_service_abc import MessageServiceABC from bot_core.configuration.bot_settings import BotSettings @@ -37,5 +29,5 @@ class BaseOnCommandEvent(OnCommandABC): self._t = translate async def on_command(self, ctx: Context): - self._logger.warn(__name__, f'Received command afk {ctx}') + print('\n', ctx.command) pass diff --git a/src/modules/base/events/base_on_message_event.py b/src/modules/base/events/base_on_message_event.py index 7dfdbabb69..f5288c50cf 100644 --- a/src/modules/base/events/base_on_message_event.py +++ b/src/modules/base/events/base_on_message_event.py @@ -2,10 +2,11 @@ from typing import Optional import discord from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_discord.events import OnMessageABC from cpl_discord.service import DiscordBotServiceABC +from bot_core.helper.log_message_helper import LogMessageHelper +from bot_core.logging.message_logger import MessageLogger from bot_data.abc.client_repository_abc import ClientRepositoryABC from bot_data.abc.server_repository_abc import ServerRepositoryABC from bot_data.abc.user_repository_abc import UserRepositoryABC @@ -18,7 +19,7 @@ class BaseOnMessageEvent(OnMessageABC): def __init__( self, - logger: LoggerABC, + logger: MessageLogger, bhs: BaseHelperABC, db: DatabaseContextABC, bot: DiscordBotServiceABC, @@ -71,6 +72,7 @@ class BaseOnMessageEvent(OnMessageABC): async def on_message(self, message: discord.Message): self._logger.debug(__name__, f'Module {type(self)} started') + self._logger.debug(__name__, f'Received message: {LogMessageHelper.get_log_string(message)}') if message is None or message.guild is None: return self._append_received_message_count(message.guild.id) diff --git a/src/modules/boot_log/boot_log_on_ready_event.py b/src/modules/boot_log/boot_log_on_ready_event.py index 403f4628ae..06e7b162e1 100644 --- a/src/modules/boot_log/boot_log_on_ready_event.py +++ b/src/modules/boot_log/boot_log_on_ready_event.py @@ -42,12 +42,12 @@ class BootLogOnReadyEvent(OnReadyABC): # print warning if initialisation took too long if init_time >= 30: self._logger.warn( - __name__, 'It takes long time to start the bot!') + __name__, 'It takes too long to start the bot!') # print error if initialisation took way too long elif init_time >= 90: self._logger.error( - __name__, 'It takes very long time to start the bot!!!') + __name__, 'It takes far too long to start the bot!!!') except Exception as e: self._logger.error(__name__, 'Init time calculation failed', e) return diff --git a/src/modules/database/database_extension.py b/src/modules/database/database_extension.py index 883752bb0b..f5d4a492e4 100644 --- a/src/modules/database/database_extension.py +++ b/src/modules/database/database_extension.py @@ -6,6 +6,7 @@ from cpl_core.dependency_injection import ServiceProviderABC from cpl_core.logging import LoggerABC from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings +from bot_core.logging.database_logger import DatabaseLogger from bot_data.service.migration_service import MigrationService @@ -18,7 +19,7 @@ class DatabaseExtension(ApplicationExtensionABC): feature_flags: FeatureFlagsSettings = config.get_configuration(FeatureFlagsSettings) if not feature_flags.database_module: return - logger: LoggerABC = services.get_service(LoggerABC) + logger: LoggerABC = services.get_service(DatabaseLogger) logger.debug(__name__, 'Database extension started') config.add_configuration('Database_StartTime', str(datetime.now())) migrations: MigrationService = services.get_service(MigrationService) diff --git a/src/modules/database/database_on_ready_event.py b/src/modules/database/database_on_ready_event.py index fab4dc0a19..9176a26c15 100644 --- a/src/modules/database/database_on_ready_event.py +++ b/src/modules/database/database_on_ready_event.py @@ -4,10 +4,10 @@ from datetime import datetime, timedelta import discord from cpl_core.configuration import ConfigurationABC from cpl_core.database.context import DatabaseContextABC -from cpl_core.logging import LoggerABC from cpl_discord.events import OnReadyABC from cpl_discord.service import DiscordBotServiceABC +from bot_core.logging.database_logger import DatabaseLogger from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe from bot_data.abc.client_repository_abc import ClientRepositoryABC from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC @@ -29,7 +29,7 @@ class DatabaseOnReadyEvent(OnReadyABC): def __init__( self, config: ConfigurationABC, - logger: LoggerABC, + logger: DatabaseLogger, bot: DiscordBotServiceABC, db_context: DatabaseContextABC, server_repo: ServerRepositoryABC, @@ -65,12 +65,12 @@ class DatabaseOnReadyEvent(OnReadyABC): # print warning if initialisation took too long if init_time >= 30: self._logger.warn( - __name__, 'It takes long time to start the bot!') + __name__, 'It takes too long to prepare the database!') # print error if initialisation took way too long elif init_time >= 90: self._logger.error( - __name__, 'It takes very long time to start the bot!!!') + __name__, 'It takes far too long to prepare the database!!!') except Exception as e: # self._logger.error(__name__, 'Database init time calculation failed', e) return From fa50123d3bb47f62c4c37766cdb4d941014763cc Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 13:22:41 +0200 Subject: [PATCH 5/8] Improved logging for migrations #45 --- src/bot_data/service/migration_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bot_data/service/migration_service.py b/src/bot_data/service/migration_service.py index fef42f2165..fc2d5913f6 100644 --- a/src/bot_data/service/migration_service.py +++ b/src/bot_data/service/migration_service.py @@ -31,11 +31,10 @@ class MigrationService: # there is a table named "tableName" self._logger.trace(__name__, f"Running SQL Command: {MigrationHistory.get_select_by_id_string(migration_id)}") migration_from_db = self._db.select(MigrationHistory.get_select_by_id_string(migration_id)) - self._logger.trace(__name__, str(migration_from_db)) if migration_from_db is not None and len(migration_from_db) > 0: continue - self._logger.debug(__name__, f"Running Migration {migration}") + self._logger.debug(__name__, f"Running Migration {migration_id}") migration_as_service: MigrationABC = self._services.get_service(migration) migration_as_service.upgrade() self._cursor.execute(MigrationHistory(migration_id).insert_string) From d5a1c15db3c62784d195ec90995b79bb0f10d2e9 Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 13:25:29 +0200 Subject: [PATCH 6/8] Prevent multiple stop_async calls --- src/bot/application.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/bot/application.py b/src/bot/application.py index 7af5db7912..aec07e0377 100644 --- a/src/bot/application.py +++ b/src/bot/application.py @@ -34,6 +34,8 @@ class Application(DiscordBotApplicationABC): self._translation: TranslationServiceABC = services.get_service(TranslationServiceABC) self._translate: TranslatePipe = services.get_service(TranslatePipe) + self._is_stopping = False + async def configure(self): self._translation.load_by_settings(self._configuration.get_configuration(TranslationSettings)) @@ -47,6 +49,10 @@ class Application(DiscordBotApplicationABC): self._logger.error(__name__, 'Start failed', e) async def stop_async(self): + if self._is_stopping: + return + + self._is_stopping = True try: self._logger.trace(__name__, f'Try to stop {DiscordBotService.__name__}') await self._bot.close() From 54db70404428bff06156a26cf1044b0fe8bc19b4 Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 23:23:26 +0200 Subject: [PATCH 7/8] Improved settings #45 --- docker-compose.yml | 29 +++++++++++++++-- src/bot/config/appsettings.development.json | 22 ++++++++++++- src/bot/config/appsettings.json | 24 ++++++++++++-- src/bot/config/appsettings.production.json | 26 +++++++++++++-- src/bot/config/appsettings.staging.json | 35 +++++++++++++++++++-- 5 files changed, 126 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 07fc8d346f..fde8d204c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,28 @@ version: "3.9" services: - kdb_1: + kdb_prod_1: image: kdb container_name: kdb_1 depends_on: - kdb_db_1 environment: + KDB_ENVIRONMENT: "production" KDB_TOKEN: "" KDB_PREFIX: "!k " restart: unless-stopped + kdb_staging_1: + image: kdb + container_name: kdb_1 + depends_on: + - kdb_db_1 + environment: + KDB_ENVIRONMENT: "staging" + KDB_TOKEN: "" + KDB_PREFIX: "!kt " + restart: unless-stopped + kdb_db_1: image: mysql:latest container_name: kdb_db_1 @@ -22,4 +34,17 @@ services: MYSQL_PASSWORD: "kd_kdb" MYSQL_DATABASE: "kd_kdb" ports: - - "3307:3306" \ No newline at end of file + - "3307:3306" + + kdb_db_2: + image: mysql:latest + container_name: kdb_db_2 + command: mysqld --default-authentication-plugin=mysql_native_password + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: "kd_kdb" + MYSQL_USER: "kd_kdb" + MYSQL_PASSWORD: "kd_kdb" + MYSQL_DATABASE: "kd_kdb" + ports: + - "3308:3306" \ No newline at end of file diff --git a/src/bot/config/appsettings.development.json b/src/bot/config/appsettings.development.json index 44dbc1664a..3278dcd71f 100644 --- a/src/bot/config/appsettings.development.json +++ b/src/bot/config/appsettings.development.json @@ -7,10 +7,30 @@ }, "LoggingSettings": { "Path": "logs/", - "Filename": "log_dev.log", + "Filename": "bot.log", "ConsoleLogLevel": "DEBUG", "FileLogLevel": "DEBUG" }, + "BotLoggingSettings": { + "Command": { + "Path": "logs/", + "Filename": "commands.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "DEBUG" + }, + "Database": { + "Path": "logs/", + "Filename": "database.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "DEBUG" + }, + "Message": { + "Path": "logs/", + "Filename": "message.log", + "ConsoleLogLevel": "DEBUG", + "FileLogLevel": "DEBUG" + } + }, "Translation": { "DefaultLanguage": "de", "Languages": [ diff --git a/src/bot/config/appsettings.json b/src/bot/config/appsettings.json index a02074ce7a..643f38ab40 100644 --- a/src/bot/config/appsettings.json +++ b/src/bot/config/appsettings.json @@ -6,9 +6,29 @@ "DateTimeLogFormat": "%Y-%m-%d_%H-%M-%S" }, "LoggingSettings": { - "Path": "logs/", - "Filename": "log_$start_time.log", + "Path": "logs/$date_now/", + "Filename": "bot.log", "ConsoleLogLevel": "ERROR", "FileLogLevel": "WARN" + }, + "BotLoggingSettings": { + "Command": { + "Path": "logs/$date_now/", + "Filename": "commands.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "WARN" + }, + "Database": { + "Path": "logs/$date_now/", + "Filename": "database.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "WARN" + }, + "Message": { + "Path": "logs/$date_now/", + "Filename": "message.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "WARN" + } } } \ No newline at end of file diff --git a/src/bot/config/appsettings.production.json b/src/bot/config/appsettings.production.json index 515c60afab..493deaba58 100644 --- a/src/bot/config/appsettings.production.json +++ b/src/bot/config/appsettings.production.json @@ -6,11 +6,31 @@ "DateTimeLogFormat": "%Y-%m-%d_%H-%M-%S" }, "LoggingSettings": { - "Path": "logs/", - "Filename": "log_$start_time.log", + "Path": "logs/$date_now/", + "Filename": "bot.log", "ConsoleLogLevel": "ERROR", "FileLogLevel": "INFO" }, + "BotLoggingSettings": { + "Command": { + "Path": "logs/$date_now/", + "Filename": "commands.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "INFO" + }, + "Database": { + "Path": "logs/$date_now/", + "Filename": "database.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "INFO" + }, + "Message": { + "Path": "logs/$date_now/", + "Filename": "message.log", + "ConsoleLogLevel": "ERROR", + "FileLogLevel": "INFO" + } + }, "Translation": { "DefaultLanguage": "de", "Languages": [ @@ -18,7 +38,7 @@ ] }, "DatabaseSettings": { - "Host": "192.168.0.40", + "Host": "localhost", "User": "kd_kdb", "Password": "a2Rfa2Ri", "Database": "kd_kdb", diff --git a/src/bot/config/appsettings.staging.json b/src/bot/config/appsettings.staging.json index 605fe7fc3e..ce9e65210c 100644 --- a/src/bot/config/appsettings.staging.json +++ b/src/bot/config/appsettings.staging.json @@ -6,17 +6,48 @@ "DateTimeLogFormat": "%Y-%m-%d_%H-%M-%S" }, "LoggingSettings": { - "Path": "logs/", - "Filename": "log_staging_$start_time.log", + "Path": "logs/$date_now/", + "Filename": "bot.log", "ConsoleLogLevel": "INFO", "FileLogLevel": "DEBUG" }, + "BotLoggingSettings": { + "Command": { + "Path": "logs/$date_now/", + "Filename": "commands.log", + "ConsoleLogLevel": "INFO", + "FileLogLevel": "DEBUG" + }, + "Database": { + "Path": "logs/$date_now/", + "Filename": "database.log", + "ConsoleLogLevel": "INFO", + "FileLogLevel": "DEBUG" + }, + "Message": { + "Path": "logs/$date_now/", + "Filename": "message.log", + "ConsoleLogLevel": "INFO", + "FileLogLevel": "DEBUG" + } + }, "Translation": { "DefaultLanguage": "de", "Languages": [ "de" ] }, + "DatabaseSettings": { + "Host": "localhost", + "User": "kd_kdb", + "Password": "a2Rfa2Ri", + "Database": "kd_kdb", + "Port": "3308", + "Charset": "utf8mb4", + "UseUnicode": "true", + "Buffered": "true", + "AuthPlugin": "mysql_native_password" + }, "FeatureFlags": { "AdminModule": true, "BaseModule": true, From cb1ebeeb0c6555224296ecdeaa2454ad9ae18d9f Mon Sep 17 00:00:00 2001 From: Sven Heidemann Date: Sun, 2 Oct 2022 23:29:23 +0200 Subject: [PATCH 8/8] Improved on command logging #45 --- .../base/events/base_on_command_event.py | 63 ++++++++++++++++++- .../base/events/base_on_message_event.py | 2 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/modules/base/events/base_on_command_event.py b/src/modules/base/events/base_on_command_event.py index bbe548e74f..e1c46a20e8 100644 --- a/src/modules/base/events/base_on_command_event.py +++ b/src/modules/base/events/base_on_command_event.py @@ -1,3 +1,7 @@ +from typing import Optional + +import discord +from cpl_core.database.context import DatabaseContextABC from cpl_core.time import TimeFormatSettings from cpl_discord.events import OnCommandABC from cpl_discord.service import DiscordBotServiceABC @@ -7,6 +11,12 @@ from discord.ext.commands import Context from bot_core.abc.message_service_abc import MessageServiceABC from bot_core.configuration.bot_settings import BotSettings from bot_core.logging.command_logger import CommandLogger +from bot_data.abc.client_repository_abc import ClientRepositoryABC +from bot_data.abc.server_repository_abc import ServerRepositoryABC +from bot_data.abc.user_repository_abc import UserRepositoryABC +from bot_data.model.user import User +from modules.base.abc.base_helper_abc import BaseHelperABC +from modules.base.configuration.base_server_settings import BaseServerSettings class BaseOnCommandEvent(OnCommandABC): @@ -18,7 +28,12 @@ class BaseOnCommandEvent(OnCommandABC): messenger: MessageServiceABC, bot_settings: BotSettings, time_format_settings: TimeFormatSettings, - translate: TranslatePipe + translate: TranslatePipe, + bhs: BaseHelperABC, + db: DatabaseContextABC, + users: UserRepositoryABC, + clients: ClientRepositoryABC, + servers: ServerRepositoryABC, ): OnCommandABC.__init__(self) self._logger = logger @@ -27,7 +42,49 @@ class BaseOnCommandEvent(OnCommandABC): self._bot_settings = bot_settings self._time_format_settings = time_format_settings self._t = translate + self._base_helper = bhs + self._db = db + self._users = users + self._clients = clients + self._servers = servers + + def _append_received_message_count(self, g_id: int): + try: + self._clients.append_received_message_count(self._bot.user.id, g_id, 1) + self._db.save_changes() + except Exception as e: + self._logger.error(__name__, f'Cannot edit client {self._bot.user.id}@{g_id}', e) + + def _handle_message_for_xp(self, message: discord.Message): + dc_user_id = message.author.id + try: + server = self._servers.get_server_by_discord_id(message.guild.id) + except Exception as e: + self._logger.error(__name__, f'Cannot get server {message.guild.id}', e) + return + + user: Optional[User] = None + try: + user = self._users.get_user_by_discord_id_and_server_id(dc_user_id, server.server_id) + except Exception as e: + self._logger.error(__name__, f'Cannot get user {dc_user_id}', e) + return + + if user is None: + self._logger.error(__name__, f'User not found {dc_user_id}') + return + + settings: BaseServerSettings = self._base_helper.get_config(message.guild.id) + old_xp = user.xp + user.xp += settings.xp_per_message + self._users.update_user(user) + self._db.save_changes() + + self._logger.debug(__name__, f'User {user} sent message. xp: from {old_xp} to {user.xp}') async def on_command(self, ctx: Context): - print('\n', ctx.command) - pass + self._logger.debug(__name__, f'Module {type(self)} started') + self._logger.info(__name__, f'Received command: {ctx.command} from {ctx.channel}') + if ctx is None or ctx.guild is None: + return + self._append_received_message_count(ctx.guild.id) diff --git a/src/modules/base/events/base_on_message_event.py b/src/modules/base/events/base_on_message_event.py index f5288c50cf..ee8fcdbc4b 100644 --- a/src/modules/base/events/base_on_message_event.py +++ b/src/modules/base/events/base_on_message_event.py @@ -72,7 +72,7 @@ class BaseOnMessageEvent(OnMessageABC): async def on_message(self, message: discord.Message): self._logger.debug(__name__, f'Module {type(self)} started') - self._logger.debug(__name__, f'Received message: {LogMessageHelper.get_log_string(message)}') + self._logger.info(__name__, f'Received message: {LogMessageHelper.get_log_string(message)}') if message is None or message.guild is None: return self._append_received_message_count(message.guild.id)