Improved logging

This commit is contained in:
Sven Heidemann 2022-10-02 13:13:23 +02:00
parent 12cd5c9045
commit e2749f6a81
24 changed files with 118 additions and 94 deletions

View File

@ -1,6 +1,6 @@
from typing import Optional, Type, Callable
from cpl_core.configuration import ConfigurationABC, ConfigurationModelABC
from cpl_core.configuration import ConfigurationABC
from cpl_core.console import Console
from cpl_core.dependency_injection import ServiceProviderABC
from cpl_core.logging import LoggerABC
@ -11,14 +11,10 @@ from cpl_translation import TranslatePipe, TranslationServiceABC, TranslationSet
from bot_core.configuration.bot_logging_settings import BotLoggingSettings
from bot_core.configuration.bot_settings import BotSettings
from bot_core.configuration.server_settings import ServerSettings
from bot_core.logging.command_logger import CommandLogger
from bot_core.logging.event_logger import EventLogger
from modules.base.configuration.base_server_settings import BaseServerSettings
from bot_core.logging.database_logger import DatabaseLogger
from modules.base.configuration.base_settings import BaseSettings
from modules.boot_log.configuration.boot_log_server_settings import BootLogServerSettings
from modules.boot_log.configuration.boot_log_settings import BootLogSettings
from modules.permission.configuration.permission_server_settings import PermissionServerSettings
from modules.permission.configuration.permission_settings import PermissionSettings
@ -28,8 +24,6 @@ class Application(DiscordBotApplicationABC):
DiscordBotApplicationABC.__init__(self, config, services)
self._services = services
self._cmd_logger: Optional[LoggerABC] = None
self._event_logger: Optional[LoggerABC] = None
# cpl-core
self._logger: LoggerABC = services.get_service(LoggerABC)
@ -40,25 +34,8 @@ class Application(DiscordBotApplicationABC):
self._translation: TranslationServiceABC = services.get_service(TranslationServiceABC)
self._translate: TranslatePipe = services.get_service(TranslatePipe)
def _configure_settings_with_sub_settings(self, settings: Type, list_atr: Callable, atr: Callable):
settings: Optional[settings] = self._configuration.get_configuration(settings)
if settings is None:
return
for sub_settings in list_atr(settings):
self._logger.trace(__name__, f'Saved config: {type(sub_settings).__name__}_{atr(sub_settings)}')
self._configuration.add_configuration(f'{type(sub_settings).__name__}_{atr(sub_settings)}', sub_settings)
async def configure(self):
self._translation.load_by_settings(self._configuration.get_configuration(TranslationSettings))
self._configure_settings_with_sub_settings(BotSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(BaseSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(BootLogSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(PermissionSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(BotLoggingSettings, lambda x: x.files, lambda x: x.key)
# custom loggers need to be loaded AFTER config is loaded correctly
self._cmd_logger: LoggerABC = self._services.get_service(CommandLogger)
self._event_logger: LoggerABC = self._services.get_service(EventLogger)
async def main(self):
try:

View File

@ -1,7 +1,7 @@
{
"LoggingSettings": {
"Path": "logs/",
"Filename": "dev.log",
"Filename": "bot.log",
"ConsoleLogLevel": "DEBUG",
"FileLogLevel": "TRACE"
},
@ -12,9 +12,15 @@
"ConsoleLogLevel": "DEBUG",
"FileLogLevel": "TRACE"
},
"Event": {
"Database": {
"Path": "logs/",
"Filename": "event.log",
"Filename": "database.log",
"ConsoleLogLevel": "DEBUG",
"FileLogLevel": "TRACE"
},
"Message": {
"Path": "logs/",
"Filename": "message.log",
"ConsoleLogLevel": "DEBUG",
"FileLogLevel": "TRACE"
}

View File

@ -10,10 +10,11 @@ from cpl_core.environment import ApplicationEnvironment
from cpl_core.logging import LoggerABC
from bot_core.abc.client_utils_service_abc import ClientUtilsServiceABC
from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC
from bot_core.abc.message_service_abc import MessageServiceABC
from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings
from bot_core.logging.command_logger import CommandLogger
from bot_core.logging.event_logger import EventLogger
from bot_core.logging.database_logger import DatabaseLogger
from bot_core.logging.message_logger import MessageLogger
from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe
from bot_core.service.client_utils_service import ClientUtilsService
from bot_core.service.message_service import MessageService
@ -45,16 +46,6 @@ class Startup(StartupABC):
self._config: Optional[ConfigurationABC] = None
def configure_configuration(self, configuration: ConfigurationABC, environment: ApplicationEnvironment) -> ConfigurationABC:
environment.set_working_directory(os.path.dirname(os.path.realpath(__file__)))
configuration.add_environment_variables('KDB_')
configuration.add_environment_variables('DISCORD_')
configuration.add_json_file(f'config/appsettings.json', optional=False)
configuration.add_json_file(f'config/appsettings.{environment.environment_name}.json', optional=True)
configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True)
configuration.add_configuration('Startup_StartTime', str(self._start_time))
self._config = configuration
return configuration
@ -62,7 +53,8 @@ class Startup(StartupABC):
services.add_logging()
# custom logging
services.add_singleton(LoggerABC, CommandLogger)
services.add_singleton(LoggerABC, EventLogger)
services.add_singleton(LoggerABC, DatabaseLogger)
services.add_singleton(LoggerABC, MessageLogger)
services.add_translation()
@ -87,4 +79,9 @@ class Startup(StartupABC):
services.add_transient(UserJoinedServerRepositoryABC, UserJoinedServerRepositoryService)
services.add_transient(UserJoinedVoiceChannelRepositoryABC, UserJoinedVoiceChannelRepositoryService)
return services.build_service_provider()
provider = services.build_service_provider()
# instantiate custom logger
for c in CustomFileLoggerABC.__subclasses__():
i: LoggerABC = provider.get_service(c)
return provider

View File

@ -1,10 +1,20 @@
import os
from datetime import datetime
from typing import Callable, Type, Optional
from cpl_core.application import StartupExtensionABC
from cpl_core.configuration import ConfigurationABC
from cpl_core.dependency_injection import ServiceCollectionABC
from cpl_core.environment import ApplicationEnvironmentABC
from cpl_core.logging import LoggerABC
from bot_core.configuration.bot_logging_settings import BotLoggingSettings
from bot_core.configuration.bot_settings import BotSettings
from bot_core.logging.command_logger import CommandLogger
from bot_core.logging.database_logger import DatabaseLogger
from modules.base.configuration.base_settings import BaseSettings
from modules.boot_log.configuration.boot_log_settings import BootLogSettings
from modules.permission.configuration.permission_settings import PermissionSettings
class StartupSettingsExtension(StartupExtensionABC):
@ -23,6 +33,20 @@ class StartupSettingsExtension(StartupExtensionABC):
configuration.add_json_file(f'config/appsettings.{environment.host_name}.json', optional=True)
configuration.add_configuration('Startup_StartTime', str(self._start_time))
self._configure_settings_with_sub_settings(configuration, BotSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(configuration, BaseSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(configuration, BootLogSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(configuration, PermissionSettings, lambda x: x.servers, lambda x: x.id)
self._configure_settings_with_sub_settings(configuration, BotLoggingSettings, lambda x: x.files, lambda x: x.key)
def configure_services(self, services: ServiceCollectionABC, env: ApplicationEnvironmentABC):
pass
@staticmethod
def _configure_settings_with_sub_settings(config: ConfigurationABC, settings: Type, list_atr: Callable, atr: Callable):
settings: Optional[settings] = config.get_configuration(settings)
if settings is None:
return
for sub_settings in list_atr(settings):
config.add_configuration(f'{type(sub_settings).__name__}_{atr(sub_settings)}', sub_settings)

View File

@ -20,15 +20,15 @@ class CustomFileLoggerABC(Logger, ABC):
def _begin_log(self):
console_level = self._console.value
self._console = LoggingLevelEnum.OFF
self.info(__name__, f'Starting...\n')
self.info(__name__, f'Starting...')
self._console = LoggingLevelEnum(console_level)
def _get_string(self, name_list_as_str: str, level: LoggingLevelEnum, message: str) -> str:
names = name_list_as_str.split(' ')
log_level = level.name
string = f'<{self._get_datetime_now()}> [ {log_level} ] '
string = f'<{self._get_datetime_now()}> [ {log_level} ]'
for name in names:
string += f'[ {name} ] '
string += f' [ {name} ]'
string += f': {message}'
return string

View File

View File

@ -0,0 +1,9 @@
import discord
class LogMessageHelper:
@staticmethod
def get_log_string(message: discord.Message):
content = message.content.replace("\n", "\n\t")
return f'{message.author} @ {message.channel} -> \n\t{content}'

View File

@ -0,0 +1,11 @@
from cpl_core.configuration import ConfigurationABC
from cpl_core.environment import ApplicationEnvironmentABC
from cpl_core.time import TimeFormatSettings
from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC
class DatabaseLogger(CustomFileLoggerABC):
def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC):
CustomFileLoggerABC.__init__(self, 'Database', config, time_format, env)

View File

@ -5,7 +5,7 @@ from cpl_core.time import TimeFormatSettings
from bot_core.abc.custom_file_logger_abc import CustomFileLoggerABC
class EventLogger(CustomFileLoggerABC):
class MessageLogger(CustomFileLoggerABC):
def __init__(self, config: ConfigurationABC, time_format: TimeFormatSettings, env: ApplicationEnvironmentABC):
CustomFileLoggerABC.__init__(self, 'Event', config, time_format, env)
CustomFileLoggerABC.__init__(self, 'Message', config, time_format, env)

View File

@ -4,19 +4,20 @@ from typing import Union
import discord
from cpl_core.configuration.configuration_abc import ConfigurationABC
from cpl_core.database.context.database_context_abc import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_discord.service import DiscordBotServiceABC
from cpl_query.extension import List
from discord.ext.commands import Context
from bot_core.abc.message_service_abc import MessageServiceABC
from bot_core.configuration.server_settings import ServerSettings
from bot_core.helper.log_message_helper import LogMessageHelper
from bot_core.logging.message_logger import MessageLogger
from bot_data.abc.client_repository_abc import ClientRepositoryABC
class MessageService(MessageServiceABC):
def __init__(self, config: ConfigurationABC, logger: LoggerABC, bot: DiscordBotServiceABC, clients: ClientRepositoryABC, db: DatabaseContextABC):
def __init__(self, config: ConfigurationABC, logger: MessageLogger, bot: DiscordBotServiceABC, clients: ClientRepositoryABC, db: DatabaseContextABC):
self._config = config
self._logger = logger
self._bot = bot
@ -35,7 +36,7 @@ class MessageService(MessageServiceABC):
server_st: ServerSettings = self._config.get_configuration(f'ServerSettings_{message.guild.id}')
if not mass_delete:
await asyncio.sleep(server_st.message_delete_timer)
self._logger.debug(__name__, f'Try to delete message:\n\t{message}\n\t{message.content}')
self._logger.debug(__name__, f'Try to delete message: {LogMessageHelper.get_log_string(message)}')
guild_id = message.guild.id
try:
await message.delete()

View File

@ -1,11 +1,12 @@
from cpl_core.database import DatabaseSettings
from cpl_core.database.context import DatabaseContext
from cpl_core.logging import LoggerABC
from bot_core.logging.database_logger import DatabaseLogger
class DBContext(DatabaseContext):
def __init__(self, logger: LoggerABC):
def __init__(self, logger: DatabaseLogger):
self._logger = logger

View File

@ -1,12 +1,11 @@
from cpl_core.logging import LoggerABC
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.migration_abc import MigrationABC
from bot_data.db_context import DBContext
class InitialMigration(MigrationABC):
def __init__(self, logger: LoggerABC, db: DBContext):
def __init__(self, logger: DatabaseLogger, db: DBContext):
self._logger = logger
self._db = db
self._cursor = db.cursor

View File

@ -1,8 +1,9 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.client_repository_abc import ClientRepositoryABC
from bot_data.abc.server_repository_abc import ServerRepositoryABC
from bot_data.model.client import Client
@ -10,7 +11,7 @@ from bot_data.model.client import Client
class ClientRepositoryService(ClientRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
self._logger = logger
self._context = db_context

View File

@ -1,16 +1,17 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC
from cpl_core.database.context import DatabaseContextABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC
from bot_data.abc.server_repository_abc import ServerRepositoryABC
from bot_data.model.known_user import KnownUser
class KnownUserRepositoryService(KnownUserRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
self._logger = logger
self._context = db_context

View File

@ -2,15 +2,15 @@ from typing import Type
from cpl_core.database.context import DatabaseContextABC
from cpl_core.dependency_injection import ServiceProviderABC
from cpl_core.logging import LoggerABC
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.migration_abc import MigrationABC
from bot_data.model.migration_history import MigrationHistory
class MigrationService:
def __init__(self, logger: LoggerABC, services: ServiceProviderABC, db: DatabaseContextABC):
def __init__(self, logger: DatabaseLogger, services: ServiceProviderABC, db: DatabaseContextABC):
self._logger = logger
self._services = services

View File

@ -1,15 +1,16 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.server_repository_abc import ServerRepositoryABC
from bot_data.model.server import Server
class ServerRepositoryService(ServerRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC):
self._logger = logger
self._context = db_context

View File

@ -1,18 +1,18 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.user_joined_server_repository_abc import \
UserJoinedServerRepositoryABC
from bot_data.abc.user_repository_abc import UserRepositoryABC
from bot_data.model.user import User
from bot_data.model.user_joined_server import UserJoinedServer
class UserJoinedServerRepositoryService(UserJoinedServerRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, users: UserRepositoryABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, users: UserRepositoryABC):
self._logger = logger
self._context = db_context

View File

@ -1,17 +1,17 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.user_joined_voice_channel_abc import UserJoinedVoiceChannelRepositoryABC
from bot_data.abc.user_repository_abc import UserRepositoryABC
from bot_data.model.user_joined_voice_channel import UserJoinedVoiceChannel
from bot_data.abc.user_joined_voice_channel_abc import UserJoinedVoiceChannelRepositoryABC
class UserJoinedVoiceChannelRepositoryService(UserJoinedVoiceChannelRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, users: UserRepositoryABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, users: UserRepositoryABC):
self._logger = logger
self._context = db_context

View File

@ -1,8 +1,9 @@
from typing import Optional
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_query.extension import List
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.abc.server_repository_abc import ServerRepositoryABC
from bot_data.abc.user_repository_abc import UserRepositoryABC
from bot_data.model.user import User
@ -10,7 +11,7 @@ from bot_data.model.user import User
class UserRepositoryService(UserRepositoryABC):
def __init__(self, logger: LoggerABC, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
def __init__(self, logger: DatabaseLogger, db_context: DatabaseContextABC, servers: ServerRepositoryABC):
self._logger = logger
self._context = db_context

View File

@ -1,16 +1,8 @@
import datetime
import traceback
import uuid
from cpl_core.time import TimeFormatSettings
from cpl_discord.events import OnCommandABC
from cpl_discord.service import DiscordBotServiceABC
from cpl_translation import TranslatePipe
from discord.ext import commands
from discord.ext.commands import Context, CommandError
from cpl_core.logging import LoggerABC
from cpl_discord.events.on_command_error_abc import OnCommandErrorABC
from discord.ext.commands import Context
from bot_core.abc.message_service_abc import MessageServiceABC
from bot_core.configuration.bot_settings import BotSettings
@ -37,5 +29,5 @@ class BaseOnCommandEvent(OnCommandABC):
self._t = translate
async def on_command(self, ctx: Context):
self._logger.warn(__name__, f'Received command afk {ctx}')
print('\n', ctx.command)
pass

View File

@ -2,10 +2,11 @@ from typing import Optional
import discord
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_discord.events import OnMessageABC
from cpl_discord.service import DiscordBotServiceABC
from bot_core.helper.log_message_helper import LogMessageHelper
from bot_core.logging.message_logger import MessageLogger
from bot_data.abc.client_repository_abc import ClientRepositoryABC
from bot_data.abc.server_repository_abc import ServerRepositoryABC
from bot_data.abc.user_repository_abc import UserRepositoryABC
@ -18,7 +19,7 @@ class BaseOnMessageEvent(OnMessageABC):
def __init__(
self,
logger: LoggerABC,
logger: MessageLogger,
bhs: BaseHelperABC,
db: DatabaseContextABC,
bot: DiscordBotServiceABC,
@ -71,6 +72,7 @@ class BaseOnMessageEvent(OnMessageABC):
async def on_message(self, message: discord.Message):
self._logger.debug(__name__, f'Module {type(self)} started')
self._logger.debug(__name__, f'Received message: {LogMessageHelper.get_log_string(message)}')
if message is None or message.guild is None:
return
self._append_received_message_count(message.guild.id)

View File

@ -42,12 +42,12 @@ class BootLogOnReadyEvent(OnReadyABC):
# print warning if initialisation took too long
if init_time >= 30:
self._logger.warn(
__name__, 'It takes long time to start the bot!')
__name__, 'It takes too long to start the bot!')
# print error if initialisation took way too long
elif init_time >= 90:
self._logger.error(
__name__, 'It takes very long time to start the bot!!!')
__name__, 'It takes far too long to start the bot!!!')
except Exception as e:
self._logger.error(__name__, 'Init time calculation failed', e)
return

View File

@ -6,6 +6,7 @@ from cpl_core.dependency_injection import ServiceProviderABC
from cpl_core.logging import LoggerABC
from bot_core.configuration.feature_flags_settings import FeatureFlagsSettings
from bot_core.logging.database_logger import DatabaseLogger
from bot_data.service.migration_service import MigrationService
@ -18,7 +19,7 @@ class DatabaseExtension(ApplicationExtensionABC):
feature_flags: FeatureFlagsSettings = config.get_configuration(FeatureFlagsSettings)
if not feature_flags.database_module:
return
logger: LoggerABC = services.get_service(LoggerABC)
logger: LoggerABC = services.get_service(DatabaseLogger)
logger.debug(__name__, 'Database extension started')
config.add_configuration('Database_StartTime', str(datetime.now()))
migrations: MigrationService = services.get_service(MigrationService)

View File

@ -4,10 +4,10 @@ from datetime import datetime, timedelta
import discord
from cpl_core.configuration import ConfigurationABC
from cpl_core.database.context import DatabaseContextABC
from cpl_core.logging import LoggerABC
from cpl_discord.events import OnReadyABC
from cpl_discord.service import DiscordBotServiceABC
from bot_core.logging.database_logger import DatabaseLogger
from bot_core.pipes.date_time_offset_pipe import DateTimeOffsetPipe
from bot_data.abc.client_repository_abc import ClientRepositoryABC
from bot_data.abc.known_user_repository_abc import KnownUserRepositoryABC
@ -29,7 +29,7 @@ class DatabaseOnReadyEvent(OnReadyABC):
def __init__(
self,
config: ConfigurationABC,
logger: LoggerABC,
logger: DatabaseLogger,
bot: DiscordBotServiceABC,
db_context: DatabaseContextABC,
server_repo: ServerRepositoryABC,
@ -65,12 +65,12 @@ class DatabaseOnReadyEvent(OnReadyABC):
# print warning if initialisation took too long
if init_time >= 30:
self._logger.warn(
__name__, 'It takes long time to start the bot!')
__name__, 'It takes too long to prepare the database!')
# print error if initialisation took way too long
elif init_time >= 90:
self._logger.error(
__name__, 'It takes very long time to start the bot!!!')
__name__, 'It takes far too long to prepare the database!!!')
except Exception as e: #
self._logger.error(__name__, 'Database init time calculation failed', e)
return