From b9ac11e15fd1e9b64c778c992cf8eb04c9c3740b Mon Sep 17 00:00:00 2001 From: edraft Date: Mon, 22 Sep 2025 23:24:46 +0200 Subject: [PATCH 1/2] Added structured and wrapped logger #187 --- src/cpl-api/cpl/api/__init__.py | 1 + src/cpl-api/cpl/api/abc/__init__.py | 2 +- src/cpl-api/cpl/api/application/__init__.py | 2 +- src/cpl-api/cpl/api/application/web_app.py | 28 +++---- src/cpl-api/cpl/api/logger.py | 9 ++- .../cpl/api/middleware/authentication.py | 20 ++--- .../cpl/api/middleware/authorization.py | 10 +-- src/cpl-api/cpl/api/middleware/logging.py | 22 ++--- src/cpl-api/cpl/api/middleware/request.py | 13 +-- src/cpl-api/cpl/api/model/__init__.py | 2 +- src/cpl-api/cpl/api/registry/__init__.py | 2 +- src/cpl-api/cpl/api/router.py | 3 +- .../cpl/application/abc/application_abc.py | 4 +- src/cpl-auth/cpl/auth/auth_logger.py | 10 +-- .../cpl/auth/keycloak/keycloak_admin.py | 6 +- .../cpl/auth/keycloak/keycloak_client.py | 6 +- src/cpl-auth/cpl/auth/permission_seeder.py | 8 +- .../schema/_administration/api_key_dao.py | 5 +- .../auth/schema/_administration/auth_user.py | 10 +-- .../schema/_administration/auth_user_dao.py | 5 +- .../_permission/api_key_permission_dao.py | 5 +- .../auth/schema/_permission/permission_dao.py | 5 +- .../cpl/auth/schema/_permission/role_dao.py | 5 +- .../schema/_permission/role_permission_dao.py | 5 +- .../auth/schema/_permission/role_user_dao.py | 5 +- src/cpl-core/cpl/core/ctx/user_context.py | 9 ++- src/cpl-core/cpl/core/log/__init__.py | 1 + .../cpl/core/log/structured_logger.py | 80 +++++++++++++++++++ src/cpl-core/cpl/core/log/wrapped_logger.py | 37 +++++++++ .../database/abc/data_access_object_abc.py | 8 +- .../cpl/database/abc/db_model_dao_abc.py | 4 +- src/cpl-database/cpl/database/db_logger.py | 10 +-- .../cpl/database/mysql/db_context.py | 35 ++++---- .../cpl/database/mysql/mysql_pool.py | 6 +- .../cpl/database/postgres/db_context.py | 36 ++++----- .../cpl/database/postgres/postgres_pool.py | 6 +- .../database/schema/executed_migration_dao.py | 5 +- .../cpl/database/service/migration_service.py | 9 +-- .../cpl/database/service/seeder_service.py | 6 +- .../cpl/dependency/service_collection.py | 11 +++ src/cpl-mail/cpl/mail/mail_logger.py | 10 +-- tests/custom/api/src/main.py | 3 +- tests/custom/database/src/model/city_dao.py | 2 +- tests/custom/database/src/model/user_dao.py | 2 +- 44 files changed, 287 insertions(+), 186 deletions(-) create mode 100644 src/cpl-core/cpl/core/log/structured_logger.py create mode 100644 src/cpl-core/cpl/core/log/wrapped_logger.py diff --git a/src/cpl-api/cpl/api/__init__.py b/src/cpl-api/cpl/api/__init__.py index 0546ce03..16c78038 100644 --- a/src/cpl-api/cpl/api/__init__.py +++ b/src/cpl-api/cpl/api/__init__.py @@ -4,6 +4,7 @@ from .error import APIError, AlreadyExists, EndpointNotImplemented, Forbidden, N from .logger import APILogger from .settings import ApiSettings + def add_api(collection: _ServiceCollection): try: from cpl.database import mysql diff --git a/src/cpl-api/cpl/api/abc/__init__.py b/src/cpl-api/cpl/api/abc/__init__.py index 857fd67f..6dee2dfc 100644 --- a/src/cpl-api/cpl/api/abc/__init__.py +++ b/src/cpl-api/cpl/api/abc/__init__.py @@ -1 +1 @@ -from .asgi_middleware_abc import ASGIMiddleware \ No newline at end of file +from .asgi_middleware_abc import ASGIMiddleware diff --git a/src/cpl-api/cpl/api/application/__init__.py b/src/cpl-api/cpl/api/application/__init__.py index d3d60652..b540ca1b 100644 --- a/src/cpl-api/cpl/api/application/__init__.py +++ b/src/cpl-api/cpl/api/application/__init__.py @@ -1 +1 @@ -from .web_app import WebApp \ No newline at end of file +from .web_app import WebApp diff --git a/src/cpl-api/cpl/api/application/web_app.py b/src/cpl-api/cpl/api/application/web_app.py index 0daccaba..9fdd7bc5 100644 --- a/src/cpl-api/cpl/api/application/web_app.py +++ b/src/cpl-api/cpl/api/application/web_app.py @@ -29,7 +29,6 @@ from cpl.application.abc.application_abc import ApplicationABC from cpl.core.configuration import Configuration from cpl.dependency.service_provider_abc import ServiceProviderABC -_logger = APILogger("API") PolicyInput = Union[dict[str, PolicyResolver], Policy] @@ -39,6 +38,8 @@ class WebApp(ApplicationABC): super().__init__(services, [auth, api]) self._app: Starlette | None = None + self._logger = services.get_service(APILogger) + self._api_settings = Configuration.get(ApiSettings) self._policies = services.get_service(PolicyRegistry) self._routes = services.get_service(RouteRegistry) @@ -52,16 +53,15 @@ class WebApp(ApplicationABC): APIError: self._handle_exception, } - @staticmethod - async def _handle_exception(request: Request, exc: Exception): + async def _handle_exception(self, request: Request, exc: Exception): if isinstance(exc, APIError): - _logger.error(exc) + self._logger.error(exc) return JSONResponse({"error": str(exc)}, status_code=exc.status_code) if hasattr(request.state, "request_id"): - _logger.error(f"Request {request.state.request_id}", exc) + self._logger.error(f"Request {request.state.request_id}", exc) else: - _logger.error("Request unknown", exc) + self._logger.error("Request unknown", exc) return JSONResponse({"error": str(exc)}, status_code=500) @@ -69,10 +69,10 @@ class WebApp(ApplicationABC): origins = self._api_settings.allowed_origins if origins is None or origins == "": - _logger.warning("No allowed origins specified, allowing all origins") + self._logger.warning("No allowed origins specified, allowing all origins") return ["*"] - _logger.debug(f"Allowed origins: {origins}") + self._logger.debug(f"Allowed origins: {origins}") return origins.split(",") def with_database(self) -> Self: @@ -191,11 +191,11 @@ class WebApp(ApplicationABC): if isinstance(policy, dict): for name, resolver in policy.items(): if not isinstance(name, str): - _logger.warning(f"Skipping policy at index {i}, name must be a string") + self._logger.warning(f"Skipping policy at index {i}, name must be a string") continue if not callable(resolver): - _logger.warning(f"Skipping policy {name}, resolver must be callable") + self._logger.warning(f"Skipping policy {name}, resolver must be callable") continue _policies.append(Policy(name, resolver)) @@ -213,10 +213,10 @@ class WebApp(ApplicationABC): for policy_name in rule["policies"]: policy = self._policies.get(policy_name) if not policy: - _logger.fatal(f"Authorization policy '{policy_name}' not found") + self._logger.fatal(f"Authorization policy '{policy_name}' not found") async def main(self): - _logger.debug(f"Preparing API") + self._logger.debug(f"Preparing API") self._validate_policies() if self._app is None: @@ -238,7 +238,7 @@ class WebApp(ApplicationABC): else: app = self._app - _logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}") + self._logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}") config = uvicorn.Config( app, host=self._api_settings.host, port=self._api_settings.port, log_config=None, loop="asyncio" @@ -246,4 +246,4 @@ class WebApp(ApplicationABC): server = uvicorn.Server(config) await server.serve() - _logger.info("Shutdown API") + self._logger.info("Shutdown API") diff --git a/src/cpl-api/cpl/api/logger.py b/src/cpl-api/cpl/api/logger.py index e7f2cfd2..d9f88a45 100644 --- a/src/cpl-api/cpl/api/logger.py +++ b/src/cpl-api/cpl/api/logger.py @@ -1,7 +1,8 @@ -from cpl.core.log.logger import Logger +from cpl.core.log import LoggerABC +from cpl.core.log.wrapped_logger import WrappedLogger -class APILogger(Logger): +class APILogger(WrappedLogger): - def __init__(self, source: str): - Logger.__init__(self, source, "api") + def __init__(self, logger: LoggerABC): + WrappedLogger.__init__(self, logger) diff --git a/src/cpl-api/cpl/api/middleware/authentication.py b/src/cpl-api/cpl/api/middleware/authentication.py index cd047706..dd6b0bd6 100644 --- a/src/cpl-api/cpl/api/middleware/authentication.py +++ b/src/cpl-api/cpl/api/middleware/authentication.py @@ -2,8 +2,8 @@ from keycloak import KeycloakAuthenticationError from starlette.types import Scope, Receive, Send from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware -from cpl.api.logger import APILogger from cpl.api.error import Unauthorized +from cpl.api.logger import APILogger from cpl.api.middleware.request import get_request from cpl.api.router import Router from cpl.auth.keycloak import KeycloakClient @@ -11,15 +11,15 @@ from cpl.auth.schema import AuthUserDao, AuthUser from cpl.core.ctx import set_user from cpl.dependency import ServiceProviderABC -_logger = APILogger(__name__) - class AuthenticationMiddleware(ASGIMiddleware): @ServiceProviderABC.inject - def __init__(self, app, keycloak: KeycloakClient, user_dao: AuthUserDao): + def __init__(self, app, logger: APILogger, keycloak: KeycloakClient, user_dao: AuthUserDao): ASGIMiddleware.__init__(self, app) + self._logger = logger + self._keycloak = keycloak self._user_dao = user_dao @@ -28,11 +28,11 @@ class AuthenticationMiddleware(ASGIMiddleware): url = request.url.path if url not in Router.get_auth_required_routes(): - _logger.trace(f"No authentication required for {url}") + self._logger.trace(f"No authentication required for {url}") return await self._app(scope, receive, send) if not request.headers.get("Authorization"): - _logger.debug(f"Unauthorized access to {url}, missing Authorization header") + self._logger.debug(f"Unauthorized access to {url}, missing Authorization header") return await Unauthorized(f"Missing header Authorization").asgi_response(scope, receive, send) auth_header = request.headers.get("Authorization", None) @@ -41,7 +41,7 @@ class AuthenticationMiddleware(ASGIMiddleware): token = auth_header.split("Bearer ")[1] if not await self._verify_login(token): - _logger.debug(f"Unauthorized access to {url}, invalid token") + self._logger.debug(f"Unauthorized access to {url}, invalid token") return await Unauthorized("Invalid token").asgi_response(scope, receive, send) # check user exists in db, if not create @@ -51,7 +51,7 @@ class AuthenticationMiddleware(ASGIMiddleware): user = await self._get_or_crate_user(keycloak_id) if user.deleted: - _logger.debug(f"Unauthorized access to {url}, user is deleted") + self._logger.debug(f"Unauthorized access to {url}, user is deleted") return await Unauthorized("User is deleted").asgi_response(scope, receive, send) request.state.user = user @@ -73,8 +73,8 @@ class AuthenticationMiddleware(ASGIMiddleware): token_info = self._keycloak.introspect(token) return token_info.get("active", False) except KeycloakAuthenticationError as e: - _logger.debug(f"Keycloak authentication error: {e}") + self._logger.debug(f"Keycloak authentication error: {e}") return False except Exception as e: - _logger.error(f"Unexpected error during token verification: {e}") + self._logger.error(f"Unexpected error during token verification: {e}") return False diff --git a/src/cpl-api/cpl/api/middleware/authorization.py b/src/cpl-api/cpl/api/middleware/authorization.py index 021017ba..4125e65d 100644 --- a/src/cpl-api/cpl/api/middleware/authorization.py +++ b/src/cpl-api/cpl/api/middleware/authorization.py @@ -11,15 +11,15 @@ from cpl.auth.schema._administration.auth_user_dao import AuthUserDao from cpl.core.ctx.user_context import get_user from cpl.dependency.service_provider_abc import ServiceProviderABC -_logger = APILogger(__name__) - class AuthorizationMiddleware(ASGIMiddleware): @ServiceProviderABC.inject - def __init__(self, app, policies: PolicyRegistry, user_dao: AuthUserDao): + def __init__(self, app, logger: APILogger, policies: PolicyRegistry, user_dao: AuthUserDao): ASGIMiddleware.__init__(self, app) + self._logger = logger + self._policies = policies self._user_dao = user_dao @@ -28,7 +28,7 @@ class AuthorizationMiddleware(ASGIMiddleware): url = request.url.path if url not in Router.get_authorization_rules_paths(): - _logger.trace(f"No authorization required for {url}") + self._logger.trace(f"No authorization required for {url}") return await self._app(scope, receive, send) user = get_user() @@ -64,7 +64,7 @@ class AuthorizationMiddleware(ASGIMiddleware): for policy_name in rule["policies"]: policy = self._policies.get(policy_name) if not policy: - _logger.warning(f"Authorization policy '{policy_name}' not found") + self._logger.warning(f"Authorization policy '{policy_name}' not found") continue if not await policy.resolve(user): diff --git a/src/cpl-api/cpl/api/middleware/logging.py b/src/cpl-api/cpl/api/middleware/logging.py index e47cbe77..6a28b22c 100644 --- a/src/cpl-api/cpl/api/middleware/logging.py +++ b/src/cpl-api/cpl/api/middleware/logging.py @@ -6,15 +6,17 @@ from starlette.types import Receive, Scope, Send from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware from cpl.api.logger import APILogger from cpl.api.middleware.request import get_request - -_logger = APILogger(__name__) +from cpl.dependency import ServiceProviderABC class LoggingMiddleware(ASGIMiddleware): - def __init__(self, app): + @ServiceProviderABC.inject + def __init__(self, app, logger: APILogger): ASGIMiddleware.__init__(self, app) + self._logger = logger + async def __call__(self, scope: Scope, receive: Receive, send: Send): if scope["type"] != "http": await self._call_next(scope, receive, send) @@ -53,9 +55,8 @@ class LoggingMiddleware(ASGIMiddleware): } return {key: value for key, value in headers.items() if key in relevant_keys} - @classmethod - async def _log_request(cls, request: Request): - _logger.debug( + async def _log_request(self, request: Request): + self._logger.debug( f"Request {getattr(request.state, 'request_id', '-')}: {request.method}@{request.url.path} from {request.client.host}" ) @@ -64,7 +65,7 @@ class LoggingMiddleware(ASGIMiddleware): user = get_user() request_info = { - "headers": cls._filter_relevant_headers(dict(request.headers)), + "headers": self._filter_relevant_headers(dict(request.headers)), "args": dict(request.query_params), "form-data": ( await request.form() @@ -78,10 +79,9 @@ class LoggingMiddleware(ASGIMiddleware): ), } - _logger.trace(f"Request {getattr(request.state, 'request_id', '-')}: {request_info}") + self._logger.trace(f"Request {getattr(request.state, 'request_id', '-')}: {request_info}") - @staticmethod - async def _log_after_request(request: Request, status_code: int, duration: float): - _logger.info( + async def _log_after_request(self, request: Request, status_code: int, duration: float): + self._logger.info( f"Request finished {getattr(request.state, 'request_id', '-')}: {status_code}-{request.method}@{request.url.path} from {request.client.host} in {duration:.2f}ms" ) diff --git a/src/cpl-api/cpl/api/middleware/request.py b/src/cpl-api/cpl/api/middleware/request.py index 5255b26c..215ff683 100644 --- a/src/cpl-api/cpl/api/middleware/request.py +++ b/src/cpl-api/cpl/api/middleware/request.py @@ -9,16 +9,19 @@ from starlette.types import Scope, Receive, Send from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware from cpl.api.logger import APILogger from cpl.api.typing import TRequest +from cpl.dependency import ServiceProviderABC _request_context: ContextVar[Union[TRequest, None]] = ContextVar("request", default=None) -_logger = APILogger(__name__) - class RequestMiddleware(ASGIMiddleware): - def __init__(self, app): + @ServiceProviderABC.inject + def __init__(self, app, logger: APILogger): ASGIMiddleware.__init__(self, app) + + self._logger = logger + self._ctx_token = None async def __call__(self, scope: Scope, receive: Receive, send: Send): @@ -33,7 +36,7 @@ class RequestMiddleware(ASGIMiddleware): async def set_request_data(self, request: TRequest): request.state.request_id = uuid4() request.state.start_time = time.time() - _logger.trace(f"Set new current request: {request.state.request_id}") + self._logger.trace(f"Set new current request: {request.state.request_id}") self._ctx_token = _request_context.set(request) @@ -45,7 +48,7 @@ class RequestMiddleware(ASGIMiddleware): if self._ctx_token is None: return - _logger.trace(f"Clearing current request: {request.state.request_id}") + self._logger.trace(f"Clearing current request: {request.state.request_id}") _request_context.reset(self._ctx_token) diff --git a/src/cpl-api/cpl/api/model/__init__.py b/src/cpl-api/cpl/api/model/__init__.py index c7d3622f..fa7235db 100644 --- a/src/cpl-api/cpl/api/model/__init__.py +++ b/src/cpl-api/cpl/api/model/__init__.py @@ -1,3 +1,3 @@ from .api_route import ApiRoute from .policy import Policy -from .validation_match import ValidationMatch \ No newline at end of file +from .validation_match import ValidationMatch diff --git a/src/cpl-api/cpl/api/registry/__init__.py b/src/cpl-api/cpl/api/registry/__init__.py index de732fa0..ffc35aa3 100644 --- a/src/cpl-api/cpl/api/registry/__init__.py +++ b/src/cpl-api/cpl/api/registry/__init__.py @@ -1,2 +1,2 @@ from .policy import PolicyRegistry -from .route import RouteRegistry \ No newline at end of file +from .route import RouteRegistry diff --git a/src/cpl-api/cpl/api/router.py b/src/cpl-api/cpl/api/router.py index d89b2458..e4beb61b 100644 --- a/src/cpl-api/cpl/api/router.py +++ b/src/cpl-api/cpl/api/router.py @@ -92,8 +92,9 @@ class Router: @classmethod def route(cls, path: str, method: HTTPMethods, registry: RouteRegistry = None, **kwargs): + from cpl.api.model.api_route import ApiRoute + if not registry: - from cpl.api.model.api_route import ApiRoute from cpl.dependency.service_provider_abc import ServiceProviderABC routes = ServiceProviderABC.get_global_service(RouteRegistry) diff --git a/src/cpl-application/cpl/application/abc/application_abc.py b/src/cpl-application/cpl/application/abc/application_abc.py index f8a2080d..f32f633e 100644 --- a/src/cpl-application/cpl/application/abc/application_abc.py +++ b/src/cpl-application/cpl/application/abc/application_abc.py @@ -2,10 +2,10 @@ from abc import ABC, abstractmethod from typing import Callable, Self from cpl.application.host import Host -from cpl.core.console.console import Console -from cpl.core.log import LogSettings from cpl.core.log.log_level import LogLevel +from cpl.core.log.log_settings import LogSettings from cpl.core.log.logger_abc import LoggerABC +from cpl.core.log.structured_logger import StructuredLogger from cpl.dependency.service_provider_abc import ServiceProviderABC diff --git a/src/cpl-auth/cpl/auth/auth_logger.py b/src/cpl-auth/cpl/auth/auth_logger.py index e3b40acb..ec9421dc 100644 --- a/src/cpl-auth/cpl/auth/auth_logger.py +++ b/src/cpl-auth/cpl/auth/auth_logger.py @@ -1,8 +1,8 @@ -from cpl.core.log import Logger -from cpl.core.typing import Source +from cpl.core.log import LoggerABC +from cpl.core.log.wrapped_logger import WrappedLogger -class AuthLogger(Logger): +class AuthLogger(WrappedLogger): - def __init__(self, source: Source): - Logger.__init__(self, source, "auth") + def __init__(self, logger: LoggerABC): + WrappedLogger.__init__(self, logger) diff --git a/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py b/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py index f2a3ef74..e5eb0722 100644 --- a/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py +++ b/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py @@ -3,13 +3,11 @@ from keycloak import KeycloakAdmin as _KeycloakAdmin, KeycloakOpenIDConnection from cpl.auth.auth_logger import AuthLogger from cpl.auth.keycloak_settings import KeycloakSettings -_logger = AuthLogger("keycloak") - class KeycloakAdmin(_KeycloakAdmin): - def __init__(self, settings: KeycloakSettings): - _logger.info("Initializing Keycloak admin") + def __init__(self, logger: AuthLogger, settings: KeycloakSettings): + logger.info("Initializing Keycloak admin") _connection = KeycloakOpenIDConnection( server_url=settings.url, client_id=settings.client_id, diff --git a/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py b/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py index f53f962f..6d62c777 100644 --- a/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py +++ b/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py @@ -5,12 +5,10 @@ from keycloak import KeycloakOpenID from cpl.auth.auth_logger import AuthLogger from cpl.auth.keycloak_settings import KeycloakSettings -_logger = AuthLogger("keycloak") - class KeycloakClient(KeycloakOpenID): - def __init__(self, settings: KeycloakSettings): + def __init__(self, logger: AuthLogger, settings: KeycloakSettings): KeycloakOpenID.__init__( self, server_url=settings.url, @@ -18,7 +16,7 @@ class KeycloakClient(KeycloakOpenID): realm_name=settings.realm, client_secret_key=settings.client_secret, ) - _logger.info("Initializing Keycloak client") + logger.info("Initializing Keycloak client") def get_user_id(self, token: str) -> Optional[str]: info = self.introspect(token) diff --git a/src/cpl-auth/cpl/auth/permission_seeder.py b/src/cpl-auth/cpl/auth/permission_seeder.py index 9ca4c50c..d326863d 100644 --- a/src/cpl-auth/cpl/auth/permission_seeder.py +++ b/src/cpl-auth/cpl/auth/permission_seeder.py @@ -16,12 +16,11 @@ from cpl.core.utils.get_value import get_value from cpl.database.abc.data_seeder_abc import DataSeederABC from cpl.database.db_logger import DBLogger -_logger = DBLogger(__name__) - class PermissionSeeder(DataSeederABC): def __init__( self, + logger: DBLogger, permission_dao: PermissionDao, role_dao: RoleDao, role_permission_dao: RolePermissionDao, @@ -29,6 +28,7 @@ class PermissionSeeder(DataSeederABC): api_key_permission_dao: ApiKeyPermissionDao, ): DataSeederABC.__init__(self) + self._logger = logger self._permission_dao = permission_dao self._role_dao = role_dao self._role_permission_dao = role_permission_dao @@ -40,7 +40,7 @@ class PermissionSeeder(DataSeederABC): possible_permissions = [permission for permission in PermissionsRegistry.get()] if len(permissions) == len(possible_permissions): - _logger.info("Permissions already existing") + self._logger.info("Permissions already existing") await self._update_missing_descriptions() return @@ -53,7 +53,7 @@ class PermissionSeeder(DataSeederABC): await self._permission_dao.delete_many(to_delete, hard_delete=True) - _logger.warning("Permissions incomplete") + self._logger.warning("Permissions incomplete") permission_names = [permission.name for permission in permissions] await self._permission_dao.create_many( [ diff --git a/src/cpl-auth/cpl/auth/schema/_administration/api_key_dao.py b/src/cpl-auth/cpl/auth/schema/_administration/api_key_dao.py index 9db9cc4c..b642747c 100644 --- a/src/cpl-auth/cpl/auth/schema/_administration/api_key_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_administration/api_key_dao.py @@ -3,15 +3,12 @@ from typing import Optional from cpl.auth.schema._administration.api_key import ApiKey from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class ApiKeyDao(DbModelDaoABC[ApiKey]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, ApiKey, TableManager.get("api_keys")) + DbModelDaoABC.__init__(self, ApiKey, TableManager.get("api_keys")) self.attribute(ApiKey.identifier, str) self.attribute(ApiKey.key, str, "keystring") diff --git a/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py b/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py index da8ec803..9cb2b373 100644 --- a/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py +++ b/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py @@ -6,14 +6,12 @@ from async_property import async_property from keycloak import KeycloakGetError from cpl.auth.keycloak import KeycloakAdmin -from cpl.auth.auth_logger import AuthLogger from cpl.auth.permission.permissions import Permissions from cpl.core.typing import SerialId from cpl.database.abc import DbModelABC +from cpl.database.db_logger import DBLogger from cpl.dependency import ServiceProviderABC -_logger = AuthLogger(__name__) - class AuthUser(DbModelABC): def __init__( @@ -43,7 +41,8 @@ class AuthUser(DbModelABC): except KeycloakGetError as e: return "UNKNOWN" except Exception as e: - _logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e) + logger = ServiceProviderABC.get_global_service(DBLogger) + logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e) return "UNKNOWN" @property @@ -57,7 +56,8 @@ class AuthUser(DbModelABC): except KeycloakGetError as e: return "UNKNOWN" except Exception as e: - _logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e) + logger = ServiceProviderABC.get_global_service(DBLogger) + logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e) return "UNKNOWN" @async_property diff --git a/src/cpl-auth/cpl/auth/schema/_administration/auth_user_dao.py b/src/cpl-auth/cpl/auth/schema/_administration/auth_user_dao.py index c219f87d..f808b746 100644 --- a/src/cpl-auth/cpl/auth/schema/_administration/auth_user_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_administration/auth_user_dao.py @@ -4,17 +4,14 @@ from cpl.auth.permission.permissions import Permissions from cpl.auth.schema._administration.auth_user import AuthUser from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger from cpl.database.external_data_temp_table_builder import ExternalDataTempTableBuilder from cpl.dependency import ServiceProviderABC -_logger = DBLogger(__name__) - class AuthUserDao(DbModelDaoABC[AuthUser]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, AuthUser, TableManager.get("auth_users")) + DbModelDaoABC.__init__(self, AuthUser, TableManager.get("auth_users")) self.attribute(AuthUser.keycloak_id, str, db_name="keycloakId") diff --git a/src/cpl-auth/cpl/auth/schema/_permission/api_key_permission_dao.py b/src/cpl-auth/cpl/auth/schema/_permission/api_key_permission_dao.py index 935928bd..781d8177 100644 --- a/src/cpl-auth/cpl/auth/schema/_permission/api_key_permission_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_permission/api_key_permission_dao.py @@ -1,15 +1,12 @@ from cpl.auth.schema._permission.api_key_permission import ApiKeyPermission from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class ApiKeyPermissionDao(DbModelDaoABC[ApiKeyPermission]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, ApiKeyPermission, TableManager.get("api_key_permissions")) + DbModelDaoABC.__init__(self, ApiKeyPermission, TableManager.get("api_key_permissions")) self.attribute(ApiKeyPermission.api_key_id, int) self.attribute(ApiKeyPermission.permission_id, int) diff --git a/src/cpl-auth/cpl/auth/schema/_permission/permission_dao.py b/src/cpl-auth/cpl/auth/schema/_permission/permission_dao.py index c75fd7bb..6a5b2fa7 100644 --- a/src/cpl-auth/cpl/auth/schema/_permission/permission_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_permission/permission_dao.py @@ -3,15 +3,12 @@ from typing import Optional from cpl.auth.schema._permission.permission import Permission from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class PermissionDao(DbModelDaoABC[Permission]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, Permission, TableManager.get("permissions")) + DbModelDaoABC.__init__(self, Permission, TableManager.get("permissions")) self.attribute(Permission.name, str) self.attribute(Permission.description, Optional[str]) diff --git a/src/cpl-auth/cpl/auth/schema/_permission/role_dao.py b/src/cpl-auth/cpl/auth/schema/_permission/role_dao.py index be620094..e1ae5e3c 100644 --- a/src/cpl-auth/cpl/auth/schema/_permission/role_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_permission/role_dao.py @@ -1,14 +1,11 @@ from cpl.auth.schema._permission.role import Role from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class RoleDao(DbModelDaoABC[Role]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, Role, TableManager.get("roles")) + DbModelDaoABC.__init__(self, Role, TableManager.get("roles")) self.attribute(Role.name, str) self.attribute(Role.description, str) diff --git a/src/cpl-auth/cpl/auth/schema/_permission/role_permission_dao.py b/src/cpl-auth/cpl/auth/schema/_permission/role_permission_dao.py index 59f60786..b350ccce 100644 --- a/src/cpl-auth/cpl/auth/schema/_permission/role_permission_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_permission/role_permission_dao.py @@ -1,15 +1,12 @@ from cpl.auth.schema._permission.role_permission import RolePermission from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class RolePermissionDao(DbModelDaoABC[RolePermission]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, RolePermission, TableManager.get("role_permissions")) + DbModelDaoABC.__init__(self, RolePermission, TableManager.get("role_permissions")) self.attribute(RolePermission.role_id, int) self.attribute(RolePermission.permission_id, int) diff --git a/src/cpl-auth/cpl/auth/schema/_permission/role_user_dao.py b/src/cpl-auth/cpl/auth/schema/_permission/role_user_dao.py index 90a0a888..8d669275 100644 --- a/src/cpl-auth/cpl/auth/schema/_permission/role_user_dao.py +++ b/src/cpl-auth/cpl/auth/schema/_permission/role_user_dao.py @@ -1,15 +1,12 @@ from cpl.auth.schema._permission.role_user import RoleUser from cpl.database import TableManager from cpl.database.abc import DbModelDaoABC -from cpl.database.db_logger import DBLogger - -_logger = DBLogger(__name__) class RoleUserDao(DbModelDaoABC[RoleUser]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, RoleUser, TableManager.get("role_users")) + DbModelDaoABC.__init__(self, RoleUser, TableManager.get("role_users")) self.attribute(RoleUser.role_id, int) self.attribute(RoleUser.user_id, int) diff --git a/src/cpl-core/cpl/core/ctx/user_context.py b/src/cpl-core/cpl/core/ctx/user_context.py index 5a4b6eae..5a3e617c 100644 --- a/src/cpl-core/cpl/core/ctx/user_context.py +++ b/src/cpl-core/cpl/core/ctx/user_context.py @@ -1,16 +1,17 @@ from contextvars import ContextVar from typing import Optional -from cpl.auth.auth_logger import AuthLogger from cpl.auth.schema._administration.auth_user import AuthUser _user_context: ContextVar[Optional[AuthUser]] = ContextVar("user", default=None) -_logger = AuthLogger(__name__) - def set_user(user_id: Optional[AuthUser]): - _logger.trace("Setting user context", user_id) + from cpl.dependency.service_provider_abc import ServiceProviderABC + from cpl.core.log.logger_abc import LoggerABC + + logger = ServiceProviderABC.get_global_service(LoggerABC) + logger.trace("Setting user context", user_id) _user_context.set(user_id) diff --git a/src/cpl-core/cpl/core/log/__init__.py b/src/cpl-core/cpl/core/log/__init__.py index f66f1426..72f60ede 100644 --- a/src/cpl-core/cpl/core/log/__init__.py +++ b/src/cpl-core/cpl/core/log/__init__.py @@ -2,3 +2,4 @@ from .logger import Logger from .logger_abc import LoggerABC from .log_level import LogLevel from .log_settings import LogSettings +from .structured_logger import StructuredLogger diff --git a/src/cpl-core/cpl/core/log/structured_logger.py b/src/cpl-core/cpl/core/log/structured_logger.py new file mode 100644 index 00000000..db67b8e2 --- /dev/null +++ b/src/cpl-core/cpl/core/log/structured_logger.py @@ -0,0 +1,80 @@ +import asyncio +import importlib.util +import traceback +from datetime import datetime + +from cpl.core.log import Logger, LogLevel +from cpl.core.typing import Source, Messages + + +class StructuredLogger(Logger): + + def __init__(self, source: Source, file_prefix: str = None): + Logger.__init__(self, source, file_prefix) + + @property + def log_file(self): + return f"logs/{self._file_prefix}_{datetime.now().strftime('%Y-%m-%d')}.jsonl" + + def _log(self, level: LogLevel, *messages: Messages): + try: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + formatted_message = self._format_message(level.value, timestamp, *messages) + + self._write_log_to_file(level, formatted_message) + self._write_to_console(level, formatted_message) + except Exception as e: + print(f"Error while logging: {e} -> {traceback.format_exc()}") + + def _get_structured_message(self, level: str, timestamp: str, messages: str) -> str: + structured_message = { + "timestamp": timestamp, + "level": level.upper(), + "source": self._source, + "messages": messages, + } + + self._enrich_message_with_request(structured_message) + self._enrich_message_with_user(structured_message) + + return str(structured_message) + + @staticmethod + def _enrich_message_with_request(message: dict): + if importlib.util.find_spec("cpl.api") is None: + return + + from cpl.api.middleware.request import get_request + from starlette.requests import Request + + request = get_request() + + if request is None: + return + + message["request"] = { + "url": str(request.url), + "method": request.method, + "scope": request.scope, + } + if isinstance(request, Request) and request.scope == "http": + request: Request = request # fix typing for IDEs + + message["request"]["data"] = asyncio.create_task(request.body()) + + @staticmethod + def _enrich_message_with_user(message: dict): + if importlib.util.find_spec("cpl-auth") is None: + return + + from cpl.core.ctx import get_user + + user = get_user() + if user is None: + return + + message["user"] = { + "id": str(user.id), + "username": user.username, + "email": user.email, + } diff --git a/src/cpl-core/cpl/core/log/wrapped_logger.py b/src/cpl-core/cpl/core/log/wrapped_logger.py new file mode 100644 index 00000000..384a88e2 --- /dev/null +++ b/src/cpl-core/cpl/core/log/wrapped_logger.py @@ -0,0 +1,37 @@ +from cpl.core.log import LoggerABC, LogLevel +from cpl.core.typing import Messages + + +class WrappedLogger(LoggerABC): + + def __init__(self, logger: LoggerABC): + LoggerABC.__init__(self) + assert isinstance(logger, LoggerABC), "logger must be an instance of LoggerABC" + self._logger = logger + + def set_level(self, level: LogLevel): + self._logger.set_level(level) + + def _format_message(self, level: str, timestamp, *messages: Messages) -> str: + return self._logger._format_message(level, timestamp, *messages) + + def header(self, string: str): + self._logger.header(string) + + def trace(self, *messages: Messages): + self._logger.trace(*messages) + + def debug(self, *messages: Messages): + self._logger.debug(*messages) + + def info(self, *messages: Messages): + self._logger.info(*messages) + + def warning(self, *messages: Messages): + self._logger.warning(*messages) + + def error(self, messages: str, e: Exception = None): + self._logger.error(messages, e) + + def fatal(self, messages: str, e: Exception = None): + self._logger.fatal(messages, e) diff --git a/src/cpl-database/cpl/database/abc/data_access_object_abc.py b/src/cpl-database/cpl/database/abc/data_access_object_abc.py index ec011ca9..70ab90ce 100644 --- a/src/cpl-database/cpl/database/abc/data_access_object_abc.py +++ b/src/cpl-database/cpl/database/abc/data_access_object_abc.py @@ -18,16 +18,12 @@ from cpl.database.typing import T_DBM, Attribute, AttributeFilters, AttributeSor class DataAccessObjectABC(ABC, Generic[T_DBM]): @abstractmethod - def __init__(self, source: str, model_type: Type[T_DBM], table_name: str): + def __init__(self, model_type: Type[T_DBM], table_name: str): from cpl.dependency.service_provider_abc import ServiceProviderABC self._db = ServiceProviderABC.get_global_service(DBContextABC) - self._logger = DBLogger(source) - self._model_type = model_type - self._table_name = table_name - - self._logger = DBLogger(source) + self._logger = ServiceProviderABC.get_global_service(DBLogger) self._model_type = model_type self._table_name = table_name diff --git a/src/cpl-database/cpl/database/abc/db_model_dao_abc.py b/src/cpl-database/cpl/database/abc/db_model_dao_abc.py index f53e6658..440f774f 100644 --- a/src/cpl-database/cpl/database/abc/db_model_dao_abc.py +++ b/src/cpl-database/cpl/database/abc/db_model_dao_abc.py @@ -10,8 +10,8 @@ from cpl.database.abc.db_model_abc import DbModelABC class DbModelDaoABC[T_DBM](DataAccessObjectABC[T_DBM]): @abstractmethod - def __init__(self, source: str, model_type: Type[T_DBM], table_name: str): - DataAccessObjectABC.__init__(self, source, model_type, table_name) + def __init__(self, model_type: Type[T_DBM], table_name: str): + DataAccessObjectABC.__init__(self, model_type, table_name) self.attribute(DbModelABC.id, int, ignore=True) self.attribute(DbModelABC.deleted, bool) diff --git a/src/cpl-database/cpl/database/db_logger.py b/src/cpl-database/cpl/database/db_logger.py index fdee274c..af0bd495 100644 --- a/src/cpl-database/cpl/database/db_logger.py +++ b/src/cpl-database/cpl/database/db_logger.py @@ -1,8 +1,8 @@ -from cpl.core.log import Logger -from cpl.core.typing import Source +from cpl.core.log import LoggerABC +from cpl.core.log.wrapped_logger import WrappedLogger -class DBLogger(Logger): +class DBLogger(WrappedLogger): - def __init__(self, source: Source): - Logger.__init__(self, source, "db") + def __init__(self, logger: LoggerABC): + WrappedLogger.__init__(self, logger) diff --git a/src/cpl-database/cpl/database/mysql/db_context.py b/src/cpl-database/cpl/database/mysql/db_context.py index 686cc4f8..f395a048 100644 --- a/src/cpl-database/cpl/database/mysql/db_context.py +++ b/src/cpl-database/cpl/database/mysql/db_context.py @@ -4,18 +4,17 @@ from typing import Any, List, Dict, Tuple, Union from mysql.connector import Error as MySQLError, PoolError from cpl.core.configuration import Configuration -from cpl.core.environment import Environment from cpl.database.abc.db_context_abc import DBContextABC from cpl.database.db_logger import DBLogger from cpl.database.model.database_settings import DatabaseSettings from cpl.database.mysql.mysql_pool import MySQLPool -_logger = DBLogger(__name__) - class DBContext(DBContextABC): - def __init__(self): + def __init__(self, logger: DBLogger): DBContextABC.__init__(self) + self._logger = logger + self._pool: MySQLPool = None self._fails = 0 @@ -23,62 +22,62 @@ class DBContext(DBContextABC): def connect(self, database_settings: DatabaseSettings): try: - _logger.debug("Connecting to database") + self._logger.debug("Connecting to database") self._pool = MySQLPool( database_settings, ) - _logger.info("Connected to database") + self._logger.info("Connected to database") except Exception as e: - _logger.fatal("Connecting to database failed", e) + self._logger.fatal("Connecting to database failed", e) async def execute(self, statement: str, args=None, multi=True) -> List[List]: - _logger.trace(f"execute {statement} with args: {args}") + self._logger.trace(f"execute {statement} with args: {args}") return await self._pool.execute(statement, args, multi) async def select_map(self, statement: str, args=None) -> List[Dict]: - _logger.trace(f"select {statement} with args: {args}") + self._logger.trace(f"select {statement} with args: {args}") try: return await self._pool.select_map(statement, args) except (MySQLError, PoolError) as e: if self._fails >= 3: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) uid = uuid.uuid4() raise Exception( f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}" ) - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) self._fails += 1 try: - _logger.debug("Retry select") + self._logger.debug("Retry select") return await self.select_map(statement, args) except Exception as e: pass return [] except Exception as e: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) raise e async def select(self, statement: str, args=None) -> Union[List[str], List[Tuple], List[Any]]: - _logger.trace(f"select {statement} with args: {args}") + self._logger.trace(f"select {statement} with args: {args}") try: return await self._pool.select(statement, args) except (MySQLError, PoolError) as e: if self._fails >= 3: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) uid = uuid.uuid4() raise Exception( f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}" ) - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) self._fails += 1 try: - _logger.debug("Retry select") + self._logger.debug("Retry select") return await self.select(statement, args) except Exception as e: pass return [] except Exception as e: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) raise e diff --git a/src/cpl-database/cpl/database/mysql/mysql_pool.py b/src/cpl-database/cpl/database/mysql/mysql_pool.py index a6b45dfd..962cf648 100644 --- a/src/cpl-database/cpl/database/mysql/mysql_pool.py +++ b/src/cpl-database/cpl/database/mysql/mysql_pool.py @@ -6,8 +6,7 @@ from mysql.connector.aio import MySQLConnectionPool from cpl.core.environment import Environment from cpl.database.db_logger import DBLogger from cpl.database.model import DatabaseSettings - -_logger = DBLogger(__name__) +from cpl.dependency import ServiceProviderABC class MySQLPool: @@ -36,7 +35,8 @@ class MySQLPool: await cursor.execute("SELECT 1") await cursor.fetchall() except Exception as e: - _logger.fatal(f"Error connecting to the database: {e}") + logger = ServiceProviderABC.get_global_service(DBLogger) + logger.fatal(f"Error connecting to the database: {e}") finally: await con.close() diff --git a/src/cpl-database/cpl/database/postgres/db_context.py b/src/cpl-database/cpl/database/postgres/db_context.py index 40fcd7f1..00fb02f6 100644 --- a/src/cpl-database/cpl/database/postgres/db_context.py +++ b/src/cpl-database/cpl/database/postgres/db_context.py @@ -7,16 +7,16 @@ from psycopg_pool import PoolTimeout from cpl.core.configuration import Configuration from cpl.core.environment import Environment from cpl.database.abc.db_context_abc import DBContextABC -from cpl.database.database_settings import DatabaseSettings from cpl.database.db_logger import DBLogger +from cpl.database.model import DatabaseSettings from cpl.database.postgres.postgres_pool import PostgresPool -_logger = DBLogger(__name__) - class DBContext(DBContextABC): - def __init__(self): + def __init__(self, logger: DBLogger): DBContextABC.__init__(self) + + self._logger = logger self._pool: PostgresPool = None self._fails = 0 @@ -24,63 +24,63 @@ class DBContext(DBContextABC): def connect(self, database_settings: DatabaseSettings): try: - _logger.debug("Connecting to database") + self._logger.debug("Connecting to database") self._pool = PostgresPool( database_settings, Environment.get("DB_POOL_SIZE", int, 1), ) - _logger.info("Connected to database") + self._logger.info("Connected to database") except Exception as e: - _logger.fatal("Connecting to database failed", e) + self._logger.fatal("Connecting to database failed", e) async def execute(self, statement: str, args=None, multi=True) -> list[list]: - _logger.trace(f"execute {statement} with args: {args}") + self._logger.trace(f"execute {statement} with args: {args}") return await self._pool.execute(statement, args, multi) async def select_map(self, statement: str, args=None) -> list[dict]: - _logger.trace(f"select {statement} with args: {args}") + self._logger.trace(f"select {statement} with args: {args}") try: return await self._pool.select_map(statement, args) except (OperationalError, PoolTimeout) as e: if self._fails >= 3: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) uid = uuid.uuid4() raise Exception( f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}" ) - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) self._fails += 1 try: - _logger.debug("Retry select") + self._logger.debug("Retry select") return await self.select_map(statement, args) except Exception as e: pass return [] except Exception as e: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) raise e async def select(self, statement: str, args=None) -> list[str] | list[tuple] | list[Any]: - _logger.trace(f"select {statement} with args: {args}") + self._logger.trace(f"select {statement} with args: {args}") try: return await self._pool.select(statement, args) except (OperationalError, PoolTimeout) as e: if self._fails >= 3: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) uid = uuid.uuid4() raise Exception( f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}" ) - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) self._fails += 1 try: - _logger.debug("Retry select") + self._logger.debug("Retry select") return await self.select(statement, args) except Exception as e: pass return [] except Exception as e: - _logger.error(f"Database error caused by `{statement}`", e) + self._logger.error(f"Database error caused by `{statement}`", e) raise e diff --git a/src/cpl-database/cpl/database/postgres/postgres_pool.py b/src/cpl-database/cpl/database/postgres/postgres_pool.py index 5bdda940..37e7ed76 100644 --- a/src/cpl-database/cpl/database/postgres/postgres_pool.py +++ b/src/cpl-database/cpl/database/postgres/postgres_pool.py @@ -7,8 +7,7 @@ from psycopg_pool import AsyncConnectionPool, PoolTimeout from cpl.core.environment import Environment from cpl.database.db_logger import DBLogger from cpl.database.model import DatabaseSettings - -_logger = DBLogger(__name__) +from cpl.dependency import ServiceProviderABC class PostgresPool: @@ -38,7 +37,8 @@ class PostgresPool: await pool.check_connection(con) except PoolTimeout as e: await pool.close() - _logger.fatal(f"Failed to connect to the database", e) + logger = ServiceProviderABC.get_global_service(DBLogger) + logger.fatal(f"Failed to connect to the database", e) self._pool = pool return self._pool diff --git a/src/cpl-database/cpl/database/schema/executed_migration_dao.py b/src/cpl-database/cpl/database/schema/executed_migration_dao.py index 89092011..0a422569 100644 --- a/src/cpl-database/cpl/database/schema/executed_migration_dao.py +++ b/src/cpl-database/cpl/database/schema/executed_migration_dao.py @@ -1,14 +1,11 @@ from cpl.database import TableManager from cpl.database.abc.data_access_object_abc import DataAccessObjectABC -from cpl.database.db_logger import DBLogger from cpl.database.schema.executed_migration import ExecutedMigration -_logger = DBLogger(__name__) - class ExecutedMigrationDao(DataAccessObjectABC[ExecutedMigration]): def __init__(self): - DataAccessObjectABC.__init__(self, __name__, ExecutedMigration, TableManager.get("executed_migrations")) + DataAccessObjectABC.__init__(self, ExecutedMigration, TableManager.get("executed_migrations")) self.attribute(ExecutedMigration.migration_id, str, primary_key=True, db_name="migrationId") diff --git a/src/cpl-database/cpl/database/service/migration_service.py b/src/cpl-database/cpl/database/service/migration_service.py index c84ee475..e797e981 100644 --- a/src/cpl-database/cpl/database/service/migration_service.py +++ b/src/cpl-database/cpl/database/service/migration_service.py @@ -8,12 +8,11 @@ from cpl.database.model.server_type import ServerType, ServerTypes from cpl.database.schema.executed_migration import ExecutedMigration from cpl.database.schema.executed_migration_dao import ExecutedMigrationDao -_logger = DBLogger(__name__) - class MigrationService: - def __init__(self, db: DBContextABC, executedMigrationDao: ExecutedMigrationDao): + def __init__(self, logger: DBLogger, db: DBContextABC, executedMigrationDao: ExecutedMigrationDao): + self._logger = logger self._db = db self._executedMigrationDao = executedMigrationDao @@ -96,13 +95,13 @@ class MigrationService: if migration_from_db is not None: continue - _logger.debug(f"Running upgrade migration: {migration.name}") + self._logger.debug(f"Running upgrade migration: {migration.name}") await self._db.execute(migration.script, multi=True) await self._executedMigrationDao.create(ExecutedMigration(migration.name), skip_editor=True) except Exception as e: - _logger.fatal( + self._logger.fatal( f"Migration failed: {migration.name}\n{active_statement}", e, ) diff --git a/src/cpl-database/cpl/database/service/seeder_service.py b/src/cpl-database/cpl/database/service/seeder_service.py index 5b0b8fba..ed34dc3f 100644 --- a/src/cpl-database/cpl/database/service/seeder_service.py +++ b/src/cpl-database/cpl/database/service/seeder_service.py @@ -3,16 +3,14 @@ from cpl.database.db_logger import DBLogger from cpl.dependency import ServiceProviderABC -_logger = DBLogger(__name__) - - class SeederService: def __init__(self, provider: ServiceProviderABC): self._provider = provider + self._logger = provider.get_service(DBLogger) async def seed(self): seeders = self._provider.get_services(DataSeederABC) - _logger.debug(f"Found {len(seeders)} seeders") + self._logger.debug(f"Found {len(seeders)} seeders") for seeder in seeders: await seeder.seed() diff --git a/src/cpl-dependency/cpl/dependency/service_collection.py b/src/cpl-dependency/cpl/dependency/service_collection.py index 3db2df36..f6f3328e 100644 --- a/src/cpl-dependency/cpl/dependency/service_collection.py +++ b/src/cpl-dependency/cpl/dependency/service_collection.py @@ -2,6 +2,8 @@ from typing import Union, Type, Callable, Self from cpl.core.log.logger import Logger from cpl.core.log.logger_abc import LoggerABC +from cpl.core.log.structured_logger import StructuredLogger +from cpl.core.log.wrapped_logger import WrappedLogger from cpl.core.typing import T, Service from cpl.dependency.service_descriptor import ServiceDescriptor from cpl.dependency.service_lifetime_enum import ServiceLifetimeEnum @@ -82,3 +84,12 @@ class ServiceCollection: def add_logging(self) -> Self: self.add_transient(LoggerABC, Logger) return self + + def add_structured_logging(self) -> Self: + self.add_transient(LoggerABC, StructuredLogger) + return self + + def add_wrapped_logging(self) -> Self: + for wrapper in WrappedLogger.__subclasses__(): + self.add_transient(wrapper) + return self diff --git a/src/cpl-mail/cpl/mail/mail_logger.py b/src/cpl-mail/cpl/mail/mail_logger.py index 3d2b778b..efdfd233 100644 --- a/src/cpl-mail/cpl/mail/mail_logger.py +++ b/src/cpl-mail/cpl/mail/mail_logger.py @@ -1,8 +1,8 @@ -from cpl.core.log.logger import Logger -from cpl.core.typing import Source +from cpl.core.log import LoggerABC +from cpl.core.log.wrapped_logger import WrappedLogger -class MailLogger(Logger): +class MailLogger(WrappedLogger): - def __init__(self, source: Source): - Logger.__init__(self, source, "mail") + def __init__(self, logger: LoggerABC): + WrappedLogger.__init__(self, logger) diff --git a/tests/custom/api/src/main.py b/tests/custom/api/src/main.py index 286a6399..1919f92e 100644 --- a/tests/custom/api/src/main.py +++ b/tests/custom/api/src/main.py @@ -16,7 +16,8 @@ def main(): Configuration.add_json_file(f"appsettings.{Environment.get_environment()}.json") Configuration.add_json_file(f"appsettings.{Environment.get_host_name()}.json", optional=True) - builder.services.add_logging() + builder.services.add_structured_logging() + builder.services.add_wrapped_logging() builder.services.add_transient(PingService) builder.services.add_module(api) diff --git a/tests/custom/database/src/model/city_dao.py b/tests/custom/database/src/model/city_dao.py index bea1ba85..144152e3 100644 --- a/tests/custom/database/src/model/city_dao.py +++ b/tests/custom/database/src/model/city_dao.py @@ -5,7 +5,7 @@ from model.city import City class CityDao(DbModelDaoABC[City]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, City, "city") + DbModelDaoABC.__init__(self, City, "city") self.attribute(City.name, str) self.attribute(City.zip, int) diff --git a/tests/custom/database/src/model/user_dao.py b/tests/custom/database/src/model/user_dao.py index 48267dec..7cd4b14a 100644 --- a/tests/custom/database/src/model/user_dao.py +++ b/tests/custom/database/src/model/user_dao.py @@ -5,7 +5,7 @@ from model.user import User class UserDao(DbModelDaoABC[User]): def __init__(self): - DbModelDaoABC.__init__(self, __name__, User, "users") + DbModelDaoABC.__init__(self, User, "users") self.attribute(User.name, str) self.attribute(User.city_id, int, db_name="CityId") From 6639946346483ee7c59404c3570eca4ccd222d4a Mon Sep 17 00:00:00 2001 From: edraft Date: Tue, 23 Sep 2025 21:06:47 +0200 Subject: [PATCH 2/2] Improved wrapped logging #187 --- .gitignore | 3 + src/cpl-api/cpl/api/application/web_app.py | 2 +- src/cpl-api/cpl/api/logger.py | 5 +- src/cpl-api/cpl/api/model/policy.py | 2 +- src/cpl-api/cpl/api/registry/route.py | 1 - .../cpl/application/abc/application_abc.py | 1 - .../abc/application_extension_abc.py | 2 +- src/cpl-auth/cpl/auth/__init__.py | 2 +- src/cpl-auth/cpl/auth/auth_logger.py | 8 --- .../cpl/auth/keycloak/keycloak_admin.py | 4 +- .../cpl/auth/keycloak/keycloak_client.py | 2 +- src/cpl-auth/cpl/auth/logger.py | 7 ++ src/cpl-auth/cpl/auth/permission_seeder.py | 2 +- .../auth/schema/_administration/auth_user.py | 10 +-- src/cpl-core/cpl/core/ctx/user_context.py | 6 +- .../cpl/core/log/structured_logger.py | 45 ++++++++++-- src/cpl-core/cpl/core/log/wrapped_logger.py | 68 +++++++++++++++++-- .../database/abc/data_access_object_abc.py | 2 +- src/cpl-database/cpl/database/db_logger.py | 8 --- src/cpl-database/cpl/database/logger.py | 7 ++ .../cpl/database/mysql/db_context.py | 2 +- .../cpl/database/mysql/mysql_pool.py | 2 +- .../cpl/database/postgres/db_context.py | 2 +- .../cpl/database/postgres/postgres_pool.py | 2 +- .../cpl/database/service/migration_service.py | 2 +- .../cpl/database/service/seeder_service.py | 2 +- .../cpl/dependency/service_collection.py | 23 ++++--- src/cpl-mail/cpl/mail/__init__.py | 2 +- src/cpl-mail/cpl/mail/email_client.py | 2 +- src/cpl-mail/cpl/mail/logger.py | 7 ++ src/cpl-mail/cpl/mail/mail_logger.py | 8 --- tests/custom/api/src/main.py | 2 +- tests/custom/api/src/routes/ping.py | 9 ++- 33 files changed, 171 insertions(+), 81 deletions(-) delete mode 100644 src/cpl-auth/cpl/auth/auth_logger.py create mode 100644 src/cpl-auth/cpl/auth/logger.py delete mode 100644 src/cpl-database/cpl/database/db_logger.py create mode 100644 src/cpl-database/cpl/database/logger.py create mode 100644 src/cpl-mail/cpl/mail/logger.py delete mode 100644 src/cpl-mail/cpl/mail/mail_logger.py diff --git a/.gitignore b/.gitignore index 1b0d611c..3651c7a9 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,6 @@ PythonImportHelper-v2-Completion.json # cpl unittest stuff unittests/test_*_playground + +# cpl logs +**/logs/*.jsonl diff --git a/src/cpl-api/cpl/api/application/web_app.py b/src/cpl-api/cpl/api/application/web_app.py index 9fdd7bc5..63631054 100644 --- a/src/cpl-api/cpl/api/application/web_app.py +++ b/src/cpl-api/cpl/api/application/web_app.py @@ -203,7 +203,7 @@ class WebApp(ApplicationABC): _policies.append(policy) - self._policies.extend_policies(_policies) + self._policies.extend(_policies) self.with_middleware(AuthorizationMiddleware) return self diff --git a/src/cpl-api/cpl/api/logger.py b/src/cpl-api/cpl/api/logger.py index d9f88a45..b3ef94a4 100644 --- a/src/cpl-api/cpl/api/logger.py +++ b/src/cpl-api/cpl/api/logger.py @@ -1,8 +1,7 @@ -from cpl.core.log import LoggerABC from cpl.core.log.wrapped_logger import WrappedLogger class APILogger(WrappedLogger): - def __init__(self, logger: LoggerABC): - WrappedLogger.__init__(self, logger) + def __init__(self): + WrappedLogger.__init__(self, "api") diff --git a/src/cpl-api/cpl/api/model/policy.py b/src/cpl-api/cpl/api/model/policy.py index ea118e0c..ac33dc4e 100644 --- a/src/cpl-api/cpl/api/model/policy.py +++ b/src/cpl-api/cpl/api/model/policy.py @@ -1,5 +1,5 @@ from asyncio import iscoroutinefunction -from typing import Optional, Any, Coroutine, Awaitable +from typing import Optional from cpl.api.typing import PolicyResolver from cpl.core.ctx import get_user diff --git a/src/cpl-api/cpl/api/registry/route.py b/src/cpl-api/cpl/api/registry/route.py index 6e9b167d..e030007b 100644 --- a/src/cpl-api/cpl/api/registry/route.py +++ b/src/cpl-api/cpl/api/registry/route.py @@ -1,6 +1,5 @@ from typing import Optional -from cpl.api.model.policy import Policy from cpl.api.model.api_route import ApiRoute from cpl.core.abc.registry_abc import RegistryABC diff --git a/src/cpl-application/cpl/application/abc/application_abc.py b/src/cpl-application/cpl/application/abc/application_abc.py index f32f633e..6ffe8317 100644 --- a/src/cpl-application/cpl/application/abc/application_abc.py +++ b/src/cpl-application/cpl/application/abc/application_abc.py @@ -5,7 +5,6 @@ from cpl.application.host import Host from cpl.core.log.log_level import LogLevel from cpl.core.log.log_settings import LogSettings from cpl.core.log.logger_abc import LoggerABC -from cpl.core.log.structured_logger import StructuredLogger from cpl.dependency.service_provider_abc import ServiceProviderABC diff --git a/src/cpl-application/cpl/application/abc/application_extension_abc.py b/src/cpl-application/cpl/application/abc/application_extension_abc.py index e4b36d43..9a2e011c 100644 --- a/src/cpl-application/cpl/application/abc/application_extension_abc.py +++ b/src/cpl-application/cpl/application/abc/application_extension_abc.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from cpl.dependency import ServiceProviderABC +from cpl.dependency.service_provider_abc import ServiceProviderABC class ApplicationExtensionABC(ABC): diff --git a/src/cpl-auth/cpl/auth/__init__.py b/src/cpl-auth/cpl/auth/__init__.py index 0f12362d..5e9d5c87 100644 --- a/src/cpl-auth/cpl/auth/__init__.py +++ b/src/cpl-auth/cpl/auth/__init__.py @@ -6,7 +6,7 @@ from cpl.auth import permission as _permission from cpl.auth.keycloak.keycloak_admin import KeycloakAdmin as _KeycloakAdmin from cpl.auth.keycloak.keycloak_client import KeycloakClient as _KeycloakClient from cpl.dependency.service_collection import ServiceCollection as _ServiceCollection -from .auth_logger import AuthLogger +from .logger import AuthLogger from .keycloak_settings import KeycloakSettings from .permission_seeder import PermissionSeeder diff --git a/src/cpl-auth/cpl/auth/auth_logger.py b/src/cpl-auth/cpl/auth/auth_logger.py deleted file mode 100644 index ec9421dc..00000000 --- a/src/cpl-auth/cpl/auth/auth_logger.py +++ /dev/null @@ -1,8 +0,0 @@ -from cpl.core.log import LoggerABC -from cpl.core.log.wrapped_logger import WrappedLogger - - -class AuthLogger(WrappedLogger): - - def __init__(self, logger: LoggerABC): - WrappedLogger.__init__(self, logger) diff --git a/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py b/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py index e5eb0722..55a1df12 100644 --- a/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py +++ b/src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py @@ -1,13 +1,13 @@ from keycloak import KeycloakAdmin as _KeycloakAdmin, KeycloakOpenIDConnection -from cpl.auth.auth_logger import AuthLogger from cpl.auth.keycloak_settings import KeycloakSettings +from cpl.auth.logger import AuthLogger class KeycloakAdmin(_KeycloakAdmin): def __init__(self, logger: AuthLogger, settings: KeycloakSettings): - logger.info("Initializing Keycloak admin") + # logger.info("Initializing Keycloak admin") _connection = KeycloakOpenIDConnection( server_url=settings.url, client_id=settings.client_id, diff --git a/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py b/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py index 6d62c777..da778a0d 100644 --- a/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py +++ b/src/cpl-auth/cpl/auth/keycloak/keycloak_client.py @@ -2,7 +2,7 @@ from typing import Optional from keycloak import KeycloakOpenID -from cpl.auth.auth_logger import AuthLogger +from cpl.auth.logger import AuthLogger from cpl.auth.keycloak_settings import KeycloakSettings diff --git a/src/cpl-auth/cpl/auth/logger.py b/src/cpl-auth/cpl/auth/logger.py new file mode 100644 index 00000000..48e993af --- /dev/null +++ b/src/cpl-auth/cpl/auth/logger.py @@ -0,0 +1,7 @@ +from cpl.core.log.wrapped_logger import WrappedLogger + + +class AuthLogger(WrappedLogger): + + def __init__(self): + WrappedLogger.__init__(self, "auth") diff --git a/src/cpl-auth/cpl/auth/permission_seeder.py b/src/cpl-auth/cpl/auth/permission_seeder.py index d326863d..d9d42cfa 100644 --- a/src/cpl-auth/cpl/auth/permission_seeder.py +++ b/src/cpl-auth/cpl/auth/permission_seeder.py @@ -14,7 +14,7 @@ from cpl.auth.schema import ( ) from cpl.core.utils.get_value import get_value from cpl.database.abc.data_seeder_abc import DataSeederABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger class PermissionSeeder(DataSeederABC): diff --git a/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py b/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py index 9cb2b373..a7a11b02 100644 --- a/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py +++ b/src/cpl-auth/cpl/auth/schema/_administration/auth_user.py @@ -9,7 +9,7 @@ from cpl.auth.keycloak import KeycloakAdmin from cpl.auth.permission.permissions import Permissions from cpl.core.typing import SerialId from cpl.database.abc import DbModelABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.dependency import ServiceProviderABC @@ -36,8 +36,8 @@ class AuthUser(DbModelABC): return "ANONYMOUS" try: - keycloak_admin: KeycloakAdmin = ServiceProviderABC.get_global_service(KeycloakAdmin) - return keycloak_admin.get_user(self._keycloak_id).get("username") + keycloak = ServiceProviderABC.get_global_service(KeycloakAdmin) + return keycloak.get_user(self._keycloak_id).get("username") except KeycloakGetError as e: return "UNKNOWN" except Exception as e: @@ -51,8 +51,8 @@ class AuthUser(DbModelABC): return "ANONYMOUS" try: - keycloak_admin: KeycloakAdmin = ServiceProviderABC.get_global_service(KeycloakAdmin) - return keycloak_admin.get_user(self._keycloak_id).get("email") + keycloak = ServiceProviderABC.get_global_service(KeycloakAdmin) + return keycloak.get_user(self._keycloak_id).get("email") except KeycloakGetError as e: return "UNKNOWN" except Exception as e: diff --git a/src/cpl-core/cpl/core/ctx/user_context.py b/src/cpl-core/cpl/core/ctx/user_context.py index 5a3e617c..eec696bc 100644 --- a/src/cpl-core/cpl/core/ctx/user_context.py +++ b/src/cpl-core/cpl/core/ctx/user_context.py @@ -6,13 +6,13 @@ from cpl.auth.schema._administration.auth_user import AuthUser _user_context: ContextVar[Optional[AuthUser]] = ContextVar("user", default=None) -def set_user(user_id: Optional[AuthUser]): +def set_user(user: Optional[AuthUser]): from cpl.dependency.service_provider_abc import ServiceProviderABC from cpl.core.log.logger_abc import LoggerABC logger = ServiceProviderABC.get_global_service(LoggerABC) - logger.trace("Setting user context", user_id) - _user_context.set(user_id) + logger.trace("Setting user context", user.id) + _user_context.set(user) def get_user() -> Optional[AuthUser]: diff --git a/src/cpl-core/cpl/core/log/structured_logger.py b/src/cpl-core/cpl/core/log/structured_logger.py index db67b8e2..e16755dd 100644 --- a/src/cpl-core/cpl/core/log/structured_logger.py +++ b/src/cpl-core/cpl/core/log/structured_logger.py @@ -1,9 +1,13 @@ import asyncio import importlib.util +import json import traceback from datetime import datetime -from cpl.core.log import Logger, LogLevel +from starlette.requests import Request + +from cpl.core.log.log_level import LogLevel +from cpl.core.log.logger import Logger from cpl.core.typing import Source, Messages @@ -20,8 +24,9 @@ class StructuredLogger(Logger): try: timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") formatted_message = self._format_message(level.value, timestamp, *messages) + structured_message = self._get_structured_message(level.value, timestamp, formatted_message) - self._write_log_to_file(level, formatted_message) + self._write_log_to_file(level, structured_message) self._write_to_console(level, formatted_message) except Exception as e: print(f"Error while logging: {e} -> {traceback.format_exc()}") @@ -37,10 +42,31 @@ class StructuredLogger(Logger): self._enrich_message_with_request(structured_message) self._enrich_message_with_user(structured_message) - return str(structured_message) + return json.dumps(structured_message, ensure_ascii=False) @staticmethod - def _enrich_message_with_request(message: dict): + def _scope_to_json(request: Request, include_headers: bool = False) -> dict: + scope = dict(request.scope) + + def convert(value): + if isinstance(value, bytes): + return value.decode("utf-8") + if isinstance(value, (list, tuple)): + return [convert(v) for v in value] + if isinstance(value, dict): + return {str(k): convert(v) for k, v in value.items()} + if not isinstance(value, (str, int, float, bool, type(None))): + return str(value) + return value + + serializable_scope = {str(k): convert(v) for k, v in scope.items()} + + if not include_headers and "headers" in serializable_scope: + serializable_scope["headers"] = "" + + return serializable_scope + + def _enrich_message_with_request(self, message: dict): if importlib.util.find_spec("cpl.api") is None: return @@ -55,7 +81,7 @@ class StructuredLogger(Logger): message["request"] = { "url": str(request.url), "method": request.method, - "scope": request.scope, + "scope": self._scope_to_json(request), } if isinstance(request, Request) and request.scope == "http": request: Request = request # fix typing for IDEs @@ -73,8 +99,13 @@ class StructuredLogger(Logger): if user is None: return + from cpl.dependency.service_provider_abc import ServiceProviderABC + from cpl.auth.keycloak.keycloak_admin import KeycloakAdmin + + keycloak = ServiceProviderABC.get_global_service(KeycloakAdmin) + kc_user = keycloak.get_user(user.keycloak_id) message["user"] = { "id": str(user.id), - "username": user.username, - "email": user.email, + "username": kc_user.get("username"), + "email": kc_user.get("email"), } diff --git a/src/cpl-core/cpl/core/log/wrapped_logger.py b/src/cpl-core/cpl/core/log/wrapped_logger.py index 384a88e2..a0bb5eb0 100644 --- a/src/cpl-core/cpl/core/log/wrapped_logger.py +++ b/src/cpl-core/cpl/core/log/wrapped_logger.py @@ -1,13 +1,28 @@ +import inspect + from cpl.core.log import LoggerABC, LogLevel -from cpl.core.typing import Messages +from cpl.core.typing import Messages, Source +from cpl.dependency.service_provider_abc import ServiceProviderABC class WrappedLogger(LoggerABC): - def __init__(self, logger: LoggerABC): + def __init__(self, file_prefix: str): LoggerABC.__init__(self) - assert isinstance(logger, LoggerABC), "logger must be an instance of LoggerABC" - self._logger = logger + assert file_prefix is not None and file_prefix != "", "file_prefix must be a non-empty string" + + t_logger = ServiceProviderABC.get_global_service(LoggerABC) + self._t_logger = type(t_logger) if t_logger is not None else None + self._source = None + self._file_prefix = file_prefix + + self._set_logger() + + def _set_logger(self): + if self._t_logger is None: + raise Exception("No LoggerABC service registered in ServiceProviderABC") + + self._logger = self._t_logger(self._source, self._file_prefix) def set_level(self, level: LogLevel): self._logger.set_level(level) @@ -15,23 +30,68 @@ class WrappedLogger(LoggerABC): def _format_message(self, level: str, timestamp, *messages: Messages) -> str: return self._logger._format_message(level, timestamp, *messages) + @staticmethod + def _get_source() -> str | None: + stack = inspect.stack() + if len(stack) <= 1: + return None + + from cpl.dependency import ServiceCollection + + ignore_classes = [ + ServiceProviderABC, + ServiceProviderABC.__subclasses__(), + ServiceCollection, + WrappedLogger, + WrappedLogger.__subclasses__(), + ] + + ignore_modules = [x.__module__ for x in ignore_classes if isinstance(x, type)] + + for i, frame_info in enumerate(stack[1:]): + module = inspect.getmodule(frame_info.frame) + if module is None: + continue + + if module.__name__ in ignore_classes or module in ignore_classes: + continue + + if module in ignore_modules or module.__name__ in ignore_modules: + continue + + if module.__name__ != __name__: + return module.__name__ + + return None + + def _set_source(self): + self._source = self._get_source() + self._set_logger() + def header(self, string: str): + self._set_source() self._logger.header(string) def trace(self, *messages: Messages): + self._set_source() self._logger.trace(*messages) def debug(self, *messages: Messages): + self._set_source() self._logger.debug(*messages) def info(self, *messages: Messages): + self._set_source() self._logger.info(*messages) def warning(self, *messages: Messages): + self._set_source() self._logger.warning(*messages) def error(self, messages: str, e: Exception = None): + self._set_source() self._logger.error(messages, e) def fatal(self, messages: str, e: Exception = None): + self._set_source() self._logger.fatal(messages, e) diff --git a/src/cpl-database/cpl/database/abc/data_access_object_abc.py b/src/cpl-database/cpl/database/abc/data_access_object_abc.py index 70ab90ce..bb153246 100644 --- a/src/cpl-database/cpl/database/abc/data_access_object_abc.py +++ b/src/cpl-database/cpl/database/abc/data_access_object_abc.py @@ -9,7 +9,7 @@ from cpl.core.utils.get_value import get_value from cpl.core.utils.string import String from cpl.database.abc.db_context_abc import DBContextABC from cpl.database.const import DATETIME_FORMAT -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.external_data_temp_table_builder import ExternalDataTempTableBuilder from cpl.database.postgres.sql_select_builder import SQLSelectBuilder from cpl.database.typing import T_DBM, Attribute, AttributeFilters, AttributeSorts diff --git a/src/cpl-database/cpl/database/db_logger.py b/src/cpl-database/cpl/database/db_logger.py deleted file mode 100644 index af0bd495..00000000 --- a/src/cpl-database/cpl/database/db_logger.py +++ /dev/null @@ -1,8 +0,0 @@ -from cpl.core.log import LoggerABC -from cpl.core.log.wrapped_logger import WrappedLogger - - -class DBLogger(WrappedLogger): - - def __init__(self, logger: LoggerABC): - WrappedLogger.__init__(self, logger) diff --git a/src/cpl-database/cpl/database/logger.py b/src/cpl-database/cpl/database/logger.py new file mode 100644 index 00000000..6ee087bc --- /dev/null +++ b/src/cpl-database/cpl/database/logger.py @@ -0,0 +1,7 @@ +from cpl.core.log.wrapped_logger import WrappedLogger + + +class DBLogger(WrappedLogger): + + def __init__(self): + WrappedLogger.__init__(self, "db") diff --git a/src/cpl-database/cpl/database/mysql/db_context.py b/src/cpl-database/cpl/database/mysql/db_context.py index f395a048..1796db7b 100644 --- a/src/cpl-database/cpl/database/mysql/db_context.py +++ b/src/cpl-database/cpl/database/mysql/db_context.py @@ -5,7 +5,7 @@ from mysql.connector import Error as MySQLError, PoolError from cpl.core.configuration import Configuration from cpl.database.abc.db_context_abc import DBContextABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.model.database_settings import DatabaseSettings from cpl.database.mysql.mysql_pool import MySQLPool diff --git a/src/cpl-database/cpl/database/mysql/mysql_pool.py b/src/cpl-database/cpl/database/mysql/mysql_pool.py index 962cf648..ba2bfacd 100644 --- a/src/cpl-database/cpl/database/mysql/mysql_pool.py +++ b/src/cpl-database/cpl/database/mysql/mysql_pool.py @@ -4,7 +4,7 @@ import sqlparse from mysql.connector.aio import MySQLConnectionPool from cpl.core.environment import Environment -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.model import DatabaseSettings from cpl.dependency import ServiceProviderABC diff --git a/src/cpl-database/cpl/database/postgres/db_context.py b/src/cpl-database/cpl/database/postgres/db_context.py index 00fb02f6..2e354f76 100644 --- a/src/cpl-database/cpl/database/postgres/db_context.py +++ b/src/cpl-database/cpl/database/postgres/db_context.py @@ -7,7 +7,7 @@ from psycopg_pool import PoolTimeout from cpl.core.configuration import Configuration from cpl.core.environment import Environment from cpl.database.abc.db_context_abc import DBContextABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.model import DatabaseSettings from cpl.database.postgres.postgres_pool import PostgresPool diff --git a/src/cpl-database/cpl/database/postgres/postgres_pool.py b/src/cpl-database/cpl/database/postgres/postgres_pool.py index 37e7ed76..91db7562 100644 --- a/src/cpl-database/cpl/database/postgres/postgres_pool.py +++ b/src/cpl-database/cpl/database/postgres/postgres_pool.py @@ -5,7 +5,7 @@ from psycopg import sql from psycopg_pool import AsyncConnectionPool, PoolTimeout from cpl.core.environment import Environment -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.model import DatabaseSettings from cpl.dependency import ServiceProviderABC diff --git a/src/cpl-database/cpl/database/service/migration_service.py b/src/cpl-database/cpl/database/service/migration_service.py index e797e981..710480c6 100644 --- a/src/cpl-database/cpl/database/service/migration_service.py +++ b/src/cpl-database/cpl/database/service/migration_service.py @@ -2,7 +2,7 @@ import glob import os from cpl.database.abc import DBContextABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.database.model import Migration from cpl.database.model.server_type import ServerType, ServerTypes from cpl.database.schema.executed_migration import ExecutedMigration diff --git a/src/cpl-database/cpl/database/service/seeder_service.py b/src/cpl-database/cpl/database/service/seeder_service.py index ed34dc3f..c23d39e3 100644 --- a/src/cpl-database/cpl/database/service/seeder_service.py +++ b/src/cpl-database/cpl/database/service/seeder_service.py @@ -1,5 +1,5 @@ from cpl.database.abc.data_seeder_abc import DataSeederABC -from cpl.database.db_logger import DBLogger +from cpl.database.logger import DBLogger from cpl.dependency import ServiceProviderABC diff --git a/src/cpl-dependency/cpl/dependency/service_collection.py b/src/cpl-dependency/cpl/dependency/service_collection.py index f6f3328e..e2088050 100644 --- a/src/cpl-dependency/cpl/dependency/service_collection.py +++ b/src/cpl-dependency/cpl/dependency/service_collection.py @@ -1,9 +1,6 @@ from typing import Union, Type, Callable, Self -from cpl.core.log.logger import Logger from cpl.core.log.logger_abc import LoggerABC -from cpl.core.log.structured_logger import StructuredLogger -from cpl.core.log.wrapped_logger import WrappedLogger from cpl.core.typing import T, Service from cpl.dependency.service_descriptor import ServiceDescriptor from cpl.dependency.service_lifetime_enum import ServiceLifetimeEnum @@ -82,14 +79,20 @@ class ServiceCollection: return self def add_logging(self) -> Self: + from cpl.core.log.logger import Logger + from cpl.core.log.wrapped_logger import WrappedLogger + self.add_transient(LoggerABC, Logger) - return self - - def add_structured_logging(self) -> Self: - self.add_transient(LoggerABC, StructuredLogger) - return self - - def add_wrapped_logging(self) -> Self: + for wrapper in WrappedLogger.__subclasses__(): + self.add_transient(wrapper) + return self + + def add_structured_logging(self) -> Self: + from cpl.core.log.structured_logger import StructuredLogger + from cpl.core.log.wrapped_logger import WrappedLogger + + self.add_transient(LoggerABC, StructuredLogger) + for wrapper in WrappedLogger.__subclasses__(): self.add_transient(wrapper) return self diff --git a/src/cpl-mail/cpl/mail/__init__.py b/src/cpl-mail/cpl/mail/__init__.py index 6e8a4f7a..37375770 100644 --- a/src/cpl-mail/cpl/mail/__init__.py +++ b/src/cpl-mail/cpl/mail/__init__.py @@ -3,7 +3,7 @@ from .abc.email_client_abc import EMailClientABC from .email_client import EMailClient from .email_client_settings import EMailClientSettings from .email_model import EMail -from .mail_logger import MailLogger +from .logger import MailLogger def add_mail(collection: _ServiceCollection): diff --git a/src/cpl-mail/cpl/mail/email_client.py b/src/cpl-mail/cpl/mail/email_client.py index 5f5669d1..a2ccad3d 100644 --- a/src/cpl-mail/cpl/mail/email_client.py +++ b/src/cpl-mail/cpl/mail/email_client.py @@ -5,7 +5,7 @@ from typing import Optional from cpl.mail.abc.email_client_abc import EMailClientABC from cpl.mail.email_client_settings import EMailClientSettings from cpl.mail.email_model import EMail -from cpl.mail.mail_logger import MailLogger +from cpl.mail.logger import MailLogger class EMailClient(EMailClientABC): diff --git a/src/cpl-mail/cpl/mail/logger.py b/src/cpl-mail/cpl/mail/logger.py new file mode 100644 index 00000000..1d929ce7 --- /dev/null +++ b/src/cpl-mail/cpl/mail/logger.py @@ -0,0 +1,7 @@ +from cpl.core.log.wrapped_logger import WrappedLogger + + +class MailLogger(WrappedLogger): + + def __init__(self): + WrappedLogger.__init__(self, "mail") diff --git a/src/cpl-mail/cpl/mail/mail_logger.py b/src/cpl-mail/cpl/mail/mail_logger.py deleted file mode 100644 index efdfd233..00000000 --- a/src/cpl-mail/cpl/mail/mail_logger.py +++ /dev/null @@ -1,8 +0,0 @@ -from cpl.core.log import LoggerABC -from cpl.core.log.wrapped_logger import WrappedLogger - - -class MailLogger(WrappedLogger): - - def __init__(self, logger: LoggerABC): - WrappedLogger.__init__(self, logger) diff --git a/tests/custom/api/src/main.py b/tests/custom/api/src/main.py index 1919f92e..4432a97e 100644 --- a/tests/custom/api/src/main.py +++ b/tests/custom/api/src/main.py @@ -16,8 +16,8 @@ def main(): Configuration.add_json_file(f"appsettings.{Environment.get_environment()}.json") Configuration.add_json_file(f"appsettings.{Environment.get_host_name()}.json", optional=True) + # builder.services.add_logging() builder.services.add_structured_logging() - builder.services.add_wrapped_logging() builder.services.add_transient(PingService) builder.services.add_module(api) diff --git a/tests/custom/api/src/routes/ping.py b/tests/custom/api/src/routes/ping.py index b77273e2..7fad7145 100644 --- a/tests/custom/api/src/routes/ping.py +++ b/tests/custom/api/src/routes/ping.py @@ -1,17 +1,16 @@ from urllib.request import Request +from service import PingService from starlette.responses import JSONResponse +from cpl.api import APILogger from cpl.api.router import Router -from cpl.auth.permission.permissions import Permissions -from cpl.core.log import Logger -from service import PingService @Router.authenticate() -@Router.authorize(permissions=[Permissions.administrator]) +# @Router.authorize(permissions=[Permissions.administrator]) # @Router.authorize(policies=["test"]) @Router.get(f"/ping") -async def ping(r: Request, ping: PingService, logger: Logger): +async def ping(r: Request, ping: PingService, logger: APILogger): logger.info(f"Ping: {ping}") return JSONResponse(ping.ping(r))