sh-edraft.de/cpl
5
0
Fork 0
Code Issues 5 Pull Requests 2 Actions Packages Projects 1 Releases 12 Wiki Activity

WIP: dev into master #184

Draft
edraft wants to merge 121 commits from dev into master
pull from: dev
merge into: sh-edraft.de:master
Conversation 1 Commits 121 Files Changed 891 +287 -186
44 changed files with 287 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b9ac11e15f - Show all commits

1
src/cpl-api/cpl/api/__init__.py
View File

@@ -4,6 +4,7 @@ from .error import APIError, AlreadyExists, EndpointNotImplemented, Forbidden, N
from .logger import APILogger
from .settings import ApiSettings
def add_api(collection: _ServiceCollection):
try:
from cpl.database import mysql

28
src/cpl-api/cpl/api/application/web_app.py
View File

@@ -29,7 +29,6 @@ from cpl.application.abc.application_abc import ApplicationABC
from cpl.core.configuration import Configuration
from cpl.dependency.service_provider_abc import ServiceProviderABC
_logger = APILogger("API")
PolicyInput = Union[dict[str, PolicyResolver], Policy]
@@ -39,6 +38,8 @@ class WebApp(ApplicationABC):
super().__init__(services, [auth, api])
self._app: Starlette | None = None
self._logger = services.get_service(APILogger)
self._api_settings = Configuration.get(ApiSettings)
self._policies = services.get_service(PolicyRegistry)
self._routes = services.get_service(RouteRegistry)
@@ -52,16 +53,15 @@ class WebApp(ApplicationABC):
APIError: self._handle_exception,
}
@staticmethod
async def _handle_exception(request: Request, exc: Exception):
async def _handle_exception(self, request: Request, exc: Exception):
if isinstance(exc, APIError):
_logger.error(exc)
self._logger.error(exc)
return JSONResponse({"error": str(exc)}, status_code=exc.status_code)
if hasattr(request.state, "request_id"):
_logger.error(f"Request {request.state.request_id}", exc)
self._logger.error(f"Request {request.state.request_id}", exc)
else:
_logger.error("Request unknown", exc)
self._logger.error("Request unknown", exc)
return JSONResponse({"error": str(exc)}, status_code=500)
@@ -69,10 +69,10 @@ class WebApp(ApplicationABC):
origins = self._api_settings.allowed_origins
if origins is None or origins == "":
_logger.warning("No allowed origins specified, allowing all origins")
self._logger.warning("No allowed origins specified, allowing all origins")
return ["*"]
_logger.debug(f"Allowed origins: {origins}")
self._logger.debug(f"Allowed origins: {origins}")
return origins.split(",")
def with_database(self) -> Self:
@@ -191,11 +191,11 @@ class WebApp(ApplicationABC):
if isinstance(policy, dict):
for name, resolver in policy.items():
if not isinstance(name, str):
_logger.warning(f"Skipping policy at index {i}, name must be a string")
self._logger.warning(f"Skipping policy at index {i}, name must be a string")
continue
if not callable(resolver):
_logger.warning(f"Skipping policy {name}, resolver must be callable")
self._logger.warning(f"Skipping policy {name}, resolver must be callable")
continue
_policies.append(Policy(name, resolver))
@@ -213,10 +213,10 @@ class WebApp(ApplicationABC):
for policy_name in rule["policies"]:
policy = self._policies.get(policy_name)
if not policy:
_logger.fatal(f"Authorization policy '{policy_name}' not found")
self._logger.fatal(f"Authorization policy '{policy_name}' not found")
async def main(self):
_logger.debug(f"Preparing API")
self._logger.debug(f"Preparing API")
self._validate_policies()
if self._app is None:
@@ -238,7 +238,7 @@ class WebApp(ApplicationABC):
else:
app = self._app
_logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}")
self._logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}")
config = uvicorn.Config(
app, host=self._api_settings.host, port=self._api_settings.port, log_config=None, loop="asyncio"
@@ -246,4 +246,4 @@ class WebApp(ApplicationABC):
server = uvicorn.Server(config)
await server.serve()
_logger.info("Shutdown API")
self._logger.info("Shutdown API")

9
src/cpl-api/cpl/api/logger.py
View File

@@ -1,7 +1,8 @@
from cpl.core.log.logger import Logger
from cpl.core.log import LoggerABC
from cpl.core.log.wrapped_logger import WrappedLogger
class APILogger(Logger):
class APILogger(WrappedLogger):
def __init__(self, source: str):
Logger.__init__(self, source, "api")
def __init__(self, logger: LoggerABC):
WrappedLogger.__init__(self, logger)

20
src/cpl-api/cpl/api/middleware/authentication.py
View File

@@ -2,8 +2,8 @@ from keycloak import KeycloakAuthenticationError
from starlette.types import Scope, Receive, Send
from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware
from cpl.api.logger import APILogger
from cpl.api.error import Unauthorized
from cpl.api.logger import APILogger
from cpl.api.middleware.request import get_request
from cpl.api.router import Router
from cpl.auth.keycloak import KeycloakClient
@@ -11,15 +11,15 @@ from cpl.auth.schema import AuthUserDao, AuthUser
from cpl.core.ctx import set_user
from cpl.dependency import ServiceProviderABC
_logger = APILogger(__name__)
class AuthenticationMiddleware(ASGIMiddleware):
@ServiceProviderABC.inject
def __init__(self, app, keycloak: KeycloakClient, user_dao: AuthUserDao):
def __init__(self, app, logger: APILogger, keycloak: KeycloakClient, user_dao: AuthUserDao):
ASGIMiddleware.__init__(self, app)
self._logger = logger
self._keycloak = keycloak
self._user_dao = user_dao
@@ -28,11 +28,11 @@ class AuthenticationMiddleware(ASGIMiddleware):
url = request.url.path
if url not in Router.get_auth_required_routes():
_logger.trace(f"No authentication required for {url}")
self._logger.trace(f"No authentication required for {url}")
return await self._app(scope, receive, send)
if not request.headers.get("Authorization"):
_logger.debug(f"Unauthorized access to {url}, missing Authorization header")
self._logger.debug(f"Unauthorized access to {url}, missing Authorization header")
return await Unauthorized(f"Missing header Authorization").asgi_response(scope, receive, send)
auth_header = request.headers.get("Authorization", None)
@@ -41,7 +41,7 @@ class AuthenticationMiddleware(ASGIMiddleware):
token = auth_header.split("Bearer ")[1]
if not await self._verify_login(token):
_logger.debug(f"Unauthorized access to {url}, invalid token")
self._logger.debug(f"Unauthorized access to {url}, invalid token")
return await Unauthorized("Invalid token").asgi_response(scope, receive, send)
# check user exists in db, if not create
@@ -51,7 +51,7 @@ class AuthenticationMiddleware(ASGIMiddleware):
user = await self._get_or_crate_user(keycloak_id)
if user.deleted:
_logger.debug(f"Unauthorized access to {url}, user is deleted")
self._logger.debug(f"Unauthorized access to {url}, user is deleted")
return await Unauthorized("User is deleted").asgi_response(scope, receive, send)
request.state.user = user
@@ -73,8 +73,8 @@ class AuthenticationMiddleware(ASGIMiddleware):
token_info = self._keycloak.introspect(token)
return token_info.get("active", False)
except KeycloakAuthenticationError as e:
_logger.debug(f"Keycloak authentication error: {e}")
self._logger.debug(f"Keycloak authentication error: {e}")
return False
except Exception as e:
_logger.error(f"Unexpected error during token verification: {e}")
self._logger.error(f"Unexpected error during token verification: {e}")
return False

10
src/cpl-api/cpl/api/middleware/authorization.py
View File

@@ -11,15 +11,15 @@ from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.core.ctx.user_context import get_user
from cpl.dependency.service_provider_abc import ServiceProviderABC
_logger = APILogger(__name__)
class AuthorizationMiddleware(ASGIMiddleware):
@ServiceProviderABC.inject
def __init__(self, app, policies: PolicyRegistry, user_dao: AuthUserDao):
def __init__(self, app, logger: APILogger, policies: PolicyRegistry, user_dao: AuthUserDao):
ASGIMiddleware.__init__(self, app)
self._logger = logger
self._policies = policies
self._user_dao = user_dao
@@ -28,7 +28,7 @@ class AuthorizationMiddleware(ASGIMiddleware):
url = request.url.path
if url not in Router.get_authorization_rules_paths():
_logger.trace(f"No authorization required for {url}")
self._logger.trace(f"No authorization required for {url}")
return await self._app(scope, receive, send)
user = get_user()
@@ -64,7 +64,7 @@ class AuthorizationMiddleware(ASGIMiddleware):
for policy_name in rule["policies"]:
policy = self._policies.get(policy_name)
if not policy:
_logger.warning(f"Authorization policy '{policy_name}' not found")
self._logger.warning(f"Authorization policy '{policy_name}' not found")
continue
if not await policy.resolve(user):

22
src/cpl-api/cpl/api/middleware/logging.py
View File

@@ -6,15 +6,17 @@ from starlette.types import Receive, Scope, Send
from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware
from cpl.api.logger import APILogger
from cpl.api.middleware.request import get_request
_logger = APILogger(__name__)
from cpl.dependency import ServiceProviderABC
class LoggingMiddleware(ASGIMiddleware):
def __init__(self, app):
@ServiceProviderABC.inject
def __init__(self, app, logger: APILogger):
ASGIMiddleware.__init__(self, app)
self._logger = logger
async def __call__(self, scope: Scope, receive: Receive, send: Send):
if scope["type"] != "http":
await self._call_next(scope, receive, send)
@@ -53,9 +55,8 @@ class LoggingMiddleware(ASGIMiddleware):
}
return {key: value for key, value in headers.items() if key in relevant_keys}
@classmethod
async def _log_request(cls, request: Request):
_logger.debug(
async def _log_request(self, request: Request):
self._logger.debug(
f"Request {getattr(request.state, 'request_id', '-')}: {request.method}@{request.url.path} from {request.client.host}"
)
@@ -64,7 +65,7 @@ class LoggingMiddleware(ASGIMiddleware):
user = get_user()
request_info = {
"headers": cls._filter_relevant_headers(dict(request.headers)),
"headers": self._filter_relevant_headers(dict(request.headers)),
"args": dict(request.query_params),
"form-data": (
await request.form()
@@ -78,10 +79,9 @@ class LoggingMiddleware(ASGIMiddleware):
),
}
_logger.trace(f"Request {getattr(request.state, 'request_id', '-')}: {request_info}")
self._logger.trace(f"Request {getattr(request.state, 'request_id', '-')}: {request_info}")
@staticmethod
async def _log_after_request(request: Request, status_code: int, duration: float):
_logger.info(
async def _log_after_request(self, request: Request, status_code: int, duration: float):
self._logger.info(
f"Request finished {getattr(request.state, 'request_id', '-')}: {status_code}-{request.method}@{request.url.path} from {request.client.host} in {duration:.2f}ms"
)

13
src/cpl-api/cpl/api/middleware/request.py
View File

@@ -9,16 +9,19 @@ from starlette.types import Scope, Receive, Send
from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware
from cpl.api.logger import APILogger
from cpl.api.typing import TRequest
from cpl.dependency import ServiceProviderABC
_request_context: ContextVar[Union[TRequest, None]] = ContextVar("request", default=None)
_logger = APILogger(__name__)
class RequestMiddleware(ASGIMiddleware):
def __init__(self, app):
@ServiceProviderABC.inject
def __init__(self, app, logger: APILogger):
ASGIMiddleware.__init__(self, app)
self._logger = logger
self._ctx_token = None
async def __call__(self, scope: Scope, receive: Receive, send: Send):
@@ -33,7 +36,7 @@ class RequestMiddleware(ASGIMiddleware):
async def set_request_data(self, request: TRequest):
request.state.request_id = uuid4()
request.state.start_time = time.time()
_logger.trace(f"Set new current request: {request.state.request_id}")
self._logger.trace(f"Set new current request: {request.state.request_id}")
self._ctx_token = _request_context.set(request)
@@ -45,7 +48,7 @@ class RequestMiddleware(ASGIMiddleware):
if self._ctx_token is None:
return
_logger.trace(f"Clearing current request: {request.state.request_id}")
self._logger.trace(f"Clearing current request: {request.state.request_id}")
_request_context.reset(self._ctx_token)

3
src/cpl-api/cpl/api/router.py
View File

@@ -92,8 +92,9 @@ class Router:
@classmethod
def route(cls, path: str, method: HTTPMethods, registry: RouteRegistry = None, **kwargs):
from cpl.api.model.api_route import ApiRoute
if not registry:
from cpl.api.model.api_route import ApiRoute
from cpl.dependency.service_provider_abc import ServiceProviderABC
routes = ServiceProviderABC.get_global_service(RouteRegistry)

4
src/cpl-application/cpl/application/abc/application_abc.py
View File

@@ -2,10 +2,10 @@ from abc import ABC, abstractmethod
from typing import Callable, Self
from cpl.application.host import Host
from cpl.core.console.console import Console
from cpl.core.log import LogSettings
from cpl.core.log.log_level import LogLevel
from cpl.core.log.log_settings import LogSettings
from cpl.core.log.logger_abc import LoggerABC
from cpl.core.log.structured_logger import StructuredLogger
from cpl.dependency.service_provider_abc import ServiceProviderABC

10
src/cpl-auth/cpl/auth/auth_logger.py
View File

@@ -1,8 +1,8 @@
from cpl.core.log import Logger
from cpl.core.typing import Source
from cpl.core.log import LoggerABC
from cpl.core.log.wrapped_logger import WrappedLogger
class AuthLogger(Logger):
class AuthLogger(WrappedLogger):
def __init__(self, source: Source):
Logger.__init__(self, source, "auth")
def __init__(self, logger: LoggerABC):
WrappedLogger.__init__(self, logger)

6
src/cpl-auth/cpl/auth/keycloak/keycloak_admin.py
View File

@@ -3,13 +3,11 @@ from keycloak import KeycloakAdmin as _KeycloakAdmin, KeycloakOpenIDConnection
from cpl.auth.auth_logger import AuthLogger
from cpl.auth.keycloak_settings import KeycloakSettings
_logger = AuthLogger("keycloak")
class KeycloakAdmin(_KeycloakAdmin):
def __init__(self, settings: KeycloakSettings):
_logger.info("Initializing Keycloak admin")
def __init__(self, logger: AuthLogger, settings: KeycloakSettings):
logger.info("Initializing Keycloak admin")
_connection = KeycloakOpenIDConnection(
server_url=settings.url,
client_id=settings.client_id,

6
src/cpl-auth/cpl/auth/keycloak/keycloak_client.py
View File

@@ -5,12 +5,10 @@ from keycloak import KeycloakOpenID
from cpl.auth.auth_logger import AuthLogger
from cpl.auth.keycloak_settings import KeycloakSettings
_logger = AuthLogger("keycloak")
class KeycloakClient(KeycloakOpenID):
def __init__(self, settings: KeycloakSettings):
def __init__(self, logger: AuthLogger, settings: KeycloakSettings):
KeycloakOpenID.__init__(
self,
server_url=settings.url,
@@ -18,7 +16,7 @@ class KeycloakClient(KeycloakOpenID):
realm_name=settings.realm,
client_secret_key=settings.client_secret,
)
_logger.info("Initializing Keycloak client")
logger.info("Initializing Keycloak client")
def get_user_id(self, token: str) -> Optional[str]:
info = self.introspect(token)

8
src/cpl-auth/cpl/auth/permission_seeder.py
View File

@@ -16,12 +16,11 @@ from cpl.core.utils.get_value import get_value
from cpl.database.abc.data_seeder_abc import DataSeederABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class PermissionSeeder(DataSeederABC):
def __init__(
self,
logger: DBLogger,
permission_dao: PermissionDao,
role_dao: RoleDao,
role_permission_dao: RolePermissionDao,
@@ -29,6 +28,7 @@ class PermissionSeeder(DataSeederABC):
api_key_permission_dao: ApiKeyPermissionDao,
):
DataSeederABC.__init__(self)
self._logger = logger
self._permission_dao = permission_dao
self._role_dao = role_dao
self._role_permission_dao = role_permission_dao
@@ -40,7 +40,7 @@ class PermissionSeeder(DataSeederABC):
possible_permissions = [permission for permission in PermissionsRegistry.get()]
if len(permissions) == len(possible_permissions):
_logger.info("Permissions already existing")
self._logger.info("Permissions already existing")
await self._update_missing_descriptions()
return
@@ -53,7 +53,7 @@ class PermissionSeeder(DataSeederABC):
await self._permission_dao.delete_many(to_delete, hard_delete=True)
_logger.warning("Permissions incomplete")
self._logger.warning("Permissions incomplete")
permission_names = [permission.name for permission in permissions]
await self._permission_dao.create_many(
[

5
src/cpl-auth/cpl/auth/schema/_administration/api_key_dao.py
View File

@@ -3,15 +3,12 @@ from typing import Optional
from cpl.auth.schema._administration.api_key import ApiKey
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class ApiKeyDao(DbModelDaoABC[ApiKey]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, ApiKey, TableManager.get("api_keys"))
DbModelDaoABC.__init__(self, ApiKey, TableManager.get("api_keys"))
self.attribute(ApiKey.identifier, str)
self.attribute(ApiKey.key, str, "keystring")

10
src/cpl-auth/cpl/auth/schema/_administration/auth_user.py
View File

@@ -6,14 +6,12 @@ from async_property import async_property
from keycloak import KeycloakGetError
from cpl.auth.keycloak import KeycloakAdmin
from cpl.auth.auth_logger import AuthLogger
from cpl.auth.permission.permissions import Permissions
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
from cpl.database.db_logger import DBLogger
from cpl.dependency import ServiceProviderABC
_logger = AuthLogger(__name__)
class AuthUser(DbModelABC):
def __init__(
@@ -43,7 +41,8 @@ class AuthUser(DbModelABC):
except KeycloakGetError as e:
return "UNKNOWN"
except Exception as e:
_logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e)
logger = ServiceProviderABC.get_global_service(DBLogger)
logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e)
return "UNKNOWN"
@property
@@ -57,7 +56,8 @@ class AuthUser(DbModelABC):
except KeycloakGetError as e:
return "UNKNOWN"
except Exception as e:
_logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e)
logger = ServiceProviderABC.get_global_service(DBLogger)
logger.error(f"Failed to get user {self._keycloak_id} from Keycloak", e)
return "UNKNOWN"
@async_property

5
src/cpl-auth/cpl/auth/schema/_administration/auth_user_dao.py
View File

@@ -4,17 +4,14 @@ from cpl.auth.permission.permissions import Permissions
from cpl.auth.schema._administration.auth_user import AuthUser
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
from cpl.database.external_data_temp_table_builder import ExternalDataTempTableBuilder
from cpl.dependency import ServiceProviderABC
_logger = DBLogger(__name__)
class AuthUserDao(DbModelDaoABC[AuthUser]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, AuthUser, TableManager.get("auth_users"))
DbModelDaoABC.__init__(self, AuthUser, TableManager.get("auth_users"))
self.attribute(AuthUser.keycloak_id, str, db_name="keycloakId")

5
src/cpl-auth/cpl/auth/schema/_permission/api_key_permission_dao.py
View File

@@ -1,15 +1,12 @@
from cpl.auth.schema._permission.api_key_permission import ApiKeyPermission
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class ApiKeyPermissionDao(DbModelDaoABC[ApiKeyPermission]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, ApiKeyPermission, TableManager.get("api_key_permissions"))
DbModelDaoABC.__init__(self, ApiKeyPermission, TableManager.get("api_key_permissions"))
self.attribute(ApiKeyPermission.api_key_id, int)
self.attribute(ApiKeyPermission.permission_id, int)

5
src/cpl-auth/cpl/auth/schema/_permission/permission_dao.py
View File

@@ -3,15 +3,12 @@ from typing import Optional
from cpl.auth.schema._permission.permission import Permission
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class PermissionDao(DbModelDaoABC[Permission]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, Permission, TableManager.get("permissions"))
DbModelDaoABC.__init__(self, Permission, TableManager.get("permissions"))
self.attribute(Permission.name, str)
self.attribute(Permission.description, Optional[str])

5
src/cpl-auth/cpl/auth/schema/_permission/role_dao.py
View File

@@ -1,14 +1,11 @@
from cpl.auth.schema._permission.role import Role
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class RoleDao(DbModelDaoABC[Role]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, Role, TableManager.get("roles"))
DbModelDaoABC.__init__(self, Role, TableManager.get("roles"))
self.attribute(Role.name, str)
self.attribute(Role.description, str)

5
src/cpl-auth/cpl/auth/schema/_permission/role_permission_dao.py
View File

@@ -1,15 +1,12 @@
from cpl.auth.schema._permission.role_permission import RolePermission
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class RolePermissionDao(DbModelDaoABC[RolePermission]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, RolePermission, TableManager.get("role_permissions"))
DbModelDaoABC.__init__(self, RolePermission, TableManager.get("role_permissions"))
self.attribute(RolePermission.role_id, int)
self.attribute(RolePermission.permission_id, int)

5
src/cpl-auth/cpl/auth/schema/_permission/role_user_dao.py
View File

@@ -1,15 +1,12 @@
from cpl.auth.schema._permission.role_user import RoleUser
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.db_logger import DBLogger
_logger = DBLogger(__name__)
class RoleUserDao(DbModelDaoABC[RoleUser]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, RoleUser, TableManager.get("role_users"))
DbModelDaoABC.__init__(self, RoleUser, TableManager.get("role_users"))
self.attribute(RoleUser.role_id, int)
self.attribute(RoleUser.user_id, int)

9
src/cpl-core/cpl/core/ctx/user_context.py
View File

@@ -1,16 +1,17 @@
from contextvars import ContextVar
from typing import Optional
from cpl.auth.auth_logger import AuthLogger
from cpl.auth.schema._administration.auth_user import AuthUser
_user_context: ContextVar[Optional[AuthUser]] = ContextVar("user", default=None)
_logger = AuthLogger(__name__)
def set_user(user_id: Optional[AuthUser]):
_logger.trace("Setting user context", user_id)
from cpl.dependency.service_provider_abc import ServiceProviderABC
from cpl.core.log.logger_abc import LoggerABC
logger = ServiceProviderABC.get_global_service(LoggerABC)
logger.trace("Setting user context", user_id)
_user_context.set(user_id)

1
src/cpl-core/cpl/core/log/__init__.py
View File

@@ -2,3 +2,4 @@ from .logger import Logger
from .logger_abc import LoggerABC
from .log_level import LogLevel
from .log_settings import LogSettings
from .structured_logger import StructuredLogger

80
src/cpl-core/cpl/core/log/structured_logger.py Normal file
View File

@@ -0,0 +1,80 @@
import asyncio
import importlib.util
import traceback
from datetime import datetime
from cpl.core.log import Logger, LogLevel
from cpl.core.typing import Source, Messages
class StructuredLogger(Logger):
def __init__(self, source: Source, file_prefix: str = None):
Logger.__init__(self, source, file_prefix)
@property
def log_file(self):
return f"logs/{self._file_prefix}_{datetime.now().strftime('%Y-%m-%d')}.jsonl"
def _log(self, level: LogLevel, *messages: Messages):
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
formatted_message = self._format_message(level.value, timestamp, *messages)
self._write_log_to_file(level, formatted_message)
self._write_to_console(level, formatted_message)
except Exception as e:
print(f"Error while logging: {e} -> {traceback.format_exc()}")
def _get_structured_message(self, level: str, timestamp: str, messages: str) -> str:
structured_message = {
"timestamp": timestamp,
"level": level.upper(),
"source": self._source,
"messages": messages,
}
self._enrich_message_with_request(structured_message)
self._enrich_message_with_user(structured_message)
return str(structured_message)
@staticmethod
def _enrich_message_with_request(message: dict):
if importlib.util.find_spec("cpl.api") is None:
return
from cpl.api.middleware.request import get_request
from starlette.requests import Request
request = get_request()
if request is None:
return
message["request"] = {
"url": str(request.url),
"method": request.method,
"scope": request.scope,
}
if isinstance(request, Request) and request.scope == "http":
request: Request = request # fix typing for IDEs
message["request"]["data"] = asyncio.create_task(request.body())
@staticmethod
def _enrich_message_with_user(message: dict):
if importlib.util.find_spec("cpl-auth") is None:
return
from cpl.core.ctx import get_user
user = get_user()
if user is None:
return
message["user"] = {
"id": str(user.id),
"username": user.username,
"email": user.email,
}

37
src/cpl-core/cpl/core/log/wrapped_logger.py Normal file
View File

@@ -0,0 +1,37 @@
from cpl.core.log import LoggerABC, LogLevel
from cpl.core.typing import Messages
class WrappedLogger(LoggerABC):
def __init__(self, logger: LoggerABC):
LoggerABC.__init__(self)
assert isinstance(logger, LoggerABC), "logger must be an instance of LoggerABC"
self._logger = logger
def set_level(self, level: LogLevel):
self._logger.set_level(level)
def _format_message(self, level: str, timestamp, *messages: Messages) -> str:
return self._logger._format_message(level, timestamp, *messages)
def header(self, string: str):
self._logger.header(string)
def trace(self, *messages: Messages):
self._logger.trace(*messages)
def debug(self, *messages: Messages):
self._logger.debug(*messages)
def info(self, *messages: Messages):
self._logger.info(*messages)
def warning(self, *messages: Messages):
self._logger.warning(*messages)
def error(self, messages: str, e: Exception = None):
self._logger.error(messages, e)
def fatal(self, messages: str, e: Exception = None):
self._logger.fatal(messages, e)

8
src/cpl-database/cpl/database/abc/data_access_object_abc.py
View File

@@ -18,16 +18,12 @@ from cpl.database.typing import T_DBM, Attribute, AttributeFilters, AttributeSor
class DataAccessObjectABC(ABC, Generic[T_DBM]):
@abstractmethod
def __init__(self, source: str, model_type: Type[T_DBM], table_name: str):
def __init__(self, model_type: Type[T_DBM], table_name: str):
from cpl.dependency.service_provider_abc import ServiceProviderABC
self._db = ServiceProviderABC.get_global_service(DBContextABC)
self._logger = DBLogger(source)
self._model_type = model_type
self._table_name = table_name
self._logger = DBLogger(source)
self._logger = ServiceProviderABC.get_global_service(DBLogger)
self._model_type = model_type
self._table_name = table_name

4
src/cpl-database/cpl/database/abc/db_model_dao_abc.py
View File

@@ -10,8 +10,8 @@ from cpl.database.abc.db_model_abc import DbModelABC
class DbModelDaoABC[T_DBM](DataAccessObjectABC[T_DBM]):
@abstractmethod
def __init__(self, source: str, model_type: Type[T_DBM], table_name: str):
DataAccessObjectABC.__init__(self, source, model_type, table_name)
def __init__(self, model_type: Type[T_DBM], table_name: str):
DataAccessObjectABC.__init__(self, model_type, table_name)
self.attribute(DbModelABC.id, int, ignore=True)
self.attribute(DbModelABC.deleted, bool)

10
src/cpl-database/cpl/database/db_logger.py
View File

@@ -1,8 +1,8 @@
from cpl.core.log import Logger
from cpl.core.typing import Source
from cpl.core.log import LoggerABC
from cpl.core.log.wrapped_logger import WrappedLogger
class DBLogger(Logger):
class DBLogger(WrappedLogger):
def __init__(self, source: Source):
Logger.__init__(self, source, "db")
def __init__(self, logger: LoggerABC):
WrappedLogger.__init__(self, logger)

35
src/cpl-database/cpl/database/mysql/db_context.py
View File

@@ -4,18 +4,17 @@ from typing import Any, List, Dict, Tuple, Union
from mysql.connector import Error as MySQLError, PoolError
from cpl.core.configuration import Configuration
from cpl.core.environment import Environment
from cpl.database.abc.db_context_abc import DBContextABC
from cpl.database.db_logger import DBLogger
from cpl.database.model.database_settings import DatabaseSettings
from cpl.database.mysql.mysql_pool import MySQLPool
_logger = DBLogger(__name__)
class DBContext(DBContextABC):
def __init__(self):
def __init__(self, logger: DBLogger):
DBContextABC.__init__(self)
self._logger = logger
self._pool: MySQLPool = None
self._fails = 0
@@ -23,62 +22,62 @@ class DBContext(DBContextABC):
def connect(self, database_settings: DatabaseSettings):
try:
_logger.debug("Connecting to database")
self._logger.debug("Connecting to database")
self._pool = MySQLPool(
database_settings,
)
_logger.info("Connected to database")
self._logger.info("Connected to database")
except Exception as e:
_logger.fatal("Connecting to database failed", e)
self._logger.fatal("Connecting to database failed", e)
async def execute(self, statement: str, args=None, multi=True) -> List[List]:
_logger.trace(f"execute {statement} with args: {args}")
self._logger.trace(f"execute {statement} with args: {args}")
return await self._pool.execute(statement, args, multi)
async def select_map(self, statement: str, args=None) -> List[Dict]:
_logger.trace(f"select {statement} with args: {args}")
self._logger.trace(f"select {statement} with args: {args}")
try:
return await self._pool.select_map(statement, args)
except (MySQLError, PoolError) as e:
if self._fails >= 3:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
uid = uuid.uuid4()
raise Exception(
f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}"
)
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
self._fails += 1
try:
_logger.debug("Retry select")
self._logger.debug("Retry select")
return await self.select_map(statement, args)
except Exception as e:
pass
return []
except Exception as e:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
raise e
async def select(self, statement: str, args=None) -> Union[List[str], List[Tuple], List[Any]]:
_logger.trace(f"select {statement} with args: {args}")
self._logger.trace(f"select {statement} with args: {args}")
try:
return await self._pool.select(statement, args)
except (MySQLError, PoolError) as e:
if self._fails >= 3:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
uid = uuid.uuid4()
raise Exception(
f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}"
)
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
self._fails += 1
try:
_logger.debug("Retry select")
self._logger.debug("Retry select")
return await self.select(statement, args)
except Exception as e:
pass
return []
except Exception as e:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
raise e

6
src/cpl-database/cpl/database/mysql/mysql_pool.py
View File

@@ -6,8 +6,7 @@ from mysql.connector.aio import MySQLConnectionPool
from cpl.core.environment import Environment
from cpl.database.db_logger import DBLogger
from cpl.database.model import DatabaseSettings
_logger = DBLogger(__name__)
from cpl.dependency import ServiceProviderABC
class MySQLPool:
@@ -36,7 +35,8 @@ class MySQLPool:
await cursor.execute("SELECT 1")
await cursor.fetchall()
except Exception as e:
_logger.fatal(f"Error connecting to the database: {e}")
logger = ServiceProviderABC.get_global_service(DBLogger)
logger.fatal(f"Error connecting to the database: {e}")
finally:
await con.close()

36
src/cpl-database/cpl/database/postgres/db_context.py
View File

@@ -7,16 +7,16 @@ from psycopg_pool import PoolTimeout
from cpl.core.configuration import Configuration
from cpl.core.environment import Environment
from cpl.database.abc.db_context_abc import DBContextABC
from cpl.database.database_settings import DatabaseSettings
from cpl.database.db_logger import DBLogger
from cpl.database.model import DatabaseSettings
from cpl.database.postgres.postgres_pool import PostgresPool
_logger = DBLogger(__name__)
class DBContext(DBContextABC):
def __init__(self):
def __init__(self, logger: DBLogger):
DBContextABC.__init__(self)
self._logger = logger
self._pool: PostgresPool = None
self._fails = 0
@@ -24,63 +24,63 @@ class DBContext(DBContextABC):
def connect(self, database_settings: DatabaseSettings):
try:
_logger.debug("Connecting to database")
self._logger.debug("Connecting to database")
self._pool = PostgresPool(
database_settings,
Environment.get("DB_POOL_SIZE", int, 1),
)
_logger.info("Connected to database")
self._logger.info("Connected to database")
except Exception as e:
_logger.fatal("Connecting to database failed", e)
self._logger.fatal("Connecting to database failed", e)
async def execute(self, statement: str, args=None, multi=True) -> list[list]:
_logger.trace(f"execute {statement} with args: {args}")
self._logger.trace(f"execute {statement} with args: {args}")
return await self._pool.execute(statement, args, multi)
async def select_map(self, statement: str, args=None) -> list[dict]:
_logger.trace(f"select {statement} with args: {args}")
self._logger.trace(f"select {statement} with args: {args}")
try:
return await self._pool.select_map(statement, args)
except (OperationalError, PoolTimeout) as e:
if self._fails >= 3:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
uid = uuid.uuid4()
raise Exception(
f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}"
)
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
self._fails += 1
try:
_logger.debug("Retry select")
self._logger.debug("Retry select")
return await self.select_map(statement, args)
except Exception as e:
pass
return []
except Exception as e:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
raise e
async def select(self, statement: str, args=None) -> list[str] | list[tuple] | list[Any]:
_logger.trace(f"select {statement} with args: {args}")
self._logger.trace(f"select {statement} with args: {args}")
try:
return await self._pool.select(statement, args)
except (OperationalError, PoolTimeout) as e:
if self._fails >= 3:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
uid = uuid.uuid4()
raise Exception(
f"Query failed three times with {type(e).__name__}. Contact an admin with the UID: {uid}"
)
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
self._fails += 1
try:
_logger.debug("Retry select")
self._logger.debug("Retry select")
return await self.select(statement, args)
except Exception as e:
pass
return []
except Exception as e:
_logger.error(f"Database error caused by `{statement}`", e)
self._logger.error(f"Database error caused by `{statement}`", e)
raise e

6
src/cpl-database/cpl/database/postgres/postgres_pool.py
View File

@@ -7,8 +7,7 @@ from psycopg_pool import AsyncConnectionPool, PoolTimeout
from cpl.core.environment import Environment
from cpl.database.db_logger import DBLogger
from cpl.database.model import DatabaseSettings
_logger = DBLogger(__name__)
from cpl.dependency import ServiceProviderABC
class PostgresPool:
@@ -38,7 +37,8 @@ class PostgresPool:
await pool.check_connection(con)
except PoolTimeout as e:
await pool.close()
_logger.fatal(f"Failed to connect to the database", e)
logger = ServiceProviderABC.get_global_service(DBLogger)
logger.fatal(f"Failed to connect to the database", e)
self._pool = pool
return self._pool

5
src/cpl-database/cpl/database/schema/executed_migration_dao.py
View File

@@ -1,14 +1,11 @@
from cpl.database import TableManager
from cpl.database.abc.data_access_object_abc import DataAccessObjectABC
from cpl.database.db_logger import DBLogger
from cpl.database.schema.executed_migration import ExecutedMigration
_logger = DBLogger(__name__)
class ExecutedMigrationDao(DataAccessObjectABC[ExecutedMigration]):
def __init__(self):
DataAccessObjectABC.__init__(self, __name__, ExecutedMigration, TableManager.get("executed_migrations"))
DataAccessObjectABC.__init__(self, ExecutedMigration, TableManager.get("executed_migrations"))
self.attribute(ExecutedMigration.migration_id, str, primary_key=True, db_name="migrationId")

9
src/cpl-database/cpl/database/service/migration_service.py
View File

@@ -8,12 +8,11 @@ from cpl.database.model.server_type import ServerType, ServerTypes
from cpl.database.schema.executed_migration import ExecutedMigration
from cpl.database.schema.executed_migration_dao import ExecutedMigrationDao
_logger = DBLogger(__name__)
class MigrationService:
def __init__(self, db: DBContextABC, executedMigrationDao: ExecutedMigrationDao):
def __init__(self, logger: DBLogger, db: DBContextABC, executedMigrationDao: ExecutedMigrationDao):
self._logger = logger
self._db = db
self._executedMigrationDao = executedMigrationDao
@@ -96,13 +95,13 @@ class MigrationService:
if migration_from_db is not None:
continue
_logger.debug(f"Running upgrade migration: {migration.name}")
self._logger.debug(f"Running upgrade migration: {migration.name}")
await self._db.execute(migration.script, multi=True)
await self._executedMigrationDao.create(ExecutedMigration(migration.name), skip_editor=True)
except Exception as e:
_logger.fatal(
self._logger.fatal(
f"Migration failed: {migration.name}\n{active_statement}",
e,
)

6
src/cpl-database/cpl/database/service/seeder_service.py
View File

@@ -3,16 +3,14 @@ from cpl.database.db_logger import DBLogger
from cpl.dependency import ServiceProviderABC
_logger = DBLogger(__name__)
class SeederService:
def __init__(self, provider: ServiceProviderABC):
self._provider = provider
self._logger = provider.get_service(DBLogger)
async def seed(self):
seeders = self._provider.get_services(DataSeederABC)
_logger.debug(f"Found {len(seeders)} seeders")
self._logger.debug(f"Found {len(seeders)} seeders")
for seeder in seeders:
await seeder.seed()

11
src/cpl-dependency/cpl/dependency/service_collection.py
View File

@@ -2,6 +2,8 @@ from typing import Union, Type, Callable, Self
from cpl.core.log.logger import Logger
from cpl.core.log.logger_abc import LoggerABC
from cpl.core.log.structured_logger import StructuredLogger
from cpl.core.log.wrapped_logger import WrappedLogger
from cpl.core.typing import T, Service
from cpl.dependency.service_descriptor import ServiceDescriptor
from cpl.dependency.service_lifetime_enum import ServiceLifetimeEnum
@@ -82,3 +84,12 @@ class ServiceCollection:
def add_logging(self) -> Self:
self.add_transient(LoggerABC, Logger)
return self
def add_structured_logging(self) -> Self:
self.add_transient(LoggerABC, StructuredLogger)
return self
def add_wrapped_logging(self) -> Self:
for wrapper in WrappedLogger.__subclasses__():
self.add_transient(wrapper)
return self

10
src/cpl-mail/cpl/mail/mail_logger.py
View File

@@ -1,8 +1,8 @@
from cpl.core.log.logger import Logger
from cpl.core.typing import Source
from cpl.core.log import LoggerABC
from cpl.core.log.wrapped_logger import WrappedLogger
class MailLogger(Logger):
class MailLogger(WrappedLogger):
def __init__(self, source: Source):
Logger.__init__(self, source, "mail")
def __init__(self, logger: LoggerABC):
WrappedLogger.__init__(self, logger)

3
tests/custom/api/src/main.py
View File

@@ -16,7 +16,8 @@ def main():
Configuration.add_json_file(f"appsettings.{Environment.get_environment()}.json")
Configuration.add_json_file(f"appsettings.{Environment.get_host_name()}.json", optional=True)
builder.services.add_logging()
builder.services.add_structured_logging()
builder.services.add_wrapped_logging()
builder.services.add_transient(PingService)
builder.services.add_module(api)

2
tests/custom/database/src/model/city_dao.py
View File

@@ -5,7 +5,7 @@ from model.city import City
class CityDao(DbModelDaoABC[City]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, City, "city")
DbModelDaoABC.__init__(self, City, "city")
self.attribute(City.name, str)
self.attribute(City.zip, int)

2
tests/custom/database/src/model/user_dao.py
View File

@@ -5,7 +5,7 @@ from model.user import User
class UserDao(DbModelDaoABC[User]):
def __init__(self):
DbModelDaoABC.__init__(self, __name__, User, "users")
DbModelDaoABC.__init__(self, User, "users")
self.attribute(User.name, str)
self.attribute(User.city_id, int, db_name="CityId")