Compare commits

..

31 Commits

Author SHA1 Message Date
c02903b009 Merge pull request 'master' (#198) from master into dev
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 7s
Build on push / prepare (push) Successful in 11s
Build on push / query (push) Successful in 19s
Build on push / core (push) Successful in 19s
Build on push / dependency (push) Successful in 18s
Build on push / mail (push) Successful in 16s
Build on push / translation (push) Successful in 19s
Build on push / database (push) Successful in 20s
Build on push / application (push) Successful in 29s
Build on push / auth (push) Successful in 15s
Build on push / api (push) Successful in 18s
Reviewed-on: #198
2025-10-08 21:31:55 +02:00
b6cf5962aa Merge pull request 'Removed tools & docs' (#197) from dev_cleanup into master
Reviewed-on: #197
2025-10-08 21:30:47 +02:00
d3084041a9 Removed tools & docs 2025-10-08 21:30:01 +02:00
d4af923e4c Merge pull request 'Added gql base #181' (#196) from #181_gql into dev
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 7s
Build on push / prepare (push) Successful in 12s
Build on push / query (push) Successful in 23s
Build on push / core (push) Successful in 23s
Build on push / dependency (push) Successful in 29s
Build on push / translation (push) Successful in 17s
Build on push / application (push) Successful in 22s
Build on push / database (push) Successful in 22s
Build on push / mail (push) Successful in 23s
Build on push / auth (push) Successful in 15s
Build on push / api (push) Successful in 15s
Reviewed-on: #196
Closes #181
2025-10-08 21:25:41 +02:00
545540d05d Added subscriptions final #181
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 7s
2025-10-08 21:22:51 +02:00
3774cef56a Updated permissions #181
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 10s
2025-10-08 17:27:46 +02:00
cdb5e4ff89 [WIP] Subscriptions
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 6s
2025-10-04 06:57:14 +02:00
e362b7fb61 Added/Fixed api_key/user/role gql #181
Some checks failed
Test before pr merge / test-lint (pull_request) Failing after 6s
2025-10-04 05:44:13 +02:00
262e26cb83 Internal api key gql #181 2025-10-04 05:44:13 +02:00
5b3872a1fe Added possibility to put auth schema to root graphs #181 2025-10-04 05:44:13 +02:00
e7e3712e08 Renamed AuthUsers -> Users & completed user gql #181 2025-10-04 05:44:13 +02:00
df69f1c725 Recursive filter #181 2025-10-04 05:44:13 +02:00
39351a5eb9 Recursive types #181 2025-10-04 05:44:13 +02:00
39d06dfe48 Added mutations #181 2025-10-04 05:44:13 +02:00
3286a95cbf require any #181 2025-10-04 05:44:13 +02:00
6f46b94998 [WIP] with authentication #181 2025-10-04 05:44:13 +02:00
20e5da5770 Recursive complex filtering #181 2025-10-04 05:44:13 +02:00
a12a4082db Dao complex filtering #181 2025-10-04 05:44:13 +02:00
d8c60defba Further gql improvements & added test data #181 2025-10-04 05:44:13 +02:00
ada50c693e Changed to strawberry #181 2025-10-04 05:44:13 +02:00
a35b44b3b5 [WIP] collection #181 2025-10-04 05:44:13 +02:00
683805137a Added arguments to field #181 2025-10-04 05:44:13 +02:00
b0f1fb9839 Removed query base #181 2025-10-04 05:44:13 +02:00
e1ab9cf0db Added gql base #181 2025-10-04 05:44:13 +02:00
685c20e3bf Added cron jobs as hosted services
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 6s
Build on push / prepare (push) Successful in 11s
Build on push / core (push) Successful in 18s
Build on push / query (push) Successful in 19s
Build on push / dependency (push) Successful in 14s
Build on push / database (push) Successful in 15s
Build on push / application (push) Successful in 18s
Build on push / mail (push) Successful in 19s
Build on push / translation (push) Successful in 29s
Build on push / auth (push) Successful in 18s
Build on push / api (push) Successful in 18s
2025-10-04 05:38:21 +02:00
e0f6e1c241 Console & file logging format msg seperate & removed timestamp from console & minor fixes to di
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 7s
Build on push / prepare (push) Successful in 10s
Build on push / core (push) Successful in 18s
Build on push / query (push) Successful in 19s
Build on push / dependency (push) Successful in 14s
Build on push / translation (push) Successful in 16s
Build on push / mail (push) Successful in 17s
Build on push / application (push) Successful in 19s
Build on push / database (push) Successful in 19s
Build on push / auth (push) Successful in 25s
Build on push / api (push) Successful in 14s
2025-09-26 15:48:33 +02:00
c410a692be Better modules
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 6s
Build on push / prepare (push) Successful in 9s
Build on push / core (push) Successful in 18s
Build on push / query (push) Successful in 19s
Build on push / dependency (push) Successful in 17s
Build on push / mail (push) Successful in 16s
Build on push / translation (push) Successful in 16s
Build on push / application (push) Successful in 18s
Build on push / database (push) Successful in 19s
Build on push / auth (push) Successful in 17s
Build on push / api (push) Successful in 14s
2025-09-26 12:55:00 +02:00
56a16cbeba Module dependencies as static var
Some checks failed
Test before pr merge / test-lint (pull_request) Failing after 6s
Build on push / prepare (push) Successful in 10s
Build on push / query (push) Successful in 19s
Build on push / core (push) Successful in 20s
Build on push / dependency (push) Successful in 17s
Build on push / application (push) Successful in 15s
Build on push / database (push) Successful in 16s
Build on push / mail (push) Successful in 18s
Build on push / translation (push) Successful in 22s
Build on push / auth (push) Successful in 18s
Build on push / api (push) Successful in 17s
2025-09-26 08:46:30 +02:00
d05d947d54 Import cleanup
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 6s
Build on push / prepare (push) Successful in 12s
Build on push / core (push) Successful in 18s
Build on push / query (push) Successful in 17s
Build on push / dependency (push) Successful in 17s
Build on push / application (push) Successful in 15s
Build on push / translation (push) Successful in 18s
Build on push / mail (push) Successful in 18s
Build on push / database (push) Successful in 20s
Build on push / auth (push) Successful in 17s
Build on push / api (push) Successful in 13s
2025-09-26 00:03:12 +02:00
0529269747 Fixed formatting
All checks were successful
Test before pr merge / test-lint (pull_request) Successful in 6s
Build on push / prepare (push) Successful in 9s
Build on push / core (push) Successful in 19s
Build on push / query (push) Successful in 19s
Build on push / dependency (push) Successful in 14s
Build on push / mail (push) Successful in 16s
Build on push / application (push) Successful in 19s
Build on push / database (push) Successful in 20s
Build on push / translation (push) Successful in 20s
Build on push / auth (push) Successful in 14s
Build on push / api (push) Successful in 15s
2025-09-25 10:37:29 +02:00
e3e1703ff8 Fixed versioning
Some checks failed
Test before pr merge / test-lint (pull_request) Failing after 6s
Build on push / prepare (push) Successful in 9s
Build on push / query (push) Successful in 19s
Build on push / core (push) Successful in 19s
Build on push / dependency (push) Successful in 14s
Build on push / api (push) Has been cancelled
Build on push / auth (push) Has been cancelled
Build on push / application (push) Has been cancelled
Build on push / mail (push) Has been cancelled
Build on push / translation (push) Has been cancelled
Build on push / database (push) Has been cancelled
2025-09-25 10:36:36 +02:00
186 changed files with 4100 additions and 621 deletions

View File

@@ -25,7 +25,11 @@ jobs:
git tag
DATE=$(date +'%Y.%m.%d')
TAG_COUNT=$(git tag -l "${DATE}.*" | wc -l)
BUILD_NUMBER=$(($TAG_COUNT + 1))
if [ "$TAG_COUNT" -eq 0 ]; then
BUILD_NUMBER=0
else
BUILD_NUMBER=$(($TAG_COUNT + 1))
fi
VERSION_SUFFIX=${{ inputs.version_suffix }}
if [ -n "$VERSION_SUFFIX" ] && [ "$VERSION_SUFFIX" = "dev" ]; then

View File

@@ -1,50 +1,117 @@
from starlette.responses import JSONResponse
from cpl import api
from cpl.api.application.web_app import WebApp
from cpl.api_module import ApiModule
from cpl.application import ApplicationBuilder
from cpl.auth.permission.permissions import Permissions
from cpl.auth.schema import AuthUser, Role
from cpl.dependency.event_bus import EventBusABC
from cpl.graphql.event_bus.memory import InMemoryEventBus
from queries.cities import CityGraphType, CityFilter, CitySort
from queries.hello import UserGraphType # , UserFilter, UserSort, UserGraphType
from queries.user import UserFilter, UserSort
from cpl.api.api_module import ApiModule
from cpl.application.application_builder import ApplicationBuilder
from cpl.auth.schema import User, Role
from cpl.core.configuration import Configuration
from cpl.core.console import Console
from cpl.core.environment import Environment
from cpl.core.utils.cache import Cache
from cpl.database.mysql.mysql_module import MySQLModule
from cpl.graphql.application.graphql_app import GraphQLApp
from cpl.graphql.auth.graphql_auth_module import GraphQLAuthModule
from cpl.graphql.graphql_module import GraphQLModule
from model.author_dao import AuthorDao
from model.author_query import AuthorGraphType, AuthorFilter, AuthorSort
from model.post_dao import PostDao
from model.post_query import PostFilter, PostSort, PostGraphType, PostMutation, PostSubscription
from permissions import PostPermissions
from queries.hello import HelloQuery
from scoped_service import ScopedService
from service import PingService
from test_data_seeder import TestDataSeeder
def main():
builder = ApplicationBuilder[WebApp](WebApp)
builder = ApplicationBuilder[GraphQLApp](GraphQLApp)
Configuration.add_json_file(f"appsettings.json")
Configuration.add_json_file(f"appsettings.{Environment.get_environment()}.json")
Configuration.add_json_file(f"appsettings.{Environment.get_host_name()}.json", optional=True)
# builder.services.add_logging()
builder.services.add_structured_logging()
builder.services.add_transient(PingService)
builder.services.add_module(MySQLModule)
builder.services.add_module(ApiModule)
builder.services.add_scoped(ScopedService)
builder.services.add_cache(AuthUser)
builder.services.add_cache(Role)
(
builder.services.add_structured_logging()
.add_transient(PingService)
.add_module(MySQLModule)
.add_module(ApiModule)
.add_module(GraphQLModule)
.add_module(GraphQLAuthModule)
.add_scoped(ScopedService)
.add_singleton(EventBusABC, InMemoryEventBus)
.add_cache(User)
.add_cache(Role)
.add_transient(CityGraphType)
.add_transient(CityFilter)
.add_transient(CitySort)
.add_transient(UserGraphType)
.add_transient(UserFilter)
.add_transient(UserSort)
# .add_transient(UserGraphType)
# .add_transient(UserFilter)
# .add_transient(UserSort)
.add_transient(HelloQuery)
# test data
.add_singleton(TestDataSeeder)
# authors
.add_transient(AuthorDao)
.add_transient(AuthorGraphType)
.add_transient(AuthorFilter)
.add_transient(AuthorSort)
# posts
.add_transient(PostDao)
.add_transient(PostGraphType)
.add_transient(PostFilter)
.add_transient(PostSort)
.add_transient(PostMutation)
.add_transient(PostSubscription)
)
app = builder.build()
app.with_logging()
app.with_database()
app.with_migrations("./scripts")
app.with_authentication()
app.with_authorization()
app.with_route(path="/route1", fn=lambda r: JSONResponse("route1"), method="GET", authentication=True, permissions=[Permissions.administrator])
app.with_route(
path="/route1",
fn=lambda r: JSONResponse("route1"),
method="GET",
# authentication=True,
# permissions=[Permissions.administrator],
)
app.with_routes_directory("routes")
schema = app.with_graphql()
schema.query.string_field("ping", resolver=lambda: "pong")
schema.query.with_query("hello", HelloQuery)
schema.query.dao_collection_field(AuthorGraphType, AuthorDao, "authors", AuthorFilter, AuthorSort)
(
schema.query.dao_collection_field(PostGraphType, PostDao, "posts", PostFilter, PostSort)
# .with_require_any_permission(PostPermissions.read)
.with_public()
)
schema.mutation.with_mutation("post", PostMutation).with_public()
schema.subscription.with_subscription(PostSubscription)
app.with_auth_root_queries(True)
app.with_auth_root_mutations(True)
app.with_playground()
app.with_graphiql()
app.with_permissions(PostPermissions)
provider = builder.service_provider
user_cache = provider.get_service(Cache[AuthUser])
user_cache = provider.get_service(Cache[User])
role_cache = provider.get_service(Cache[Role])
if role_cache == user_cache:

View File

View File

@@ -0,0 +1,30 @@
from datetime import datetime
from typing import Self
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
class Author(DbModelABC[Self]):
def __init__(
self,
id: int,
first_name: str,
last_name: str,
deleted: bool = False,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._first_name = first_name
self._last_name = last_name
@property
def first_name(self) -> str:
return self._first_name
@property
def last_name(self) -> str:
return self._last_name

View File

@@ -0,0 +1,11 @@
from cpl.database.abc import DbModelDaoABC
from model.author import Author
class AuthorDao(DbModelDaoABC):
def __init__(self):
DbModelDaoABC.__init__(self, Author, "authors")
self.attribute(Author.first_name, str, db_name="firstname")
self.attribute(Author.last_name, str, db_name="lastname")

View File

@@ -0,0 +1,37 @@
from cpl.graphql.schema.db_model_graph_type import DbModelGraphType
from cpl.graphql.schema.filter.db_model_filter import DbModelFilter
from cpl.graphql.schema.sort.sort import Sort
from cpl.graphql.schema.sort.sort_order import SortOrder
from model.author import Author
class AuthorFilter(DbModelFilter[Author]):
def __init__(self):
DbModelFilter.__init__(self, public=True)
self.int_field("id")
self.string_field("firstName")
self.string_field("lastName")
class AuthorSort(Sort[Author]):
def __init__(self):
Sort.__init__(self)
self.field("id", SortOrder)
self.field("firstName", SortOrder)
self.field("lastName", SortOrder)
class AuthorGraphType(DbModelGraphType[Author]):
def __init__(self):
DbModelGraphType.__init__(self, public=True)
self.int_field(
"id",
resolver=lambda root: root.id,
).with_public(True)
self.string_field(
"firstName",
resolver=lambda root: root.first_name,
).with_public(True)
self.string_field(
"lastName",
resolver=lambda root: root.last_name,
).with_public(True)

View File

@@ -0,0 +1,44 @@
from datetime import datetime
from typing import Self
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
class Post(DbModelABC[Self]):
def __init__(
self,
id: int,
author_id: SerialId,
title: str,
content: str,
deleted: bool = False,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._author_id = author_id
self._title = title
self._content = content
@property
def author_id(self) -> SerialId:
return self._author_id
@property
def title(self) -> str:
return self._title
@title.setter
def title(self, value: str):
self._title = value
@property
def content(self) -> str:
return self._content
@content.setter
def content(self, value: str):
self._content = value

View File

@@ -0,0 +1,15 @@
from cpl.database.abc import DbModelDaoABC
from model.author_dao import AuthorDao
from model.post import Post
class PostDao(DbModelDaoABC[Post]):
def __init__(self, authors: AuthorDao):
DbModelDaoABC.__init__(self, Post, "posts")
self.attribute(Post.author_id, int, db_name="authorId")
self.reference("author", "id", Post.author_id, "authors", authors)
self.attribute(Post.title, str)
self.attribute(Post.content, str)

View File

@@ -0,0 +1,148 @@
from cpl.dependency.event_bus import EventBusABC
from cpl.graphql.query_context import QueryContext
from cpl.graphql.schema.db_model_graph_type import DbModelGraphType
from cpl.graphql.schema.filter.db_model_filter import DbModelFilter
from cpl.graphql.schema.input import Input
from cpl.graphql.schema.mutation import Mutation
from cpl.graphql.schema.sort.sort import Sort
from cpl.graphql.schema.sort.sort_order import SortOrder
from cpl.graphql.schema.subscription import Subscription
from model.author_dao import AuthorDao
from model.author_query import AuthorGraphType, AuthorFilter
from model.post import Post
from model.post_dao import PostDao
class PostFilter(DbModelFilter[Post]):
def __init__(self):
DbModelFilter.__init__(self, public=True)
self.int_field("id")
self.filter_field("author", AuthorFilter)
self.string_field("title")
self.string_field("content")
class PostSort(Sort[Post]):
def __init__(self):
Sort.__init__(self)
self.field("id", SortOrder)
self.field("title", SortOrder)
self.field("content", SortOrder)
class PostGraphType(DbModelGraphType[Post]):
def __init__(self, authors: AuthorDao):
DbModelGraphType.__init__(self, public=True)
self.int_field(
"id",
resolver=lambda root: root.id,
).with_optional().with_public(True)
async def _a(root: Post):
return await authors.get_by_id(root.author_id)
def r_name(ctx: QueryContext):
return ctx.user.username == "admin"
self.object_field("author", AuthorGraphType, resolver=_a).with_public(True) # .with_require_any([], [r_name]))
self.string_field(
"title",
resolver=lambda root: root.title,
).with_public(True)
self.string_field(
"content",
resolver=lambda root: root.content,
).with_public(True)
class PostCreateInput(Input[Post]):
title: str
content: str
author_id: int
def __init__(self):
Input.__init__(self)
self.string_field("title").with_required()
self.string_field("content").with_required()
self.int_field("author_id").with_required()
class PostUpdateInput(Input[Post]):
title: str
content: str
author_id: int
def __init__(self):
Input.__init__(self)
self.int_field("id").with_required()
self.string_field("title").with_required(False)
self.string_field("content").with_required(False)
class PostSubscription(Subscription):
def __init__(self, bus: EventBusABC):
Subscription.__init__(self)
self._bus = bus
def selector(event: Post, info) -> bool:
return event.id == 101
self.subscription_field("postChange", PostGraphType, selector).with_public()
class PostMutation(Mutation):
def __init__(self, posts: PostDao, authors: AuthorDao, bus: EventBusABC):
Mutation.__init__(self)
self._posts = posts
self._authors = authors
self._bus = bus
self.field("create", int, resolver=self.create_post).with_public().with_required().with_argument(
"input",
PostCreateInput,
).with_required()
self.field("update", bool, resolver=self.update_post).with_public().with_required().with_argument(
"input",
PostUpdateInput,
).with_required()
self.field("delete", bool, resolver=self.delete_post).with_public().with_required().with_argument(
"id",
int,
).with_required()
self.field("restore", bool, resolver=self.restore_post).with_public().with_required().with_argument(
"id",
int,
).with_required()
async def create_post(self, input: PostCreateInput) -> int:
return await self._posts.create(Post(0, input.author_id, input.title, input.content))
async def update_post(self, input: PostUpdateInput) -> bool:
post = await self._posts.get_by_id(input.id)
if post is None:
return False
post.title = input.title if input.title is not None else post.title
post.content = input.content if input.content is not None else post.content
await self._posts.update(post)
await self._bus.publish("postChange", post)
return True
async def delete_post(self, id: int) -> bool:
post = await self._posts.get_by_id(id)
if post is None:
return False
await self._posts.delete(post)
return True
async def restore_post(self, id: int) -> bool:
post = await self._posts.get_by_id(id)
if post is None:
return False
await self._posts.restore(post)
return True

View File

@@ -0,0 +1,8 @@
from enum import Enum
class PostPermissions(Enum):
read = "post.read"
write = "post.write"
delete = "post.delete"

View File

View File

@@ -0,0 +1,39 @@
from cpl.graphql.schema.filter.filter import Filter
from cpl.graphql.schema.graph_type import GraphType
from cpl.graphql.schema.sort.sort import Sort
from cpl.graphql.schema.sort.sort_order import SortOrder
class City:
def __init__(self, id: int, name: str):
self.id = id
self.name = name
class CityFilter(Filter[City]):
def __init__(self):
Filter.__init__(self)
self.field("id", int)
self.field("name", str)
class CitySort(Sort[City]):
def __init__(self):
Sort.__init__(self)
self.field("id", SortOrder)
self.field("name", SortOrder)
class CityGraphType(GraphType[City]):
def __init__(self):
GraphType.__init__(self)
self.int_field(
"id",
resolver=lambda root: root.id,
)
self.string_field(
"name",
resolver=lambda root: root.name,
)

View File

@@ -0,0 +1,70 @@
from queries.cities import CityFilter, CitySort, CityGraphType, City
from queries.user import User, UserFilter, UserSort, UserGraphType
from cpl.api.middleware.request import get_request
from cpl.auth.schema import UserDao, User
from cpl.graphql.schema.filter.filter import Filter
from cpl.graphql.schema.graph_type import GraphType
from cpl.graphql.schema.query import Query
from cpl.graphql.schema.sort.sort import Sort
from cpl.graphql.schema.sort.sort_order import SortOrder
users = [User(i, f"User {i}") for i in range(1, 101)]
cities = [City(i, f"City {i}") for i in range(1, 101)]
# class UserFilter(Filter[User]):
# def __init__(self):
# Filter.__init__(self)
# self.field("id", int)
# self.field("username", str)
#
#
# class UserSort(Sort[User]):
# def __init__(self):
# Sort.__init__(self)
# self.field("id", SortOrder)
# self.field("username", SortOrder)
#
# class UserGraphType(GraphType[User]):
#
# def __init__(self):
# GraphType.__init__(self)
#
# self.int_field(
# "id",
# resolver=lambda root: root.id,
# )
# self.string_field(
# "username",
# resolver=lambda root: root.username,
# )
class HelloQuery(Query):
def __init__(self):
Query.__init__(self)
self.string_field(
"message",
resolver=lambda name: f"Hello {name} {get_request().state.request_id}",
).with_argument("name", str, "Name to greet", "world")
self.collection_field(
UserGraphType,
"users",
UserFilter,
UserSort,
resolver=lambda: users,
)
self.collection_field(
CityGraphType,
"cities",
CityFilter,
CitySort,
resolver=lambda: cities,
)
# self.dao_collection_field(
# UserGraphType,
# UserDao,
# "Users",
# UserFilter,
# UserSort,
# )

View File

@@ -0,0 +1,39 @@
from cpl.graphql.schema.filter.filter import Filter
from cpl.graphql.schema.graph_type import GraphType
from cpl.graphql.schema.sort.sort import Sort
from cpl.graphql.schema.sort.sort_order import SortOrder
class User:
def __init__(self, id: int, name: str):
self.id = id
self.name = name
class UserFilter(Filter[User]):
def __init__(self):
Filter.__init__(self)
self.field("id", int)
self.field("name", str)
class UserSort(Sort[User]):
def __init__(self):
Sort.__init__(self)
self.field("id", SortOrder)
self.field("name", SortOrder)
class UserGraphType(GraphType[User]):
def __init__(self):
GraphType.__init__(self)
self.int_field(
"id",
resolver=lambda root: root.id,
)
self.string_field(
"name",
resolver=lambda root: root.name,
)

View File

@@ -0,0 +1,22 @@
CREATE TABLE IF NOT EXISTS `authors` (
`id` INT(30) NOT NULL AUTO_INCREMENT,
`firstname` VARCHAR(64) NOT NULL,
`lastname` VARCHAR(64) NOT NULL,
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL,
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY(`id`)
);
CREATE TABLE IF NOT EXISTS `posts` (
`id` INT(30) NOT NULL AUTO_INCREMENT,
`authorId` INT(30) NOT NULL REFERENCES `authors`(`id`) ON DELETE CASCADE,
`title` TEXT NOT NULL,
`content` TEXT NOT NULL,
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL,
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY(`id`)
);

View File

@@ -0,0 +1,48 @@
from faker import Faker
from cpl.database.abc import DataSeederABC
from cpl.query import Enumerable
from model.author import Author
from model.author_dao import AuthorDao
from model.post import Post
from model.post_dao import PostDao
fake = Faker()
class TestDataSeeder(DataSeederABC):
def __init__(self, authors: AuthorDao, posts: PostDao):
DataSeederABC.__init__(self)
self._authors = authors
self._posts = posts
async def seed(self):
if await self._authors.count() == 0:
await self._seed_authors()
if await self._posts.count() == 0:
await self._seed_posts()
async def _seed_authors(self):
authors = Enumerable.range(0, 35).select(
lambda x: Author(
0,
fake.first_name(),
fake.last_name(),
)
).to_list()
await self._authors.create_many(authors, skip_editor=True)
async def _seed_posts(self):
posts = Enumerable.range(0, 100).select(
lambda x: Post(
id=0,
author_id=fake.random_int(min=1, max=35),
title=fake.sentence(nb_words=6),
content=fake.paragraph(nb_sentences=6),
)
).to_list()
await self._posts.create_many(posts, skip_editor=True)

View File

@@ -4,6 +4,7 @@ from cpl.core.console import Console
from cpl.core.environment import Environment
from cpl.core.log import LoggerABC
from cpl.dependency import ServiceProvider
from cpl.dependency.typing import Modules
from model.city import City
from model.city_dao import CityDao
from model.user import User
@@ -11,8 +12,8 @@ from model.user_dao import UserDao
class Application(ApplicationABC):
def __init__(self, services: ServiceProvider):
ApplicationABC.__init__(self, services)
def __init__(self, services: ServiceProvider, modules: Modules):
ApplicationABC.__init__(self, services, modules)
self._logger = services.get_service(LoggerABC)

View File

@@ -3,6 +3,7 @@ from cpl.application import ApplicationBuilder
from cpl.auth.permission.permissions_registry import PermissionsRegistry
from cpl.core.console import Console
from cpl.core.log import LogLevel
from cpl.database import DatabaseModule
from custom_permissions import CustomPermissions
from startup import Startup
@@ -10,13 +11,12 @@ from startup import Startup
def main():
builder = ApplicationBuilder(Application).with_startup(Startup)
builder.services.add_logging()
app = builder.build()
app.with_logging(LogLevel.trace)
app.with_permissions(CustomPermissions)
app.with_migrations("./scripts")
app.with_seeders()
# app.with_seeders()
Console.write_line(CustomPermissions.test.value in PermissionsRegistry.get())
app.run()

View File

@@ -5,16 +5,16 @@ from cpl.core.typing import SerialId
from cpl.database.abc.db_model_abc import DbModelABC
class City(DbModelABC):
class City(DbModelABC[Self]):
def __init__(
self,
id: int,
name: str,
zip: str,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None= None,
updated: datetime | None= None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._name = name

View File

@@ -5,7 +5,7 @@ from cpl.core.typing import SerialId
from cpl.database.abc.db_model_abc import DbModelABC
class User(DbModelABC):
class User(DbModelABC[Self]):
def __init__(
self,
@@ -13,9 +13,9 @@ class User(DbModelABC):
name: str,
city_id: int = 0,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None= None,
updated: datetime | None= None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._name = name

View File

@@ -6,7 +6,7 @@ from cpl.auth.permission.permission_module import PermissionsModule
from cpl.core.configuration import Configuration
from cpl.core.environment import Environment
from cpl.core.log import Logger, LoggerABC
from cpl.database import mysql
from cpl.database import mysql, DatabaseModule
from cpl.database.abc.data_access_object_abc import DataAccessObjectABC
from cpl.database.mysql.mysql_module import MySQLModule
from cpl.dependency import ServiceCollection
@@ -25,6 +25,7 @@ class Startup(StartupABC):
@staticmethod
async def configure_services(services: ServiceCollection):
services.add_module(MySQLModule)
services.add_module(DatabaseModule)
services.add_module(AuthModule)
services.add_module(PermissionsModule)

View File

@@ -1,3 +1,4 @@
import asyncio
import time
from cpl.application.abc import ApplicationABC
@@ -7,6 +8,7 @@ from cpl.core.environment import Environment
from cpl.core.log import LoggerABC
from cpl.core.pipes import IPAddressPipe
from cpl.dependency import ServiceProvider
from cpl.dependency.typing import Modules
from cpl.mail import EMail, EMailClientABC
from cpl.query import List
from scoped_service import ScopedService
@@ -16,8 +18,8 @@ from test_settings import TestSettings
class Application(ApplicationABC):
def __init__(self, services: ServiceProvider):
ApplicationABC.__init__(self, services)
def __init__(self, services: ServiceProvider, modules: Modules):
ApplicationABC.__init__(self, services, modules)
self._logger = self._services.get_service(LoggerABC)
self._mailer = self._services.get_service(EMailClientABC)
@@ -35,7 +37,7 @@ class Application(ApplicationABC):
def _wait(time_ms: int):
time.sleep(time_ms)
def main(self):
async def main(self):
self._logger.debug(f"Host: {Environment.get_host_name()}")
self._logger.debug(f"Environment: {Environment.get_environment()}")
Console.write_line(List(range(0, 10)).select(lambda x: f"x={x}").to_list())
@@ -75,7 +77,7 @@ class Application(ApplicationABC):
# self.test_send_mail()
x = 0
while x < 5:
while x < 500:
Console.write_line("Running...")
x += 1
time.sleep(5)
await asyncio.sleep(5)

View File

@@ -1,7 +1,9 @@
import asyncio
import time
from datetime import datetime
from cpl.core.console import Console
from cpl.core.time.cron import Cron
from cpl.dependency.hosted.cronjob import CronjobABC
from cpl.dependency.hosted.hosted_service import HostedService
@@ -17,4 +19,12 @@ class Hosted(HostedService):
async def stop(self):
Console.write_line("Hosted Service Stopped")
self._stopped = True
self._stopped = True
class MyCronJob(CronjobABC):
def __init__(self):
CronjobABC.__init__(self, Cron("*/1 * * * *")) # Every minute
async def loop(self):
Console.write_line(f"[{datetime.now()}] Hello from Cronjob!")

View File

@@ -1,11 +1,10 @@
from cpl import mail
from cpl.application.abc import StartupABC
from cpl.core.configuration import Configuration
from cpl.core.environment import Environment
from cpl.core.pipes import IPAddressPipe
from cpl.dependency import ServiceCollection
from cpl.mail.mail_module import MailModule
from hosted_service import Hosted
from hosted_service import Hosted, MyCronJob
from scoped_service import ScopedService
from test_service import TestService
@@ -26,3 +25,4 @@ class Startup(StartupABC):
services.add_singleton(TestService)
services.add_scoped(ScopedService)
services.add_hosted_service(Hosted)
services.add_hosted_service(MyCronJob)

View File

@@ -1,4 +1,4 @@
from .error import APIError, AlreadyExists, EndpointNotImplemented, Forbidden, NotFound, Unauthorized
from .logger import APILogger
from .settings import ApiSettings
from .api_module import ApiModule

View File

@@ -0,0 +1,45 @@
from abc import ABC
from enum import Enum
from typing import Self
from starlette.applications import Starlette
from cpl.api.model.api_route import ApiRoute
from cpl.api.model.validation_match import ValidationMatch
from cpl.api.typing import HTTPMethods, PartialMiddleware, TEndpoint, PolicyInput
from cpl.application.abc.application_abc import ApplicationABC
from cpl.dependency.service_provider import ServiceProvider
from cpl.dependency.typing import Modules
class WebAppABC(ApplicationABC, ABC):
def __init__(self, services: ServiceProvider, modules: Modules, required_modules: list[str | object] = None):
ApplicationABC.__init__(self, services, modules, required_modules)
def with_routes_directory(self, directory: str) -> Self: ...
def with_app(self, app: Starlette) -> Self: ...
def with_routes(
self,
routes: list[ApiRoute],
method: HTTPMethods,
authentication: bool = False,
roles: list[str | Enum] = None,
permissions: list[str | Enum] = None,
policies: list[str] = None,
match: ValidationMatch = None,
) -> Self: ...
def with_route(
self,
path: str,
fn: TEndpoint,
method: HTTPMethods,
authentication: bool = False,
roles: list[str | Enum] = None,
permissions: list[str | Enum] = None,
policies: list[str] = None,
match: ValidationMatch = None,
) -> Self: ...
def with_middleware(self, middleware: PartialMiddleware) -> Self: ...
def with_authentication(self) -> Self: ...
def with_authorization(self, *policies: list[PolicyInput] | PolicyInput) -> Self: ...

View File

@@ -0,0 +1,22 @@
from cpl.api import ApiSettings
from cpl.api.registry.policy import PolicyRegistry
from cpl.api.registry.route import RouteRegistry
from cpl.auth.auth_module import AuthModule
from cpl.auth.permission.permission_module import PermissionsModule
from cpl.database.database_module import DatabaseModule
from cpl.dependency import ServiceCollection
from cpl.dependency.module.module import Module
class ApiModule(Module):
config = [ApiSettings]
singleton = [
PolicyRegistry,
RouteRegistry,
]
@staticmethod
def register(collection: ServiceCollection):
collection.add_module(DatabaseModule)
collection.add_module(AuthModule)
collection.add_module(PermissionsModule)

View File

@@ -1,6 +1,6 @@
import os
from enum import Enum
from typing import Mapping, Any, Callable, Self, Union
from typing import Mapping, Any, Self
import uvicorn
from starlette.applications import Starlette
@@ -10,7 +10,8 @@ from starlette.requests import Request
from starlette.responses import JSONResponse
from starlette.types import ExceptionHandler
from cpl import api, auth
from cpl.api.abc.web_app_abc import WebAppABC
from cpl.api.api_module import ApiModule
from cpl.api.error import APIError
from cpl.api.logger import APILogger
from cpl.api.middleware.authentication import AuthenticationMiddleware
@@ -24,21 +25,20 @@ from cpl.api.registry.policy import PolicyRegistry
from cpl.api.registry.route import RouteRegistry
from cpl.api.router import Router
from cpl.api.settings import ApiSettings
from cpl.api.typing import HTTPMethods, PartialMiddleware, PolicyResolver
from cpl.api_module import ApiModule
from cpl.application.abc.application_abc import ApplicationABC
from cpl.api.typing import HTTPMethods, PartialMiddleware, TEndpoint, PolicyInput
from cpl.auth.auth_module import AuthModule
from cpl.auth.permission.permission_module import PermissionsModule
from cpl.core.configuration import Configuration
from cpl.core.configuration.configuration import Configuration
from cpl.dependency.inject import inject
from cpl.dependency.service_provider import ServiceProvider
PolicyInput = Union[dict[str, PolicyResolver], Policy]
from cpl.dependency.typing import Modules
class WebApp(ApplicationABC):
def __init__(self, services: ServiceProvider):
super().__init__(services, [AuthModule, PermissionsModule, ApiModule])
class WebApp(WebAppABC):
def __init__(self, services: ServiceProvider, modules: Modules, required_modules: list[str | object] = None):
WebAppABC.__init__(
self, services, modules, [AuthModule, PermissionsModule, ApiModule] + (required_modules or [])
)
self._app: Starlette | None = None
self._logger = services.get_service(APILogger)
@@ -78,21 +78,17 @@ class WebApp(ApplicationABC):
self._logger.debug(f"Allowed origins: {origins}")
return origins.split(",")
def with_database(self) -> Self:
self.with_migrations()
self.with_seeders()
return self
def with_app(self, app: Starlette) -> Self:
assert app is not None, "app must not be None"
assert isinstance(app, Starlette), "app must be an instance of Starlette"
self._app = app
return self
def _check_for_app(self):
if self._app is not None:
raise ValueError("App is already set, cannot add routes or middleware")
def _validate_policies(self):
for rule in Router.get_authorization_rules():
for policy_name in rule["policies"]:
policy = self._policies.get(policy_name)
if not policy:
self._logger.fatal(f"Authorization policy '{policy_name}' not found")
def with_routes_directory(self, directory: str) -> Self:
self._check_for_app()
assert directory is not None, "directory must not be None"
@@ -107,6 +103,12 @@ class WebApp(ApplicationABC):
return self
def with_app(self, app: Starlette) -> Self:
assert app is not None, "app must not be None"
assert isinstance(app, Starlette), "app must be an instance of Starlette"
self._app = app
return self
def with_routes(
self,
routes: list[ApiRoute],
@@ -136,7 +138,7 @@ class WebApp(ApplicationABC):
def with_route(
self,
path: str,
fn: Callable[[Request], Any],
fn: TEndpoint,
method: HTTPMethods,
authentication: bool = False,
roles: list[str | Enum] = None,
@@ -167,6 +169,30 @@ class WebApp(ApplicationABC):
return self
def with_websocket(
self,
path: str,
fn: TEndpoint,
authentication: bool = False,
roles: list[str | Enum] = None,
permissions: list[str | Enum] = None,
policies: list[str] = None,
match: ValidationMatch = None,
) -> Self:
self._check_for_app()
assert path is not None, "path must not be None"
assert fn is not None, "fn must not be None"
Router.websocket(path, registry=self._routes)(fn)
if authentication:
Router.authenticate()(fn)
if roles or permissions or policies:
Router.authorize(roles, permissions, policies, match)(fn)
return self
def with_middleware(self, middleware: PartialMiddleware) -> Self:
self._check_for_app()
@@ -184,6 +210,7 @@ class WebApp(ApplicationABC):
return self
def with_authorization(self, *policies: list[PolicyInput] | PolicyInput) -> Self:
self._check_for_app()
if policies:
_policies = []
@@ -211,12 +238,8 @@ class WebApp(ApplicationABC):
self.with_middleware(AuthorizationMiddleware)
return self
def _validate_policies(self):
for rule in Router.get_authorization_rules():
for policy_name in rule["policies"]:
policy = self._policies.get(policy_name)
if not policy:
self._logger.fatal(f"Authorization policy '{policy_name}' not found")
async def _log_before_startup(self):
self._logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}")
async def main(self):
self._logger.debug(f"Preparing API")
@@ -241,7 +264,7 @@ class WebApp(ApplicationABC):
else:
app = self._app
self._logger.info(f"Start API on {self._api_settings.host}:{self._api_settings.port}")
await self._log_before_startup()
config = uvicorn.Config(
app, host=self._api_settings.host, port=self._api_settings.port, log_config=None, loop="asyncio"

View File

@@ -8,7 +8,7 @@ class APIError(HTTPException):
status_code = 500
def __init__(self, message: str = ""):
super().__init__(self.status_code, message)
HTTPException.__init__(self, self.status_code, message)
self._message = message
@property

View File

@@ -7,13 +7,13 @@ from cpl.api.logger import APILogger
from cpl.api.middleware.request import get_request
from cpl.api.router import Router
from cpl.auth.keycloak import KeycloakClient
from cpl.auth.schema import AuthUserDao, AuthUser
from cpl.auth.schema import UserDao, User
from cpl.core.ctx import set_user
class AuthenticationMiddleware(ASGIMiddleware):
def __init__(self, app, logger: APILogger, keycloak: KeycloakClient, user_dao: AuthUserDao):
def __init__(self, app, logger: APILogger, keycloak: KeycloakClient, user_dao: UserDao):
ASGIMiddleware.__init__(self, app)
self._logger = logger
@@ -25,6 +25,21 @@ class AuthenticationMiddleware(ASGIMiddleware):
request = get_request()
url = request.url.path
if url not in Router.get_auth_required_routes():
self._logger.trace(f"No authentication required for {url}")
return await self._app(scope, receive, send)
user = getattr(request.state, "user", None)
if not user or user.deleted:
self._logger.debug(f"Unauthorized access to {url}, user missing or deleted")
return await Unauthorized("Unauthorized").asgi_response(scope, receive, send)
return await self._call_next(scope, receive, send)
async def _old_call__(self, scope: Scope, receive: Receive, send: Send):
request = get_request()
url = request.url.path
if url not in Router.get_auth_required_routes():
self._logger.trace(f"No authentication required for {url}")
return await self._app(scope, receive, send)
@@ -57,12 +72,12 @@ class AuthenticationMiddleware(ASGIMiddleware):
return await self._call_next(scope, receive, send)
async def _get_or_crate_user(self, keycloak_id: str) -> AuthUser:
async def _get_or_crate_user(self, keycloak_id: str) -> User:
existing = await self._user_dao.find_by_keycloak_id(keycloak_id)
if existing is not None:
return existing
user = AuthUser(0, keycloak_id)
user = User(0, keycloak_id)
uid = await self._user_dao.create(user)
return await self._user_dao.get_by_id(uid)

View File

@@ -7,13 +7,13 @@ from cpl.api.middleware.request import get_request
from cpl.api.model.validation_match import ValidationMatch
from cpl.api.registry.policy import PolicyRegistry
from cpl.api.router import Router
from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.auth.schema._administration.user_dao import UserDao
from cpl.core.ctx.user_context import get_user
class AuthorizationMiddleware(ASGIMiddleware):
def __init__(self, app, logger: APILogger, policies: PolicyRegistry, user_dao: AuthUserDao):
def __init__(self, app, logger: APILogger, policies: PolicyRegistry, user_dao: UserDao):
ASGIMiddleware.__init__(self, app)
self._logger = logger

View File

@@ -5,10 +5,15 @@ from uuid import uuid4
from starlette.requests import Request
from starlette.types import Scope, Receive, Send
from starlette.websockets import WebSocket
from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware
from cpl.api.logger import APILogger
from cpl.api.typing import TRequest
from cpl.auth.keycloak.keycloak_client import KeycloakClient
from cpl.auth.schema import User
from cpl.auth.schema._administration.user_dao import UserDao
from cpl.core.ctx import set_user
from cpl.dependency.inject import inject
from cpl.dependency.service_provider import ServiceProvider
@@ -17,19 +22,23 @@ _request_context: ContextVar[Union[TRequest, None]] = ContextVar("request", defa
class RequestMiddleware(ASGIMiddleware):
def __init__(self, app, provider: ServiceProvider, logger: APILogger):
def __init__(self, app, provider: ServiceProvider, logger: APILogger, keycloak: KeycloakClient, user_dao: UserDao):
ASGIMiddleware.__init__(self, app)
self._provider = provider
self._logger = logger
self._keycloak = keycloak
self._user_dao = user_dao
self._ctx_token = None
async def __call__(self, scope: Scope, receive: Receive, send: Send):
request = Request(scope, receive, send)
request = Request(scope, receive, send) if scope["type"] != "websocket" else WebSocket(scope, receive, send)
await self.set_request_data(request)
try:
await self._try_set_user(request)
with self._provider.create_scope():
inject(await self._app(scope, receive, send))
finally:
@@ -53,6 +62,37 @@ class RequestMiddleware(ASGIMiddleware):
self._logger.trace(f"Clearing current request: {request.state.request_id}")
_request_context.reset(self._ctx_token)
async def _try_set_user(self, request: Request):
auth_header = request.headers.get("Authorization")
if not auth_header or not auth_header.startswith("Bearer "):
return
token = auth_header.split("Bearer ")[1]
try:
token_info = self._keycloak.introspect(token)
if not token_info.get("active", False):
return
keycloak_id = self._keycloak.get_user_id(token)
if not keycloak_id:
return
user = await self._user_dao.find_by_keycloak_id(keycloak_id)
if not user:
user = User(0, keycloak_id)
uid = await self._user_dao.create(user)
user = await self._user_dao.get_by_id(uid)
if user.deleted:
return
request.state.user = user
set_user(user)
self._logger.trace(f"User {user.id} bound to request {request.state.request_id}")
except Exception as e:
self._logger.debug(f"Silent user binding failed: {e}")
def get_request() -> Optional[TRequest]:
return _request_context.get()

View File

@@ -0,0 +1,31 @@
from typing import Callable
import starlette.routing
class WebSocketRoute:
def __init__(self, path: str, fn: Callable, **kwargs):
self._path = path
self._fn = fn
self._kwargs = kwargs
@property
def name(self) -> str:
return self._fn.__name__
@property
def fn(self) -> Callable:
return self._fn
@property
def path(self) -> str:
return self._path
@property
def kwargs(self) -> dict:
return self._kwargs
def to_starlette(self, *args) -> starlette.routing.WebSocketRoute:
return starlette.routing.WebSocketRoute(self._path, self._fn)

View File

@@ -1,32 +1,35 @@
from typing import Optional
from typing import Optional, Union
from cpl.api.model.api_route import ApiRoute
from cpl.api.model.websocket_route import WebSocketRoute
from cpl.core.abc.registry_abc import RegistryABC
TRoute = Union[ApiRoute, WebSocketRoute]
class RouteRegistry(RegistryABC):
def __init__(self):
RegistryABC.__init__(self)
def extend(self, items: list[ApiRoute]):
def extend(self, items: list[TRoute]):
for policy in items:
self.add(policy)
def add(self, item: ApiRoute):
assert isinstance(item, ApiRoute), "route must be an instance of ApiRoute"
def add(self, item: TRoute):
assert isinstance(item, (ApiRoute, WebSocketRoute)), "route must be an instance of ApiRoute"
if item.path in self._items:
raise ValueError(f"ApiRoute {item.path} is already registered")
self._items[item.path] = item
def set(self, item: ApiRoute):
def set(self, item: TRoute):
assert isinstance(item, ApiRoute), "route must be an instance of ApiRoute"
self._items[item.path] = item
def get(self, key: str) -> Optional[ApiRoute]:
def get(self, key: str) -> Optional[TRoute]:
return self._items.get(key)
def all(self) -> list[ApiRoute]:
def all(self) -> list[TRoute]:
return list(self._items.values())

View File

@@ -91,6 +91,22 @@ class Router:
return inner
@classmethod
def websocket(cls, path: str, registry: RouteRegistry = None, **kwargs):
from cpl.api.model.websocket_route import WebSocketRoute
if not registry:
routes = get_provider().get_service(RouteRegistry)
else:
routes = registry
def inner(fn):
routes.add(WebSocketRoute(path, fn, **kwargs))
setattr(fn, "_route_path", path)
return fn
return inner
@classmethod
def route(cls, path: str, method: HTTPMethods, registry: RouteRegistry = None, **kwargs):
from cpl.api.model.api_route import ApiRoute

View File

@@ -6,7 +6,7 @@ from cpl.core.configuration import ConfigurationModelABC
class ApiSettings(ConfigurationModelABC):
def __init__(self, src: Optional[dict] = None):
super().__init__(src)
ConfigurationModelABC.__init__(self, src)
self.option("host", str, "0.0.0.0")
self.option("port", int, 5000)

View File

@@ -2,13 +2,15 @@ from typing import Union, Literal, Callable, Type, Awaitable
from urllib.request import Request
from starlette.middleware import Middleware
from starlette.responses import Response
from starlette.types import ASGIApp
from starlette.websockets import WebSocket
from cpl.api.abc.asgi_middleware_abc import ASGIMiddleware
from cpl.auth.schema import AuthUser
from cpl.auth.schema import User
TRequest = Union[Request, WebSocket]
TEndpoint = Callable[[TRequest, ...], Awaitable[Response]] | Callable[[TRequest, ...], Response]
HTTPMethods = Literal["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
PartialMiddleware = Union[
ASGIMiddleware,
@@ -16,4 +18,5 @@ PartialMiddleware = Union[
Middleware,
Callable[[ASGIApp], ASGIApp],
]
PolicyResolver = Callable[[AuthUser], bool | Awaitable[bool]]
PolicyResolver = Callable[[User], bool | Awaitable[bool]]
PolicyInput = Union[dict[str, PolicyResolver], "Policy"]

View File

@@ -1,26 +0,0 @@
from cpl.api.registry.policy import PolicyRegistry
from cpl.api.registry.route import RouteRegistry
from cpl.auth.auth_module import AuthModule
from cpl.auth.permission.permission_module import PermissionsModule
from cpl.core.errors import dependency_error
from cpl.database.database_module import DatabaseModule
from cpl.database.model.server_type import ServerType, ServerTypes
from cpl.database.mysql.mysql_module import MySQLModule
from cpl.dependency.module import Module, TModule
class ApiModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
return [AuthModule, DatabaseModule, PermissionsModule]
@staticmethod
def register(collection: "ServiceCollection"):
collection.add_module(DatabaseModule)
collection.add_module(AuthModule)
collection.add_module(PermissionsModule)
collection.add_singleton(PolicyRegistry)
collection.add_singleton(RouteRegistry)

View File

@@ -1,2 +1,2 @@
from .application_builder import ApplicationBuilder
from .host import Host
from .host import Host

View File

@@ -2,10 +2,12 @@ from abc import ABC, abstractmethod
from typing import Callable, Self
from cpl.application.host import Host
from cpl.core.errors import module_dependency_error
from cpl.core.log.log_level import LogLevel
from cpl.core.log.log_settings import LogSettings
from cpl.core.log.logger_abc import LoggerABC
from cpl.dependency.service_provider import ServiceProvider
from cpl.dependency.typing import TModule
def __not_implemented__(package: str, func: Callable):
@@ -20,17 +22,6 @@ class ApplicationABC(ABC):
Contains instances of prepared objects
"""
@abstractmethod
def __init__(self, services: ServiceProvider, required_modules: list[str | object] = None):
self._services = services
self._required_modules = (
[x.__name__ if not isinstance(x, str) else x for x in required_modules] if required_modules else []
)
@property
def required_modules(self) -> list[str]:
return self._required_modules
@classmethod
def extend(cls, name: str | Callable, func: Callable[[Self], Self]):
r"""Extend the Application with a custom method
@@ -47,6 +38,30 @@ class ApplicationABC(ABC):
setattr(cls, name, func)
return cls
@abstractmethod
def __init__(
self, services: ServiceProvider, loaded_modules: set[TModule], required_modules: list[str | object] = None
):
self._services = services
self._modules = loaded_modules
self._required_modules = (
[x.__name__ if not isinstance(x, str) else x for x in required_modules] if required_modules else []
)
def validate_app_required_modules(self):
modules_names = {x.__name__ for x in self._modules}
for module in self._required_modules:
if module in modules_names:
continue
module_dependency_error(
type(self).__name__,
module.__name__ if not isinstance(module, str) else module,
ImportError(
f"Required module '{module}' for application '{self.__class__.__name__}' is not loaded. Load using 'add_module({module})' method."
),
)
def with_logging(self, level: LogLevel = None):
if level is None:
from cpl.core.configuration.configuration import Configuration
@@ -57,14 +72,21 @@ class ApplicationABC(ABC):
logger = self._services.get_service(LoggerABC)
logger.set_level(level)
def with_permissions(self, *args, **kwargs):
__not_implemented__("cpl-auth", self.with_permissions)
def with_permissions(self, *args):
try:
from cpl.auth import AuthModule
def with_migrations(self, *args, **kwargs):
__not_implemented__("cpl-database", self.with_migrations)
AuthModule.with_permissions(*args)
except ImportError:
__not_implemented__("cpl-auth", self.with_permissions)
def with_seeders(self, *args, **kwargs):
__not_implemented__("cpl-database", self.with_seeders)
def with_migrations(self, *args):
try:
from cpl.database.database_module import DatabaseModule
DatabaseModule.with_migrations(self._services, *args)
except ImportError:
__not_implemented__("cpl-database", self.with_migrations)
def with_extension(self, func: Callable[[Self, ...], None], *args, **kwargs):
r"""Extend the Application with a custom method
@@ -84,9 +106,17 @@ class ApplicationABC(ABC):
Called by custom Application.main
"""
try:
for module in self._modules:
if not hasattr(module, "configure") and not callable(getattr(module, "configure")):
continue
module.configure(self._services)
Host.run_app(self.main)
except KeyboardInterrupt:
pass
finally:
logger = self._services.get_service(LoggerABC)
logger.info("Application shutdown")
@abstractmethod
def main(self): ...

View File

@@ -6,7 +6,6 @@ from cpl.application.abc.application_extension_abc import ApplicationExtensionAB
from cpl.application.abc.startup_abc import StartupABC
from cpl.application.abc.startup_extension_abc import StartupExtensionABC
from cpl.application.host import Host
from cpl.core.errors import dependency_error
from cpl.dependency.context import get_provider, use_root_provider
from cpl.dependency.service_collection import ServiceCollection
@@ -43,19 +42,6 @@ class ApplicationBuilder(Generic[TApp]):
return provider
def validate_app_required_modules(self, app: ApplicationABC):
for module in app.required_modules:
if module in self._services.loaded_modules:
continue
dependency_error(
type(app).__name__,
module,
ImportError(
f"Required module '{module}' for application '{app.__class__.__name__}' is not loaded. Load using 'add_module({module})' method."
),
)
def with_startup(self, startup: Type[StartupABC]) -> "ApplicationBuilder":
self._startup = startup
return self
@@ -84,6 +70,6 @@ class ApplicationBuilder(Generic[TApp]):
Host.run(extension.run, self.service_provider)
use_root_provider(self._services.build())
app = self._app(self.service_provider)
self.validate_app_required_modules(app)
app = self._app(self.service_provider, self._services.loaded_modules)
app.validate_app_required_modules()
return app

View File

@@ -57,14 +57,9 @@ class Host:
async def runner():
try:
if asyncio.iscoroutinefunction(func):
app_task = asyncio.create_task(func(*args, **kwargs))
await func(*args, **kwargs)
else:
app_task = cls.get_loop().run_in_executor(None, func, *args, **kwargs)
await asyncio.wait(
[app_task, *cls._tasks.values()],
return_when=asyncio.FIRST_COMPLETED,
)
func(*args, **kwargs)
except (KeyboardInterrupt, asyncio.CancelledError):
pass
finally:
@@ -77,4 +72,4 @@ class Host:
if asyncio.iscoroutinefunction(func):
return cls.get_loop().run_until_complete(func(*args, **kwargs))
return func(*args, **kwargs)
return func(*args, **kwargs)

View File

@@ -1,21 +1,6 @@
from enum import Enum
from typing import Type
from cpl.application.abc import ApplicationABC as _ApplicationABC
from cpl.auth import permission as _permission
from cpl.auth.keycloak.keycloak_admin import KeycloakAdmin as _KeycloakAdmin
from cpl.auth.keycloak.keycloak_client import KeycloakClient as _KeycloakClient
from .auth_module import AuthModule
from .keycloak_settings import KeycloakSettings
from .logger import AuthLogger
def _with_permissions(self: _ApplicationABC, *permissions: Type[Enum]) -> _ApplicationABC:
from cpl.auth.permission.permissions_registry import PermissionsRegistry
for perm in permissions:
PermissionsRegistry.with_enum(perm)
return self
_ApplicationABC.extend(_ApplicationABC.with_permissions, _with_permissions)

View File

@@ -1,14 +1,18 @@
import os
from enum import Enum
from typing import Type
from cpl.auth.keycloak.keycloak_admin import KeycloakAdmin as _KeycloakAdmin
from cpl.auth.keycloak.keycloak_client import KeycloakClient as _KeycloakClient
from cpl.auth.keycloak_settings import KeycloakSettings
from cpl.database.database_module import DatabaseModule
from cpl.database.model.server_type import ServerType, ServerTypes
from cpl.database.service.migration_service import MigrationService
from cpl.dependency.module import Module, TModule
from cpl.dependency.service_collection import ServiceCollection
from cpl.database.mysql.mysql_module import MySQLModule
from cpl.database.postgres.postgres_module import PostgresModule
from cpl.dependency.module.module import Module
from cpl.dependency.service_provider import ServiceProvider
from .keycloak.keycloak_admin import KeycloakAdmin
from .keycloak.keycloak_client import KeycloakClient
from .schema._administration.api_key_dao import ApiKeyDao
from .schema._administration.auth_user_dao import AuthUserDao
from .schema._administration.user_dao import UserDao
from .schema._permission.api_key_permission_dao import ApiKeyPermissionDao
from .schema._permission.permission_dao import PermissionDao
from .schema._permission.role_dao import RoleDao
@@ -17,28 +21,36 @@ from .schema._permission.role_user_dao import RoleUserDao
class AuthModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
return [DatabaseModule]
dependencies = [DatabaseModule, (MySQLModule, PostgresModule)]
config = [KeycloakSettings]
singleton = [
KeycloakClient,
KeycloakAdmin,
UserDao,
ApiKeyDao,
ApiKeyPermissionDao,
PermissionDao,
RoleDao,
RolePermissionDao,
RoleUserDao,
]
scoped = []
transient = []
@staticmethod
def register(collection: ServiceCollection):
collection.add_singleton(_KeycloakClient)
collection.add_singleton(_KeycloakAdmin)
def configure(provider: ServiceProvider):
paths = {
ServerTypes.POSTGRES: "scripts/postgres",
ServerTypes.MYSQL: "scripts/mysql",
}
collection.add_singleton(AuthUserDao)
collection.add_singleton(ApiKeyDao)
collection.add_singleton(ApiKeyPermissionDao)
collection.add_singleton(PermissionDao)
collection.add_singleton(RoleDao)
collection.add_singleton(RolePermissionDao)
collection.add_singleton(RoleUserDao)
DatabaseModule.with_migrations(
provider, str(os.path.join(os.path.dirname(os.path.realpath(__file__)), paths[ServerType.server_type]))
)
provider = collection.build()
migration_service: MigrationService = provider.get_service(MigrationService)
if ServerType.server_type == ServerTypes.POSTGRES:
migration_service.with_directory(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts/postgres")
)
elif ServerType.server_type == ServerTypes.MYSQL:
migration_service.with_directory(os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts/mysql"))
@staticmethod
def with_permissions(*permissions: Type[Enum]):
from cpl.auth.permission.permissions_registry import PermissionsRegistry
for perm in permissions:
PermissionsRegistry.with_enum(perm)

View File

@@ -0,0 +1,4 @@
from .permission_module import PermissionsModule
from .permission_seeder import PermissionSeeder
from .permissions import Permissions
from .permissions_registry import PermissionsRegistry

View File

@@ -2,19 +2,17 @@ from cpl.auth.auth_module import AuthModule
from cpl.auth.permission.permission_seeder import PermissionSeeder
from cpl.auth.permission.permissions import Permissions
from cpl.auth.permission.permissions_registry import PermissionsRegistry
from cpl.auth.permission.role_seeder import RoleSeeder
from cpl.database.abc.data_seeder_abc import DataSeederABC
from cpl.dependency.module import Module, TModule
from cpl.database.database_module import DatabaseModule
from cpl.dependency.module.module import Module
from cpl.dependency.service_collection import ServiceCollection
class PermissionsModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
from cpl.database.database_module import DatabaseModule
return [DatabaseModule, AuthModule]
dependencies = [DatabaseModule, AuthModule]
transient = [(DataSeederABC, PermissionSeeder), (DataSeederABC, RoleSeeder)]
@staticmethod
def register(collection: ServiceCollection):
collection.add_singleton(DataSeederABC, PermissionSeeder)
PermissionsRegistry.with_enum(Permissions)

View File

@@ -1,4 +1,3 @@
from cpl.auth.permission.permissions import Permissions
from cpl.auth.permission.permissions_registry import PermissionsRegistry
from cpl.auth.schema import (
Permission,

View File

@@ -0,0 +1,60 @@
from cpl.auth.schema import (
Role,
RolePermission,
PermissionDao,
RoleDao,
RolePermissionDao,
ApiKeyDao,
ApiKeyPermissionDao,
UserDao,
RoleUserDao,
RoleUser,
)
from cpl.database.abc.data_seeder_abc import DataSeederABC
from cpl.database.logger import DBLogger
class RoleSeeder(DataSeederABC):
def __init__(
self,
logger: DBLogger,
permission_dao: PermissionDao,
role_dao: RoleDao,
role_permission_dao: RolePermissionDao,
api_key_dao: ApiKeyDao,
api_key_permission_dao: ApiKeyPermissionDao,
user_dao: UserDao,
role_user_dao: RoleUserDao,
):
DataSeederABC.__init__(self)
self._logger = logger
self._permission_dao = permission_dao
self._role_dao = role_dao
self._role_permission_dao = role_permission_dao
self._api_key_dao = api_key_dao
self._api_key_permission_dao = api_key_permission_dao
self._user_dao = user_dao
self._role_user_dao = role_user_dao
async def seed(self):
self._logger.info("Creating admin role")
roles = await self._role_dao.get_all()
if len(roles) == 0:
rid = await self._role_dao.create(Role(0, "admin", "Default admin role"))
permissions = await self._permission_dao.get_all()
await self._role_permission_dao.create_many(
[RolePermission(0, rid, permission.id) for permission in permissions]
)
role = await self._role_dao.get_by_name("admin")
if len(await role.users) > 0:
return
users = await self._user_dao.get_all()
if len(users) == 0:
return
user = users[0]
self._logger.warning(f"Assigning admin role to first user {user.id}")
await self._role_user_dao.create(RoleUser(0, role.id, user.id))

View File

@@ -1,7 +1,7 @@
from ._administration.api_key import ApiKey
from ._administration.api_key_dao import ApiKeyDao
from ._administration.auth_user import AuthUser
from ._administration.auth_user_dao import AuthUserDao
from ._administration.user import User
from ._administration.user_dao import UserDao
from ._permission.api_key_permission import ApiKeyPermission
from ._permission.api_key_permission_dao import ApiKeyPermissionDao

View File

@@ -1,6 +1,6 @@
import secrets
from datetime import datetime
from typing import Optional, Union
from typing import Optional, Union, Self
from async_property import async_property
@@ -16,7 +16,7 @@ from cpl.dependency.service_provider import ServiceProvider
_logger = Logger(__name__)
class ApiKey(DbModelABC):
class ApiKey(DbModelABC[Self]):
def __init__(
self,
@@ -25,8 +25,8 @@ class ApiKey(DbModelABC):
key: Union[str, bytes],
deleted: bool = False,
editor_id: Optional[Id] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._identifier = identifier

View File

@@ -1,6 +1,6 @@
import uuid
from datetime import datetime
from typing import Optional
from typing import Optional, Self
from async_property import async_property
from keycloak import KeycloakGetError
@@ -10,18 +10,18 @@ from cpl.auth.permission.permissions import Permissions
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
from cpl.database.logger import DBLogger
from cpl.dependency import ServiceProvider
from cpl.dependency import get_provider
class AuthUser(DbModelABC):
class User(DbModelABC[Self]):
def __init__(
self,
id: SerialId,
keycloak_id: str,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._keycloak_id = keycloak_id
@@ -69,21 +69,21 @@ class AuthUser(DbModelABC):
@async_property
async def permissions(self):
from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.auth.schema._administration.user_dao import UserDao
auth_user_dao: AuthUserDao = get_provider().get_service(AuthUserDao)
return await auth_user_dao.get_permissions(self.id)
user_dao: UserDao = get_provider().get_service(UserDao)
return await user_dao.get_permissions(self.id)
async def has_permission(self, permission: Permissions) -> bool:
from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.auth.schema._administration.user_dao import UserDao
auth_user_dao: AuthUserDao = get_provider().get_service(AuthUserDao)
return await auth_user_dao.has_permission(self.id, permission)
user_dao: UserDao = get_provider().get_service(UserDao)
return await user_dao.has_permission(self.id, permission)
async def anonymize(self):
from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.auth.schema._administration.user_dao import UserDao
auth_user_dao: AuthUserDao = get_provider().get_service(AuthUserDao)
user_dao: UserDao = get_provider().get_service(UserDao)
self._keycloak_id = str(uuid.UUID(int=0))
await auth_user_dao.update(self)
await user_dao.update(self)

View File

@@ -1,19 +1,23 @@
from typing import Optional, Union
from cpl.auth.permission.permissions import Permissions
from cpl.auth.schema._administration.auth_user import AuthUser
from cpl.auth.schema._permission.permission_dao import PermissionDao
from cpl.auth.schema._permission.permission import Permission
from cpl.auth.schema._administration.user import User
from cpl.database import TableManager
from cpl.database.abc import DbModelDaoABC
from cpl.database.external_data_temp_table_builder import ExternalDataTempTableBuilder
from cpl.dependency import ServiceProvider
from cpl.dependency.context import get_provider
class AuthUserDao(DbModelDaoABC[AuthUser]):
class UserDao(DbModelDaoABC[User]):
def __init__(self):
DbModelDaoABC.__init__(self, AuthUser, TableManager.get("auth_users"))
def __init__(self, permission_dao: PermissionDao):
DbModelDaoABC.__init__(self, User, TableManager.get("users"))
self.attribute(AuthUser.keycloak_id, str, db_name="keycloakId")
self._permissions = permission_dao
self.attribute(User.keycloak_id, str)
async def get_users():
return [(x.id, x.username, x.email) for x in await self.get_all()]
@@ -27,11 +31,11 @@ class AuthUserDao(DbModelDaoABC[AuthUser]):
.with_value_getter(get_users)
)
async def get_by_keycloak_id(self, keycloak_id: str) -> AuthUser:
return await self.get_single_by({AuthUser.keycloak_id: keycloak_id})
async def get_by_keycloak_id(self, keycloak_id: str) -> User:
return await self.get_single_by({User.keycloak_id: keycloak_id})
async def find_by_keycloak_id(self, keycloak_id: str) -> Optional[AuthUser]:
return await self.find_single_by({AuthUser.keycloak_id: keycloak_id})
async def find_by_keycloak_id(self, keycloak_id: str) -> Optional[User]:
return await self.find_single_by({User.keycloak_id: keycloak_id})
async def has_permission(self, user_id: int, permission: Union[Permissions, str]) -> bool:
from cpl.auth.schema._permission.permission_dao import PermissionDao
@@ -54,7 +58,7 @@ class AuthUserDao(DbModelDaoABC[AuthUser]):
return result[0]["count"] > 0
async def get_permissions(self, user_id: int) -> list[Permissions]:
async def get_permissions(self, user_id: int) -> list[Permission]:
result = await self._db.select_map(
f"""
SELECT p.*
@@ -66,4 +70,4 @@ class AuthUserDao(DbModelDaoABC[AuthUser]):
AND ru.deleted = FALSE;
"""
)
return [Permissions(p["name"]) for p in result]
return [self._permissions.to_object(x) for x in result]

View File

@@ -15,9 +15,9 @@ class ApiKeyPermission(DbJoinModelABC):
api_key_id: SerialId,
permission_id: SerialId,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbJoinModelABC.__init__(self, api_key_id, permission_id, id, deleted, editor_id, created, updated)
self._api_key_id = api_key_id

View File

@@ -1,20 +1,20 @@
from datetime import datetime
from typing import Optional
from typing import Optional, Self
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
class Permission(DbModelABC):
class Permission(DbModelABC[Self]):
def __init__(
self,
id: SerialId,
name: str,
description: str,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._name = name

View File

@@ -1,24 +1,24 @@
from datetime import datetime
from typing import Optional
from typing import Optional, Self
from async_property import async_property
from cpl.auth.permission.permissions import Permissions
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
from cpl.dependency import ServiceProvider
from cpl.dependency import ServiceProvider, get_provider
class Role(DbModelABC):
class Role(DbModelABC[Self]):
def __init__(
self,
id: SerialId,
name: str,
description: str,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._name = name

View File

@@ -1,46 +1,44 @@
from datetime import datetime
from typing import Optional
from typing import Self
from async_property import async_property
from cpl.core.typing import SerialId
from cpl.database.abc import DbModelABC
from cpl.dependency import ServiceProvider
from cpl.database.abc import DbJoinModelABC
from cpl.dependency import get_provider
class RolePermission(DbModelABC):
class RolePermission(DbJoinModelABC[Self]):
def __init__(
self,
id: SerialId,
role_id: SerialId,
permission_id: SerialId,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)
self._role_id = role_id
self._permission_id = permission_id
DbJoinModelABC.__init__(self, id, role_id, permission_id, deleted, editor_id, created, updated)
@property
def role_id(self) -> int:
return self._role_id
return self._source_id
@async_property
async def role(self):
from cpl.auth.schema._permission.role_dao import RoleDao
role_dao: RoleDao = get_provider().get_service(RoleDao)
return await role_dao.get_by_id(self._role_id)
return await role_dao.get_by_id(self._source_id)
@property
def permission_id(self) -> int:
return self._permission_id
return self._foreign_id
@async_property
async def permission(self):
from cpl.auth.schema._permission.permission_dao import PermissionDao
permission_dao: PermissionDao = get_provider().get_service(PermissionDao)
return await permission_dao.get_by_id(self._permission_id)
return await permission_dao.get_by_id(self._foreign_id)

View File

@@ -5,7 +5,7 @@ from async_property import async_property
from cpl.core.typing import SerialId
from cpl.database.abc import DbJoinModelABC
from cpl.dependency import ServiceProvider
from cpl.dependency import ServiceProvider, get_provider
class RoleUser(DbJoinModelABC):
@@ -15,9 +15,9 @@ class RoleUser(DbJoinModelABC):
user_id: SerialId,
role_id: SerialId,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbJoinModelABC.__init__(self, id, user_id, role_id, deleted, editor_id, created, updated)
self._user_id = user_id
@@ -29,10 +29,10 @@ class RoleUser(DbJoinModelABC):
@async_property
async def user(self):
from cpl.auth.schema._administration.auth_user_dao import AuthUserDao
from cpl.auth.schema._administration.user_dao import UserDao
auth_user_dao: AuthUserDao = get_provider().get_service(AuthUserDao)
return await auth_user_dao.get_by_id(self._user_id)
user_dao: UserDao = get_provider().get_service(UserDao)
return await user_dao.get_by_id(self._user_id)
@property
def role_id(self) -> int:

View File

@@ -1,4 +1,4 @@
CREATE TABLE IF NOT EXISTS administration_auth_users
CREATE TABLE IF NOT EXISTS administration_users
(
id INT AUTO_INCREMENT PRIMARY KEY,
keycloakId CHAR(36) NOT NULL,
@@ -9,10 +9,10 @@ CREATE TABLE IF NOT EXISTS administration_auth_users
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT UC_KeycloakId UNIQUE (keycloakId),
CONSTRAINT FK_EditorId FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_EditorId FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS administration_auth_users_history
CREATE TABLE IF NOT EXISTS administration_users_history
(
id INT NOT NULL,
keycloakId CHAR(36) NOT NULL,
@@ -23,22 +23,22 @@ CREATE TABLE IF NOT EXISTS administration_auth_users_history
updated TIMESTAMP NOT NULL
);
CREATE TRIGGER TR_administration_auth_usersUpdate
CREATE TRIGGER TR_administration_usersUpdate
AFTER UPDATE
ON administration_auth_users
ON administration_users
FOR EACH ROW
BEGIN
INSERT INTO administration_auth_users_history
INSERT INTO administration_users_history
(id, keycloakId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.keycloakId, OLD.deleted, OLD.editorId, OLD.created, NOW());
END;
CREATE TRIGGER TR_administration_auth_usersDelete
CREATE TRIGGER TR_administration_usersDelete
AFTER DELETE
ON administration_auth_users
ON administration_users
FOR EACH ROW
BEGIN
INSERT INTO administration_auth_users_history
INSERT INTO administration_users_history
(id, keycloakId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.keycloakId, 1, OLD.editorId, OLD.created, NOW());
END;

View File

@@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS administration_api_keys
CONSTRAINT UC_Identifier_Key UNIQUE (identifier, keyString),
CONSTRAINT UC_Key UNIQUE (keyString),
CONSTRAINT FK_ApiKeys_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_ApiKeys_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS administration_api_keys_history

View File

@@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS permission_permissions
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT UQ_PermissionName UNIQUE (name),
CONSTRAINT FK_Permissions_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_Permissions_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS permission_permissions_history
@@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS permission_roles
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT UQ_RoleName UNIQUE (name),
CONSTRAINT FK_Roles_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_Roles_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS permission_roles_history
@@ -89,22 +89,22 @@ END;
CREATE TABLE IF NOT EXISTS permission_role_permissions
(
id INT AUTO_INCREMENT PRIMARY KEY,
RoleId INT NOT NULL,
roleId INT NOT NULL,
permissionId INT NOT NULL,
deleted BOOL NOT NULL DEFAULT FALSE,
editorId INT NULL,
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT UQ_RolePermission UNIQUE (RoleId, permissionId),
CONSTRAINT FK_RolePermissions_Role FOREIGN KEY (RoleId) REFERENCES permission_roles (id) ON DELETE CASCADE,
CONSTRAINT UQ_RolePermission UNIQUE (roleId, permissionId),
CONSTRAINT FK_RolePermissions_Role FOREIGN KEY (roleId) REFERENCES permission_roles (id) ON DELETE CASCADE,
CONSTRAINT FK_RolePermissions_Permission FOREIGN KEY (permissionId) REFERENCES permission_permissions (id) ON DELETE CASCADE,
CONSTRAINT FK_RolePermissions_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_RolePermissions_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS permission_role_permissions_history
(
id INT NOT NULL,
RoleId INT NOT NULL,
roleId INT NOT NULL,
permissionId INT NOT NULL,
deleted BOOL NOT NULL,
editorId INT NULL,
@@ -118,8 +118,8 @@ CREATE TRIGGER TR_RolePermissionsUpdate
FOR EACH ROW
BEGIN
INSERT INTO permission_role_permissions_history
(id, RoleId, permissionId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.RoleId, OLD.permissionId, OLD.deleted, OLD.editorId, OLD.created, NOW());
(id, roleId, permissionId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.roleId, OLD.permissionId, OLD.deleted, OLD.editorId, OLD.created, NOW());
END;
CREATE TRIGGER TR_RolePermissionsDelete
@@ -128,52 +128,52 @@ CREATE TRIGGER TR_RolePermissionsDelete
FOR EACH ROW
BEGIN
INSERT INTO permission_role_permissions_history
(id, RoleId, permissionId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.RoleId, OLD.permissionId, 1, OLD.editorId, OLD.created, NOW());
(id, roleId, permissionId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.roleId, OLD.permissionId, 1, OLD.editorId, OLD.created, NOW());
END;
CREATE TABLE IF NOT EXISTS permission_role_auth_users
CREATE TABLE IF NOT EXISTS permission_role_users
(
id INT AUTO_INCREMENT PRIMARY KEY,
RoleId INT NOT NULL,
UserId INT NOT NULL,
roleId INT NOT NULL,
userId INT NOT NULL,
deleted BOOL NOT NULL DEFAULT FALSE,
editorId INT NULL,
created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT UQ_RoleUser UNIQUE (RoleId, UserId),
CONSTRAINT FK_Roleauth_users_Role FOREIGN KEY (RoleId) REFERENCES permission_roles (id) ON DELETE CASCADE,
CONSTRAINT FK_Roleauth_users_User FOREIGN KEY (UserId) REFERENCES administration_auth_users (id) ON DELETE CASCADE,
CONSTRAINT FK_Roleauth_users_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT UQ_RoleUser UNIQUE (roleId, userId),
CONSTRAINT FK_Roleusers_Role FOREIGN KEY (roleId) REFERENCES permission_roles (id) ON DELETE CASCADE,
CONSTRAINT FK_Roleusers_User FOREIGN KEY (userId) REFERENCES administration_users (id) ON DELETE CASCADE,
CONSTRAINT FK_Roleusers_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS permission_role_auth_users_history
CREATE TABLE IF NOT EXISTS permission_role_users_history
(
id INT NOT NULL,
RoleId INT NOT NULL,
UserId INT NOT NULL,
roleId INT NOT NULL,
userId INT NOT NULL,
deleted BOOL NOT NULL,
editorId INT NULL,
created TIMESTAMP NOT NULL,
updated TIMESTAMP NOT NULL
);
CREATE TRIGGER TR_Roleauth_usersUpdate
CREATE TRIGGER TR_RoleusersUpdate
AFTER UPDATE
ON permission_role_auth_users
ON permission_role_users
FOR EACH ROW
BEGIN
INSERT INTO permission_role_auth_users_history
(id, RoleId, UserId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.RoleId, OLD.UserId, OLD.deleted, OLD.editorId, OLD.created, NOW());
INSERT INTO permission_role_users_history
(id, roleId, userId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.roleId, OLD.userId, OLD.deleted, OLD.editorId, OLD.created, NOW());
END;
CREATE TRIGGER TR_Roleauth_usersDelete
CREATE TRIGGER TR_RoleusersDelete
AFTER DELETE
ON permission_role_auth_users
ON permission_role_users
FOR EACH ROW
BEGIN
INSERT INTO permission_role_auth_users_history
(id, RoleId, UserId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.RoleId, OLD.UserId, 1, OLD.editorId, OLD.created, NOW());
INSERT INTO permission_role_users_history
(id, roleId, userId, deleted, editorId, created, updated)
VALUES (OLD.id, OLD.roleId, OLD.userId, 1, OLD.editorId, OLD.created, NOW());
END;

View File

@@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS permission_api_key_permissions
CONSTRAINT UQ_ApiKeyPermission UNIQUE (apiKeyId, permissionId),
CONSTRAINT FK_ApiKeyPermissions_ApiKey FOREIGN KEY (apiKeyId) REFERENCES administration_api_keys (id) ON DELETE CASCADE,
CONSTRAINT FK_ApiKeyPermissions_Permission FOREIGN KEY (permissionId) REFERENCES permission_permissions (id) ON DELETE CASCADE,
CONSTRAINT FK_ApiKeyPermissions_Editor FOREIGN KEY (editorId) REFERENCES administration_auth_users (id)
CONSTRAINT FK_ApiKeyPermissions_Editor FOREIGN KEY (editorId) REFERENCES administration_users (id)
);
CREATE TABLE IF NOT EXISTS permission_api_key_permissions_history

View File

@@ -1,26 +1,26 @@
CREATE SCHEMA IF NOT EXISTS administration;
CREATE TABLE IF NOT EXISTS administration.auth_users
CREATE TABLE IF NOT EXISTS administration.users
(
id SERIAL PRIMARY KEY,
keycloakId UUID NOT NULL,
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UC_KeycloakId UNIQUE (keycloakId)
);
CREATE TABLE IF NOT EXISTS administration.auth_users_history
CREATE TABLE IF NOT EXISTS administration.users_history
(
LIKE administration.auth_users
LIKE administration.users
);
CREATE TRIGGER users_history_trigger
BEFORE INSERT OR UPDATE OR DELETE
ON administration.auth_users
ON administration.users
FOR EACH ROW
EXECUTE FUNCTION public.history_trigger_function();

View File

@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS administration.api_keys
keyString VARCHAR(255) NOT NULL,
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),

View File

@@ -9,7 +9,7 @@ CREATE TABLE permission.permissions
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UQ_PermissionName UNIQUE (name)
@@ -35,7 +35,7 @@ CREATE TABLE permission.roles
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UQ_RoleName UNIQUE (name)
@@ -61,7 +61,7 @@ CREATE TABLE permission.role_permissions
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UQ_RolePermission UNIQUE (RoleId, permissionId)
@@ -83,11 +83,11 @@ CREATE TABLE permission.role_users
(
id SERIAL PRIMARY KEY,
RoleId INT NOT NULL REFERENCES permission.roles (id) ON DELETE CASCADE,
UserId INT NOT NULL REFERENCES administration.auth_users (id) ON DELETE CASCADE,
UserId INT NOT NULL REFERENCES administration.users (id) ON DELETE CASCADE,
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UQ_RoleUser UNIQUE (RoleId, UserId)

View File

@@ -6,7 +6,7 @@ CREATE TABLE permission.api_key_permissions
-- for history
deleted BOOLEAN NOT NULL DEFAULT FALSE,
editorId INT NULL REFERENCES administration.auth_users (id),
editorId INT NULL REFERENCES administration.users (id),
created timestamptz NOT NULL DEFAULT NOW(),
updated timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT UQ_ApiKeyPermission UNIQUE (apiKeyId, permissionId)

View File

@@ -1,13 +1,13 @@
from contextvars import ContextVar
from typing import Optional
from cpl.auth.schema._administration.auth_user import AuthUser
from cpl.auth.schema._administration.user import User
from cpl.dependency import get_provider
_user_context: ContextVar[Optional[AuthUser]] = ContextVar("user", default=None)
_user_context: ContextVar[Optional[User]] = ContextVar("user", default=None)
def set_user(user: Optional[AuthUser]):
def set_user(user: Optional[User]):
from cpl.core.log.logger_abc import LoggerABC
logger = get_provider().get_service(LoggerABC)
@@ -15,5 +15,5 @@ def set_user(user: Optional[AuthUser]):
_user_context.set(user)
def get_user() -> Optional[AuthUser]:
def get_user() -> Optional[User]:
return _user_context.get()

View File

@@ -7,20 +7,21 @@ def dependency_error(src: str, package_name: str, e: ImportError = None) -> None
Console.error(f"'{package_name}' is required to use feature: {src}. Please install it and try again.")
tb = traceback.format_exc()
if not tb.startswith("NoneType: None"):
Console.write_line("->", tb)
Console.error("->", tb)
elif e is not None:
Console.write_line("->", str(e))
Console.error(f"-> {str(e)}")
exit(1)
def module_dependency_error(src: str, module: str, e: ImportError = None) -> None:
Console.error(f"'{module}' is required to use feature: {src}. Please initialize it with `add_module({module})`.")
Console.error(f"'{module}' is required by '{src}'. Please initialize it with `add_module({module})`.")
tb = traceback.format_exc()
if not tb.startswith("NoneType: None"):
Console.write_line("->", tb)
Console.error("->", tb)
elif e is not None:
Console.write_line("->", str(e))
Console.error(f"-> {str(e)}")
exit(1)
exit(1)

View File

@@ -93,14 +93,13 @@ class Logger(LoggerABC):
def _log(self, level: LogLevel, *messages: Messages):
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
formatted_message = self._format_message(level.value, timestamp, *messages)
self._write_log_to_file(level, formatted_message)
self._write_to_console(level, formatted_message)
self._write_log_to_file(level, self._file_format_message(level.value, timestamp, *messages))
self._write_to_console(level, self._console_format_message(level.value, timestamp, *messages))
except Exception as e:
print(f"Error while logging: {e} -> {traceback.format_exc()}")
def _format_message(self, level: str, timestamp, *messages: Messages) -> str:
def _file_format_message(self, level: str, timestamp, *messages: Messages) -> str:
if isinstance(messages, tuple):
messages = list(messages)
@@ -119,6 +118,24 @@ class Logger(LoggerABC):
return message
def _console_format_message(self, level: str, timestamp, *messages: Messages) -> str:
if isinstance(messages, tuple):
messages = list(messages)
if not isinstance(messages, list):
messages = [messages]
messages = [str(message) for message in messages if message is not None]
message = f"[{level.upper():^3}]"
message += f" [{self._file_prefix}]"
if self._source is not None:
message += f" - [{self._source}]"
message += f": {' '.join(messages)}"
return message
def header(self, string: str):
self._log(LogLevel.info, string)

View File

@@ -11,7 +11,10 @@ class LoggerABC(ABC):
def set_level(self, level: LogLevel): ...
@abstractmethod
def _format_message(self, level: str, timestamp, *messages: Messages) -> str: ...
def _file_format_message(self, level: str, timestamp, *messages: Messages) -> str: ...
@abstractmethod
def _console_format_message(self, level: str, timestamp, *messages: Messages) -> str: ...
@abstractmethod
def header(self, string: str):

View File

@@ -1,15 +1,13 @@
import asyncio
import importlib.util
import json
import traceback
from datetime import datetime
from starlette.requests import Request
from cpl.core.log.log_level import LogLevel
from cpl.core.log.logger import Logger
from cpl.core.typing import Source, Messages
from cpl.dependency import get_provider
from cpl.dependency.context import get_provider
class StructuredLogger(Logger):
@@ -21,18 +19,7 @@ class StructuredLogger(Logger):
def log_file(self):
return f"logs/{self._file_prefix}_{datetime.now().strftime('%Y-%m-%d')}.jsonl"
def _log(self, level: LogLevel, *messages: Messages):
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
formatted_message = self._format_message(level.value, timestamp, *messages)
structured_message = self._get_structured_message(level.value, timestamp, formatted_message)
self._write_log_to_file(level, structured_message)
self._write_to_console(level, formatted_message)
except Exception as e:
print(f"Error while logging: {e} -> {traceback.format_exc()}")
def _get_structured_message(self, level: str, timestamp: str, messages: str) -> str:
def _file_format_message(self, level: str, timestamp: str, *messages: Messages) -> str:
structured_message = {
"timestamp": timestamp,
"level": level.upper(),
@@ -81,7 +68,7 @@ class StructuredLogger(Logger):
message["request"] = {
"url": str(request.url),
"method": request.method,
"method": request.method if request.scope == "http" else "websocket",
"scope": self._scope_to_json(request),
}
if isinstance(request, Request) and request.scope == "http":

View File

@@ -1,7 +1,7 @@
import inspect
from typing import Type
from cpl.core.log import LoggerABC, LogLevel
from cpl.core.log import LoggerABC, LogLevel, StructuredLogger
from cpl.core.typing import Messages
from cpl.dependency.inject import inject
from cpl.dependency.service_provider import ServiceProvider
@@ -31,8 +31,11 @@ class WrappedLogger(LoggerABC):
def set_level(self, level: LogLevel):
self._logger.set_level(level)
def _format_message(self, level: str, timestamp, *messages: Messages) -> str:
return self._logger._format_message(level, timestamp, *messages)
def _file_format_message(self, level: str, timestamp, *messages: Messages) -> str:
return self._logger._file_format_message(level, timestamp, *messages)
def _console_format_message(self, level: str, timestamp, *messages: Messages) -> str:
return self._logger._console_format_message(level, timestamp, *messages)
@staticmethod
def _get_source() -> str | None:
@@ -48,6 +51,7 @@ class WrappedLogger(LoggerABC):
ServiceCollection,
WrappedLogger,
WrappedLogger.__subclasses__(),
StructuredLogger,
]
ignore_modules = [x.__module__ for x in ignore_classes if isinstance(x, type)]

View File

@@ -1,2 +1,2 @@
from .time_format_settings import TimeFormatSettings
from .time_format_settings_names_enum import TimeFormatSettingsNamesEnum
from .cron import Cron

View File

@@ -0,0 +1,13 @@
from datetime import datetime
import croniter
class Cron:
def __init__(self, cron_expression: str, start_time: datetime = None):
self._cron_expression = cron_expression
self._start_time = start_time or datetime.now()
self._iter = croniter.croniter(cron_expression, self._start_time)
def next(self) -> datetime:
return self._iter.get_next(datetime)

View File

@@ -13,7 +13,7 @@ class TimeFormatSettings(ConfigurationModelABC):
date_time_format: str = None,
date_time_log_format: str = None,
):
ConfigurationModelABC.__init__(self)
ConfigurationModelABC.__init__(self, readonly=False)
self._date_format: Optional[str] = date_format
self._time_format: Optional[str] = time_format
self._date_time_format: Optional[str] = date_time_format

View File

@@ -1,8 +0,0 @@
from enum import Enum
class TimeFormatSettingsNamesEnum(Enum):
date_format = "DateFormat"
time_format = "TimeFormat"
date_time_format = "DateTimeFormat"
date_time_log_format = "DateTimeLogFormat"

View File

@@ -2,10 +2,6 @@ import os
from cryptography.fernet import Fernet
from cpl.core.log.logger import Logger
_logger = Logger(__name__)
class CredentialManager:
r"""Handles credential encryption and decryption"""
@@ -14,6 +10,8 @@ class CredentialManager:
@classmethod
def with_secret(cls, file: str = None):
from cpl.core.log import Logger
if file is None:
file = ".secret"
@@ -25,12 +23,12 @@ class CredentialManager:
with open(file, "w") as secret_file:
secret_file.write(Fernet.generate_key().decode())
secret_file.close()
_logger.warning("Secret file not found, regenerating")
Logger(__name__).warning("Secret file not found, regenerating")
with open(file, "r") as secret_file:
secret = secret_file.read().strip()
if secret == "" or secret is None:
_logger.fatal("No secret found in .secret file.")
Logger(__name__).fatal("No secret found in .secret file.")
cls._secret = str(secret)

View File

@@ -3,3 +3,4 @@ colorama==0.4.6
tabulate==0.9.0
termcolor==3.1.0
pynput==1.8.1
croniter==6.0.0

View File

@@ -1,34 +1,5 @@
import os
from cpl.application.abc import ApplicationABC as _ApplicationABC
from . import mysql as _mysql
from . import postgres as _postgres
from .database_module import DatabaseModule
from .logger import DBLogger
from .table_manager import TableManager
def _with_migrations(self: _ApplicationABC, *paths: str | list[str]) -> _ApplicationABC:
from cpl.database.service.migration_service import MigrationService
migration_service = self._services.get_service(MigrationService)
migration_service.with_directory(os.path.join(os.path.dirname(os.path.abspath(__file__)), "scripts"))
if isinstance(paths, str):
paths = [paths]
for path in paths:
migration_service.with_directory(path)
return self
def _with_seeders(self: _ApplicationABC) -> _ApplicationABC:
from cpl.database.service.seeder_service import SeederService
from cpl.application.host import Host
seeder_service: SeederService = self._services.get_service(SeederService)
Host.run(seeder_service.seed)
return self
_ApplicationABC.extend(_ApplicationABC.with_migrations, _with_migrations)
_ApplicationABC.extend(_ApplicationABC.with_seeders, _with_seeders)

View File

@@ -1,4 +1,6 @@
from .connection_abc import ConnectionABC
from .data_access_object_abc import DataAccessObjectABC
from .data_seeder_abc import DataSeederABC
from .db_context_abc import DBContextABC
from .db_join_model_abc import DbJoinModelABC
from .db_model_abc import DbModelABC

View File

@@ -14,7 +14,7 @@ from cpl.database.logger import DBLogger
from cpl.database.model.server_type import ServerType, ServerTypes
from cpl.database.postgres.sql_select_builder import SQLSelectBuilder
from cpl.database.typing import T_DBM, Attribute, AttributeFilters, AttributeSorts
from cpl.dependency import get_provider
from cpl.dependency.context import get_provider
class DataAccessObjectABC(ABC, Generic[T_DBM]):
@@ -46,6 +46,10 @@ class DataAccessObjectABC(ABC, Generic[T_DBM]):
def table_name(self) -> str:
return self._table_name
@property
def type(self) -> Type[T_DBM]:
return self._model_type
def has_attribute(self, attr_name: Attribute) -> bool:
"""
Check if the attribute exists in the DAO
@@ -81,7 +85,7 @@ class DataAccessObjectABC(ABC, Generic[T_DBM]):
self.__ignored_attributes.add(attr_name)
if not db_name:
db_name = attr_name.lower().replace("_", "")
db_name = String.to_camel_case(attr_name)
self.__db_names[attr_name] = db_name
self.__db_names[db_name] = db_name
@@ -490,16 +494,16 @@ class DataAccessObjectABC(ABC, Generic[T_DBM]):
table, join_condition = self.__foreign_tables[attr]
builder.with_left_join(table, join_condition)
if filters:
if filters is not None:
await self._build_conditions(builder, filters, external_table_deps)
if sorts:
if sorts is not None:
self._build_sorts(builder, sorts, external_table_deps)
if take:
if take is not None:
builder.with_limit(take)
if skip:
if skip is not None:
builder.with_offset(skip)
for external_table in external_table_deps:

View File

@@ -12,9 +12,9 @@ class DbJoinModelABC[T](DbModelABC[T]):
source_id: Id,
foreign_id: Id,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
DbModelABC.__init__(self, id, deleted, editor_id, created, updated)

View File

@@ -2,7 +2,10 @@ from abc import ABC
from datetime import datetime, timezone
from typing import Optional, Generic
from async_property import async_property
from cpl.core.typing import Id, SerialId, T
from cpl.dependency import get_provider
class DbModelABC(ABC, Generic[T]):
@@ -10,9 +13,9 @@ class DbModelABC(ABC, Generic[T]):
self,
id: Id,
deleted: bool = False,
editor_id: Optional[SerialId] = None,
created: Optional[datetime] = None,
updated: Optional[datetime] = None,
editor_id: SerialId | None = None,
created: datetime | None = None,
updated: datetime | None = None,
):
self._id = id
self._deleted = deleted
@@ -41,14 +44,16 @@ class DbModelABC(ABC, Generic[T]):
def editor_id(self, value: SerialId):
self._editor_id = value
# @async_property
# async def editor(self):
# if self._editor_id is None:
# return None
#
# from data.schemas.administration.user_dao import userDao
#
# return await userDao.get_by_id(self._editor_id)
@async_property
async def editor(self):
if self._editor_id is None:
return None
from cpl.auth.schema import UserDao
user_dao = get_provider().get_service(UserDao)
return await user_dao.get_by_id(self._editor_id)
@property
def created(self) -> datetime:

View File

@@ -2,7 +2,7 @@ from abc import abstractmethod
from datetime import datetime
from typing import Type
from cpl.database import TableManager
from cpl.database.table_manager import TableManager
from cpl.database.abc.data_access_object_abc import DataAccessObjectABC
from cpl.database.abc.db_model_abc import DbModelABC
@@ -18,7 +18,7 @@ class DbModelDaoABC[T_DBM](DataAccessObjectABC[T_DBM]):
self.attribute(DbModelABC.editor_id, int, db_name="editorId", ignore=True) # handled by db trigger
self.reference(
"editor", "id", DbModelABC.editor_id, TableManager.get("auth_users")
"editor", "id", DbModelABC.editor_id, TableManager.get("users")
) # not relevant for updates due to editor_id
self.attribute(DbModelABC.created, datetime, ignore=True) # handled by db trigger

View File

@@ -1,22 +1,33 @@
from cpl.core.errors import module_dependency_error
from cpl.database.model.server_type import ServerType
from cpl.database.model.database_settings import DatabaseSettings
from cpl.database.mysql.mysql_module import MySQLModule
from cpl.database.postgres.postgres_module import PostgresModule
from cpl.database.schema.executed_migration_dao import ExecutedMigrationDao
from cpl.database.service.migration_service import MigrationService
from cpl.database.service.seeder_service import SeederService
from cpl.dependency.module import Module, TModule
from cpl.dependency.service_collection import ServiceCollection
from cpl.dependency.module.module import Module
from cpl.dependency.service_provider import ServiceProvider
class DatabaseModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
if not ServerType.has_server_type:
module_dependency_error(__name__, "MySQLModule or PostgresModule")
dependencies = [(MySQLModule, PostgresModule)]
config = [DatabaseSettings]
singleton = [
ExecutedMigrationDao,
MigrationService,
SeederService,
]
return []
@classmethod
def configure(cls, provider: ServiceProvider): ...
@staticmethod
def register(collection: ServiceCollection):
collection.add_singleton(ExecutedMigrationDao)
collection.add_singleton(MigrationService)
collection.add_singleton(SeederService)
def with_migrations(services: ServiceProvider, *paths: str | list[str]):
from cpl.database.service.migration_service import MigrationService
migration_service = services.get_service(MigrationService)
if isinstance(paths, str):
paths = [paths]
for path in paths:
migration_service.with_directory(path)

View File

@@ -1,6 +1,6 @@
from typing import Optional
from cpl.core.configuration import Configuration
from cpl.core.configuration.configuration import Configuration
from cpl.core.configuration.configuration_model_abc import ConfigurationModelABC
@@ -21,4 +21,4 @@ class DatabaseSettings(ConfigurationModelABC):
self.option("use_unicode", bool, False)
self.option("buffered", bool, False)
self.option("auth_plugin", str, "caching_sha2_password")
self.option("ssl_disabled", bool, False)
self.option("ssl_disabled", bool, True)

View File

@@ -0,0 +1,4 @@
from .connection import DatabaseConnection
from .db_context import DBContext
from .mysql_module import MySQLModule
from .mysql_pool import MySQLPool

View File

@@ -1,19 +1,17 @@
from cpl.core.configuration.configuration import Configuration
from cpl.database.abc.db_context_abc import DBContextABC
from cpl.database.model.database_settings import DatabaseSettings
from cpl.database.model.server_type import ServerTypes, ServerType
from cpl.database.mysql.db_context import DBContext
from cpl.dependency.module import Module, TModule
from cpl.dependency.module.module import Module
from cpl.dependency.service_collection import ServiceCollection
class MySQLModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
return []
config = [DatabaseSettings]
singleton = [(DBContextABC, DBContext)]
@staticmethod
def register(collection: ServiceCollection):
ServerType.set_server_type(ServerTypes(ServerTypes.MYSQL.value))
Configuration.set("DB_DEFAULT_PORT", 3306)
collection.add_singleton(DBContextABC, DBContext)

View File

@@ -1,6 +1,8 @@
from typing import Optional, Any
import sqlparse
import asyncio
from mysql.connector import errors, PoolError
from mysql.connector.aio import MySQLConnectionPool
from cpl.core.environment import Environment
@@ -10,7 +12,6 @@ from cpl.dependency.context import get_provider
class MySQLPool:
def __init__(self, database_settings: DatabaseSettings):
self._dbconfig = {
"host": database_settings.host,
@@ -22,62 +23,90 @@ class MySQLPool:
"use_unicode": database_settings.use_unicode,
"buffered": database_settings.buffered,
"auth_plugin": database_settings.auth_plugin,
"ssl_disabled": False,
"ssl_disabled": database_settings.ssl_disabled,
}
self._pool: Optional[MySQLConnectionPool] = None
self._pool_lock = asyncio.Lock()
async def _get_pool(self):
async def _get_pool(self) -> MySQLConnectionPool:
if self._pool is None:
self._pool = MySQLConnectionPool(
pool_name="mypool", pool_size=Environment.get("DB_POOL_SIZE", int, 1), **self._dbconfig
)
await self._pool.initialize_pool()
async with self._pool_lock:
if self._pool is None:
try:
self._pool = MySQLConnectionPool(
pool_name="cplpool",
pool_size=Environment.get("DB_POOL_SIZE", int, 20),
**self._dbconfig,
)
await self._pool.initialize_pool()
con = await self._pool.get_connection()
try:
async with await con.cursor() as cursor:
await cursor.execute("SELECT 1")
await cursor.fetchall()
except Exception as e:
logger = get_provider().get_service(DBLogger)
logger.fatal(f"Error connecting to the database: {e}")
finally:
await con.close()
# Testverbindung (Ping)
con = await self._pool.get_connection()
try:
async with await con.cursor() as cursor:
await cursor.execute("SELECT 1")
await cursor.fetchall()
finally:
await con.close()
except Exception as e:
logger = get_provider().get_service(DBLogger)
logger.fatal("Error connecting to the database", e)
raise
return self._pool
async def _get_connection(self, retries: int = 3, delay: float = 0.5):
"""Stabiler Connection-Getter mit Retry und Ping"""
pool = await self._get_pool()
for attempt in range(retries):
try:
con = await pool.get_connection()
# Verbindungs-Check (Ping)
try:
async with await con.cursor() as cursor:
await cursor.execute("SELECT 1")
await cursor.fetchall()
except errors.OperationalError:
await con.close()
raise
return con
except PoolError:
if attempt == retries - 1:
raise
await asyncio.sleep(delay)
@staticmethod
async def _exec_sql(cursor: Any, query: str, args=None, multi=True):
result = []
if multi:
queries = [str(stmt).strip() for stmt in sqlparse.parse(query) if str(stmt).strip()]
for q in queries:
if q.strip() == "":
continue
await cursor.execute(q, args)
if cursor.description is not None:
result = await cursor.fetchall()
if q:
await cursor.execute(q, args)
if cursor.description is not None:
result = await cursor.fetchall()
else:
await cursor.execute(query, args)
if cursor.description is not None:
result = await cursor.fetchall()
return result
async def execute(self, query: str, args=None, multi=True) -> list[list]:
pool = await self._get_pool()
con = await pool.get_connection()
async def execute(self, query: str, args=None, multi=True) -> list[str]:
con = await self._get_connection()
try:
async with await con.cursor() as cursor:
result = await self._exec_sql(cursor, query, args, multi)
res = await self._exec_sql(cursor, query, args, multi)
await con.commit()
return result
return list(res)
finally:
await con.close()
async def select(self, query: str, args=None, multi=True) -> list[str]:
pool = await self._get_pool()
con = await pool.get_connection()
con = await self._get_connection()
try:
async with await con.cursor() as cursor:
res = await self._exec_sql(cursor, query, args, multi)
@@ -86,11 +115,17 @@ class MySQLPool:
await con.close()
async def select_map(self, query: str, args=None, multi=True) -> list[dict]:
pool = await self._get_pool()
con = await pool.get_connection()
con = await self._get_connection()
try:
async with await con.cursor(dictionary=True) as cursor:
res = await self._exec_sql(cursor, query, args, multi)
return list(res)
decoded_res = []
for row in res:
decoded_row = {
k: (v.decode("utf-8") if isinstance(v, (bytes, bytearray)) else v) for k, v in row.items()
}
decoded_res.append(decoded_row)
return decoded_res
finally:
await con.close()

View File

@@ -0,0 +1,4 @@
from .db_context import DBContext
from .postgres_module import PostgresModule
from .postgres_pool import PostgresPool
from .sql_select_builder import SQLSelectBuilder

View File

@@ -1,20 +1,17 @@
from cpl.core.configuration.configuration import Configuration
from cpl.database.abc.db_context_abc import DBContextABC
from cpl.database.database_module import DatabaseModule
from cpl.database.model.database_settings import DatabaseSettings
from cpl.database.model.server_type import ServerTypes, ServerType
from cpl.database.postgres.db_context import DBContext
from cpl.dependency.module import Module, TModule
from cpl.dependency.module.module import Module
from cpl.dependency.service_collection import ServiceCollection
class PostgresModule(Module):
@staticmethod
def dependencies() -> list[TModule]:
return [DatabaseModule]
config = [DatabaseSettings]
singleton = [(DBContextABC, DBContext)]
@staticmethod
def register(collection: ServiceCollection):
ServerType.set_server_type(ServerTypes(ServerTypes.POSTGRES.value))
Configuration.set("DB_DEFAULT_PORT", 5432)
collection.add_singleton(DBContextABC, DBContext)

View File

@@ -7,7 +7,7 @@ from psycopg_pool import AsyncConnectionPool, PoolTimeout
from cpl.core.environment import Environment
from cpl.database.logger import DBLogger
from cpl.database.model import DatabaseSettings
from cpl.dependency import ServiceProvider
from cpl.dependency.context import get_provider
class PostgresPool:
@@ -27,19 +27,20 @@ class PostgresPool:
self._pool: Optional[AsyncConnectionPool] = None
async def _get_pool(self):
if self._pool is None:
if self._pool is None or self._pool.closed:
pool = AsyncConnectionPool(
conninfo=self._conninfo, open=False, min_size=1, max_size=Environment.get("DB_POOL_SIZE", int, 1)
)
await pool.open()
try:
await pool.open()
async with pool.connection() as con:
await pool.check_connection(con)
self._pool = pool
except PoolTimeout as e:
await pool.close()
logger = get_provider().get_service(DBLogger)
logger.fatal(f"Failed to connect to the database", e)
self._pool = pool
return self._pool

View File

@@ -0,0 +1,2 @@
from .executed_migration import ExecutedMigration
from .executed_migration_dao import ExecutedMigrationDao

View File

@@ -1,15 +1,15 @@
from datetime import datetime
from typing import Optional
from typing import Optional, Self
from cpl.database.abc import DbModelABC
class ExecutedMigration(DbModelABC):
class ExecutedMigration(DbModelABC[Self]):
def __init__(
self,
migration_id: str,
created: Optional[datetime] = None,
modified: Optional[datetime] = None,
created: datetime | None = None,
modified: datetime | None = None,
):
DbModelABC.__init__(self, migration_id, False, created, modified)

View File

@@ -1,4 +1,4 @@
from cpl.database import TableManager
from cpl.database.table_manager import TableManager
from cpl.database.abc.data_access_object_abc import DataAccessObjectABC
from cpl.database.schema.executed_migration import ExecutedMigration

View File

@@ -0,0 +1,2 @@
from .seeder_service import SeederService
from .migration_service import MigrationService

View File

@@ -1,19 +1,18 @@
import glob
import os
from cpl.database.abc import DBContextABC
from cpl.database.abc.db_context_abc import DBContextABC
from cpl.database.logger import DBLogger
from cpl.database.model import Migration
from cpl.database.model.migration import Migration
from cpl.database.model.server_type import ServerType, ServerTypes
from cpl.database.schema.executed_migration import ExecutedMigration
from cpl.database.schema.executed_migration_dao import ExecutedMigrationDao
from cpl.dependency.hosted.startup_task import StartupTask
from cpl.dependency.hosted import StartupTask
class MigrationService(StartupTask):
def __init__(self, logger: DBLogger, db: DBContextABC, executed_migration_dao: ExecutedMigrationDao):
StartupTask.__init__(self)
self._logger = logger
self._db = db
self._executed_migration_dao = executed_migration_dao
@@ -24,12 +23,23 @@ class MigrationService(StartupTask):
self.with_directory(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../scripts/postgres"))
elif ServerType.server_type == ServerTypes.MYSQL:
self.with_directory(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../scripts/mysql"))
else:
raise Exception("Unsupported database type")
async def run(self):
await self._execute(self._load_scripts())
def with_directory(self, directory: str) -> "MigrationService":
self._script_directories.append(directory)
cpl_rel_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../..")
cpl_abs_path = os.path.abspath(cpl_rel_path)
if directory.startswith(cpl_abs_path) or os.path.abspath(directory).startswith(cpl_abs_path):
if len(self._script_directories) > 0:
self._script_directories.insert(1, directory)
else:
self._script_directories.append(directory)
else:
self._script_directories.append(directory)
return self
async def _get_migration_history(self) -> list[ExecutedMigration]:

View File

@@ -1,15 +1,17 @@
from cpl.database.abc.data_seeder_abc import DataSeederABC
from cpl.database.logger import DBLogger
from cpl.dependency import ServiceProvider
from cpl.dependency.hosted import StartupTask
class SeederService:
class SeederService(StartupTask):
def __init__(self, provider: ServiceProvider):
StartupTask.__init__(self)
self._provider = provider
self._logger = provider.get_service(DBLogger)
async def seed(self):
async def run(self):
seeders = self._provider.get_services(DataSeederABC)
self._logger.debug(f"Found {len(seeders)} seeders")
for seeder in seeders:

View File

@@ -7,9 +7,9 @@ class TableManager:
ServerTypes.POSTGRES: "system._executed_migrations",
ServerTypes.MYSQL: "system__executed_migrations",
},
"auth_users": {
ServerTypes.POSTGRES: "administration.auth_users",
ServerTypes.MYSQL: "administration_auth_users",
"users": {
ServerTypes.POSTGRES: "administration.users",
ServerTypes.MYSQL: "administration_users",
},
"api_keys": {
ServerTypes.POSTGRES: "administration.api_keys",
@@ -33,7 +33,7 @@ class TableManager:
},
"role_users": {
ServerTypes.POSTGRES: "permission.role_users",
ServerTypes.MYSQL: "permission_role_auth_users",
ServerTypes.MYSQL: "permission_role_users",
},
}

View File

@@ -0,0 +1,10 @@
from abc import abstractmethod, ABC
from typing import Any, AsyncGenerator
class EventBusABC(ABC):
@abstractmethod
async def publish(self, channel: str, event: Any) -> None: ...
@abstractmethod
async def subscribe(self, channel: str) -> AsyncGenerator[Any, None]: ...

View File

@@ -1,2 +1,3 @@
from .hosted_service import HostedService
from .startup_task import StartupTask
from .startup_task import StartupTask
from .cronjob import CronjobABC

Some files were not shown because too many files have changed in this diff Show More