Improved data integrity service loop #446

This commit is contained in:
Sven Heidemann 2023-12-03 13:14:10 +01:00
parent 4233e089f8
commit 94732b7227
6 changed files with 225 additions and 348 deletions

View File

@ -1,5 +1,5 @@
from datetime import datetime
from typing import Callable
from typing import Callable, Union
import discord
from cpl_core.configuration import ConfigurationABC
@ -25,6 +25,7 @@ from bot_data.abc.user_joined_voice_channel_repository_abc import (
from bot_data.abc.user_message_count_per_hour_repository_abc import (
UserMessageCountPerHourRepositoryABC,
)
from bot_data.model.auto_role_rule import AutoRoleRule
from bot_data.model.server_config import ServerConfig
from bot_data.model.user import User
from bot_data.model.user_message_count_per_hour import UserMessageCountPerHour
@ -218,7 +219,7 @@ class ClientUtilsService(ClientUtilsABC):
guild: Guild = self._bot.guilds.where(lambda g: g == guild).single()
channel = guild.get_channel(discord_channel_id)
message = await channel.fetch_message(discord_message_id)
emoji = List(discord.Emoji, guild.emojis).where(lambda x: x.name == rule.emoji_name).single()
emoji = List(discord.Emoji, [x for x in guild.emojis if x.name == rule.emoji_name]).single()
if emoji is None:
self._logger.debug(__name__, f"Emoji {rule.emoji_name} not found")

View File

@ -1,9 +1,8 @@
from datetime import datetime, timedelta
from typing import Union
import discord
from cpl_core.configuration import ConfigurationABC
from cpl_core.database.context import DatabaseContextABC
from cpl_discord.container import Member, Guild
from cpl_discord.service import DiscordBotServiceABC
from bot_core.abc.client_utils_abc import ClientUtilsABC
@ -66,356 +65,255 @@ class DataIntegrityService:
self._is_for_shutdown = False
def _check_known_users(self):
self._logger.debug(__name__, f"Start checking KnownUsers table, {len(self._bot.users)}")
for u in self._bot.users:
u: discord.User = u
try:
if u.bot:
self._logger.trace(__name__, f"User {u.id} is ignored, because its a bot")
continue
async def check_data_integrity(self, is_for_shutdown=False):
self._logger.info(__name__, f"Data integrity service started")
if is_for_shutdown != self._is_for_shutdown:
self._is_for_shutdown = is_for_shutdown
user = self._known_users.find_user_by_discord_id(u.id)
if user is not None:
continue
try:
for g in self._bot.guilds:
self._logger.debug(__name__, f"Start check for server: {g.id}")
s = self._get_or_create_server(g)
self._logger.debug(__name__, f"Start check for clients")
self._check_clients(g.id, s)
self._logger.warn(__name__, f"Unknown user: {u.id}")
self._logger.debug(__name__, f"Add user: {u.id}")
self._known_users.add_user(KnownUser(u.id))
self._db_context.save_changes()
for m in [m for m in g.members if not m.bot]:
await self._check_default_role(m)
self._check_known_user(m.id)
user = self._known_users.find_user_by_discord_id(u.id)
if user is None:
self._logger.fatal(__name__, f"Cannot add user: {u.id}")
self._logger.debug(__name__, f"Start check for member: {g.id}@{m.id}")
u = self._get_or_create_user(s, m.id)
self._logger.debug(__name__, f"Added user: {u.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get user", e)
self._logger.debug(__name__, f"Start check for user joined server: {g.id}@{m.id}")
self._check_user_join(g, m, u)
def check_servers(self):
self._logger.debug(__name__, f"Start checking Servers table")
for g in self._bot.guilds:
g: discord.Guild = g
try:
server = self._servers.find_server_by_discord_id(g.id)
if server is not None:
continue
self._logger.debug(__name__, f"Start check for user joined voice channels: {g.id}@{m.id}")
self._check_user_joined_vc(g.id, m, u)
self._logger.warn(__name__, f"Server not found in database: {g.id}")
self._logger.debug(__name__, f"Add server: {g.id}")
self._servers.add_server(Server(g.id))
self._db_context.save_changes()
self._logger.debug(__name__, f"Start check for user joined game servers: {g.id}@{m.id}")
self._check_user_joined_gs(g.id, m.id, u)
server = self._servers.find_server_by_discord_id(g.id)
if server is None:
self._logger.fatal(__name__, f"Cannot add server: {g.id}")
self._logger.debug(__name__, f"Start check for user got achievements: {g.id}@{m.id}")
await self._check_for_user_achievements(u)
self._logger.debug(__name__, f"Added server: {g.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get server", e)
for m in [m for m in g.members if m.bot]:
u = self._users.find_user_by_discord_id_and_server_id(m.id, s.id)
if u is None:
continue
self._remove_bot(u)
self._logger.info(__name__, f"Data integrity service finished")
except Exception as e:
self._logger.fatal(__name__, f"Checking data integrity failed", e)
def _get_or_create_server(self, guild: Guild) -> Server:
try:
server = self._servers.find_server_by_discord_id(guild.id)
if server is not None:
return server
self._logger.warn(__name__, f"Server not found in database: {guild.id}")
self._logger.debug(__name__, f"Add server: {guild.id}")
self._servers.add_server(Server(guild.id))
self._db_context.save_changes()
server = self._servers.find_server_by_discord_id(guild.id)
if server is None:
self._logger.fatal(__name__, f"Cannot add server: {guild.id}")
self._logger.trace(__name__, f"Added server: {guild.id}")
return server
except Exception as e:
self._logger.error(__name__, f"Cannot get server", e)
def _check_clients(self, guild_id: int, server: Server):
try:
client = self._clients.find_client_by_server_id(server.id)
if client is not None:
return
self._logger.warn(
__name__,
f"Client for server {guild_id} not found in database: {self._bot.user.id}",
)
self._logger.debug(__name__, f"Add client: {self._bot.user.id}")
self._clients.add_client(Client(self._bot.user.id, 0, 0, 0, 0, 0, server))
self._db_context.save_changes()
client = self._clients.find_client_by_server_id(server.id)
if client is None:
self._logger.fatal(
__name__,
f"Cannot add client {self._bot.user.id} for server {guild_id}",
)
self._logger.trace(__name__, f"Added client: {guild_id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get client", e)
results = self._servers.get_servers()
if results is None or len(results) == 0:
self._logger.error(__name__, f"Table Servers is empty!")
def _check_clients(self):
self._logger.debug(__name__, f"Start checking Clients table")
for g in self._bot.guilds:
g: discord.Guild = g
try:
server: Server = self._servers.find_server_by_discord_id(g.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {g.id}")
def _check_known_user(self, member_id: int):
try:
if self._known_users.find_user_by_discord_id(member_id) is not None:
return
client = self._clients.find_client_by_server_id(server.id)
if client is not None:
self._logger.warn(__name__, f"Unknown user: {member_id}")
self._logger.trace(__name__, f"Add known user: {member_id}")
self._known_users.add_user(KnownUser(member_id))
self._db_context.save_changes()
user = self._known_users.find_user_by_discord_id(member_id)
if user is None:
self._logger.fatal(__name__, f"Cannot add user: {member_id}")
self._logger.trace(__name__, f"Added known user: {member_id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get user", e)
def _get_or_create_user(self, server: Server, member_id: int) -> User:
try:
user = self._users.find_user_by_discord_id_and_server_id(member_id, server.id)
if user is not None:
return user
self._logger.warn(__name__, f"User not found in database: {member_id}")
self._logger.debug(__name__, f"Add user: {member_id}")
self._users.add_user(User(member_id, 0, 0, 0, None, server))
self._db_context.save_changes()
self._logger.trace(__name__, f"Added User: {member_id}")
return self._users.get_user_by_discord_id_and_server_id(member_id, server.id)
except Exception as e:
self._logger.error(__name__, f"Cannot get User", e)
results = self._users.get_users()
if results is None or len(results) == 0:
self._logger.error(__name__, f"Table Users is empty!")
def _check_user_join(self, guild: Guild, member: Member, user: User):
try:
join = self._user_joins.find_active_user_joined_server_by_user_id(user.id)
if join is not None:
return
self._logger.warn(
__name__,
f"Active UserJoinedServer not found in database: {guild.id}:{member.id}@{member.joined_at}",
)
self._logger.debug(
__name__,
f"Add UserJoinedServer: {guild.id}:{member.id}@{member.joined_at}",
)
self._user_joins.add_user_joined_server(UserJoinedServer(user, self._dtp.transform(member.joined_at), None))
self._db_context.save_changes()
self._logger.trace(__name__, f"Added UserJoinedServer: {member.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedServer", e)
try:
joins = self._user_joins.get_user_joined_servers()
for join in [x for x in joins if x.user.server.discord_id == guild.id and x.leaved_on is None]:
dc_user = guild.get_member(join.user.discord_id)
if dc_user is not None:
continue
self._logger.warn(
__name__,
f"Client for server {g.id} not found in database: {self._bot.user.id}",
f"User {join.user.discord_id} already left the server.",
)
self._logger.debug(__name__, f"Add client: {self._bot.user.id}")
self._clients.add_client(Client(self._bot.user.id, 0, 0, 0, 0, 0, server))
join.leaved_on = datetime.now()
self._user_joins.update_user_joined_server(join)
self._db_context.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot update UserJoinedServer", e)
client = self._clients.find_client_by_server_id(server.id)
if client is None:
self._logger.fatal(
__name__,
f"Cannot add client {self._bot.user.id} for server {g.id}",
)
def _check_user_joined_vc(self, guild_id: int, member: Member, user: User):
settings: ServerConfig = self._config.get_configuration(f"ServerConfig_{guild_id}")
self._logger.debug(__name__, f"Added client: {g.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get client", e)
try:
# close open voice states
joins = self._user_joins_vc.find_active_user_joined_voice_channels_by_user_id(user.id)
if joins is None or len(joins) == 0:
return
results = self._servers.get_servers()
if results is None or len(results) == 0:
self._logger.error(__name__, f"Table Servers is empty!")
def _check_users(self):
self._logger.debug(__name__, f"Start checking Users table")
for g in self._bot.guilds:
g: discord.Guild = g
try:
server = self._servers.find_server_by_discord_id(g.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {g.id}")
for u in g.members:
u: Union[discord.Member, discord.User] = u
if u.bot:
self._logger.trace(__name__, f"User {u.id} is ignored, because its a bot")
continue
user = self._users.find_user_by_discord_id_and_server_id(u.id, server.id)
if user is not None:
continue
self._logger.warn(__name__, f"User not found in database: {u.id}")
self._logger.debug(__name__, f"Add user: {u.id}")
self._users.add_user(User(u.id, 0, 0, 0, None, server))
self._db_context.save_changes()
self._logger.debug(__name__, f"Added User: {u.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get User", e)
results = self._users.get_users()
if results is None or len(results) == 0:
self._logger.error(__name__, f"Table Users is empty!")
def _check_user_joins(self):
self._logger.debug(__name__, f"Start checking UserJoinedServers table")
for guild in self._bot.guilds:
guild: discord.Guild = guild
server = self._servers.find_server_by_discord_id(guild.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {guild.id}")
try:
for u in guild.members:
u: discord.User = u
if u.bot:
self._logger.trace(__name__, f"User {u.id} is ignored, because its a bot")
continue
user = self._users.find_user_by_discord_id_and_server_id(u.id, server.id)
if user is None:
self._logger.fatal(__name__, f"User not found in database: {u.id}")
join = self._user_joins.find_active_user_joined_server_by_user_id(user.id)
if join is not None:
continue
m: discord.Member = u
self._logger.warn(
__name__,
f"Active UserJoinedServer not found in database: {guild.id}:{u.id}@{m.joined_at}",
)
self._logger.debug(
__name__,
f"Add UserJoinedServer: {guild.id}:{u.id}@{m.joined_at}",
)
self._user_joins.add_user_joined_server(
UserJoinedServer(user, self._dtp.transform(m.joined_at), None)
)
self._db_context.save_changes()
self._logger.debug(__name__, f"Added UserJoinedServer: {u.id}")
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedServer", e)
results = self._users.get_users()
if results is None or len(results) == 0:
self._logger.error(__name__, f"Table Users is empty!")
joins = self._user_joins.get_user_joined_servers()
for join in joins:
join: UserJoinedServer = join
if join.user.server.discord_id != guild.id:
continue
self._logger.warn(
__name__,
f"Active UserJoinedVoiceChannel found in database: {guild_id}:{member.id}@{join.joined_on}",
)
join.leaved_on = datetime.now()
if join.leaved_on is not None:
continue
if ((join.leaved_on - join.joined_on).total_seconds() / 60 / 60) > settings.max_voice_state_hours:
join.leaved_on = join.joined_on + timedelta(hours=settings.max_voice_state_hours)
dc_user = guild.get_member(join.user.discord_id)
if dc_user is None:
self._logger.warn(
__name__,
f"User {join.user.discord_id} already left the server.",
)
join.leaved_on = datetime.now()
self._user_joins.update_user_joined_server(join)
self._user_joins_vc.update_user_joined_voice_channel(join)
self._db_context.save_changes()
if self._is_for_shutdown:
user.xp += round(join.time * settings.xp_per_ontime_hour)
self._users.update_user(user)
def _check_user_joins_vc(self):
self._logger.debug(__name__, f"Start checking UserJoinedVoiceChannel table")
for guild in self._bot.guilds:
guild: discord.Guild = guild
settings: ServerConfig = self._config.get_configuration(f"ServerConfig_{guild.id}")
server = self._servers.find_server_by_discord_id(guild.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {guild.id}")
try:
# close open voice states
for member in guild.members:
if member.bot:
self._logger.trace(__name__, f"User {member.id} is ignored, because its a bot")
continue
user = self._users.find_user_by_discord_id_and_server_id(member.id, server.id)
if user is None:
self._logger.fatal(__name__, f"User not found in database: {member.id}")
joins = self._user_joins_vc.find_active_user_joined_voice_channels_by_user_id(user.id)
if joins is None or len(joins) == 0:
continue
for join in joins:
self._logger.warn(
__name__,
f"Active UserJoinedVoiceChannel found in database: {guild.id}:{member.id}@{join.joined_on}",
)
join.leaved_on = datetime.now()
if (
(join.leaved_on - join.joined_on).total_seconds() / 60 / 60
) > settings.max_voice_state_hours:
join.leaved_on = join.joined_on + timedelta(hours=settings.max_voice_state_hours)
self._user_joins_vc.update_user_joined_voice_channel(join)
if self._is_for_shutdown:
user.xp += round(join.time * settings.xp_per_ontime_hour)
self._users.update_user(user)
self._db_context.save_changes()
if self._is_for_shutdown:
return
# add open voice states
for member in guild.members:
if member.bot:
self._logger.trace(__name__, f"User {member.id} is ignored, because its a bot")
continue
if member.voice is None or member.voice.channel.id in settings.afk_channel_ids:
continue
user = self._users.find_user_by_discord_id_and_server_id(member.id, server.id)
if user is None:
self._logger.fatal(__name__, f"User not found in database: {member.id}")
join = UserJoinedVoiceChannel(user, member.voice.channel.id, datetime.now())
self._user_joins_vc.add_user_joined_voice_channel(join)
self._db_context.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedVoiceChannel", e)
def _check_user_joined_gs(self):
self._logger.debug(__name__, f"Start checking UserJoinedGameServer table")
for guild in self._bot.guilds:
guild: discord.Guild = guild
server = self._servers.find_server_by_discord_id(guild.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {guild.id}")
try:
for member in guild.members:
if member.bot:
self._logger.trace(__name__, f"User {member.id} is ignored, because its a bot")
continue
user = self._users.find_user_by_discord_id_and_server_id(member.id, server.id)
if user is None:
self._logger.fatal(__name__, f"User not found in database: {member.id}")
joins = self._user_joined_gs.find_active_user_joined_game_servers_by_user_id(user.id)
if joins is None or len(joins) == 0:
continue
for join in joins:
self._logger.warn(
__name__,
f"Active UserJoinedGameServer found in database: {guild.id}:{member.id}@{join.joined_on}",
)
join.leaved_on = datetime.now()
settings: ServerConfig = self._config.get_configuration(f"ServerConfig_{guild.id}")
if (
(join.leaved_on - join.joined_on).total_seconds() / 60 / 60
) > settings.max_voice_state_hours:
join.leaved_on = join.joined_on + timedelta(hours=settings.max_voice_state_hours)
self._user_joined_gs.update_user_joined_game_server(join)
if self._is_for_shutdown:
user.xp += round(join.time * settings.xp_per_ontime_hour)
self._users.update_user(user)
self._db_context.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedGameServer", e)
async def _check_for_user_achievements(self):
self._logger.debug(__name__, f"Start checking UserGotAchievement table")
for guild in self._bot.guilds:
server = self._servers.find_server_by_discord_id(guild.id)
if server is None:
self._logger.fatal(__name__, f"Server not found in database: {guild.id}")
for member in guild.members:
if member.bot:
self._logger.trace(__name__, f"User {member.id} is ignored, because its a bot")
continue
user = self._users.find_user_by_discord_id_and_server_id(member.id, server.id)
if user is None:
self._logger.fatal(__name__, f"User not found in database: {member.id}")
await self._achievements.validate_achievements_for_user(user)
async def _check_default_role(self):
for guild in self._bot.guilds:
for member in guild.members:
await self._client_utils.check_default_role(member)
def _check_for_bots(self):
for guild in self._bot.guilds:
server = self._servers.get_server_by_discord_id(guild.id)
for member in guild.members.where(lambda x: x.bot):
user = self._users.find_user_by_discord_id_and_server_id(member.id, server.id)
if user is None:
continue
for join in self._user_joins.get_user_joined_servers_by_user_id(user.id):
self._user_joins.delete_user_joined_server(join)
self._user_joins_vc.delete_user_joined_voice_channel_by_user_id(user.id)
self._users.delete_user(user)
self._db_context.save_changes()
if self._is_for_shutdown:
return
async def check_data_integrity(self, is_for_shutdown=False):
if is_for_shutdown != self._is_for_shutdown:
self._is_for_shutdown = is_for_shutdown
# add open voice states
if member.voice is None or member.voice.channel.id in settings.afk_channel_ids:
return
await self._check_default_role()
self._check_known_users()
self.check_servers()
self._check_clients()
self._check_users()
self._check_user_joins()
self._check_user_joins_vc()
self._check_user_joined_gs()
await self._check_for_user_achievements()
self._check_for_bots()
join = UserJoinedVoiceChannel(user, member.voice.channel.id, datetime.now())
self._user_joins_vc.add_user_joined_voice_channel(join)
self._db_context.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedVoiceChannel", e)
def _check_user_joined_gs(self, guild_id: int, member_id: int, user: User):
try:
joins = self._user_joined_gs.find_active_user_joined_game_servers_by_user_id(user.id)
if joins is None or len(joins) == 0:
return
for join in joins:
self._logger.warn(
__name__,
f"Active UserJoinedGameServer found in database: {guild_id}:{member_id}@{join.joined_on}",
)
join.leaved_on = datetime.now()
settings: ServerConfig = self._config.get_configuration(f"ServerConfig_{guild_id}")
if join.time > settings.max_voice_state_hours:
join.leaved_on = join.joined_on + timedelta(hours=settings.max_voice_state_hours)
self._user_joined_gs.update_user_joined_game_server(join)
if self._is_for_shutdown:
user.xp += round(join.time * settings.xp_per_ontime_hour)
self._users.update_user(user)
self._db_context.save_changes()
except Exception as e:
self._logger.error(__name__, f"Cannot get UserJoinedGameServer", e)
async def _check_for_user_achievements(self, user: User):
try:
await self._achievements.validate_achievements_for_user(user)
except Exception as e:
self._logger.error(__name__, f"Cannot check UserGotAchievement for {user.id}", e)
async def _check_default_role(self, member: Member):
await self._client_utils.check_default_role(member)
def _remove_bot(self, user: User):
known_user = self._known_users.find_user_by_discord_id(user.discord_id)
if known_user is not None:
self._known_users.delete_user(known_user)
for join in self._user_joins.get_user_joined_servers_by_user_id(user.id):
self._user_joins.delete_user_joined_server(join)
self._user_joins_vc.delete_user_joined_voice_channel_by_user_id(user.id)
self._users.delete_user(user)
self._db_context.save_changes()

View File

@ -19,10 +19,6 @@ class UserJoinedServerRepositoryABC(ABC):
def get_user_joined_server_by_id(self, id: int) -> UserJoinedServer:
pass
@abstractmethod
def get_user_joined_server_by_server_id(self, server_id: int) -> UserJoinedServer:
pass
@abstractmethod
def get_user_joined_servers_by_user_id(self, user_id: int) -> list[UserJoinedServer]:
pass

View File

@ -67,15 +67,6 @@ class UserJoinedServer(TableABC):
"""
)
@staticmethod
def get_select_by_server_id_string(id: int) -> str:
return str(
f"""
SELECT * FROM `UserJoinedServers`
WHERE `ServerId` = {id};
"""
)
@staticmethod
def get_select_by_user_id_string(id: int) -> str:
return str(

View File

@ -48,13 +48,6 @@ class UserJoinedServerRepositoryService(UserJoinedServerRepositoryABC):
)
return self.__from_result(self._context.select(UserJoinedServer.get_select_by_id_string(id))[0])
def get_user_joined_server_by_server_id(self, server_id: int) -> UserJoinedServer:
self._logger.trace(
__name__,
f"Send SQL command: {UserJoinedServer.get_select_by_server_id_string(server_id)}",
)
return self.__from_result(self._context.select(UserJoinedServer.get_select_by_id_string(server_id))[0])
def get_user_joined_servers_by_user_id(self, user_id: int) -> List[UserJoinedServer]:
self._logger.trace(
__name__,

View File

@ -28,7 +28,5 @@ class ConfigOnReadyEvent(OnReadyABC):
self._data_integrity_service = data_integrity_service
async def on_ready(self):
self._data_integrity_service.check_servers()
for guild in self._bot.guilds:
await self._config_service.reload_server_config(self._servers.get_server_by_discord_id(guild.id))