diff --git a/api/src/api_graphql/query.py b/api/src/api_graphql/query.py index b3c01a1..6e999f6 100644 --- a/api/src/api_graphql/query.py +++ b/api/src/api_graphql/query.py @@ -193,7 +193,7 @@ class Query(QueryABC): if "key" in kwargs: return await userSettingsDao.find_by( - {UserSetting.user_id: user.id, UserSetting.key: kwargs["key"]} + [{UserSetting.user_id: user.id}, {UserSetting.key: kwargs["key"]}] ) return await userSettingsDao.find_by({UserSetting.user_id: user.id}) diff --git a/api/src/core/database/abc/data_access_object_abc.py b/api/src/core/database/abc/data_access_object_abc.py index 98f4a62..55867cc 100644 --- a/api/src/core/database/abc/data_access_object_abc.py +++ b/api/src/core/database/abc/data_access_object_abc.py @@ -7,6 +7,7 @@ from typing import Generic, Optional, Union, TypeVar, Any, Type from core.const import DATETIME_FORMAT from core.database.abc.db_model_abc import DbModelABC from core.database.database import Database +from core.database.external_data_temp_table_builder import ExternalDataTempTableBuilder from core.get_value import get_value from core.logger import DBLogger from core.string import camel_to_snake @@ -16,6 +17,7 @@ T_DBM = TypeVar("T_DBM", bound=DbModelABC) class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): + _external_fields: dict[str, ExternalDataTempTableBuilder] = {} @abstractmethod def __init__(self, source: str, model_type: Type[T_DBM], table_name: str): @@ -30,6 +32,7 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): self.__db_names: dict[str, str] = {} self.__foreign_tables: dict[str, str] = {} + self.__foreign_table_keys: dict[str, str] = {} self.__date_attributes: set[str] = set() self.__ignored_attributes: set[str] = set() @@ -42,12 +45,12 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return self._table_name def attribute( - self, - attr_name: Attribute, - attr_type: type, - db_name: str = None, - ignore=False, - primary_key=False, + self, + attr_name: Attribute, + attr_type: type, + db_name: str = None, + ignore=False, + primary_key=False, ): """ Add an attribute for db and object mapping to the data access object @@ -77,21 +80,20 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): self.__date_attributes.add(db_name) def reference( - self, - attr: Attribute, - primary_attr: Attribute, - foreign_attr: Attribute, - table_name: str, + self, + attr: Attribute, + primary_attr: Attribute, + foreign_attr: Attribute, + table_name: str, ): """ Add a reference to another table for the given attribute + :param Attribute attr: Name of the attribute in the object :param str primary_attr: Name of the primary key in the foreign object :param str foreign_attr: Name of the foreign key in the object :param str table_name: Name of the table to reference :return: """ - if table_name == self._table_name: - return if isinstance(attr, property): attr = attr.fget.__name__ @@ -105,11 +107,18 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): foreign_attr = foreign_attr.lower().replace("_", "") + self.__foreign_table_keys[attr] = foreign_attr + if table_name == self._table_name: + return + self.__joins[foreign_attr] = ( f"LEFT JOIN {table_name} ON {table_name}.{primary_attr} = {self._table_name}.{foreign_attr}" ) self.__foreign_tables[attr] = table_name + def use_external_fields(self, builder: ExternalDataTempTableBuilder): + self._external_fields[builder.table_name] = builder + def to_object(self, result: dict) -> T_DBM: """ Convert a result from the database to an object @@ -136,7 +145,11 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): query += f" {self.__joins[join]}" if filters is not None and (not isinstance(filters, list) or len(filters) > 0): - query += f" WHERE {self._build_conditions(filters)}" + conditions, external_table_deps = await self._build_conditions(filters) + query = await self._handle_query_external_temp_tables( + query, external_table_deps, ignore_fields=True + ) + query += f" WHERE {conditions};" result = await self._db.select_map(query) if len(result) == 0: @@ -168,11 +181,11 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return self.to_object(result[0]) async def get_by( - self, - filters: AttributeFilters = None, - sorts: AttributeSorts = None, - take: int = None, - skip: int = None, + self, + filters: AttributeFilters = None, + sorts: AttributeSorts = None, + take: int = None, + skip: int = None, ) -> list[T_DBM]: """ Get all objects by the given filters @@ -185,7 +198,7 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): :raises ValueError: When no result is found """ result = await self._db.select_map( - self._build_conditional_query(filters, sorts, take, skip) + await self._build_conditional_query(filters, sorts, take, skip) ) if not result or len(result) == 0: raise ValueError("No result found") @@ -193,11 +206,11 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return [self.to_object(x) for x in result] async def get_single_by( - self, - filters: AttributeFilters = None, - sorts: AttributeSorts = None, - take: int = None, - skip: int = None, + self, + filters: AttributeFilters = None, + sorts: AttributeSorts = None, + take: int = None, + skip: int = None, ) -> T_DBM: """ Get a single object by the given filters @@ -218,11 +231,11 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return result[0] async def find_by( - self, - filters: AttributeFilters = None, - sorts: AttributeSorts = None, - take: int = None, - skip: int = None, + self, + filters: AttributeFilters = None, + sorts: AttributeSorts = None, + take: int = None, + skip: int = None, ) -> list[Optional[T_DBM]]: """ Find all objects by the given filters @@ -234,7 +247,7 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): :rtype: list[Optional[T_DBM]] """ result = await self._db.select_map( - self._build_conditional_query(filters, sorts, take, skip) + await self._build_conditional_query(filters, sorts, take, skip) ) if not result or len(result) == 0: return [] @@ -242,11 +255,11 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return [self.to_object(x) for x in result] async def find_single_by( - self, - filters: AttributeFilters = None, - sorts: AttributeSorts = None, - take: int = None, - skip: int = None, + self, + filters: AttributeFilters = None, + sorts: AttributeSorts = None, + take: int = None, + skip: int = None, ) -> Optional[T_DBM]: """ Find a single object by the given filters @@ -346,7 +359,7 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): await self._db.execute(query) async def _build_delete_statement( - self, obj: T_DBM, hard_delete: bool = False + self, obj: T_DBM, hard_delete: bool = False ) -> str: if hard_delete: return f""" @@ -424,7 +437,7 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return "NULL" if isinstance(value, Enum): - return str(value.value) + return f"'{value.value}'" if isinstance(value, bool): return "true" if value else "false" @@ -461,77 +474,115 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): return cast_type(value) - def _build_conditional_query( - self, - filters: AttributeFilters = None, - sorts: AttributeSorts = None, - take: int = None, - skip: int = None, + async def _handle_query_external_temp_tables( + self, query: str, external_table_deps: list[str], ignore_fields=False + ) -> str: + for dep in external_table_deps: + temp_table = self._external_fields[dep] + temp_table_sql = await temp_table.build() + + if not ignore_fields: + query = query.replace( + " FROM", + f", {','.join([f'{temp_table.table_name}.{x}' for x in temp_table.fields.keys() if x not in self.__db_names])} FROM", + ) + + query = f"{temp_table_sql}\n{query}" + query += f" LEFT JOIN {temp_table.table_name} ON {temp_table.join_ref_table}.{self.__primary_key} = {temp_table.table_name}.{temp_table.primary_key}" + + return query + + async def _build_conditional_query( + self, + filters: AttributeFilters = None, + sorts: AttributeSorts = None, + take: int = None, + skip: int = None, ) -> str: query = f"SELECT {self._table_name}.* FROM {self._table_name}" - for join in self.__joins: query += f" {self.__joins[join]}" if filters is not None and (not isinstance(filters, list) or len(filters) > 0): - query += f" WHERE {self._build_conditions(filters)}" + conditions, external_table_deps = await self._build_conditions(filters) + query = await self._handle_query_external_temp_tables( + query, external_table_deps + ) + query += f" WHERE {conditions}" if sorts is not None and (not isinstance(sorts, list) or len(sorts) > 0): query += f" ORDER BY {self._build_order_by(sorts)}" if take is not None: query += f" LIMIT {take}" if skip is not None: query += f" OFFSET {skip}" + + if not query.endswith(";"): + query += ";" return query - def _build_conditions(self, filters: AttributeFilters) -> str: + def _get_external_field_key(self, field_name: str) -> Optional[str]: + """ + Returns the key to get the external field if found, otherwise None. + :param str field_name: The name of the field to search for. + :return: The key if found, otherwise None. + :rtype: Optional[str] + """ + for key, builder in self._external_fields.items(): + if field_name in builder.fields and field_name not in self.__db_names: + return key + return None + + async def _build_conditions(self, filters: AttributeFilters) -> (str, list[str]): """ Build SQL conditions from the given filters :param filters: - :return: + :return: SQL conditions & External field table dependencies """ + external_field_table_deps = [] if not isinstance(filters, list): filters = [filters] conditions = [] for f in filters: + f_conditions = [] + for attr, values in f.items(): if isinstance(attr, property): attr = attr.fget.__name__ if attr in self.__foreign_tables: foreign_table = self.__foreign_tables[attr] - conditions.extend( - self._build_foreign_conditions(foreign_table, values) - ) + cons, eftd = self._build_foreign_conditions(foreign_table, values) + if eftd: + external_field_table_deps.extend(eftd) + + f_conditions.extend(cons) continue if attr == "fuzzy": - conditions.append( - " OR ".join( - self._build_fuzzy_conditions( - [ - ( - self.__db_names[x] - if x in self.__db_names - else self.__db_names[camel_to_snake(x)] - ) - for x in get_value(values, "fields", list[str]) - ], - get_value(values, "term", str), - get_value(values, "threshold", int, 5), - ) - ) + self._handle_fuzzy_filter_conditions( + f_conditions, external_field_table_deps, values ) continue - db_name = self.__db_names[attr] + external_fields_table_name = self._get_external_field_key(attr) + if external_fields_table_name is not None: + external_fields_table = self._external_fields[ + external_fields_table_name + ] + db_name = f"{external_fields_table.table_name}.{attr}" + external_field_table_deps.append(external_fields_table.table_name) + elif ( + isinstance(values, dict) or isinstance(values, list) + ) and not attr in self.__foreign_tables: + db_name = f"{self._table_name}.{self.__db_names[attr]}" + else: + db_name = self.__db_names[attr] if isinstance(values, dict): for operator, value in values.items(): - conditions.append( - self._build_condition( - f"{self._table_name}.{db_name}", operator, value - ) + f_conditions.append( + self._build_condition(f"{db_name}", operator, value) ) elif isinstance(values, list): sub_conditions = [] @@ -539,38 +590,42 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): if isinstance(value, dict): for operator, val in value.items(): sub_conditions.append( - self._build_condition( - f"{self._table_name}.{db_name}", operator, val - ) + self._build_condition(f"{db_name}", operator, val) ) else: sub_conditions.append( self._get_value_validation_sql(db_name, value) ) - conditions.append(f"({' OR '.join(sub_conditions)})") + f_conditions.append(f"({' OR '.join(sub_conditions)})") else: - conditions.append(self._get_value_validation_sql(db_name, values)) + f_conditions.append(self._get_value_validation_sql(db_name, values)) - return " AND ".join(conditions) + conditions.append(f"({' OR '.join(f_conditions)})") + return " AND ".join(conditions), external_field_table_deps + + @staticmethod def _build_fuzzy_conditions( - self, fields: list[str], term: str, threshold: int = 10 + fields: list[str], term: str, threshold: int = 10 ) -> list[str]: conditions = [] for field in fields: conditions.append( - f"levenshtein({field}, '{term}') <= {threshold}" + f"levenshtein({field}::TEXT, '{term}') <= {threshold}" ) # Adjust the threshold as needed return conditions - def _build_foreign_conditions(self, table: str, values: dict) -> list[str]: + def _build_foreign_conditions( + self, table: str, values: dict + ) -> (list[str], list[str]): """ Build SQL conditions for foreign key references :param table: Foreign table name :param values: Filter values - :return: List of conditions + :return: List of conditions, List of external field tables """ + external_field_table_deps = [] conditions = [] for attr, sub_values in values.items(): if isinstance(attr, property): @@ -578,25 +633,43 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): if attr in self.__foreign_tables: foreign_table = self.__foreign_tables[attr] - conditions.extend( - self._build_foreign_conditions(foreign_table, sub_values) + sub_conditions, eftd = self._build_foreign_conditions( + foreign_table, sub_values + ) + if len(eftd) > 0: + external_field_table_deps.extend(eftd) + + conditions.extend(sub_conditions) + continue + + if attr == "fuzzy": + self._handle_fuzzy_filter_conditions( + conditions, external_field_table_deps, sub_values ) continue - db_name = f"{table}.{attr.lower().replace('_', '')}" + external_fields_table_name = self._get_external_field_key(attr) + if external_fields_table_name is not None: + external_fields_table = self._external_fields[ + external_fields_table_name + ] + db_name = f"{external_fields_table.table_name}.{attr}" + external_field_table_deps.append(external_fields_table.table_name) + else: + db_name = f"{table}.{attr.lower().replace('_', '')}" if isinstance(sub_values, dict): for operator, value in sub_values.items(): conditions.append( - f"({self._build_condition(db_name, operator, value)} OR {self._build_condition(db_name, "isNull", None)})") + f"{self._build_condition(db_name, operator, value)}" + ) elif isinstance(sub_values, list): sub_conditions = [] for value in sub_values: if isinstance(value, dict): for operator, val in value.items(): sub_conditions.append( - f"({self._build_condition(db_name, operator, val)} OR {self._build_condition(db_name, "isNull", None)})" - + f"{self._build_condition(db_name, operator, val)}" ) else: sub_conditions.append( @@ -606,14 +679,55 @@ class DataAccessObjectABC(ABC, Database, Generic[T_DBM]): else: conditions.append(self._get_value_validation_sql(db_name, sub_values)) - return conditions + return conditions, external_field_table_deps + + def _handle_fuzzy_filter_conditions( + self, conditions, external_field_table_deps, sub_values + ): + fuzzy_fields = get_value(sub_values, "fields", list[str]) + fuzzy_fields_db_names = [] + for fuzzy_field in fuzzy_fields: + external_fields_table_name = self._get_external_field_key(fuzzy_field) + if external_fields_table_name is not None: + external_fields_table = self._external_fields[ + external_fields_table_name + ] + fuzzy_fields_db_names.append( + f"{external_fields_table.table_name}.{fuzzy_field}" + ) + external_field_table_deps.append(external_fields_table.table_name) + elif fuzzy_field in self.__db_names: + fuzzy_fields_db_names.append( + f"{self._table_name}.{self.__db_names[fuzzy_field]}" + ) + elif fuzzy_field in self.__foreign_tables: + fuzzy_fields_db_names.append( + f"{self._table_name}.{self.__foreign_table_keys[fuzzy_field]}" + ) + else: + fuzzy_fields_db_names.append( + self.__db_names[camel_to_snake(fuzzy_field)] + ) + conditions.append( + f"({' OR '.join( + self._build_fuzzy_conditions( + [x for x in fuzzy_fields_db_names], + get_value(sub_values, "term", str), + get_value(sub_values, "threshold", int, 5), + ) + ) + })" + ) def _get_value_validation_sql(self, field: str, value: Any): value = self._get_value_sql(value) + field_selector = f"{self._table_name}.{field}" + if field in self.__foreign_tables: + field_selector = self.__db_names[field] if value == "NULL": - return f"{self._table_name}.{field} IS NULL" - return f"{self._table_name}.{field} = {value}" + return f"{field_selector} IS NULL" + return f"{field_selector} = {value}" def _build_condition(self, db_name: str, operator: str, value: Any) -> str: """ diff --git a/api/src/core/database/external_data_temp_table_builder.py b/api/src/core/database/external_data_temp_table_builder.py new file mode 100644 index 0000000..dea48cc --- /dev/null +++ b/api/src/core/database/external_data_temp_table_builder.py @@ -0,0 +1,72 @@ +import textwrap +from typing import Callable + + +class ExternalDataTempTableBuilder: + + def __init__(self): + self._table_name = None + self._fields: dict[str, str] = {} + self._primary_key = "id" + self._join_ref_table = None + self._value_getter = None + + @property + def table_name(self) -> str: + return self._table_name + + @property + def fields(self) -> dict[str, str]: + return self._fields + + @property + def primary_key(self) -> str: + return self._primary_key + + @property + def join_ref_table(self) -> str: + return self._join_ref_table + + def with_table_name(self, table_name: str) -> "ExternalDataTempTableBuilder": + self._join_ref_table = table_name + + if "." in table_name: + table_name = table_name.split(".")[-1] + + if not table_name.endswith("_temp"): + table_name = f"{table_name}_temp" + + self._table_name = table_name + return self + + def with_field( + self, name: str, sql_type: str, primary=False + ) -> "ExternalDataTempTableBuilder": + if primary: + sql_type += " PRIMARY KEY" + self._primary_key = name + self._fields[name] = sql_type + return self + + def with_value_getter( + self, value_getter: Callable + ) -> "ExternalDataTempTableBuilder": + self._value_getter = value_getter + return self + + async def build(self) -> str: + assert self._table_name is not None, "Table name is required" + assert self._value_getter is not None, "Value getter is required" + + values_str = ", ".join([f"{value}" for value in await self._value_getter()]) + + return textwrap.dedent( + f""" + DROP TABLE IF EXISTS {self._table_name}; + CREATE TEMP TABLE {self._table_name} ( + {", ".join([f"{k} {v}" for k, v in self._fields.items()])} + ); + + INSERT INTO {self._table_name} VALUES {values_str}; + """ + ) diff --git a/web/src/app/modules/shared/components/menu-bar/menu-bar.component.html b/web/src/app/modules/shared/components/menu-bar/menu-bar.component.html index 7c6d6c0..632f922 100644 --- a/web/src/app/modules/shared/components/menu-bar/menu-bar.component.html +++ b/web/src/app/modules/shared/components/menu-bar/menu-bar.component.html @@ -1,12 +1,37 @@