Improved data validation for import

bot/src/bot_graphql/mutations/level_mutation.py

@@ -41,6 +41,9 @@ class LevelMutation(QueryABC):
             int(input["permissions"]),
             server,
         )
+        levels = self._levels.get_levels_by_server_id(server.id)
+        if levels.where(lambda x: x.name == level.name).count() > 0:
+            raise ValueError(f"Level with name {level.name} already exists")
         self._levels.add_level(level)
         self._db.save_changes()
 

web/src/app/base/component-with-table.ts

@@ -1,5 +1,3 @@
-import { Level } from "../models/data/level.model";
-
 export interface Column {
   key: string;
   name: string;
@@ -18,10 +16,19 @@ export class ComponentWithTable {
 
   public name: string = "";
   public columns: Column[] = [];
+  public isEditingNew: boolean = false;
+
+  public callback = (data: any[], isNew: boolean) => {
+    this.save(data, isNew);
+  };
+  public validator: (oldElement: any, newElement: any) => boolean = (oldElement: any, newElement: any) => {
+    return true;
+  };
 
   constructor(
     name: string,
-        columns: string[]
+    columns: string[],
+    validator?: (oldElement: any, newElement: any) => boolean
   ) {
     this.name = name;
     this.columns = columns.map(column => {
@@ -33,6 +40,9 @@ export class ComponentWithTable {
       hiddenColumns = localStorage.getItem("hiddenColumns") ?? JSON.stringify([{}]);
     }
     this._hiddenColumns = JSON.parse(hiddenColumns);
+    if (validator) {
+      this.validator = validator;
+    }
   }
 
   private getKey(column: string): string {
@@ -43,15 +53,12 @@ export class ComponentWithTable {
     return !this._hiddenColumns.map(column => column.key).includes(this.getKey(column));
   }
 
-    public callback = (data: any[]) => {
-        this.save(data);
-    };
-
   public onRowEditSave(newLevel: any, index: number) {
   }
 
-    public save(data: any[]) {
+  public save(data: any[], isNew: boolean) {
     for (let i = 0; i < data.length; i++) {
+      this.isEditingNew = isNew;
       this.onRowEditSave(data[i], i);
     }
   }

web/src/app/modules/admin/auth-users/components/auth-user/auth-user.component.ts

@@ -31,7 +31,6 @@ export class AuthUserComponent extends ComponentWithTable implements OnInit, OnD
   activityValues: number[] = [0, 100];
 
   clonedUsers: { [s: string]: AuthUserDTO; } = {};
-  isEditingNew: boolean = false;
 
   authRoles = [
     { label: AuthRoles[AuthRoles.Normal].toString(), value: AuthRoles.Normal },

web/src/app/modules/shared/components/data-import-and-export/data-import-and-export.component.ts

@@ -1,6 +1,7 @@
 import { Component, EventEmitter, Input, Output, ViewChild } from "@angular/core";
 import { ToastService } from "../../../../services/toast/toast.service";
 import { TranslateService } from "@ngx-translate/core";
+import { elementAt } from "rxjs";
 
 interface UploadEvent {
   originalEvent: Event;
@@ -31,7 +32,10 @@ export class DataImportAndExportComponent {
   }
 
   @Output() dataChange: EventEmitter<any[]> = new EventEmitter<any[]>();
-    @Input() callback!: (data: any[]) => void;
+  @Input() callback!: (data: any[], isNew: boolean) => void;
+  @Input() validator: (oldElement: any, newElement: any) => boolean = (oldElement: any, newElement: any) => {
+    return true;
+  };
 
   public constructor(
     private toastService: ToastService,
@@ -56,9 +60,25 @@ export class DataImportAndExportComponent {
     const fileReader = new FileReader();
     fileReader.onload = () => {
       if (!fileReader.result) return;
-            this.data = JSON.parse(fileReader.result.toString());
+      const newData: any[] = JSON.parse(fileReader.result.toString());
       this.upload.clear();
-            this.callback(this.data);
+      newData.forEach(element => {
+        element.id = 0;
+      });
+      this.data.forEach(element => {
+        const existingElement = newData.find(x => this.validator(x, element));
+        if (existingElement) {
+          const index = this.data.indexOf(element);
+          const oldId = element.id;
+          element = existingElement;
+          element.id = oldId;
+          this.data[index] = element;
+          newData.splice(newData.indexOf(existingElement), 1);
+        }
+      });
+      this.callback(this.data, false);
+      this.callback(newData, true);
+      this.data.push(...newData);
       this.toastService.success(this.translate.instant("common.file.uploaded"), this.translate.instant("common.file.uploaded"));
     };
     fileReader.readAsText(file, "UTF-8");

web/src/app/modules/view/server/achievements/components/achievement/achievement.component.html

@@ -28,7 +28,7 @@
                     class="icon-btn btn" (click)="resetFilters()">
             </button>
             <app-data-import-and-export name="achievements" [(data)]="achievements"
-                                        [callback]="callback"></app-data-import-and-export>
+                                        [callback]="callback" [validator]="validator"></app-data-import-and-export>
           </div>
         </div>
       </ng-template>

web/src/app/modules/view/server/achievements/components/achievement/achievement.component.ts

@@ -15,7 +15,13 @@ import { TranslateService } from "@ngx-translate/core";
 import { DataService } from "../../../../../../services/data/data.service";
 import { SidebarService } from "../../../../../../services/sidebar/sidebar.service";
 import { ActivatedRoute } from "@angular/router";
-import { AchievementListQuery, AchievementTypeQuery, GameServerListQuery, LevelListQuery, Query } from "../../../../../../models/graphql/query.model";
+import {
+  AchievementListQuery,
+  AchievementTypeQuery,
+  GameServerListQuery,
+  LevelListQuery,
+  Query
+} from "../../../../../../models/graphql/query.model";
 import { catchError, debounceTime, takeUntil } from "rxjs/operators";
 import { LazyLoadEvent, MenuItem } from "primeng/api";
 import { Table } from "primeng/table";
@@ -33,8 +39,6 @@ export class AchievementComponent extends ComponentWithTable implements OnInit,
   public achievements: Achievement[] = [];
   public loading = true;
 
-  public isEditingNew: boolean = false;
-
   public filterForm!: FormGroup<{
     id: FormControl<number | null>,
     name: FormControl<string | null>,
@@ -81,7 +85,10 @@ export class AchievementComponent extends ComponentWithTable implements OnInit,
     private data: DataService,
     private sidebar: SidebarService,
     private route: ActivatedRoute) {
-    super("achievement", ["id", "name", "description", "attribute", "operator", "value"]);
+    super("achievement", ["id", "name", "description", "attribute", "operator", "value"],
+      (oldElement: Achievement, newElement: Achievement) => {
+        return oldElement.name === newElement.name;
+      });
   }
 
   public ngOnInit(): void {

web/src/app/modules/view/server/auto-role/components/auto-roles-rules/auto-roles-rules.component.html

@@ -27,7 +27,7 @@
                     class="icon-btn btn" (click)="resetFilters()">
             </button>
             <app-data-import-and-export name="achievements" [(data)]="rules"
-                                        [callback]="callback"></app-data-import-and-export>
+                                        [callback]="callback" [validator]="validator"></app-data-import-and-export>
           </div>
         </div>
       </ng-template>

web/src/app/modules/view/server/auto-role/components/auto-roles-rules/auto-roles-rules.component.ts

@@ -1,7 +1,7 @@
 import { Component, OnDestroy, OnInit } from "@angular/core";
 import { DataService } from "../../../../../../services/data/data.service";
 import { ActivatedRoute, Router } from "@angular/router";
-import { AutoRoleRule, AutoRoleRuleFilter } from "../../../../../../models/data/auto_role.model";
+import { AutoRole, AutoRoleRule, AutoRoleRuleFilter } from "../../../../../../models/data/auto_role.model";
 import { Guild } from "../../../../../../models/data/discord.model";
 import { LazyLoadEvent, MenuItem } from "primeng/api";
 import { User } from "../../../../../../models/data/user.model";
@@ -40,7 +40,6 @@ export class AutoRolesRulesComponent extends ComponentWithTable implements OnIni
   autoRoleId!: number;
 
   clonedUsers: { [s: string]: User; } = {};
-  isEditingNew: boolean = false;
 
   newAutoRoleTemplate: AutoRoleRule = {
     createdAt: "",
@@ -81,7 +80,10 @@ export class AutoRolesRulesComponent extends ComponentWithTable implements OnIni
     private route: ActivatedRoute,
     private router: Router
   ) {
-    super("auto-role-rules", ["id", "role", "emoji"]);
+    super("auto-role-rules", ["id", "role", "emoji"], (oldElement: AutoRoleRule, newElement: AutoRoleRule) => {
+      return oldElement.autoRole?.id === newElement.autoRole?.id &&
+        oldElement.roleId === newElement.roleId;
+    });
   }
 
   public getEmojiUrl(name: string): string {

web/src/app/modules/view/server/auto-role/components/auto-roles/auto-roles.component.html

@@ -28,7 +28,7 @@
                     class="icon-btn btn" (click)="resetFilters()">
             </button>
             <app-data-import-and-export name="achievements" [(data)]="auto_roles"
-                                        [callback]="callback"></app-data-import-and-export>
+                                        [callback]="callback" [validator]="validator"></app-data-import-and-export>
           </div>
         </div>
       </ng-template>

web/src/app/modules/view/server/auto-role/components/auto-roles/auto-roles.component.ts

@@ -36,7 +36,6 @@ export class AutoRolesComponent extends ComponentWithTable implements OnInit, On
   loading = true;
 
   clonedUsers: { [s: string]: User; } = {};
-  isEditingNew: boolean = false;
 
   newAutoRoleTemplate: AutoRole = {
     createdAt: "",
@@ -77,7 +76,10 @@ export class AutoRolesComponent extends ComponentWithTable implements OnInit, On
     private sidebar: SidebarService,
     private route: ActivatedRoute
   ) {
-    super("auto-role", ["id", "channel_id", "channel_name", "message_id", "rule_count"]);
+    super("auto-role", ["id", "channel_id", "channel_name", "message_id", "rule_count"], (oldElement: AutoRole, newElement: AutoRole) => {
+      return oldElement.channelId === newElement.channelId &&
+        oldElement.messageId === newElement.messageId;
+    });
   }
 
   public ngOnInit(): void {

web/src/app/modules/view/server/levels/components/levels/levels.component.html

@@ -31,7 +31,7 @@
                                 class="icon-btn btn" (click)="resetFilters()">
                         </button>
                         <app-data-import-and-export name="levels" [(data)]="levels"
-                                                    [callback]="callback"></app-data-import-and-export>
+                                                    [callback]="callback" [validator]="validator"></app-data-import-and-export>
                     </div>
                 </div>
             </ng-template>

web/src/app/modules/view/server/levels/components/levels/levels.component.ts

@@ -19,7 +19,7 @@ import { Table } from "primeng/table";
 import { User } from "../../../../../../models/data/user.model";
 import { LevelMutationResult, UpdateUserMutationResult } from "../../../../../../models/graphql/result.model";
 import { Mutations } from "../../../../../../models/graphql/mutations.model";
-import { Subject, throwError } from "rxjs";
+import { forkJoin, Subject, throwError } from "rxjs";
 import { Server } from "../../../../../../models/data/server.model";
 import { UserDTO } from "../../../../../../models/auth/auth-user.dto";
 import { ComponentWithTable } from "../../../../../../base/component-with-table";
@@ -33,8 +33,6 @@ export class LevelsComponent extends ComponentWithTable implements OnInit, OnDes
   public levels: Level[] = [];
   public loading = true;
 
-    public isEditingNew: boolean = false;
-
   public filterForm!: FormGroup<{
     id: FormControl<number | null>,
     name: FormControl<string | null>,
@@ -74,7 +72,9 @@ export class LevelsComponent extends ComponentWithTable implements OnInit, OnDes
     private sidebar: SidebarService,
     private route: ActivatedRoute
   ) {
-        super("level", ["id", "name", "color", "min_xp", "permissions"]);
+    super("level", ["id", "name", "color", "min_xp", "permissions"], (oldElement: Level, newElement: Level) => {
+      return oldElement.name === newElement.name;
+    });
   }
 
   public ngOnInit(): void {

web/src/app/modules/view/server/members/members.component.ts

@@ -34,7 +34,6 @@ export class MembersComponent extends ComponentWithTable implements OnInit, OnDe
   loading = true;
 
   clonedUsers: { [s: string]: User; } = {};
-  isEditingNew: boolean = false;
 
   newUserTemplate: User = {
     discordId: 0,

web/src/app/modules/view/server/short-role-name/components/short-role-names/short-role-names.component.html

@@ -28,7 +28,7 @@
                     class="icon-btn btn" (click)="resetFilters()">
             </button>
             <app-data-import-and-export name="achievements" [(data)]="shortRoleNames"
-                                        [callback]="callback"></app-data-import-and-export>
+                                        [callback]="callback" [validator]="validator"></app-data-import-and-export>
           </div>
         </div>
       </ng-template>

web/src/app/modules/view/server/short-role-name/components/short-role-names/short-role-names.component.ts

@@ -33,8 +33,6 @@ export class ShortRoleNamesComponent extends ComponentWithTable implements OnIni
   public shortRoleNames: ShortRoleName[] = [];
   public loading = true;
 
-  public isEditingNew: boolean = false;
-
   public filterForm!: FormGroup<{
     id: FormControl<number | null>,
     shortName: FormControl<string | null>,
@@ -75,7 +73,10 @@ export class ShortRoleNamesComponent extends ComponentWithTable implements OnIni
     private sidebar: SidebarService,
     private route: ActivatedRoute
   ) {
-    super("short-role-names", ["id", "name", "role", "position"]);
+    super("short-role-names", ["id", "name", "role", "position"], (oldElement: ShortRoleName, newElement: ShortRoleName) => {
+      return oldElement.shortName === newElement.shortName &&
+        oldElement.roleId === newElement.roleId;
+    });
   }
 
   public ngOnInit(): void {