Skip to content

Commit

Permalink
feat: table trash (#1141)
Browse files Browse the repository at this point in the history
* feat: table trash

* chore: db migration

* chore: linting types

* fix: e2e testing
  • Loading branch information
Sky-FE authored Dec 6, 2024
1 parent 3bdb38f commit ee6ae9e
Show file tree
Hide file tree
Showing 44 changed files with 1,448 additions and 124 deletions.
2 changes: 2 additions & 0 deletions apps/nestjs-backend/src/cache/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ export interface IUndoRedoOperationBase {
name: OperationName;
params: Record<string, unknown>;
result?: unknown;
userId?: string;
operationId?: string;
}

export interface IUpdateRecordsOperation extends IUndoRedoOperationBase {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { FieldKeyType, FieldOpBuilder, FieldType, IFieldRo } from '@teable/core';
import {
FieldKeyType,
FieldOpBuilder,
FieldType,
generateOperationId,
IFieldRo,
} from '@teable/core';
import type {
IFieldVo,
IConvertFieldRo,
Expand Down Expand Up @@ -291,6 +297,7 @@ export class FieldOpenApiService {
});

this.eventEmitterService.emitAsync(Events.OPERATION_FIELDS_DELETE, {
operationId: generateOperationId(),
windowId,
tableId,
userId: this.cls.get('user.id'),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
import type { IAttachmentCellValue, IAttachmentItem, IMakeOptional } from '@teable/core';
import { FieldKeyType, FieldType } from '@teable/core';
import { FieldKeyType, FieldType, generateOperationId } from '@teable/core';
import { PrismaService } from '@teable/db-main-prisma';
import { UploadType } from '@teable/openapi';
import type {
Expand Down Expand Up @@ -56,11 +56,12 @@ export class RecordOpenApiService {

async multipleCreateRecords(
tableId: string,
createRecordsRo: ICreateRecordsRo
createRecordsRo: ICreateRecordsRo,
ignoreMissingFields: boolean = false
): Promise<ICreateRecordsVo> {
return await this.prismaService.$tx(
async () => {
return await this.createRecords(tableId, createRecordsRo);
return await this.createRecords(tableId, createRecordsRo, ignoreMissingFields);
},
{
timeout: this.thresholdConfig.bigTransactionTimeout,
Expand Down Expand Up @@ -128,15 +129,17 @@ export class RecordOpenApiService {
tableId: string,
createRecordsRo: ICreateRecordsRo & {
records: IMakeOptional<IRecordInnerRo, 'id'>[];
}
},
ignoreMissingFields: boolean = false
): Promise<ICreateRecordsVo> {
const { fieldKeyType = FieldKeyType.Name, records, typecast, order } = createRecordsRo;
const chunkSize = this.thresholdConfig.calcChunkSize;
const typecastRecords = await this.validateFieldsAndTypecast(
tableId,
records,
fieldKeyType,
typecast
typecast,
ignoreMissingFields
);

const preparedRecords = await this.appendRecordOrderIndexes(tableId, typecastRecords, order);
Expand Down Expand Up @@ -175,7 +178,8 @@ export class RecordOpenApiService {
private async getEffectFieldInstances(
tableId: string,
recordsFields: Record<string, unknown>[],
fieldKeyType: FieldKeyType = FieldKeyType.Name
fieldKeyType: FieldKeyType = FieldKeyType.Name,
ignoreMissingFields: boolean = false
) {
const fieldIdsOrNamesSet = recordsFields.reduce<Set<string>>((acc, recordFields) => {
const fieldIds = Object.keys(recordFields);
Expand All @@ -193,7 +197,7 @@ export class RecordOpenApiService {
},
});

if (usedFields.length !== usedFieldIdsOrNames.length) {
if (!ignoreMissingFields && usedFields.length !== usedFieldIdsOrNames.length) {
const usedSet = new Set(map(usedFields, fieldKeyType));
const missedFields = usedFieldIdsOrNames.filter(
(fieldIdOrName) => !usedSet.has(fieldIdOrName)
Expand All @@ -211,13 +215,15 @@ export class RecordOpenApiService {
tableId: string,
records: T[],
fieldKeyType: FieldKeyType = FieldKeyType.Name,
typecast?: boolean
typecast: boolean = false,
ignoreMissingFields: boolean = false
): Promise<T[]> {
const recordsFields = map(records, 'fields');
const effectFieldInstance = await this.getEffectFieldInstances(
tableId,
recordsFields,
fieldKeyType
fieldKeyType,
ignoreMissingFields
);

const newRecordsFields: Record<string, unknown>[] = recordsFields.map(() => ({}));
Expand Down Expand Up @@ -385,17 +391,16 @@ export class RecordOpenApiService {
return { records, orders };
});

if (windowId) {
this.eventEmitterService.emitAsync(Events.OPERATION_RECORDS_DELETE, {
windowId,
tableId,
userId: this.cls.get('user.id'),
records: records.records.map((record, index) => ({
...record,
order: orders?.[index],
})),
});
}
this.eventEmitterService.emitAsync(Events.OPERATION_RECORDS_DELETE, {
operationId: generateOperationId(),
windowId,
tableId,
userId: this.cls.get('user.id'),
records: records.records.map((record, index) => ({
...record,
order: orders?.[index],
})),
});

return records;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,16 @@ export class TableOpenApiService {
await this.prismaService.txClient().trash.deleteMany({
where: { resourceId: { in: tableIds }, resourceType: ResourceType.Table },
});

// clean table trash
await this.prismaService.txClient().tableTrash.deleteMany({
where: { tableId: { in: tableIds } },
});

// clean record trash
await this.prismaService.txClient().recordTrash.deleteMany({
where: { tableId: { in: tableIds } },
});
}

async deleteTable(baseId: string, tableId: string) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { generateRecordTrashId } from '@teable/core';
import { PrismaService } from '@teable/db-main-prisma';
import { ResourceType } from '@teable/openapi';
import { Knex } from 'knex';
import { InjectModel } from 'nest-knexjs';
import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config';
import { Events } from '../../../event-emitter/events';
import { IDeleteFieldsPayload } from '../../undo-redo/operations/delete-fields.operation';
import { IDeleteRecordsPayload } from '../../undo-redo/operations/delete-records.operation';
import { IDeleteViewPayload } from '../../undo-redo/operations/delete-view.operation';

@Injectable()
export class TableTrashListener {
constructor(
private readonly prismaService: PrismaService,
@InjectModel('CUSTOM_KNEX') private readonly knex: Knex,
@ThresholdConfig() private readonly thresholdConfig: IThresholdConfig
) {}

@OnEvent(Events.OPERATION_RECORDS_DELETE, { async: true })
async recordDeleteListener(payload: IDeleteRecordsPayload) {
const { operationId, userId, tableId, records } = payload;

if (!operationId) return;

const recordIds = records.map((record) => record.id);

await this.prismaService.$tx(
async (prisma) => {
await prisma.tableTrash.create({
data: {
id: operationId,
tableId,
createdBy: userId,
resourceType: ResourceType.Record,
snapshot: JSON.stringify(recordIds),
},
});

const batchSize = 5000;
for (let i = 0; i < records.length; i += batchSize) {
const batch = records.slice(i, i + batchSize);
const recordTrashData = batch.map((record) => ({
id: generateRecordTrashId(),
table_id: tableId,
record_id: record.id,
snapshot: JSON.stringify(record),
created_by: userId,
}));

const query = this.knex.insert(recordTrashData).into('record_trash').toQuery();
await prisma.$executeRawUnsafe(query);
}
},
{
timeout: this.thresholdConfig.bigTransactionTimeout,
}
);
}

@OnEvent(Events.OPERATION_FIELDS_DELETE, { async: true })
async fieldDeleteListener(payload: IDeleteFieldsPayload) {
const { userId, tableId, fields, records, operationId } = payload;

if (!operationId) return;

await this.prismaService.tableTrash.create({
data: {
id: operationId,
tableId,
createdBy: userId,
resourceType: ResourceType.Field,
snapshot: JSON.stringify({ fields, records }),
},
});
}

@OnEvent(Events.OPERATION_VIEW_DELETE, { async: true })
async viewDeleteListener(payload: IDeleteViewPayload) {
const { operationId, tableId, viewId, userId } = payload;

if (!operationId) return;

await this.prismaService.tableTrash.create({
data: {
id: operationId,
tableId,
createdBy: userId,
resourceType: ResourceType.View,
snapshot: JSON.stringify([viewId]),
},
});
}
}
15 changes: 13 additions & 2 deletions apps/nestjs-backend/src/features/trash/trash.module.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,25 @@
import { Module } from '@nestjs/common';
import { BaseModule } from '../base/base.module';
import { FieldOpenApiModule } from '../field/open-api/field-open-api.module';
import { RecordOpenApiModule } from '../record/open-api/record-open-api.module';
import { TableOpenApiModule } from '../table/open-api/table-open-api.module';
import { UserModule } from '../user/user.module';
import { ViewModule } from '../view/view.module';
import { TableTrashListener } from './listener/table-trash.listener';
import { TrashController } from './trash.controller';
import { TrashService } from './trash.service';

@Module({
imports: [UserModule, BaseModule, TableOpenApiModule],
imports: [
UserModule,
BaseModule,
TableOpenApiModule,
FieldOpenApiModule,
RecordOpenApiModule,
ViewModule,
],
controllers: [TrashController],
providers: [TrashService],
providers: [TrashService, TableTrashListener],
exports: [TrashService],
})
export class TrashModule {}
Loading

0 comments on commit ee6ae9e

Please sign in to comment.