diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 9b258544..290725a6 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -23,6 +23,7 @@ import { NotificationsModule } from "./notifications/notifications.module"; import { AqiApiModule } from "./aqi_api/aqi_api.module"; import { FileParseValidateModule } from "./file_parse_and_validation/file_parse_and_validation.module"; import { FtpModule } from "./ftp/ftp.module"; +import { FileValidationModule } from './file_validation/file_validation.module'; const DB_HOST = process.env.POSTGRES_HOST || "localhost"; const DB_USER = process.env.POSTGRES_USER || "postgres"; @@ -69,6 +70,7 @@ function getMiddlewares() { FileParseValidateModule, AqiApiModule, FtpModule, + FileValidationModule, ], controllers: [AppController, MetricsController, HealthController], providers: [AppService, CronJobService], diff --git a/backend/src/file_submissions/file_submissions.module.ts b/backend/src/file_submissions/file_submissions.module.ts index 97eec9f5..1560f275 100644 --- a/backend/src/file_submissions/file_submissions.module.ts +++ b/backend/src/file_submissions/file_submissions.module.ts @@ -1,11 +1,11 @@ -import { Module } from '@nestjs/common'; -import { FileSubmissionsService } from './file_submissions.service'; -import { FileSubmissionsController } from './file_submissions.controller'; -import { SanitizeService } from 'src/sanitize/sanitize.service'; +import { Module } from "@nestjs/common"; +import { FileSubmissionsService } from "./file_submissions.service"; +import { FileSubmissionsController } from "./file_submissions.controller"; +import { SanitizeService } from "src/sanitize/sanitize.service"; @Module({ controllers: [FileSubmissionsController], providers: [FileSubmissionsService, SanitizeService], - imports: [] + exports: [FileSubmissionsService], }) export class FileSubmissionsModule {} diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 3762a326..5dd43fb4 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -9,8 +9,7 @@ import { randomUUID } from "crypto"; @Injectable() export class FileSubmissionsService { - constructor( - private prisma: PrismaService) {} + constructor(private prisma: PrismaService) {} async create(body: any, file: Express.Multer.File) { const createFileSubmissionDto = new CreateFileSubmissionDto(); @@ -25,14 +24,14 @@ export class FileSubmissionsService { // Call to function that makes API call to save file in the S3 bucket via COMS let comsSubmissionID = await saveToS3(body.token, file); - const path = require('path'); - const extention = path.extname(file.originalname) - const baseName = path.basename(file.originalname, extention) - const newFileName = `${baseName}-${comsSubmissionID}${extention}` + const path = require("path"); + const extention = path.extname(file.originalname); + const baseName = path.basename(file.originalname, extention); + const newFileName = `${baseName}-${comsSubmissionID}${extention}`; // Creating file DTO and inserting it in the database with the file GUID from the S3 bucket createFileSubmissionDto.submission_id = comsSubmissionID; - createFileSubmissionDto.filename = newFileName;; + createFileSubmissionDto.filename = newFileName; createFileSubmissionDto.submission_date = new Date(); createFileSubmissionDto.submitter_user_id = body.userID; createFileSubmissionDto.submission_status_code = ( @@ -252,7 +251,7 @@ async function saveToS3(token: any, file: Express.Multer.File) { return fileGUID; } -async function getFromS3(submission_id: string){ +async function getFromS3(submission_id: string) { const axios = require("axios"); let config = { @@ -260,12 +259,12 @@ async function getFromS3(submission_id: string){ maxBodyLength: Infinity, url: `${process.env.COMS_URI}/v1/object/${submission_id}`, headers: { - "Accept": "application/json", + Accept: "application/json", }, }; await axios.request(config).then((response) => { - console.log(response) + console.log(response); }); return null; diff --git a/backend/src/file_validation/file_validation.module.ts b/backend/src/file_validation/file_validation.module.ts new file mode 100644 index 00000000..7db1219f --- /dev/null +++ b/backend/src/file_validation/file_validation.module.ts @@ -0,0 +1,10 @@ +import { Module } from "@nestjs/common"; +import { FileValidationService } from "./file_validation.service"; +import { NotificationsModule } from "src/notifications/notifications.module"; + +@Module({ + imports: [NotificationsModule], + providers: [FileValidationService], + exports: [FileValidationService], +}) +export class FileValidationModule {} diff --git a/backend/src/file_validation/file_validation.service.spec.ts b/backend/src/file_validation/file_validation.service.spec.ts new file mode 100644 index 00000000..95ffe944 --- /dev/null +++ b/backend/src/file_validation/file_validation.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { FileValidationService } from './file_validation.service'; + +describe('FileValidationService', () => { + let service: FileValidationService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [FileValidationService], + }).compile(); + + service = module.get(FileValidationService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/backend/src/file_validation/file_validation.service.ts b/backend/src/file_validation/file_validation.service.ts new file mode 100644 index 00000000..e94342b5 --- /dev/null +++ b/backend/src/file_validation/file_validation.service.ts @@ -0,0 +1,452 @@ +import { Injectable, Logger } from "@nestjs/common"; +import path from "path"; +import * as XLSX from "xlsx"; +import { + mandatoryHeaders, + conditionallyMandatoryHeaders, + optionalHeaders, + conditionallyOptionalHeaders, +} from "src/utils/constants"; +import { PrismaService } from "nestjs-prisma"; +import { NotificationsService } from "src/notifications/notifications.service"; + +@Injectable() +export class FileValidationService { + private readonly logger = new Logger(FileValidationService.name); + + constructor( + private notificationService: NotificationsService, + private prisma: PrismaService, + ) {} + + async processFile( + fileBuffer: Buffer, + filePath: string, + username: string, + filename: string, + ): Promise { + const errors: string[] = []; + + // Determine file type using headers + let fileType = await this.getFileType(fileBuffer); + + // csv and txt are not detected using file headers, use file extension as fallback + const fileExtension = path.extname(filePath).toLowerCase().replace(".", ""); + if (!fileType) fileType = fileExtension; + + // Check row count and headers + const checkFileErrors: string[] = await this.checkFileErrors( + fileType, + fileBuffer, + ); + if (checkFileErrors.length > 0) + checkFileErrors.forEach((err) => errors.push(err)); + + if (errors.length > 0) { + // const ministryContact = ""; // should be obtained from file somehow + // await this.notificationService.notifyFtpUserOfError( + // username, + // filename, + // errors, + // ministryContact, + // ); + this.logger.log(errors); + } else { + // continue to file splitting + } + } + + private async getFileType(fileBuffer: Buffer): Promise { + const fileType = await import("file-type"); + const type: any = await fileType.fileTypeFromBuffer(fileBuffer); + return type?.ext; + } + + private async checkFileErrors( + fileType: string, + fileBuffer: Buffer, + ): Promise { + switch (fileType) { + case "csv": + return this.validateCsv(fileBuffer); + case "xlsx": + return this.validateXlsx(fileBuffer); + case "txt": + return this.validateTxt(fileBuffer); + default: + return ["Unsupported file type."]; + } + } + + /** + * XLSX file validation: 10k row limit, check headers + * @param fileBuffer + * @param mandatoryHeaders + * @returns + */ + private async validateXlsx(fileBuffer: Buffer): Promise { + const errors: string[] = []; + try { + const workbook = XLSX.read(fileBuffer, { type: "buffer" }); + const sheetName = workbook.SheetNames[0]; + const sheet = workbook.Sheets[sheetName]; + let jsonData: any = XLSX.utils.sheet_to_json(sheet, { header: 1 }); + // Extract headers and data + const headers = jsonData[0]; + const adjustedHeaders = headers.map((header: string) => + header.toLowerCase().split(" ").join(""), + ); + const dataRows = jsonData.slice(1); + + // Combine headers with each row + const result = dataRows.map((row) => { + let obj = {}; + row.forEach((value, index) => { + obj[adjustedHeaders[index]] = value; + }); + return obj; + }); + + // if the file is too large, stop processing here and return the error + if (result.length > 10000) { + errors.push("XLSX file exceeds the row limit of 10,000."); + return errors; + } + + const fileErrors = await this.validateHeadersAndData( + result, + adjustedHeaders, + ); + if (fileErrors.length > 0) fileErrors.forEach((err) => errors.push(err)); + + return errors; + } catch (error) { + return [`Error processing XLSX file: ${error.message}`]; + } + } + + /** + * CSV file validation: 10k row limit, check headers + * @param fileBuffer + * @param mandatoryHeaders + * @returns + */ + private async validateCsv(fileBuffer: Buffer): Promise { + // TODO + const errors: string[] = []; + try { + const workbook = XLSX.read(fileBuffer, { type: "buffer" }); + const sheetName = workbook.SheetNames[0]; + const sheet = workbook.Sheets[sheetName]; + let jsonData: any = XLSX.utils.sheet_to_json(sheet); + // convert the headers to lowercase + jsonData = jsonData.map((obj: any) => { + return Object.keys(obj).reduce((acc: any, key: string) => { + acc[key.toLowerCase()] = obj[key]; + return acc; + }, {}); + }); + + // if the file is too large, stop processing here and return the error + if (jsonData.length > 10001) { + errors.push("CSV file exceeds the row limit of 10,000."); + return errors; + } + + // const fileErrors = await this.validateHeadersAndData(jsonData); + // if (fileErrors.length > 0) fileErrors.forEach((err) => errors.push(err)); + + return errors; + } catch (error) { + return [`Error processing CSV file: ${error.message}`]; + } + } + + /** + * Text file validation, currently expects tab-separated values + * Unsure if accepting text files is necessary, might delete or adjust later. + * @param fileBuffer + * @returns + */ + private async validateTxt(fileBuffer: Buffer): Promise { + const errors: string[] = []; + try { + const fileContent: string = fileBuffer.toString("utf8"); + const lines: string[] = fileContent.split("\n"); + + // if the file is too large, stop processing here and return the error + if (lines.length > 10001) { + errors.push("TXT file exceeds the row limit of 10,000."); + return errors; + } + + // TODO - txt header validation + // Assuming values are separated by tabs... + // const headers: string[] = lines[0].split("\t"); + // const headersFound = new Set(); + // headers.forEach((header) => headersFound.add(header)); + + // const missingHeaders = mandatoryHeaders.filter( + // (header) => !headersFound.has(header), + // ); + // if (missingHeaders.length > 0) { + // errors.push( + // `Missing headers in TXT file: ${missingHeaders.join(", ")}`, + // ); + // } + return errors; + } catch (error) { + return [`Error processing TXT file: ${error.message}`]; + } + } + + /** + * using jsonData and headers, checks for mandatory and conditionally mandatory headers + * @param jsonData - array of data rows: [{header1: 'value1', header2: 'value2'}, {header1: 'value3', header2: 'value4'}] + * @param headers - array of lowercase headers with spaces removed: ['header1', 'header2'] + * @returns + */ + private async validateHeadersAndData( + jsonData: any[], + headers: string[], + ): Promise { + const errors: string[] = []; + const headersFound = new Set(headers); + + const missingHeaders = mandatoryHeaders.filter( + (header) => !headersFound.has(header.toLowerCase().split(" ").join("")), + ); + if (missingHeaders.length > 0) { + errors.push( + `Missing mandatory headers in XLSX file: ${missingHeaders.join(", ")}`, + ); + } + + // Check conditionally mandatory headers + const cmErrors = await this.checkMissingCmHeaders(jsonData, headersFound); + if (cmErrors) errors.push(cmErrors); + + return errors; + } + + /** + * Checks conditionally mandatory headers + * @param jsonData - array of data rows: [{header1: 'value1', header2: 'value2'}, {header1: 'value3', header2: 'value4'}] + * @param headersFound - set of headers found in the file + * @returns + */ + private async checkMissingCmHeaders( + jsonData: any[], + headersFound: Set, + ): Promise { + const missingCmHeaders = conditionallyMandatoryHeaders.filter( + (header) => !headersFound.has(header.toLowerCase().split(" ").join("")), + ); + const missingCmHeadersWithError = []; + if (missingCmHeaders.length > 0) { + for (const header of missingCmHeaders) { + switch (header) { + case "Depth Unit": // if Depth is filled in + if ( + headersFound.has("depth") || + headersFound.has("depthupper") || + headersFound.has("depthlower") + ) { + // check jsonData for any data in these columns, if found, add error + for (const row of jsonData) { + // check if depth column exists, check if it has data in this row + if ( + row.hasOwnProperty("depth") && + row["depth"] !== null && + row["depth"] !== undefined && + row["depth"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + // check if depth upper column exists, check if it has data in this row + if ( + row.hasOwnProperty("depthupper") && + row["depthupper"] !== null && + row["depthupper"] !== undefined && + row["depthupper"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + // check if depth lower column exists, check if it has data in this row + if ( + row.hasOwnProperty("depthlower") && + row["depthlower"] !== null && + row["depthlower"] !== undefined && + row["depthlower"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Result Unit": // if Result Value, Lab: MDL, or Lab: MRL are filled in + if ( + headersFound.has("resultvalue") || + headersFound.has("methoddetectionlimit") || + headersFound.has("methodreportinglimit") + ) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("resultvalue") && + row["resultvalue"] !== null && + row["resultvalue"] !== undefined && + row["resultvalue"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + if ( + row.hasOwnProperty("methoddetectionlimit") && + row["methoddetectionlimit"] !== null && + row["methoddetectionlimit"] !== undefined && + row["methoddetectionlimit"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + if ( + row.hasOwnProperty("methodreportinglimit") && + row["methodreportinglimit"] !== null && + row["methodreportinglimit"] !== undefined && + row["methodreportinglimit"] !== "" + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Rounded Value": // if source of rounded value = 'PROVIDED BY USER' + if (headersFound.has("sourceofroundedvalue")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("sourceofroundedvalue") && + row["sourceofroundedvalue"] !== null && + row["sourceofroundedvalue"] !== undefined && + row["sourceofroundedvalue"] === "PROVIDED BY USER" + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Specimen Name": // if data classification = LAB, FIELD_SURVEY, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "FIELD_SURVEY" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Method Detection Limit": // if data classification = LAB, FIELD_RESULT, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "FIELD_RESULT" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Method Reporting Limit": // if data classification = LAB, FIELD_RESULT, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "FIELD_RESULT" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Lab Arrival Date and Time": // if data classification = LAB, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "Fraction": // if data classification = LAB, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + missingCmHeadersWithError.push(header); + break; + } + } + } + break; + case "From Laboratory": // if data classification = LAB, SURROGATE_RESULT + if (headersFound.has("dataclassification")) { + for (const row of jsonData) { + if ( + row.hasOwnProperty("dataclassification") && + row["dataclassification"] !== null && + row["dataclassification"] !== undefined && + (row["dataclassification"] === "LAB" || + row["dataclassification"] === "SURROGATE_RESULT") + ) { + break; + } + } + } + break; + case "QC Source Sample ID": // if QC Type not blank use activity name(???) + // TODO: Need to confirm this logic + break; + } + } + } + const missingCmHeaderErrors = + missingCmHeadersWithError.length > 0 + ? `Missing conditionally mandatory headers: ${missingCmHeadersWithError.join(", ")}` + : null; + return missingCmHeaderErrors; + } +} diff --git a/backend/src/ftp/ftp.module.ts b/backend/src/ftp/ftp.module.ts index b1f49aa2..17dc16cf 100644 --- a/backend/src/ftp/ftp.module.ts +++ b/backend/src/ftp/ftp.module.ts @@ -1,12 +1,13 @@ import { Module } from "@nestjs/common"; import { FtpService } from "./ftp.service"; import { FtpController } from "./ftp.controller"; -import { FtpFileValidationService } from "./ftp_file_validation.service"; import { NotificationsModule } from "src/notifications/notifications.module"; +import { FileValidationModule } from "src/file_validation/file_validation.module"; +import { FileSubmissionsModule } from "src/file_submissions/file_submissions.module"; @Module({ - imports: [NotificationsModule], - providers: [FtpService, FtpFileValidationService], + imports: [NotificationsModule, FileValidationModule, FileSubmissionsModule], + providers: [FtpService], controllers: [FtpController], }) export class FtpModule {} diff --git a/backend/src/ftp/ftp.service.ts b/backend/src/ftp/ftp.service.ts index 2e29fa97..3e1cfe91 100644 --- a/backend/src/ftp/ftp.service.ts +++ b/backend/src/ftp/ftp.service.ts @@ -4,9 +4,10 @@ import * as ftp from "basic-ftp"; import * as path from "path"; import * as dotenv from "dotenv"; import { Writable } from "stream"; -import { FtpFileValidationService } from "./ftp_file_validation.service"; import { NotificationsService } from "src/notifications/notifications.service"; import { PrismaService } from "nestjs-prisma"; +import { FileValidationService } from "src/file_validation/file_validation.service"; +import { FileSubmissionsService } from "src/file_submissions/file_submissions.service"; dotenv.config(); @@ -17,9 +18,9 @@ export class FtpService { private remoteBasePath: string; constructor( - private ftpFileValidationService: FtpFileValidationService, - private notificationService: NotificationsService, - private prisma: PrismaService, + private fileValidationService: FileValidationService, + private notificationsService: NotificationsService, + private fileSubmissionsService: FileSubmissionsService, ) { this.client = new ftp.Client(); this.client.ftp.verbose = true; @@ -82,6 +83,14 @@ export class FtpService { // if the file is > 10MB, don't download it if (file.size > 10 * 1024 * 1024) { errors = ["File size exceeds the limit of 10MB."]; + // don't download the file, just send out a notification of the error + const ministryContact = ""; // should be obtained from file somehow + await this.notificationsService.notifyFtpUserOfError( + folder.name, + file.name, + errors, + ministryContact, + ); } else { const dataBuffer = []; // download file to a stream that puts chunks into an array @@ -95,30 +104,39 @@ export class FtpService { await this.client.downloadTo(writableStream, filePath); // convert chunk array to buffer const fileBuffer = Buffer.concat(dataBuffer); - // pass file buffer to validation - errors = await this.ftpFileValidationService.processFile( + // pass file buffer to file submission service to be uploaded to comms & have submission entry created + // await this.fileSubmissionsService.parseFileFromFtp( + // fileBuffer, + // folder.name, + // file.name, + // filePath, + // ); + // debug + await this.fileValidationService.processFile( fileBuffer, filePath, - ); - } - if (errors.length > 0) { - this.logger.log(`Validation failure for: ${filePath}`); - errors.forEach((error) => this.logger.log(error)); - this.logger.log(``); - // send out a notification to the file submitter & ministry contact outlining the errors - const ministryContact = ""; // should be obtained from file somehow - await this.notifyUserOfError( - folder.name, + folder.name, // username file.name, - errors, - ministryContact, ); - } else { - this.logger.log(`Validation success for: ${filePath}`); - this.logger.log(``); - // pass to file validation service - // await this.validationService.handleFile(file); // made up function call } + // if (errors.length > 0) { + // this.logger.log(`Validation failure for: ${filePath}`); + // errors.forEach((error) => this.logger.log(error)); + // this.logger.log(``); + // // send out a notification to the file submitter & ministry contact outlining the errors + // const ministryContact = ""; // should be obtained from file somehow + // await this.notifyUserOfError( + // folder.name, + // file.name, + // errors, + // ministryContact, + // ); + // } else { + // this.logger.log(`Validation success for: ${filePath}`); + // this.logger.log(``); + // // pass to file validation service + // // await this.validationService.handleFile(file); // made up function call + // } // this.logger.log(`Cleaning up file: ${filePath}`); // await this.client.remove(filePath); } catch (error) { @@ -138,54 +156,8 @@ export class FtpService { } } - /** - * Notifies the Data Submitter & Ministry Contact of the file validation errors. - * @param username - * @param fileName - * @param errors - * @param ministryContact - */ - async notifyUserOfError( - username: string, - fileName: string, - errors: string[], - ministryContact: string, - ) { - const ftpUser = await this.prisma.ftp_users.findUnique({ - where: { username: username }, - }); - const notificationVars = { - file_name: fileName, - user_account_name: username, - location_ids: [], - file_status: "FAILED", - errors: errors.join(","), - warnings: "", - }; - - // Notify the Data Submitter - if (this.isValidEmail(ftpUser.email)) { - await this.notificationService.sendDataSubmitterNotification( - ftpUser.email, - notificationVars, - ); - } - // Notify the Ministry Contact (if they have not disabled notifications) - if (this.isValidEmail(ministryContact)) { - await this.notificationService.sendContactNotification( - ministryContact, - notificationVars, - ); - } - } - - isValidEmail(email: string): boolean { - const emailRegex = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/; - return emailRegex.test(email); - } - // @Cron("0 */5 * * * *") // every 5 minutes - // @Cron("0,30 * * * * *") // every 30s + @Cron("0,15,30,45 * * * * *") // every 15s async handleCron() { this.logger.log("START ################"); this.logger.log("######################"); diff --git a/backend/src/ftp/ftp_file_validation.service.ts b/backend/src/ftp/ftp_file_validation.service.ts deleted file mode 100644 index a7cb0434..00000000 --- a/backend/src/ftp/ftp_file_validation.service.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Injectable, Logger } from "@nestjs/common"; -import path from "path"; -import * as XLSX from "xlsx"; -@Injectable() -export class FtpFileValidationService { - private readonly logger = new Logger(FtpFileValidationService.name); - - async processFile(fileBuffer: Buffer, filePath: string): Promise { - const errors = []; - // Required column headers - const requiredHeaders = ["test1", "test2", "test3"]; - - // Check file size is under 10 MB - if (fileBuffer.length > 10 * 1024 * 1024) { - errors.push("File size exceeds the limit of 10 MB."); - } - // Determine file type using headers - let fileType = await this.getFileType(fileBuffer); - // csv and txt are not detected using file headers, use file extension as fallback - const fileExtension = path.extname(filePath).toLowerCase().replace(".", ""); - if (!fileType) fileType = fileExtension; - // Check row count and headers - let headerError: string | null = null; - switch (fileType) { - case "csv": - headerError = await this.validateCsvHeaders( - fileBuffer, - requiredHeaders, - ); - if (headerError) errors.push(headerError); - break; - case "xlsx": - headerError = await this.validateXlsxHeaders( - fileBuffer, - requiredHeaders, - ); - if (headerError) errors.push(headerError); - break; - case "txt": - headerError = await this.validateTxtHeaders( - fileBuffer, - requiredHeaders, - ); - if (headerError) errors.push(headerError); - break; - default: - errors.push("Unsupported file type."); - } - return errors; - } - - private async getFileType(fileBuffer: Buffer): Promise { - const fileType = await import("file-type"); - const type: any = await fileType.fileTypeFromBuffer(fileBuffer); - return type?.ext; - } - - /** - * CSV file validation: 10k row limit, check headers - * @param fileBuffer - * @param requiredHeaders - * @returns - */ - private async validateCsvHeaders( - fileBuffer: Buffer, - requiredHeaders: string[], - ): Promise { - try { - const workbook = XLSX.read(fileBuffer, { type: "buffer" }); - const sheetName = workbook.SheetNames[0]; - const sheet = workbook.Sheets[sheetName]; - const jsonData: any = XLSX.utils.sheet_to_json(sheet); - - // Get headers - const headersFound = new Set(); - if (jsonData.length > 0) { - const headers: string[] = Object.keys(jsonData[0]); - headers.forEach((header) => headersFound.add(header)); - } - - const missingHeaders = requiredHeaders.filter( - (header) => !headersFound.has(header), - ); - if (missingHeaders.length > 0) { - return `Missing headers in CSV file: ${missingHeaders.join(", ")}`; - } else { - return null; - } - } catch (error) { - return `Error processing CSV file: ${error.message}`; - } - } - - /** - * XLSX file validation: 10k row limit, check headers - * @param fileBuffer - * @param requiredHeaders - * @returns - */ - private async validateXlsxHeaders( - fileBuffer: Buffer, - requiredHeaders: string[], - ): Promise { - try { - const workbook = XLSX.read(fileBuffer, { type: "buffer" }); - const sheetName = workbook.SheetNames[0]; - const sheet = workbook.Sheets[sheetName]; - const jsonData: any = XLSX.utils.sheet_to_json(sheet, { header: 1 }); - - // Get headers - const headersFound = new Set(); - - if (jsonData.length > 0) { - const headers: string[] = jsonData[0]; - headers.forEach((header) => headersFound.add(header)); - } - - const missingHeaders = requiredHeaders.filter( - (header) => !headersFound.has(header), - ); - if (missingHeaders.length > 0) { - return `Missing headers in XLSX file: ${missingHeaders.join(", ")}`; - } else { - return null; - } - } catch (error) { - return `Error processing XLSX file: ${error.message}`; - } - } - - /** - * Text file validation, currently expects tab-separated values - * Unsure if accepting text files is necessary, might delete or adjust later. - * @param fileBuffer - * @param requiredHeaders - * @returns - */ - private async validateTxtHeaders( - fileBuffer: Buffer, - requiredHeaders: string[], - ): Promise { - try { - const fileContent: string = fileBuffer.toString("utf8"); - const lines: string[] = fileContent.split("\n"); - - // Assuming values are separated by tabs... - const headers: string[] = lines[0].split("\t"); - const headersFound = new Set(); - headers.forEach((header) => headersFound.add(header)); - - const missingHeaders = requiredHeaders.filter( - (header) => !headersFound.has(header), - ); - if (missingHeaders.length > 0) { - return `Missing headers in TXT file: ${missingHeaders.join(", ")}`; - } - - return null; - } catch (error) { - return `Error processing TXT file: ${error.message}`; - } - } -} diff --git a/backend/src/notifications/notifications.service.ts b/backend/src/notifications/notifications.service.ts index 8cbd2e5a..f79b6f48 100644 --- a/backend/src/notifications/notifications.service.ts +++ b/backend/src/notifications/notifications.service.ts @@ -13,7 +13,7 @@ export class NotificationsService { constructor( private readonly httpService: HttpService, - private prisma: PrismaService + private prisma: PrismaService, ) {} /** @@ -32,7 +32,7 @@ export class NotificationsService { */ async createNotificationEntry( email: string, - username: string + username: string, ): Promise { const createNotificationDto = new CreateNotificationEntryDto(); createNotificationDto.email = email; @@ -63,7 +63,7 @@ export class NotificationsService { async updateNotificationEntry( email: string, username: string, - enabled: boolean + enabled: boolean, ): Promise { const updateNotificationDto = new UpdateNotificationEntryDto(); updateNotificationDto.enabled = enabled; @@ -162,7 +162,7 @@ export class NotificationsService { file_status: string; errors: string; warnings: string; - } + }, ): Promise { let body = `

Status: {{file_status}}

@@ -222,11 +222,11 @@ export class NotificationsService { file_status: string; warnings: string; errors: string; - } + }, ): Promise { const notificationInfo = await this.getNotificationStatus( email, - variables.user_account_name + variables.user_account_name, ); // check that notifications are enabled before continuing if (notificationInfo.enabled === false) { @@ -299,7 +299,7 @@ export class NotificationsService { warnings: string; sys_time: string; status_string: string; - } + }, ): Promise { const chesToken = await this.getChesToken(); @@ -360,6 +360,70 @@ export class NotificationsService { } } + /** + * Notifies the FTP Data Submitter & Ministry Contact of the file validation errors. + * @param email + * @param username + * @param fileName + * @param errors + * @param ministryContact + */ + async notifyUserOfError( + email: string, + username: string, + fileName: string, + errors: string[], + ministryContact: string, + ) { + const notificationVars = { + file_name: fileName, + user_account_name: username, + location_ids: [], + file_status: "FAILED", + errors: errors.join(","), + warnings: "", + }; + + // Notify the Data Submitter + if (this.isValidEmail(email)) { + await this.sendDataSubmitterNotification(email, notificationVars); + } + // Notify the Ministry Contact (if they have not disabled notifications) + if (this.isValidEmail(ministryContact)) { + await this.sendContactNotification(ministryContact, notificationVars); + } + } + + /** + * Notifies the FTP Data Submitter & Ministry Contact of the file validation errors. + * @param username + * @param fileName + * @param errors + * @param ministryContact + */ + async notifyFtpUserOfError( + username: string, + fileName: string, + errors: string[], + ministryContact: string, + ) { + const ftpUser = await this.prisma.ftp_users.findUnique({ + where: { username: username }, + }); + await this.notifyUserOfError( + ftpUser.email, + username, + fileName, + errors, + ministryContact, + ); + } + + isValidEmail(email: string): boolean { + const emailRegex = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/; + return emailRegex.test(email); + } + /** * Takes an array of email addresses, filters out the ones * with notifications disabled, and returns the remaining ones @@ -397,7 +461,7 @@ export class NotificationsService { async getChesToken(): Promise { const url = process.env.ches_token_url; const encodedToken = Buffer.from( - `${process.env.ches_client_id}:${process.env.ches_client_secret}` + `${process.env.ches_client_id}:${process.env.ches_client_secret}`, ).toString("base64"); const headers = { @@ -410,7 +474,7 @@ export class NotificationsService { try { const response = await lastValueFrom( - this.httpService.post(url, grantTypeParam.toString(), { headers }) + this.httpService.post(url, grantTypeParam.toString(), { headers }), ); return response.data.access_token; } catch (error) { @@ -419,7 +483,7 @@ export class NotificationsService { "Response:", error.response.data, error.response.status, - error.response.headers + error.response.headers, ); } else if (error.request) { this.logger.log("Request:", error.request); diff --git a/backend/src/utils/constants.ts b/backend/src/utils/constants.ts new file mode 100644 index 00000000..f484034f --- /dev/null +++ b/backend/src/utils/constants.ts @@ -0,0 +1,65 @@ +// expected headers, originally from EMSEDT-186, adjusted to match the Aug 9, 2024 example file +export const mandatoryHeaders = [ + "Ministry Contact", + "Sampling Agency", + "Project", + "Location ID", + "Observed Property ID", + "Field Visit Start Time", + "Observed DateTime", + "Analyzed DateTime", + "Data Classification", + "Result Status", + "Medium", + "Collection Method", + "Analysis Method", // Lab: + "Detection Condition", // Lab: + "Limit Type", // Lab: +]; +export const conditionallyMandatoryHeaders = [ + "Depth Unit", // if Depth is filled in + "Result Unit", // if Result Value, Lab: MDL, or Lab: MRL are filled in + "Rounded Value", // if source of rounded value = 'PROVIDED BY USER' + "Specimen Name", // if data classification = LAB, FIELD_SURVEY, SURROGATE_RESULT + "Method Detection Limit", // if data classification = LAB, FIELD_RESULT, SURROGATE_RESULT // Lab: + "Method Reporting Limit", // if data classification = LAB, FIELD_RESULT, SURROGATE_RESULT // Lab: + "Lab Arrival Date and Time", // if data classification = LAB, SURROGATE_RESULT // Lab: + "Fraction", // if data classification = LAB, SURROGATE_RESULT // Lab: + "From Laboratory", // if data classification = LAB, SURROGATE_RESULT // Lab: + "QC Source Sample ID", // if QC Type not blank use activity name(???) // QC: +]; + +export const optionalHeaders = [ + "Observation ID", + "Work Order Number", + "Field Visit End Time", + "Observed Date Time End", + "Depth", + "Depth Lower", + "Result Value", + "Source of Rounded Value", + "Rounding Specification", + "Result Grade", + "Medium Detail", + "Activity ID", + "Activity Name", + "Field: Participants", + "Field: Filter", + "Field: Filter Comment", + "Field: Preservative", + "Field: Device ID", + "Field: Device Type", + "Field: Comment", + "Lab Arrival Temperature", + "Composite Stat", + "Sampling Context Tag", +]; + +export const conditionallyOptionalHeaders = [ + "Quality Flag", // if data classification = LAB, FIELD_RESULT, SURROGATE_RESULT // Lab: + "Prepared DateTime", // if data classification = LAB, SURROGATE_RESULT // Lab: + "Sample ID", // if data classification = LAB, SURROGATE_RESULT // Lab: + "Dilution Factor", // if data classification = LAB, SURROGATE_RESULT // Lab: + "Comment", // if data classification = LAB, SURROGATE_RESULT // Lab: + "Type", // if blank, assumed to be 'ROUTINE' // QC: +];