Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use s3 for temporary storage #177

Closed
wants to merge 1 commit into from
Closed

Use s3 for temporary storage #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@ SSH_HOST=${SSH_HOSTNAME}
# e.g. ./keys/host.key
SSH_HOST_KEY_PATH=${SSH_HOST_KEY_PATH}

# The location of an instance of upload-service
# e.g. 'localhost:3000'
UPLOAD_SERVICE_API_BASE_PATH=

# The S3 bucket to use for temporary file storage
# e.g. 'permanent-local'
TEMPORARY_FILE_S3_BUCKET=
TEMPORARY_FILE_S3_BUCKET_REGION=

# Any sub-path within the `local` bucket relevant to your environment
# e.g. '_YourNameHere'
TEMPORARY_FILE_S3_SUBDIRECTORY=

# S3 Credentials
AWS_ACCESS_KEY_ID=
AWS_ACCESS_SECRET=

# This is for local dev
export NODE_TLS_REJECT_UNAUTHORIZED=0

Expand Down
15,955 changes: 9,644 additions & 6,311 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@
"typescript": "^5.0.4"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.332.0",
"@fusionauth/typescript-client": "^1.45.0",
"@permanentorg/sdk": "^0.5.4",
"@permanentorg/sdk": "^0.6.0",
"@sentry/node": "^7.52.0",
"dotenv": "^16.0.3",
"logform": "^2.3.2",
Expand Down
14 changes: 6 additions & 8 deletions src/classes/PermanentFileSystem.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import {
getArchiveSlugFromPath,
getOriginalFileForArchiveRecord,
} from '../utils';
import type { Readable } from 'stream';
import type {
Archive,
ClientConfiguration,
Expand All @@ -42,6 +41,7 @@ import type {
Attributes,
FileEntry,
} from 'ssh2';
import type { TemporaryFile } from './TemporaryFile';

const isRootPath = (requestedPath: string): boolean => (
requestedPath === '/'
Expand Down Expand Up @@ -225,19 +225,17 @@ export class PermanentFileSystem {
}

public async createFile(
requestedPath: string,
dataStream: Readable,
size: number,
temporaryFile: TemporaryFile,
): Promise<void> {
const parentPath = path.dirname(requestedPath);
const archiveRecordName = path.basename(requestedPath);
const parentPath = path.dirname(temporaryFile.permanentFileSystemPath);
const archiveRecordName = path.basename(temporaryFile.permanentFileSystemPath);
const parentFolder = await this.loadFolder(parentPath);
await createArchiveRecord(
this.getClientConfiguration(),
dataStream,
temporaryFile.url,
{
contentType: 'application/octet-stream',
size,
size: temporaryFile.size,
},
{
displayName: archiveRecordName,
Expand Down
183 changes: 84 additions & 99 deletions src/classes/SftpSessionHandler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ import path from 'path';
import fetch from 'node-fetch';
import { v4 as uuidv4 } from 'uuid';
import ssh2 from 'ssh2';
import tmp from 'tmp';
import { logger } from '../logger';
import { generateFileEntry } from '../utils';
import { NotFoundError } from '../errors';
import { TemporaryFile } from './TemporaryFile';
import type { AuthenticationSession } from './AuthenticationSession';
import type { PermanentFileSystem } from './PermanentFileSystem';
import type { PermanentFileSystemManager } from './PermanentFileSystemManager';
import type { FileResult } from 'tmp';
import type {
Attributes,
FileEntry,
Expand All @@ -23,10 +23,6 @@ const SFTP_STATUS_CODE = ssh2.utils.sftp.STATUS_CODE;

const generateHandle = (): string => uuidv4();

interface TemporaryFile extends FileResult {
virtualPath: string;
}

export class SftpSessionHandler {
private readonly sftpConnection: SFTPWrapper;

Expand Down Expand Up @@ -240,7 +236,7 @@ export class SftpSessionHandler {
);
const temporaryFile = this.openTemporaryFiles.get(handle.toString());
if (!temporaryFile) {
logger.debug('There is no open temporary file associated with this handle', { reqId, handle });
logger.debug('There is no open file associated with this handle', { reqId, handle });
logger.verbose(
'Response: Status (FAILURE)',
{
Expand All @@ -251,36 +247,31 @@ export class SftpSessionHandler {
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
return;
}
fs.write(
temporaryFile.fd,
data,
0,
(err, written, buffer) => {
if (err) {
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: temporaryFile.virtualPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
return;
}
logger.debug('Successful Write.', { reqId, handle, written });
logger.silly('Written Data:', { buffer });
temporaryFile.append(data)
.then(() => {
logger.debug('Successful Write.', { reqId, handle });
logger.silly('Written Data:', { data });
logger.verbose(
'Response: Status (OK)',
{
reqId,
code: SFTP_STATUS_CODE.OK,
path: temporaryFile.virtualPath,
path: temporaryFile.permanentFileSystemPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.OK);
},
);
})
.catch(() => {
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: temporaryFile.permanentFileSystemPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
});
}

/**
Expand Down Expand Up @@ -350,52 +341,35 @@ export class SftpSessionHandler {
);
const temporaryFile = this.openTemporaryFiles.get(handle.toString());
if (temporaryFile) {
fs.stat(
temporaryFile.name,
(statError, stats) => {
if (statError) {
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: temporaryFile.virtualPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
return;
}
const { size } = stats;
this.getCurrentPermanentFileSystem().createFile(
temporaryFile.virtualPath,
fs.createReadStream(temporaryFile.name),
size,
temporaryFile.close()
.then(async () => {
await this.getCurrentPermanentFileSystem().createFile(
temporaryFile,
).then(() => {
temporaryFile.removeCallback();
this.openTemporaryFiles.delete(handle.toString());
logger.verbose(
'Response: Status (OK)',
{
reqId,
code: SFTP_STATUS_CODE.OK,
path: temporaryFile.virtualPath,
path: temporaryFile.permanentFileSystemPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.OK);
}).catch((err) => {
logger.verbose(err);
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: temporaryFile.virtualPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
});
},
);
})
.catch((e) => {
logger.debug(e);
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: temporaryFile.permanentFileSystemPath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
});
return;
}
this.openFiles.delete(handle.toString());
Expand Down Expand Up @@ -893,40 +867,38 @@ export class SftpSessionHandler {
case 'xa+': // append and read (file must not exist)
case 'a': // append
{
tmp.file((err, name, fd, removeCallback) => {
if (err) {
const temporaryFile = new TemporaryFile(filePath);
temporaryFile.open()
.then(() => {
this.openTemporaryFiles.set(
handle,
temporaryFile,
);
logger.verbose(
'Response: Handle',
{
reqId,
handle,
path: filePath,
},
);
this.sftpConnection.handle(
reqId,
Buffer.from(handle),
);
})
.catch((e) => {
logger.debug(e);
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: filePath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
return;
}
const temporaryFile = {
name,
fd,
removeCallback,
};
this.openTemporaryFiles.set(handle, {
...temporaryFile,
virtualPath: filePath,
});
logger.verbose(
'Response: Handle',
{
reqId,
handle,
path: filePath,
},
);
this.sftpConnection.handle(
reqId,
Buffer.from(handle),
);
});
break;
}
case 'r+': // read and write (error if doesn't exist)
Expand All @@ -944,16 +916,29 @@ export class SftpSessionHandler {
break;
}
})
.catch(() => {
logger.verbose(
'Response: Status (NO_SUCH_FILE)',
{
reqId,
code: SFTP_STATUS_CODE.NO_SUCH_FILE,
path: filePath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.NO_SUCH_FILE);
.catch((e) => {
if (e instanceof NotFoundError) {
logger.verbose(
'Response: Status (NO_SUCH_FILE)',
{
reqId,
code: SFTP_STATUS_CODE.NO_SUCH_FILE,
path: filePath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.NO_SUCH_FILE);
} else {
logger.debug(e);
logger.verbose(
'Response: Status (FAILURE)',
{
reqId,
code: SFTP_STATUS_CODE.FAILURE,
path: filePath,
},
);
this.sftpConnection.status(reqId, SFTP_STATUS_CODE.FAILURE);
}
});
}

Expand Down
Loading