From 20cacf5091b109315f24ff82f2be8e338f69b93e Mon Sep 17 00:00:00 2001 From: loren Date: Thu, 11 Jul 2024 14:31:38 -0400 Subject: [PATCH 1/5] Add partialMetadata to options. --- lib/index.d.ts | 1 + lib/index.test-d.ts | 3 + lib/upload.js | 6 ++ test/spec/test-parallel-uploads.js | 160 +++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) diff --git a/lib/index.d.ts b/lib/index.d.ts index 57b04197..697e14d4 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -26,6 +26,7 @@ interface UploadOptions { uploadUrl?: string | null metadata?: { [key: string]: string } + partialMetadata?: { [key: string]: string } fingerprint?: (file: File, options: UploadOptions) => Promise uploadSize?: number | null diff --git a/lib/index.test-d.ts b/lib/index.test-d.ts index 1209dedc..9d3c6ee3 100644 --- a/lib/index.test-d.ts +++ b/lib/index.test-d.ts @@ -19,6 +19,9 @@ const upload = new tus.Upload(file, { metadata: { filename: 'foo.txt', }, + partialMetadata: { + userId: "foo123bar", + }, onProgress: (bytesSent: number, bytesTotal: number) => { const percentage = ((bytesSent / bytesTotal) * 100).toFixed(2) console.log(bytesSent, bytesTotal, `${percentage}%`) diff --git a/lib/upload.js b/lib/upload.js index d561831a..180d6284 100644 --- a/lib/upload.js +++ b/lib/upload.js @@ -12,6 +12,7 @@ const defaultOptions = { uploadUrl: null, metadata: {}, + partialMetadata: {}, fingerprint: null, uploadSize: null, @@ -587,6 +588,11 @@ class BaseUpload { const metadata = encodeMetadata(this.options.metadata) if (metadata !== '') { req.setHeader('Upload-Metadata', metadata) + } else { + const partialMetadata = encodeMetadata(this.options.partialMetadata) + if (partialMetadata != '') { + req.setHeader('Upload-Metadata', partialMetadata) + } } let promise diff --git a/test/spec/test-parallel-uploads.js b/test/spec/test-parallel-uploads.js index fe4cb529..16f971c9 100644 --- a/test/spec/test-parallel-uploads.js +++ b/test/spec/test-parallel-uploads.js @@ -204,6 +204,166 @@ describe('tus', () => { expect(options.onProgress).toHaveBeenCalledWith(11, 11) expect(testUrlStorage.removeUpload).toHaveBeenCalled() }) + it('should add partialMetadata option to Upload-Metadata', async () => { + const testStack = new TestHttpStack() + + const testUrlStorage = { + addUpload: (fingerprint, upload) => { + expect(fingerprint).toBe('fingerprinted') + expect(upload.uploadUrl).toBeUndefined() + expect(upload.size).toBe(11) + expect(upload.parallelUploadUrls).toEqual([ + 'https://tus.io/uploads/upload1', + 'https://tus.io/uploads/upload2', + ]) + + return Promise.resolve('tus::fingerprinted::1337') + }, + removeUpload: (urlStorageKey) => { + expect(urlStorageKey).toBe('tus::fingerprinted::1337') + return Promise.resolve() + }, + } + spyOn(testUrlStorage, 'removeUpload').and.callThrough() + spyOn(testUrlStorage, 'addUpload').and.callThrough() + + const file = getBlob('hello world') + const options = { + httpStack: testStack, + urlStorage: testUrlStorage, + storeFingerprintForResuming: true, + removeFingerprintOnSuccess: true, + parallelUploads: 2, + retryDelays: [10], + endpoint: 'https://tus.io/uploads', + metadata: { + foo: 'hello', + }, + partialMetadata: { + bar: 'baz', + }, + onProgress() {}, + onSuccess: waitableFunction(), + fingerprint: () => Promise.resolve('fingerprinted'), + } + spyOn(options, 'onProgress') + + const upload = new tus.Upload(file, options) + upload.start() + + let req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Length']).toBe(5) + expect(req.requestHeaders['Upload-Concat']).toBe('partial') + expect(req.requestHeaders['Upload-Metadata']).toBe('bar YmF6') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload1', + }, + }) + + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Length']).toBe(6) + expect(req.requestHeaders['Upload-Concat']).toBe('partial') + expect(req.requestHeaders['Upload-Metadata']).toBe('bar YmF6') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload2', + }, + }) + + req = await testStack.nextRequest() + + // Assert that the URLs have been stored. + expect(testUrlStorage.addUpload).toHaveBeenCalled() + + expect(req.url).toBe('https://tus.io/uploads/upload1') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Offset']).toBe(0) + expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') + expect(req.body.size).toBe(5) + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Offset': 5, + }, + }) + + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Offset']).toBe(0) + expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') + expect(req.body.size).toBe(6) + + // Return an error to ensure that the individual partial upload is properly retried. + req.respondWith({ + status: 500, + }) + + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('HEAD') + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Length': 11, + 'Upload-Offset': 0, + }, + }) + + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Offset']).toBe(0) + expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') + expect(req.body.size).toBe(6) + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Offset': 6, + }, + }) + + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') + expect(req.requestHeaders['Upload-Length']).toBeUndefined() + expect(req.requestHeaders['Upload-Concat']).toBe( + 'final;https://tus.io/uploads/upload1 https://tus.io/uploads/upload2', + ) + expect(req.requestHeaders['Upload-Metadata']).toBe('foo aGVsbG8=') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload3', + }, + }) + + await options.onSuccess.toBeCalled + + expect(upload.url).toBe('https://tus.io/uploads/upload3') + expect(options.onProgress).toHaveBeenCalledWith(5, 11) + expect(options.onProgress).toHaveBeenCalledWith(11, 11) + expect(testUrlStorage.removeUpload).toHaveBeenCalled() + }) it('should split a file into multiple parts based on custom `parallelUploadBoundaries`', async () => { const testStack = new TestHttpStack() From b03022638ea141ddff1a2e95d7a596c5bf4805e3 Mon Sep 17 00:00:00 2001 From: loren Date: Thu, 11 Jul 2024 14:50:23 -0400 Subject: [PATCH 2/5] Add api docs for partialMetadata. --- docs/api.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/api.md b/docs/api.md index 04e1eac7..50a4d57f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -156,6 +156,18 @@ metadata: { } ``` +#### partialMetadata + +_Default value:_ `{}` + +An object with string values used as additional meta data for partial uploads which will be passed along to the server when creating a new partial upload. This is separate from the metadata option above, which is not passed along for partial uploads. Can be used for filenames, file types etc, for example: + +```js +partialMetadata: { + userId: "1234567" +} +``` + #### uploadUrl _Default value:_ `null` From b46c1b05653221a422e5ef629b69fca9f565917a Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Mon, 9 Sep 2024 17:05:00 +0200 Subject: [PATCH 3/5] Rename to `metadataForPartialUploads` --- docs/api.md | 4 ++-- lib/index.d.ts | 2 +- lib/index.test-d.ts | 4 ++-- lib/upload.js | 8 ++++---- test/spec/test-parallel-uploads.js | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/api.md b/docs/api.md index fcf690ee..184a4641 100644 --- a/docs/api.md +++ b/docs/api.md @@ -166,14 +166,14 @@ metadata: { } ``` -#### partialMetadata +#### metadataForPartialUploads _Default value:_ `{}` An object with string values used as additional meta data for partial uploads which will be passed along to the server when creating a new partial upload. This is separate from the metadata option above, which is not passed along for partial uploads. Can be used for filenames, file types etc, for example: ```js -partialMetadata: { +metadataForPartialUploads: { userId: "1234567" } ``` diff --git a/lib/index.d.ts b/lib/index.d.ts index d0206dd1..3ef96c9f 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -26,7 +26,7 @@ interface UploadOptions { uploadUrl?: string | null metadata?: { [key: string]: string } - partialMetadata?: { [key: string]: string } + metadataForPartialUploads?: { [key: string]: string } fingerprint?: (file: File, options: UploadOptions) => Promise uploadSize?: number | null diff --git a/lib/index.test-d.ts b/lib/index.test-d.ts index 9d3c6ee3..6f1f399d 100644 --- a/lib/index.test-d.ts +++ b/lib/index.test-d.ts @@ -19,8 +19,8 @@ const upload = new tus.Upload(file, { metadata: { filename: 'foo.txt', }, - partialMetadata: { - userId: "foo123bar", + metadataForPartialUploads: { + userId: 'foo123bar', }, onProgress: (bytesSent: number, bytesTotal: number) => { const percentage = ((bytesSent / bytesTotal) * 100).toFixed(2) diff --git a/lib/upload.js b/lib/upload.js index 9fc6e85a..6ca7d042 100644 --- a/lib/upload.js +++ b/lib/upload.js @@ -12,7 +12,7 @@ const defaultOptions = { uploadUrl: null, metadata: {}, - partialMetadata: {}, + metadataForPartialUploads: {}, fingerprint: null, uploadSize: null, @@ -590,9 +590,9 @@ class BaseUpload { if (metadata !== '') { req.setHeader('Upload-Metadata', metadata) } else { - const partialMetadata = encodeMetadata(this.options.partialMetadata) - if (partialMetadata != '') { - req.setHeader('Upload-Metadata', partialMetadata) + const metadataForPartialUploads = encodeMetadata(this.options.metadataForPartialUploads) + if (metadataForPartialUploads != '') { + req.setHeader('Upload-Metadata', metadataForPartialUploads) } } diff --git a/test/spec/test-parallel-uploads.js b/test/spec/test-parallel-uploads.js index 16f971c9..d4961971 100644 --- a/test/spec/test-parallel-uploads.js +++ b/test/spec/test-parallel-uploads.js @@ -204,7 +204,7 @@ describe('tus', () => { expect(options.onProgress).toHaveBeenCalledWith(11, 11) expect(testUrlStorage.removeUpload).toHaveBeenCalled() }) - it('should add partialMetadata option to Upload-Metadata', async () => { + it('should add metadataForPartialUploads option to Upload-Metadata', async () => { const testStack = new TestHttpStack() const testUrlStorage = { @@ -239,7 +239,7 @@ describe('tus', () => { metadata: { foo: 'hello', }, - partialMetadata: { + metadataForPartialUploads: { bar: 'baz', }, onProgress() {}, From 63eaeb8dfa9232721302f9124cd68e4ac9fda7f7 Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Mon, 9 Sep 2024 17:23:18 +0200 Subject: [PATCH 4/5] Simplify implementation and tests --- lib/upload.js | 7 +- test/spec/test-parallel-uploads.js | 169 +---------------------------- 2 files changed, 7 insertions(+), 169 deletions(-) diff --git a/lib/upload.js b/lib/upload.js index 6ca7d042..6c0218e1 100644 --- a/lib/upload.js +++ b/lib/upload.js @@ -332,7 +332,7 @@ class BaseUpload { parallelUploads: 1, // Reset this option as we are not doing a parallel upload. parallelUploadBoundaries: null, - metadata: {}, + metadata: this.options.metadataForPartialUploads, // Add the header to indicate the this is a partial upload. headers: { ...this.options.headers, @@ -589,11 +589,6 @@ class BaseUpload { const metadata = encodeMetadata(this.options.metadata) if (metadata !== '') { req.setHeader('Upload-Metadata', metadata) - } else { - const metadataForPartialUploads = encodeMetadata(this.options.metadataForPartialUploads) - if (metadataForPartialUploads != '') { - req.setHeader('Upload-Metadata', metadataForPartialUploads) - } } let promise diff --git a/test/spec/test-parallel-uploads.js b/test/spec/test-parallel-uploads.js index d4961971..db7d46d9 100644 --- a/test/spec/test-parallel-uploads.js +++ b/test/spec/test-parallel-uploads.js @@ -76,6 +76,9 @@ describe('tus', () => { metadata: { foo: 'hello', }, + metadataForPartialUploads: { + test: 'world', + }, onProgress() {}, onSuccess: waitableFunction(), fingerprint: () => Promise.resolve('fingerprinted'), @@ -92,7 +95,7 @@ describe('tus', () => { expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') expect(req.requestHeaders['Upload-Length']).toBe(5) expect(req.requestHeaders['Upload-Concat']).toBe('partial') - expect(req.requestHeaders['Upload-Metadata']).toBeUndefined() + expect(req.requestHeaders['Upload-Metadata']).toBe('test d29ybGQ=') // world req.respondWith({ status: 201, @@ -108,7 +111,7 @@ describe('tus', () => { expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') expect(req.requestHeaders['Upload-Length']).toBe(6) expect(req.requestHeaders['Upload-Concat']).toBe('partial') - expect(req.requestHeaders['Upload-Metadata']).toBeUndefined() + expect(req.requestHeaders['Upload-Metadata']).toBe('test d29ybGQ=') // world req.respondWith({ status: 201, @@ -188,167 +191,7 @@ describe('tus', () => { expect(req.requestHeaders['Upload-Concat']).toBe( 'final;https://tus.io/uploads/upload1 https://tus.io/uploads/upload2', ) - expect(req.requestHeaders['Upload-Metadata']).toBe('foo aGVsbG8=') - - req.respondWith({ - status: 201, - responseHeaders: { - Location: 'https://tus.io/uploads/upload3', - }, - }) - - await options.onSuccess.toBeCalled - - expect(upload.url).toBe('https://tus.io/uploads/upload3') - expect(options.onProgress).toHaveBeenCalledWith(5, 11) - expect(options.onProgress).toHaveBeenCalledWith(11, 11) - expect(testUrlStorage.removeUpload).toHaveBeenCalled() - }) - it('should add metadataForPartialUploads option to Upload-Metadata', async () => { - const testStack = new TestHttpStack() - - const testUrlStorage = { - addUpload: (fingerprint, upload) => { - expect(fingerprint).toBe('fingerprinted') - expect(upload.uploadUrl).toBeUndefined() - expect(upload.size).toBe(11) - expect(upload.parallelUploadUrls).toEqual([ - 'https://tus.io/uploads/upload1', - 'https://tus.io/uploads/upload2', - ]) - - return Promise.resolve('tus::fingerprinted::1337') - }, - removeUpload: (urlStorageKey) => { - expect(urlStorageKey).toBe('tus::fingerprinted::1337') - return Promise.resolve() - }, - } - spyOn(testUrlStorage, 'removeUpload').and.callThrough() - spyOn(testUrlStorage, 'addUpload').and.callThrough() - - const file = getBlob('hello world') - const options = { - httpStack: testStack, - urlStorage: testUrlStorage, - storeFingerprintForResuming: true, - removeFingerprintOnSuccess: true, - parallelUploads: 2, - retryDelays: [10], - endpoint: 'https://tus.io/uploads', - metadata: { - foo: 'hello', - }, - metadataForPartialUploads: { - bar: 'baz', - }, - onProgress() {}, - onSuccess: waitableFunction(), - fingerprint: () => Promise.resolve('fingerprinted'), - } - spyOn(options, 'onProgress') - - const upload = new tus.Upload(file, options) - upload.start() - - let req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads') - expect(req.method).toBe('POST') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Length']).toBe(5) - expect(req.requestHeaders['Upload-Concat']).toBe('partial') - expect(req.requestHeaders['Upload-Metadata']).toBe('bar YmF6') - - req.respondWith({ - status: 201, - responseHeaders: { - Location: 'https://tus.io/uploads/upload1', - }, - }) - - req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads') - expect(req.method).toBe('POST') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Length']).toBe(6) - expect(req.requestHeaders['Upload-Concat']).toBe('partial') - expect(req.requestHeaders['Upload-Metadata']).toBe('bar YmF6') - - req.respondWith({ - status: 201, - responseHeaders: { - Location: 'https://tus.io/uploads/upload2', - }, - }) - - req = await testStack.nextRequest() - - // Assert that the URLs have been stored. - expect(testUrlStorage.addUpload).toHaveBeenCalled() - - expect(req.url).toBe('https://tus.io/uploads/upload1') - expect(req.method).toBe('PATCH') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Offset']).toBe(0) - expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') - expect(req.body.size).toBe(5) - - req.respondWith({ - status: 204, - responseHeaders: { - 'Upload-Offset': 5, - }, - }) - - req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads/upload2') - expect(req.method).toBe('PATCH') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Offset']).toBe(0) - expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') - expect(req.body.size).toBe(6) - - // Return an error to ensure that the individual partial upload is properly retried. - req.respondWith({ - status: 500, - }) - - req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads/upload2') - expect(req.method).toBe('HEAD') - - req.respondWith({ - status: 204, - responseHeaders: { - 'Upload-Length': 11, - 'Upload-Offset': 0, - }, - }) - - req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads/upload2') - expect(req.method).toBe('PATCH') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Offset']).toBe(0) - expect(req.requestHeaders['Content-Type']).toBe('application/offset+octet-stream') - expect(req.body.size).toBe(6) - - req.respondWith({ - status: 204, - responseHeaders: { - 'Upload-Offset': 6, - }, - }) - - req = await testStack.nextRequest() - expect(req.url).toBe('https://tus.io/uploads') - expect(req.method).toBe('POST') - expect(req.requestHeaders['Tus-Resumable']).toBe('1.0.0') - expect(req.requestHeaders['Upload-Length']).toBeUndefined() - expect(req.requestHeaders['Upload-Concat']).toBe( - 'final;https://tus.io/uploads/upload1 https://tus.io/uploads/upload2', - ) - expect(req.requestHeaders['Upload-Metadata']).toBe('foo aGVsbG8=') + expect(req.requestHeaders['Upload-Metadata']).toBe('foo aGVsbG8=') // hello req.respondWith({ status: 201, From bb19b1cee82c7740a31513f784fd92759a2479b7 Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Mon, 9 Sep 2024 17:34:22 +0200 Subject: [PATCH 5/5] Update documentation --- docs/api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 184a4641..5d28a921 100644 --- a/docs/api.md +++ b/docs/api.md @@ -170,7 +170,7 @@ metadata: { _Default value:_ `{}` -An object with string values used as additional meta data for partial uploads which will be passed along to the server when creating a new partial upload. This is separate from the metadata option above, which is not passed along for partial uploads. Can be used for filenames, file types etc, for example: +An object with string values used as additional meta data for partial uploads. When parallel uploads are enabled via `parallelUploads`, tus-js-client creates multiple partial uploads. The values from `metadata` are not passed to these partial uploads but only passed to the final upload, which is the concatentation of the partial uploads. In contrast, the values from `metadataForPartialUploads` are only passed to the partial uploads and not the final upload. This option has no effect if parallel uploads are not enabled. Can be used to associate partial uploads to a user, for example: ```js metadataForPartialUploads: { @@ -246,7 +246,7 @@ X-Request-ID: fe51f777-f23e-4ed9-97d7-2785cc69f961 _Default value:_ `1` -A number indicating how many parts should be uploaded in parallel. If this number is not `1`, the input file will be split into multiple parts, where each part is uploaded individually in parallel. The value of `parallelUploads` determines the number of parts. Using `parallelUploadBoundaries` the size of each part can be changed. After all parts have been uploaded, the [`concatenation` extension](https://tus.io/protocols/resumable-upload.html#concatenation) will be used to concatenate all the parts together on the server-side, so the tus server must support this extension. This option should not be used if the input file is a streaming resource. +A number indicating how many parts should be uploaded in parallel. If this number is not `1`, the input file will be split into multiple parts, where each part is uploaded individually in parallel. The value of `parallelUploads` determines the number of parts. Using `parallelUploadBoundaries` the size of each part can be changed. After all parts have been uploaded, the [`concatenation` extension](https://tus.io/protocols/resumable-upload.html#concatenation) will be used to concatenate all the parts together on the server-side, so the tus server must support this extension. This option should not be used if the input file is a streaming resource. By default, the values from `metadata` are not passed to the partial uploads and only used for the final upload where the parts are concatenated together again. The `metadataForPartialUploads` option can be used to set meta data specifically for partial uploads. The idea behind this option is that you can use multiple HTTP requests in parallel to better utilize the full capacity of the network connection to the tus server. If you want to use it, please evaluate it under real world situations to see if it actually improves your upload performance. In common browser session, we were not able to find a performance improve for the average user.