diff --git a/src/Pages/Repositories/ContentListTable/components/UploadContent/UploadContent.tsx b/src/Pages/Repositories/ContentListTable/components/UploadContent/UploadContent.tsx index ce249370..3fe2c016 100644 --- a/src/Pages/Repositories/ContentListTable/components/UploadContent/UploadContent.tsx +++ b/src/Pages/Repositories/ContentListTable/components/UploadContent/UploadContent.tsx @@ -25,7 +25,7 @@ const useStyles = createUseStyles({ const UploadContent = () => { const classes = useStyles(); const { repoUUID: uuid } = useParams(); - const [fileUUIDs, setFileUUIDs] = useState<{ sha256: string; uuid: string }[]>([]); + const [fileUUIDs, setFileUUIDs] = useState<{ sha256: string; uuid: string; href: string }[]>([]); const [confirmModal, setConfirmModal] = useState(false); const rootPath = useRootPath(); @@ -37,7 +37,18 @@ const UploadContent = () => { const { mutateAsync: uploadItems, isLoading } = useAddUploadsQuery({ repoUUID: uuid!, - uploads: fileUUIDs, + uploads: fileUUIDs + .filter(({ href }) => !href) + .map(({ sha256, uuid }) => ({ + sha256, + uuid, + })), + artifacts: fileUUIDs + .filter(({ href }) => href) + .map(({ sha256, href }) => ({ + sha256, + href, + })), }); return ( diff --git a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/FileUploader.tsx b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/FileUploader.tsx index a2f150bf..6c8a66a4 100644 --- a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/FileUploader.tsx +++ b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/FileUploader.tsx @@ -3,15 +3,24 @@ import { MultipleFileUpload, MultipleFileUploadMain, MultipleFileUploadStatus, + Tooltip, type DropEvent, } from '@patternfly/react-core'; import UploadStatusItem from './UploadStatusItem'; import StatusIcon from 'Pages/Repositories/AdminTaskTable/components/StatusIcon'; -import { getFileChecksumSHA256, type Chunk, type FileInfo } from './helpers'; +import { + BATCH_SIZE, + getFileChecksumSHA256, + MAX_CHUNK_SIZE, + MAX_RETRY_COUNT, + type Chunk, + type FileInfo, +} from './helpers'; import { createUseStyles } from 'react-jss'; import { createUpload, uploadChunk } from 'services/Content/ContentApi'; import Loader from 'components/Loader'; -import { UploadIcon } from '@patternfly/react-icons'; +import { PlayIcon, RecycleIcon, UploadIcon } from '@patternfly/react-icons'; +import { global_success_color_100 } from '@patternfly/react-tokens'; const useStyles = createUseStyles({ mainDropzone: { @@ -20,14 +29,10 @@ const useStyles = createUseStyles({ }, }); -export const MAX_CHUNK_SIZE = 1048576 * 3; // MB - -export const BATCH_SIZE = 5; - -export const MAX_RETRY_COUNT = 3; - interface Props { - setFileUUIDs: React.Dispatch>; + setFileUUIDs: React.Dispatch< + React.SetStateAction<{ sha256: string; uuid: string; href: string }[]> + >; isLoading: boolean; } @@ -53,9 +58,10 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { useEffect(() => { if (completedCount === fileCount) { - const items = Object.values(currentFiles).map(({ uuid, checksum }) => ({ + const items = Object.values(currentFiles).map(({ uuid, checksum, artifact }) => ({ sha256: checksum, uuid, + href: artifact, })); setFileUUIDs(items); @@ -76,8 +82,8 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { if (currentFiles[name]) { const targetIndexes = new Set( currentFiles[name].chunks - .map(({ queued }, index) => ({ index, queued })) - .filter(({ queued }) => !queued) + .map(({ queued, completed }, index) => ({ index, queued, completed })) + .filter(({ queued, completed }) => !completed && !queued) .map(({ index }) => index), ); @@ -86,18 +92,16 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { const result = await Promise.all( itemsForBatch.map(async (targetIndex) => { if (!currentFiles[name]?.chunks[targetIndex]) return; - const { start, end } = currentFiles[name].chunks[targetIndex]; + const { start, end, chunkRange, sha256, slice } = + currentFiles[name].chunks[targetIndex]; currentFiles[name].chunks[targetIndex].queued = true; setCurrentFiles((prev) => ({ ...prev, [name]: currentFiles[name] })); - const slice = currentFiles[name].file.slice(start, end + 1); - - const chunkRange = `bytes ${start}-${end}/${currentFiles[name].file.size}`; try { await uploadChunk({ chunkRange: chunkRange, created: currentFiles[name].created, - sha256: await getFileChecksumSHA256(new File([slice], name + chunkRange)), + sha256, file: slice, upload_uuid: currentFiles[name].uuid, }); @@ -169,7 +173,7 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { ? file.size / MAX_CHUNK_SIZE : Math.floor(file.size / MAX_CHUNK_SIZE) + 1; - const chunks: Chunk[] = []; + let chunks: Chunk[] = []; for (let index = 0; index < totalCount; index++) { const start = index ? index * MAX_CHUNK_SIZE : 0; @@ -178,10 +182,22 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { end = file.size - 1; } - chunks.push({ start, end, queued: false, completed: false, retryCount: 0 }); + const chunkRange = `bytes ${start}-${end}/${file.size}`; + const slice = file.slice(start, end + 1); + + chunks.push({ + slice, + start, + end, + queued: false, + completed: false, + retryCount: 0, + sha256: await getFileChecksumSHA256(new File([slice], file.name + chunkRange)), + chunkRange, + }); } - let checksum: string; + let checksum: string = ''; let error: string | undefined = undefined; try { @@ -192,18 +208,40 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { let uuid: string = ''; let created: string = ''; + let artifact: string = ''; + let completedChunkChecksums = new Set(); if (!error) { try { - const res = await createUpload(file.size); - uuid = res.upload_uuid; - created = res.created; + const res = await createUpload(file.size, checksum); + if (res.upload_uuid) uuid = res.upload_uuid; + if (res.created) created = res.created; + if (res.completed_checksums) + completedChunkChecksums = new Set(res.completed_checksums); + if (res.artifact_href) artifact = res.artifact_href; } catch (err) { error = 'Failed to create upload file: ' + (err as Error).message; } } + chunks = chunks.map((chunk) => ({ + ...chunk, + completed: !!artifact || completedChunkChecksums.has(chunk.sha256), + })); + setCurrentFiles((prev) => { - prev[file.name] = { uuid, created, chunks, file, checksum, error, failed: !!error }; + prev[file.name] = { + uuid, + artifact, + created, + chunks, + file, + checksum, + error, + failed: !!error, + completed: !!artifact || chunks.every(({ completed }) => completed), + isResumed: chunks.some(({ completed }) => completed), + }; + return { ...prev }; }); }; @@ -309,33 +347,54 @@ export default function FileUploader({ setFileUUIDs, isLoading }: Props) { statusToggleText={`${completedCount} of ${fileCount} files are ready to be added to the repository${failedCount ? `, ${failedCount} failed` : ''}`} statusToggleIcon={statusIcon} > - {Object.values(currentFiles).map(({ checksum, chunks, error, file, failed }) => { - const completedChunks = chunks.filter(({ completed }) => completed).length; - const progressValue = Math.round((completedChunks / chunks.length) * 100); - - return ( - { - switch (true) { - case failed: - return 'danger'; - case progressValue >= 100: - return 'success'; - default: - break; - } - })()} - retry={checksum ? () => retryItem(file.name) : undefined} - progressHelperText={error} - progressValue={progressValue} - deleteButtonDisabled={!failed && progressValue < 100 && progressValue > 0} - onClearClick={() => removeItem(file.name)} - /> - ); - })} + {Object.values(currentFiles).map( + ({ checksum, chunks, error, file, failed, artifact, isResumed }) => { + const completedChunks = chunks.filter(({ completed }) => completed).length; + const progressValue = Math.round((completedChunks / chunks.length) * 100); + + return ( + { + switch (true) { + case failed: + return 'danger'; + case progressValue >= 100: + return 'success'; + default: + break; + } + })()} + retry={checksum ? () => retryItem(file.name) : undefined} + progressHelperText={error} + progressValue={progressValue} + progressLabel={(() => { + switch (true) { + case !!artifact: + return ( + + + + ); + + case isResumed: + return ( + + + + ); + default: + return ''; + } + })()} + deleteButtonDisabled={!failed && progressValue < 100 && progressValue > 0} + onClearClick={() => removeItem(file.name)} + /> + ); + }, + )} )} diff --git a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/UploadStatusItem.tsx b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/UploadStatusItem.tsx index 5cf39d5f..e749702c 100644 --- a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/UploadStatusItem.tsx +++ b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/UploadStatusItem.tsx @@ -37,6 +37,7 @@ export interface MultipleFileUploadStatusItemProps extends React.HTMLProps } value={progressValue} + label={progressLabel} variant={progressVariant} aria-label={progressAriaLabel} aria-labelledby={progressAriaLabelledBy} diff --git a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/helpers.tsx b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/helpers.tsx index 19f3ac24..d3c11cde 100644 --- a/src/Pages/Repositories/ContentListTable/components/UploadContent/components/helpers.tsx +++ b/src/Pages/Repositories/ContentListTable/components/UploadContent/components/helpers.tsx @@ -1,5 +1,11 @@ import CryptoJS from 'crypto-js'; +export const MAX_CHUNK_SIZE = 1048576 * 3; // MB + +export const BATCH_SIZE = 5; + +export const MAX_RETRY_COUNT = 3; + const readSlice = (file: File, start: number, size: number): Promise => new Promise((resolve, reject) => { const fileReader = new FileReader(); @@ -28,6 +34,9 @@ export const getFileChecksumSHA256 = async (file: File): Promise => { }; export type Chunk = { + slice: Blob; + sha256: string; + chunkRange: string; start: number; end: number; queued: boolean; @@ -38,10 +47,12 @@ export type Chunk = { export type FileInfo = { uuid: string; created: string; + artifact: string; chunks: Chunk[]; checksum: string; + file: File; error?: string; completed?: boolean; failed?: boolean; - file: File; + isResumed?: boolean; }; diff --git a/src/services/Content/ContentApi.ts b/src/services/Content/ContentApi.ts index cfffddb6..74ef5e25 100644 --- a/src/services/Content/ContentApi.ts +++ b/src/services/Content/ContentApi.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import { objectToUrlParams } from 'helpers'; import { AdminTask } from '../AdminTasks/AdminTaskApi'; +import { MAX_CHUNK_SIZE } from 'Pages/Repositories/ContentListTable/components/UploadContent/components/helpers'; export interface ContentItem { uuid: string; @@ -241,11 +242,13 @@ export type IntrospectRepositoryRequestItem = { }; export interface UploadResponse { - completed: string; - created: string; - last_updated: string; + completed_checksums?: string[]; + artifact_href?: string; + completed?: string; + created?: string; + last_updated?: string; size: number; - upload_uuid: string; + upload_uuid?: string; } export interface UploadChunkRequest { @@ -258,6 +261,7 @@ export interface UploadChunkRequest { export interface AddUploadRequest { uploads: { sha256: string; uuid: string }[]; + artifacts: { sha256: string; href: string }[]; repoUUID: string; } @@ -544,8 +548,15 @@ export const getSnapshotErrata: ( return data; }; -export const createUpload: (size: number) => Promise = async (size) => { - const { data } = await axios.post('/api/content-sources/v1.0/repositories/uploads/', { size }); +export const createUpload: (size: number, sha256: string) => Promise = async ( + size, + sha256, +) => { + const { data } = await axios.post('/api/content-sources/v1.0/repositories/uploads/', { + size, + sha256, + chunkSize: MAX_CHUNK_SIZE, + }); return data; }; @@ -568,11 +579,12 @@ export const uploadChunk: (chunkRequest: UploadChunkRequest) => Promise Promise = async ({ uploads, + artifacts, repoUUID, }) => { const { data } = await axios.post( `/api/content-sources/v1.0/repositories/${repoUUID}/add_uploads/`, - { uploads }, + { uploads, artifacts }, ); return data; }; diff --git a/src/services/Content/ContentQueries.ts b/src/services/Content/ContentQueries.ts index 1cdaa1ca..02eb11a9 100644 --- a/src/services/Content/ContentQueries.ts +++ b/src/services/Content/ContentQueries.ts @@ -202,11 +202,12 @@ export const useAddUploadsQuery = (request: AddUploadRequest) => { const { notify } = useNotification(); return useMutation(() => addUploads(request), { onSuccess: (data) => { + const uploadCount = (request?.uploads?.length || 0) + (request?.artifacts?.length || 0); notify({ variant: AlertVariant.success, title: - request.uploads.length > 1 - ? `${request.uploads.length} rpms successfully uploaded to ${data.object_name}` + uploadCount > 1 + ? `${uploadCount} rpms successfully uploaded to ${data.object_name}` : `One rpm successfully uploaded to ${data.object_name}`, description: 'This repository will be snapshotted shortly', id: 'add-upload-success',