more file submission ops

CHANGELOG.md

@@ -42,6 +42,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - add message broker publishing to workflow
   - add rabbitmq + default config to integration tests
 - File operator that does database operations for files #148
+  - introduced `/v1/submissions/{submissionId}/files` to update and remove files in a submission #633
+  - file flagged for deletion also removed from submission and check files have the status ready when being read from the submission #633
+  - prevent publish if files have in submission have status added (added but no metadata object) or failed (failed in ingestion, completion, or for any other reason) #633
 - Mongo indexes for `file` schema #148
 - `/files` endpoint to retrieve files attached to a project #148 #627
 - option to add additional members to `application/problem+json` #642

docs/specification.yml

@@ -1409,6 +1409,179 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/405MethodNotAllowed"
+  /v1/submissions/{submissionId}/files:
+    post:
+      tags:
+        - Submission
+      summary: Add new files to a submission.
+      parameters:
+        - name: submissionId
+          in: path
+          description: ID of the object submission.
+          schema:
+            type: string
+          required: true
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                type: object
+                properties:
+                  accessionId:
+                    type: string
+                    title: Accession Id for file
+                  version:
+                    type: integer
+                    title: Version of the file
+      responses:
+        204:
+          description: No Content
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/400BadRequest"
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/401Unauthorized"
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/403Forbidden"
+    get:
+      tags:
+        - Query
+      summary: Retrieve all files with detailed info specific to a submission.
+      parameters:
+        - name: submissionId
+          in: path
+          description: ID of the object submission.
+          schema:
+            type: string
+          required: true
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/File"
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/400BadRequest"
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/401Unauthorized"
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/403Forbidden"
+    put:
+      tags:
+        - Manage
+      summary: Update Files for the submission with a specified submission ID.
+      parameters:
+        - name: submissionId
+          in: path
+          description: ID of the object submission.
+          schema:
+            type: string
+          required: true
+      requestBody:
+        content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/SubmissionFile"
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SubmissionCreated"
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/400BadRequest"
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/401Unauthorized"
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/403Forbidden"
+        405:
+          description: Method Not Allowed
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/405MethodNotAllowed"
+  /v1/submissions/{submissionId}/files/{fileId}:
+    delete:
+      tags:
+        - Manage
+      summary: Remove file from a submission.
+      parameters:
+        - name: submissionId
+          in: path
+          description: ID of the object submission.
+          schema:
+            type: string
+          required: true
+        - name: fileId
+          in: path
+          description: ID of the object submission.
+          schema:
+            type: string
+          required: true
+      responses:
+        204:
+          description: No Content
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/400BadRequest"
+        401:
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/401Unauthorized"
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/403Forbidden"
   /v1/publish/{submissionId}:
     patch:
       tags:
@@ -1603,7 +1776,7 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/REMS"
+                  $ref: "#/components/schemas/File"
         400:
           description: Bad Request
           content:
@@ -1747,7 +1920,7 @@ components:
         - dateUpdated
       additionalProperties: true
       properties:
-        accessioniId:
+        accessionId:
           type: string
           description: accession id generated to identify an object
         publishedDate:
@@ -2956,3 +3129,31 @@ components:
           $ref: "#/components/schemas/Checksum"
         unencrypted_checksums:
           $ref: "#/components/schemas/Checksum"
+    SubmissionFile:
+      type: object
+      description: Describes a submission file information that can be attached to a submission
+      properties:
+        accessionId:
+          type: string
+          title: Accession Id for file
+        version:
+          type: integer
+          title: Version of the file
+        status:
+          type: string
+          title: Status of the file
+          enum: ["added","ready","verified","completed","failed"]
+        objectId:
+          type: object
+          title: File size in bytes
+          properties:
+            accessionId:
+              type: string
+              description: accession id generated to identify an object
+            schema:
+                type: string
+                description: type of schema this Accession ID relates to and was added in submit
+        error:
+          type: string
+          title: Error for file if status is failed
+          description: required only if status is failed

metadata_backend/api/handlers/publish.py

@@ -516,9 +516,12 @@ async def publish_submission(self, req: Request) -> Response:
         # Publish to external services - must already have DOI and Metax ID
         publish_status = {}
         datacite_study = {}
+        # check first if all the files are ready, if not return HTTPBadRequest
+        await file_operator.check_submission_files_ready(submission_id)
         if "messageBroker" in workflow.endpoints:
 
-            files = await file_operator.read_submission_files(submission_id)
+            # we will only publish the files which are ready
+            files = await file_operator.read_submission_files(submission_id, ["ready"])
             for file in files:
                 ingest_msg = {
                     "type": "ingest",

metadata_backend/api/handlers/submission.py

@@ -297,19 +297,37 @@ async def put_submission_path(self, req: Request) -> Response:
         elif req.path.endswith("rems"):
             schema = "rems"
             await self.check_rems_ok({"rems": data})
+        elif req.path.endswith("files"):
+            schema = "files"
         else:
             raise web.HTTPNotFound(reason=f"'{req.path}' does not exist")
 
-        submission[schema] = data
-        JSONValidator(submission, "submission").validate
+        if schema == "files":
+            file_operator = FileOperator(db_client)
+            # we expect to get a list of dict for files
+            # that matches the json schema when added to submission object
+            submission[schema] = data
+            JSONValidator(submission, "submission").validate
+            for file in data:
+                if "accessionId" not in file:
+                    reason = f"Updating {submission_id} failed. Files require an accessionId."
+                    LOG.error(reason)
+                    raise web.HTTPBadRequest(reason=reason)
+                _file_accessionId = file.pop("accessionId")
+                _file_update_op = {f"files.$.{k}": v for k, v in file.items()}
+                await file_operator.update_file_submission(_file_accessionId, submission_id, _file_update_op)
+        else:
+            op = "add"
+            if schema in submission:
+                op = "replace"
+            patch = [
+                {"op": op, "path": f"/{schema}", "value": data},
+            ]
+
+            submission[schema] = data
+            JSONValidator(submission, "submission").validate
 
-        op = "add"
-        if schema in submission:
-            op = "replace"
-        patch = [
-            {"op": op, "path": f"/{schema}", "value": data},
-        ]
-        upd_submission = await operator.update_submission(submission_id, patch)
+            upd_submission = await operator.update_submission(submission_id, patch)
 
         body = ujson.dumps({"submissionId": upd_submission}, escape_forward_slashes=False)
         LOG.info("PUT %r in submission with ID: %r was successful.", schema, submission_id)
@@ -363,7 +381,7 @@ async def add_submission_files(self, req: Request) -> Response:
         data: List[Dict] = await req.json()
 
         if all("accessionId" in d and "version" in d for d in data):
-            # add status to
+            # set status to file as added
             data = [{**item, "status": "added"} for item in data]
             await file_operator.add_files_submission(data, submission_id)
             LOG.info("Adding files to submission with ID: %r was successful.", submission_id)
@@ -372,3 +390,28 @@ async def add_submission_files(self, req: Request) -> Response:
         reason = "Request does not contain a list of Objects each with `accessionId` and `version`"
         LOG.error(reason)
         raise web.HTTPBadRequest(reason=reason)
+
+    async def delete_submission_files(self, req: Request) -> Response:
+        """Remove files from a submission.
+
+        Body needs to contain a list of accessionId for files.
+
+        :param req: DELETE request
+        :returns: HTTP No Content response
+        """
+        submission_id = req.match_info["submissionId"]
+        file_accession_id = req.match_info["fileId"]
+        db_client = req.app["db_client"]
+        submission_operator = SubmissionOperator(db_client)
+
+        # Check submission exists and is not already published
+        await submission_operator.check_submission_exists(submission_id)
+        await submission_operator.check_submission_published(submission_id, req.method)
+
+        await self._handle_check_ownership(req, "submission", submission_id)
+
+        file_operator = FileOperator(db_client)
+
+        await file_operator.remove_file_submission(file_accession_id, "accessionId", submission_id)
+        LOG.info("Removing file: %r from submission with ID: %r was successful.", file_accession_id, submission_id)
+        return web.HTTPNoContent()