From 0d2d89ae3d9709c2a4b7216291f316ee38c896f9 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 5 Apr 2023 15:58:14 +0200 Subject: [PATCH 01/28] feat: ws' status API to require auth token (#1423) * feat: ws' status API to require auth token * feat: ws' status API pass auth token to hook validator --- .../CommitHistoryChangesSpec.scala | 4 +- .../acceptancetests/CommitSyncFlowsSpec.scala | 4 +- .../acceptancetests/EventFlowsSpec.scala | 6 +- .../EventsProcessingStatusSpec.scala | 29 +++++---- .../ProjectReProvisioningSpec.scala | 2 +- .../flows/TSProvisioning.scala | 30 ++++++---- .../DatasetsResourcesSpec.scala | 2 +- .../knowledgegraph/LineageResourcesSpec.scala | 2 +- .../tooling/ServicesClients.scala | 6 +- .../webhookservice/MicroserviceRoutes.scala | 8 +-- .../webhookservice/eventstatus/Endpoint.scala | 22 +++---- .../hookvalidation/ProjectHookVerifier.scala | 6 +- .../MicroserviceRoutesSpec.scala | 59 ++++++++++++++----- .../eventstatus/EndpointSpec.scala | 35 ++++++----- 14 files changed, 133 insertions(+), 82 deletions(-) diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitHistoryChangesSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitHistoryChangesSpec.scala index 5345deea45..dd96783d5c 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitHistoryChangesSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitHistoryChangesSpec.scala @@ -86,7 +86,7 @@ class CommitHistoryChangesSpec sleep((10 seconds).toMillis) - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) eventually { EventLog.findEvents(project.id, events.EventStatus.TriplesStore).toSet shouldBe newCommits.toList.toSet @@ -125,7 +125,7 @@ class CommitHistoryChangesSpec sleep((1 second).toMillis) - `check no hook exists`(project.id) + `check no hook exists`(project.id, user.accessToken) Then("the project and its datasets should be removed from the knowledge-graph") diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitSyncFlowsSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitSyncFlowsSpec.scala index 1fb9127f27..577c8f29c0 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitSyncFlowsSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/CommitSyncFlowsSpec.scala @@ -69,7 +69,7 @@ class CommitSyncFlowsSpec extends AcceptanceSpec with ApplicationServices with T .status shouldBe Accepted And("relevant commit events are processed") - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) Then("the non missed events should be in the Triples Store") eventually { @@ -83,7 +83,7 @@ class CommitSyncFlowsSpec extends AcceptanceSpec with ApplicationServices with T EventLog.forceCategoryEventTriggering(CategoryName("COMMIT_SYNC"), project.id) And("commit events for the missed event are created and processed") - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) Then("triples for both of the project's commits should be in the Triples Store") eventually { diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventFlowsSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventFlowsSpec.scala index 699d7b7e57..7d4fbf8b45 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventFlowsSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventFlowsSpec.scala @@ -59,7 +59,7 @@ class EventFlowsSpec extends AcceptanceSpec with ApplicationServices with TSProv .status shouldBe Accepted And("commit events are processed") - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) Then(s"all the events should get the $TriplesStore status in the Event Log") EventLog.findEvents(project.id).map(_._2).toSet shouldBe Set(TriplesStore) @@ -89,7 +89,7 @@ class EventFlowsSpec extends AcceptanceSpec with ApplicationServices with TSProv .status shouldBe Accepted And("relevant commit events are processed") - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) And(s"all the events should get the $GenerationNonRecoverableFailure status in the Event Log") EventLog.findEvents(project.id).map(_._2).toSet shouldBe Set(GenerationNonRecoverableFailure) @@ -151,7 +151,7 @@ class EventFlowsSpec extends AcceptanceSpec with ApplicationServices with TSProv .status shouldBe Accepted And("relevant commit events are processed") - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) Then(s"all the events should get the $TransformationNonRecoverableFailure status in the Event Log") EventLog.findEvents(project.id).map(_._2).toSet shouldBe Set(TransformationNonRecoverableFailure) diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventsProcessingStatusSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventsProcessingStatusSpec.scala index 6849b7f247..fdc4a8ccca 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventsProcessingStatusSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/EventsProcessingStatusSpec.scala @@ -18,9 +18,8 @@ package io.renku.graph.acceptancetests -import cats.syntax.all._ -import data.Project.Statistics.CommitsCount import data._ +import data.Project.Statistics.CommitsCount import eu.timepit.refined.api.Refined import eu.timepit.refined.auto._ import eu.timepit.refined.numeric.Positive @@ -57,35 +56,39 @@ class EventsProcessingStatusSpec CommitsCount(numberOfEvents.value) ).map(addMemberWithId(user.id)).generateOne + gitLabStub.addAuthenticated(user) + When("there's no webhook for a given project in GitLab") - Then("the status endpoint should return NOT_FOUND") - webhookServiceClient.fetchProcessingStatus(project.id).status shouldBe NotFound + Then("the status endpoint should return OK with 'activated' = false") + + val response = webhookServiceClient.fetchProcessingStatus(project.id, user.accessToken) + response.status shouldBe Ok + response.jsonBody.hcursor.downField("activated").as[Boolean].value shouldBe false - When("there is a webhook created") + When("a webhook created for the project") And("there are events under processing") val allCommitIds = commitIds.generateNonEmptyList(min = numberOfEvents, max = numberOfEvents) - gitLabStub.addAuthenticated(user) gitLabStub.setupProject(project, allCommitIds.toList: _*) mockCommitDataOnTripleGenerator(project, toPayloadJsonLD(project), allCommitIds) `data in the Triples Store`(project, allCommitIds, user.accessToken) Then("the status endpoint should return OK with some progress info") eventually { - val response = webhookServiceClient.fetchProcessingStatus(project.id) + val response = webhookServiceClient.fetchProcessingStatus(project.id, user.accessToken) response.status shouldBe Ok val responseJson = response.jsonBody.hcursor - responseJson.downField("activated").as[Boolean] shouldBe true.asRight + responseJson.downField("activated").as[Boolean].value shouldBe true val progressObjCursor = responseJson.downField("progress").as[Json].fold(throw _, identity).hcursor - progressObjCursor.downField("done").as[Int] shouldBe EventStatusProgress.Stage.Final.value.asRight - progressObjCursor.downField("total").as[Int] shouldBe EventStatusProgress.Stage.Final.value.asRight - progressObjCursor.downField("percentage").as[Float] shouldBe 100f.asRight + progressObjCursor.downField("done").as[Int].value shouldBe EventStatusProgress.Stage.Final.value + progressObjCursor.downField("total").as[Int].value shouldBe EventStatusProgress.Stage.Final.value + progressObjCursor.downField("percentage").as[Float].value shouldBe 100f val detailsObjCursor = responseJson.downField("details").as[Json].fold(throw _, identity).hcursor - detailsObjCursor.downField("status").as[String] shouldBe "success".asRight - detailsObjCursor.downField("message").as[String] shouldBe "triples store".asRight + detailsObjCursor.downField("status").as[String].value shouldBe "success" + detailsObjCursor.downField("message").as[String].value shouldBe "triples store" } } } diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/ProjectReProvisioningSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/ProjectReProvisioningSpec.scala index b52d039471..449b488a33 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/ProjectReProvisioningSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/ProjectReProvisioningSpec.scala @@ -87,7 +87,7 @@ class ProjectReProvisioningSpec extends AcceptanceSpec with ApplicationServices Then("the old data in the TS should be replaced with the new") sleep((10 seconds).toMillis) - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, user.accessToken) eventually { knowledgeGraphClient diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/flows/TSProvisioning.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/flows/TSProvisioning.scala index e88b3aa495..34dfcc8f64 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/flows/TSProvisioning.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/flows/TSProvisioning.scala @@ -33,7 +33,7 @@ import io.renku.http.client.AccessToken import io.renku.testtools.IOSpec import io.renku.webhookservice.model.HookToken import org.http4s.Status._ -import org.scalatest.Assertion +import org.scalatest.{Assertion, EitherValues} import org.scalatest.concurrent.Eventually import org.scalatest.matchers.should @@ -46,7 +46,9 @@ trait TSProvisioning with AccessTokenPresence with Eventually with AcceptanceTestPatience - with should.Matchers { + with should.Matchers + with EitherValues { + self: ApplicationServices with IOSpec => def `data in the Triples Store`( @@ -72,21 +74,25 @@ trait TSProvisioning sleep((5 seconds).toMillis) } - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, accessToken) } - def `wait for events to be processed`(projectId: projects.GitLabId): Assertion = eventually { - val response = fetchProcessingStatus(projectId) - response.status shouldBe Ok - response.jsonBody.hcursor.downField("progress").downField("percentage").as[Double] shouldBe Right(100d) - } + def `wait for events to be processed`(projectId: projects.GitLabId, accessToken: AccessToken): Assertion = + eventually { + val response = fetchProcessingStatus(projectId, accessToken) + response.status shouldBe Ok + response.jsonBody.hcursor.downField("activated").as[Boolean].value shouldBe true + response.jsonBody.hcursor.downField("progress").downField("percentage").as[Double].value shouldBe 100d + } - def `check no hook exists`(projectId: projects.GitLabId): Assertion = eventually { - fetchProcessingStatus(projectId).status shouldBe NotFound + def `check no hook exists`(projectId: projects.GitLabId, accessToken: AccessToken): Assertion = eventually { + val response = fetchProcessingStatus(projectId, accessToken) + response.status shouldBe Ok + response.jsonBody.hcursor.downField("activated").as[Boolean].value shouldBe false } - private def fetchProcessingStatus(projectId: projects.GitLabId) = - webhookServiceClient.fetchProcessingStatus(projectId) + private def fetchProcessingStatus(projectId: projects.GitLabId, accessToken: AccessToken) = + webhookServiceClient.fetchProcessingStatus(projectId, accessToken) def `wait for the Fast Tract event`(projectId: projects.GitLabId)(implicit ioRuntime: IORuntime): Unit = eventually { diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/DatasetsResourcesSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/DatasetsResourcesSpec.scala index b5b7a021d7..362fcc7caa 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/DatasetsResourcesSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/DatasetsResourcesSpec.scala @@ -477,7 +477,7 @@ class DatasetsResourcesSpec gitLabStub.setupProject(project, commitId) mockCommitDataOnTripleGenerator(project, toPayloadJsonLD(project), commitId) `data in the Triples Store`(project, commitId, creator.accessToken) - `wait for events to be processed`(project.id) + `wait for events to be processed`(project.id, creator.accessToken) When("an authenticated and authorised user fetches dataset details through GET knowledge-graph/datasets/:id") val detailsResponse = diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/LineageResourcesSpec.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/LineageResourcesSpec.scala index 060929eafd..e20137fb06 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/LineageResourcesSpec.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/LineageResourcesSpec.scala @@ -86,8 +86,8 @@ class LineageResourcesSpec extends AcceptanceSpec with ApplicationServices with Then("they should get Ok response with project lineage in Json") response.status shouldBe Ok - val lineageJson = response.jsonBody.hcursor + val lineageJson = response.jsonBody.hcursor lineageJson.downField("edges").as[List[Json]].map(_.toSet) shouldBe theExpectedEdges(exemplarData) lineageJson.downField("nodes").as[List[Json]].map(_.toSet) shouldBe theExpectedNodes(exemplarData) } diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/ServicesClients.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/ServicesClients.scala index 64541051e1..7ad8dc9c83 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/ServicesClients.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/ServicesClients.scala @@ -65,8 +65,10 @@ object WebhookServiceClient { } yield response }.unsafeRunSync() - def fetchProcessingStatus(projectId: projects.GitLabId)(implicit ioRuntime: IORuntime): ClientResponse = - GET((uri"projects" / projectId / "events" / "status").renderString) + def fetchProcessingStatus(projectId: projects.GitLabId, accessToken: AccessToken)(implicit + ior: IORuntime + ): ClientResponse = + GET((uri"projects" / projectId / "events" / "status").renderString, accessToken) } } diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala b/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala index a96aa34c9a..59da710ae1 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala @@ -61,16 +61,16 @@ private class MicroserviceRoutes[F[_]: MonadThrow]( // format: off private lazy val authorizedRoutes: HttpRoutes[F] = authMiddleware { AuthedRoutes.of { + case GET -> Root / "projects" / ProjectId(projectId) / "events" / "status" as authUser => fetchProcessingStatus(projectId, authUser) case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => createHook(projectId, authUser) case DELETE -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => deleteHook(projectId, authUser) case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" / "validation" as authUser => validateHook(projectId, authUser) } } - lazy val nonAuthorizedRoutes: HttpRoutes[F] = HttpRoutes.of[F] { - case GET -> Root / "ping" => Ok("pong") - case request @ POST -> Root / "webhooks" / "events" => processPushEvent(request) - case GET -> Root / "projects" / ProjectId(projectId) / "events" / "status" => fetchProcessingStatus(projectId) + private lazy val nonAuthorizedRoutes: HttpRoutes[F] = HttpRoutes.of[F] { + case GET -> Root / "ping" => Ok("pong") + case request @ POST -> Root / "webhooks" / "events" => processPushEvent(request) } // format: on diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala index f73e6c3bc7..bc0afd68fb 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala @@ -25,9 +25,10 @@ import cats.syntax.all._ import io.circe.syntax._ import io.renku.graph.model.projects import io.renku.graph.model.projects.GitLabId +import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.http.ErrorMessage._ import io.renku.http.client.GitLabClient -import io.renku.http.{ErrorMessage, InfoMessage} +import io.renku.http.server.security.model.AuthUser import io.renku.logging.ExecutionTimeRecorder import io.renku.webhookservice.hookvalidation import io.renku.webhookservice.hookvalidation.HookValidator @@ -39,7 +40,7 @@ import org.http4s.dsl.Http4sDsl import org.typelevel.log4cats.Logger trait Endpoint[F[_]] { - def fetchProcessingStatus(projectId: GitLabId): F[Response[F]] + def fetchProcessingStatus(projectId: GitLabId, authUser: AuthUser): F[Response[F]] } private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( @@ -52,8 +53,8 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( private val executionTimeRecorder = ExecutionTimeRecorder[F] import executionTimeRecorder._ - def fetchProcessingStatus(projectId: GitLabId): F[Response[F]] = measureExecutionTime { - validateHook(projectId) + def fetchProcessingStatus(projectId: GitLabId, authUser: AuthUser): F[Response[F]] = measureExecutionTime { + validateHook(projectId, authUser) .semiflatMap { case HookExists => findStatus(projectId) case HookMissing => Ok(StatusInfo.NotActivated.asJson) @@ -62,12 +63,13 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( .recoverWith(internalServerError(projectId)) } map logExecutionTime(withMessage = show"Finding status info for project '$projectId' finished") - private def validateHook(projectId: GitLabId): EitherT[F, Response[F], HookValidationResult] = EitherT { - hookValidator - .validateHook(projectId, maybeAccessToken = None) - .map(_.asRight[Response[F]]) - .recoverWith(noAccessTokenToNotFound) - } + private def validateHook(projectId: GitLabId, authUser: AuthUser): EitherT[F, Response[F], HookValidationResult] = + EitherT { + hookValidator + .validateHook(projectId, authUser.accessToken.some) + .map(_.asRight[Response[F]]) + .recoverWith(noAccessTokenToNotFound) + } private lazy val noAccessTokenToNotFound: PartialFunction[Throwable, F[Either[Response[F], HookValidationResult]]] = { case _: NoAccessTokenException => NotFound(InfoMessage("Info about project cannot be found")).map(_.asLeft) diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala index 2e72a2499f..458fc32b4c 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala @@ -44,9 +44,9 @@ private class ProjectHookVerifierImpl[F[_]: Async: Logger]( ) extends ProjectHookVerifier[F] { override def checkHookPresence(projectHookId: HookIdentifier, accessToken: AccessToken): F[Boolean] = - projectHookFetcher.fetchProjectHooks(projectHookId.projectId, accessToken) map checkProjectHookExists( - projectHookId.projectHookUrl - ) + projectHookFetcher + .fetchProjectHooks(projectHookId.projectId, accessToken) + .map(checkProjectHookExists(projectHookId.projectHookUrl)) private def checkProjectHookExists(urlToFind: ProjectHookUrl): List[HookIdAndUrl] => Boolean = hooksIdsAndUrls => hooksIdsAndUrls.map(_.url.value) contains urlToFind.value diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala index 6abf3a0507..e79783bb9a 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala @@ -34,9 +34,9 @@ import io.renku.testtools.IOSpec import io.renku.webhookservice.hookcreation.HookCreationEndpoint import io.renku.webhookservice.hookdeletion.HookDeletionEndpoint import io.renku.webhookservice.hookvalidation.HookValidationEndpoint +import org.http4s._ import org.http4s.Method.GET import org.http4s.Status._ -import org.http4s._ import org.http4s.implicits._ import org.scalacheck.Gen import org.scalamock.scalatest.MockFactory @@ -53,9 +53,9 @@ class MicroserviceRoutesSpec with should.Matchers with IOSpec { - "routes" should { + "GET /ping" should { - "define a GET /ping endpoint returning OK with 'pong' body" in new TestCase { + "return OK with 'pong' body" in new TestCase { val response = routes.call( Request(Method.GET, uri"/ping") @@ -64,8 +64,12 @@ class MicroserviceRoutesSpec response.status shouldBe Ok response.body[String] shouldBe "pong" } + } + + "GET /metrics" should { + + "return OK with prometheus metrics" in new TestCase { - "define a GET /metrics endpoint returning OK with some prometheus metrics" in new TestCase { val response = routes.call( Request(Method.GET, uri"/metrics") ) @@ -73,8 +77,11 @@ class MicroserviceRoutesSpec response.status shouldBe Ok response.body[String] should include("server_response_duration_seconds") } + } + + "POST webhooks/events" should { - "define a POST webhooks/events endpoint returning response from the endpoint" in new TestCase { + "pass the request to the endpoint and return the received response" in new TestCase { val responseStatus = Gen.oneOf(Ok, BadRequest).generateOne val request = Request[IO](Method.POST, uri"/webhooks/events") @@ -85,7 +92,11 @@ class MicroserviceRoutesSpec routes.call(request).status shouldBe responseStatus } - "define a DELETE webhooks/events endpoint returning response from the endpoint" in new TestCase { + } + + "DELETE webhooks/events" should { + + "return response from the endpoint" in new TestCase { val projectId = projectIds.generateOne val responseStatus = Gen.oneOf(Ok, BadRequest).generateOne @@ -97,25 +108,37 @@ class MicroserviceRoutesSpec routes.call(request).status shouldBe responseStatus } + } + + "GET projects/:id/events/status" should { - "define a GET projects/:id/events/status endpoint returning response from the endpoint" in new TestCase { + "return Ok response from the endpoint" in new TestCase { val projectId = projectIds.generateOne val request = Request[IO](Method.GET, uri"/projects" / projectId / "events" / "status") val responseStatus = Gen.oneOf(Ok, BadRequest).generateOne (eventStatusEndpoint - .fetchProcessingStatus(_: projects.GitLabId)) - .expects(projectId) + .fetchProcessingStatus(_: projects.GitLabId, _: AuthUser)) + .expects(projectId, authUser) .returning(IO.pure(Response[IO](responseStatus))) routes.call(request).status shouldBe responseStatus } - s"define a GET projects/:id/events/status endpoint returning $NotFound when no :id path parameter given" in new TestCase { + "return NotFound when no :id path parameter given" in new TestCase { routes.call(Request(Method.GET, uri"/projects/")).status shouldBe NotFound } - "define a POST projects/:id/webhooks endpoint returning response from the endpoint" in new TestCase { + "return Unauthorized when user is not authorized" in new TestCase { + + override val authenticationResponse = OptionT.none[IO, AuthUser] + + val request = Request[IO](Method.GET, uri"/projects" / projectIds.generateOne / "events" / "status") + + routes.call(request).status shouldBe Unauthorized + } + + "return response from the endpoint" in new TestCase { val projectId = projectIds.generateOne val request = Request[IO](Method.POST, uri"/projects" / projectId / "webhooks") @@ -127,8 +150,12 @@ class MicroserviceRoutesSpec routes.call(request).status shouldBe responseStatus } + } + + "POST projects/:id/webhooks" should { + + "return Unauthorized when the user is not authorized" in new TestCase { - s"define a POST projects/:id/webhooks endpoint returning $Unauthorized when user is not authorized" in new TestCase { override val authenticationResponse = OptionT.none[IO, AuthUser] val projectId = projectIds.generateOne @@ -136,8 +163,11 @@ class MicroserviceRoutesSpec routes.call(request).status shouldBe Unauthorized } + } + + "POST projects/:id/webhooks/validation" should { - "define a POST projects/:id/webhooks/validation endpoint returning response from the endpoint" in new TestCase { + "return response from the endpoint" in new TestCase { val projectId = projectIds.generateOne val request = Request[IO](Method.POST, uri"/projects" / projectId / "webhooks" / "validation") @@ -150,7 +180,8 @@ class MicroserviceRoutesSpec routes.call(request).status shouldBe responseStatus } - s"define a POST projects/:id/webhooks/validation endpoint returning $Unauthorized when user is not authorized" in new TestCase { + "return Unauthorized when user is not authorized" in new TestCase { + override val authenticationResponse = OptionT.none[IO, AuthUser] val projectId = projectIds.generateOne diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala index 94fd0658c2..842f36864b 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala @@ -29,6 +29,7 @@ import hookvalidation.HookValidator.HookValidationResult.{HookExists, HookMissin import hookvalidation.HookValidator.NoAccessTokenException import io.circe.Json import io.circe.syntax._ +import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators.exceptions import io.renku.graph.model.GraphModelGenerators.projectIds @@ -38,6 +39,7 @@ import io.renku.http.ErrorMessage._ import io.renku.http.client.AccessToken import io.renku.http.server.EndpointTester._ import io.renku.http.{ErrorMessage, InfoMessage} +import io.renku.http.server.security.model.AuthUser import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.{Error, Warn} import io.renku.logging.TestExecutionTimeRecorder @@ -55,12 +57,12 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return OK with the status info if webhook for the project exists" in new TestCase { - givenHookValidation(projectId, returning = HookExists.pure[IO]) + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val statusInfo = statusInfos.generateOne givenStatusInfoFinding(projectId, returning = rightT(statusInfo)) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe Ok response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -73,9 +75,9 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return OK with activated = false if the webhook does not exist" in new TestCase { - givenHookValidation(projectId, returning = HookMissing.pure[IO]) + givenHookValidation(projectId, authUser, returning = HookMissing.pure[IO]) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe Ok response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -85,9 +87,9 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return NOT_FOUND if no Access Token found for the project" in new TestCase { val exception = NoAccessTokenException("error") - givenHookValidation(projectId, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) + givenHookValidation(projectId, authUser, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe NotFound response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -97,9 +99,9 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return INTERNAL_SERVER_ERROR when checking if the webhook exists fails" in new TestCase { val exception = exceptions.generateOne - givenHookValidation(projectId, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) + givenHookValidation(projectId, authUser, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe InternalServerError response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -110,12 +112,12 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return INTERNAL_SERVER_ERROR when finding status returns a failure" in new TestCase { - givenHookValidation(projectId, returning = HookExists.pure[IO]) + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne givenStatusInfoFinding(projectId, returning = leftT(exception)) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe InternalServerError response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -126,12 +128,12 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return INTERNAL_SERVER_ERROR when finding status info fails" in new TestCase { - givenHookValidation(projectId, returning = HookExists.pure[IO]) + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne givenStatusInfoFinding(projectId, returning = right(exception.raiseError[IO, StatusInfo])) - val response = endpoint.fetchProcessingStatus(projectId).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() response.status shouldBe InternalServerError response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -142,6 +144,8 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit } private trait TestCase { + + val authUser = authUsers.generateOne val projectId = projectIds.generateOne private val hookValidator = mock[HookValidator[IO]] @@ -152,10 +156,13 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit lazy val statusInfoFindingErrorMessage = show"Finding status info for project '$projectId' failed" - def givenHookValidation(projectId: projects.GitLabId, returning: IO[HookValidator.HookValidationResult]) = + def givenHookValidation(projectId: projects.GitLabId, + authUser: AuthUser, + returning: IO[HookValidator.HookValidationResult] + ) = (hookValidator .validateHook(_: GitLabId, _: Option[AccessToken])) - .expects(projectId, None) + .expects(projectId, authUser.accessToken.some) .returning(returning) def givenStatusInfoFinding(projectId: projects.GitLabId, returning: EitherT[IO, Throwable, StatusInfo]) = From a55565f82f5dfc538277a3ec2d787608772c6350 Mon Sep 17 00:00:00 2001 From: eikek Date: Thu, 6 Apr 2023 13:52:03 +0200 Subject: [PATCH 02/28] fix: Retain gitlab images when converting from cli payload (#1425) The images must be taken from gitlab's project info as well when converting a cli payload. --- .../model/entities/CliProjectConverter.scala | 3 ++- .../graph/model/entities/ProjectSpec.scala | 22 +++++++++++++++++++ .../triplesgenerated/EntityBuilder.scala | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/CliProjectConverter.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/CliProjectConverter.scala index ff2224cee0..dad19d18bf 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/CliProjectConverter.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/CliProjectConverter.scala @@ -52,6 +52,7 @@ private[entities] object CliProjectConverter { case s => s } val dateCreated = (gitLabInfo.dateCreated :: cliProject.dateCreated :: Nil).min + val gitlabImage = gitLabInfo.avatarUrl.map(Image.projectImage(ResourceId(gitLabInfo.path), _)) val all = (creatorV, allPersonV, datasetV, activityV, planV).mapN(Tuple5.apply) all.andThen { case (creator, persons, datasets, activities, plans) => newProject( @@ -65,7 +66,7 @@ private[entities] object CliProjectConverter { activities.sortBy(_.startTime), datasets, plans, - cliProject.images + (cliProject.images ::: gitlabImage.toList).distinct ) } } diff --git a/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala index c38255f2d3..e1e8805463 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala @@ -33,6 +33,7 @@ import io.renku.graph.model._ import io.renku.graph.model.entities.Generators.{compositePlanNonEmptyMappings, stepPlanGenFactory} import io.renku.graph.model.entities.Project.ProjectMember.{ProjectMemberNoEmail, ProjectMemberWithEmail} import io.renku.graph.model.entities.Project.{GitLabProjectInfo, ProjectMember} +import io.renku.graph.model.images.Image import io.renku.graph.model.projects.ForksCount import io.renku.graph.model.testentities.RenkuProject.CreateCompositePlan import io.renku.graph.model.testentities.generators.EntitiesGenerators @@ -78,6 +79,27 @@ class ProjectSpec "fromCli" should { + "add images from gitlab project avatar" in new TestCase { + val projectInfo = + gitLabProjectInfos.map(projectInfoMaybeParent.set(None)).suchThat(_.avatarUrl.isDefined).generateOne + val testProject: testentities.Project = + createRenkuProject(projectInfo, cliVersion, schemaVersion) + .asInstanceOf[testentities.RenkuProject.WithoutParent] + .copy(images = Nil) + + val cliProject = testProject.to[CliProject] + cliProject.images shouldBe Nil + + val modelProject = entities.Project + .fromCli(cliProject, Set.empty, projectInfo) + .toEither + .fold(errs => sys.error(errs.toString()), identity) + + modelProject.images shouldBe List( + Image.projectImage(modelProject.resourceId, projectInfo.avatarUrl.get) + ) + } + "turn CliProject entity without parent into the Project object" in new TestCase { forAll(gitLabProjectInfos.map(projectInfoMaybeParent.set(None))) { projectInfo => val creator = projectMembersWithEmail.generateOne diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/triplesgenerated/EntityBuilder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/triplesgenerated/EntityBuilder.scala index dc0b28aaf5..68e74eff5e 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/triplesgenerated/EntityBuilder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/triplesgenerated/EntityBuilder.scala @@ -48,7 +48,6 @@ private class EntityBuilderImpl[F[_]: MonadThrow]( private val applicative: Applicative[F] = Applicative[F] import applicative._ - import projectInfoFinder._ override def buildEntity(event: TriplesGeneratedEvent)(implicit maybeAccessToken: Option[AccessToken] @@ -59,7 +58,7 @@ private class EntityBuilderImpl[F[_]: MonadThrow]( private def findValidProjectInfo(event: TriplesGeneratedEvent)(implicit maybeAccessToken: Option[AccessToken] - ) = findProjectInfo(event.project.path) semiflatMap { + ) = projectInfoFinder.findProjectInfo(event.project.path) semiflatMap { case Some(projectInfo) => projectInfo.pure[F] case None => ProcessingNonRecoverableError From 0a43f6e22570a663ee48210df1c6be6e6a5db7d0 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Fri, 14 Apr 2023 09:10:03 +0200 Subject: [PATCH 03/28] chore: Update diffx-scalatest-should from 0.8.2 to 0.8.3 (#1426) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e577971cef..555367da3a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,7 +10,7 @@ object Dependencies { val circeCore = "0.14.5" val circeGenericExtras = "0.14.3" val circeOptics = "0.14.1" - val diffx = "0.8.2" + val diffx = "0.8.3" val http4s = "0.23.18" val http4sEmber = "0.23.18" val http4sPrometheus = "0.24.3" From 78b3ddfdffa8b4fdce908b8996a397b3669c6791 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Sat, 15 Apr 2023 20:13:05 +0200 Subject: [PATCH 04/28] chore: Update pureconfig, pureconfig-cats from 0.17.2 to 0.17.3 (#1427) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 555367da3a..501e0eb8fc 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -22,7 +22,7 @@ object Dependencies { val luceneQueryParser = "9.5.0" val monocle = "2.1.0" val owlapi = "5.5.0" - val pureconfig = "0.17.2" + val pureconfig = "0.17.3" val rdf4jQueryParserSparql = "4.2.3" val refined = "0.10.3" val refinedPureconfig = "0.10.3" From 53db599489ded759aeb1add9b79a70a4905d0d9f Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:11:22 +0200 Subject: [PATCH 05/28] chore: Update cats-effect from 3.4.8 to 3.4.9 (#1429) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 501e0eb8fc..97489e6851 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ object Dependencies { object V { val ammonite = "2.4.1" val catsCore = "2.9.0" - val catsEffect = "3.4.8" + val catsEffect = "3.4.9" val circeCore = "0.14.5" val circeGenericExtras = "0.14.3" val circeOptics = "0.14.1" From 3d6559d2af04cdab393d538c20b1f77946cf3a3d Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:11:41 +0200 Subject: [PATCH 06/28] chore: Update testcontainers-scala-postgresql, ... from 0.40.14 to 0.40.15 (#1428) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 97489e6851..bfd89b10db 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -33,7 +33,7 @@ object Dependencies { val sentryLogback = "6.17.0" val skunk = "0.5.1" val swaggerParser = "2.1.13" - val testContainersScala = "0.40.14" + val testContainersScala = "0.40.15" val widoco = "1.4.17" val wiremock = "2.35.0" } From c098b4e28c6b07ec22472b37a36e5a13cf64efdc Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Thu, 20 Apr 2023 10:37:22 +0200 Subject: [PATCH 07/28] chore: Update logback-classic from 1.4.6 to 1.4.7 (#1431) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- token-repository/build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index bfd89b10db..550324b9a6 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,7 +18,7 @@ object Dependencies { val jsonld4s = "0.10.0" val log4cats = "2.5.0" val log4jCore = "2.20.0" - val logback = "1.4.6" + val logback = "1.4.7" val luceneQueryParser = "9.5.0" val monocle = "2.1.0" val owlapi = "5.5.0" diff --git a/token-repository/build.sbt b/token-repository/build.sbt index ddec2bdc4b..b754387c7a 100644 --- a/token-repository/build.sbt +++ b/token-repository/build.sbt @@ -20,4 +20,4 @@ name := "token-repository" Test / fork := true -libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.6" +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.4.7" From 81f5dbc5d6c3e004b98e99f6e9dd012dc823fd03 Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Fri, 21 Apr 2023 14:43:57 +0200 Subject: [PATCH 08/28] fix: refactoring around capacity finder and fix flaky test (#1433) * Refactor capacity finder The query can be left as an impl. detail and the caller only needs to specify the status for which to find capacity info for. * Fix copy-paste error resulting in a flaky test --- .../events/producers/CapacityFinder.scala | 41 +++++++++++-------- .../SubscriptionCategory.scala | 12 +++--- .../SubscriptionCategory.scala | 12 +++--- .../eventlog/InMemoryEventLogDbSpec.scala | 7 ++-- .../events/producers/CapacityFinderSpec.scala | 3 +- .../SubscriptionCategorySpec.scala | 2 +- .../SubscriptionCategorySpec.scala | 8 ++-- 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/event-log/src/main/scala/io/renku/eventlog/events/producers/CapacityFinder.scala b/event-log/src/main/scala/io/renku/eventlog/events/producers/CapacityFinder.scala index 31e3b9f299..8785a0b47e 100644 --- a/event-log/src/main/scala/io/renku/eventlog/events/producers/CapacityFinder.scala +++ b/event-log/src/main/scala/io/renku/eventlog/events/producers/CapacityFinder.scala @@ -24,6 +24,7 @@ import cats.syntax.all._ import io.renku.db.{DbClient, SqlStatement} import io.renku.eventlog.metrics.QueriesExecutionTimes import io.renku.eventlog.EventLogDB.SessionResource +import io.renku.graph.model.events.EventStatus private trait CapacityFinder[F[_]] { def findUsedCapacity: F[UsedCapacity] @@ -35,26 +36,32 @@ private object CapacityFinder { override lazy val findUsedCapacity = UsedCapacity.zero.pure[F] } - def queryBased[F[_]: Async: SessionResource: QueriesExecutionTimes](query: String): CapacityFinder[F] = - new QueryBasedCapacityFinder[F](query) -} + def ofStatus[F[_]: Async: SessionResource: QueriesExecutionTimes](status: EventStatus): CapacityFinder[F] = + new StatusCapacityFinder[F](status) + + private class StatusCapacityFinder[F[_]: Async: SessionResource: QueriesExecutionTimes](status: EventStatus) + extends DbClient[F](Some(QueriesExecutionTimes[F])) + with CapacityFinder[F] { -private class QueryBasedCapacityFinder[F[_]: Async: SessionResource: QueriesExecutionTimes](query: String) - extends DbClient(Some(QueriesExecutionTimes[F])) - with CapacityFinder[F] { + import skunk.Encoder + import skunk.codec.all.varchar + import skunk.codec.numeric._ + import skunk.implicits._ - import skunk._ - import skunk.codec.numeric._ - import skunk.implicits._ + implicit val statusVarchar: Encoder[EventStatus] = + varchar.values.contramap(_.value) - override def findUsedCapacity: F[UsedCapacity] = SessionResource[F].useK(statement) + val statement = measureExecutionTime { + SqlStatement + .named(s"find capacity for ${status.value}") + .select[EventStatus, Long]( + sql"SELECT COUNT(event_id) FROM event WHERE status = $statusVarchar".query(int8) + ) + .arguments(status) + .build[Id](_.unique) + .mapResult(v => UsedCapacity(v.toInt)) + } - private lazy val statement = measureExecutionTime { - SqlStatement - .named("find capacity") - .select[Void, Long](sql"""#$query""".query(int8)) - .arguments(Void) - .build[Id](_.unique) - .mapResult(v => UsedCapacity(v.toInt)) + override def findUsedCapacity: F[UsedCapacity] = SessionResource[F].useK(statement) } } diff --git a/event-log/src/main/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategory.scala b/event-log/src/main/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategory.scala index c8b4cd3ce6..baff19b95f 100644 --- a/event-log/src/main/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategory.scala +++ b/event-log/src/main/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategory.scala @@ -56,12 +56,10 @@ private[producers] object SubscriptionCategory { EventEncoder(encodeEvent, encodePayload), dispatchRecovery ) - } yield new SubscriptionCategoryImpl[F, DefaultSubscriber](categoryName, - subscribers, - eventsDistributor, - CapacityFinder.queryBased(capacityFindingQuery) + } yield new SubscriptionCategoryImpl[F, DefaultSubscriber]( + categoryName, + subscribers, + eventsDistributor, + CapacityFinder.ofStatus(EventStatus.GeneratingTriples) ) - - private[awaitinggeneration] val capacityFindingQuery = - s"SELECT COUNT(event_id) FROM event WHERE status='${EventStatus.GeneratingTriples.value}'" } diff --git a/event-log/src/main/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategory.scala b/event-log/src/main/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategory.scala index 35f86b8291..7f5a5b3e16 100644 --- a/event-log/src/main/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategory.scala +++ b/event-log/src/main/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategory.scala @@ -54,12 +54,10 @@ private[producers] object SubscriptionCategory { EventEncoder(encodeEvent, encodePayload), dispatchRecovery ) - } yield new SubscriptionCategoryImpl[F, DefaultSubscriber](categoryName, - subscribers, - eventsDistributor, - CapacityFinder.queryBased(capacityFindingQuery) + } yield new SubscriptionCategoryImpl[F, DefaultSubscriber]( + categoryName, + subscribers, + eventsDistributor, + CapacityFinder.ofStatus(EventStatus.TransformingTriples) ) - - private[triplesgenerated] val capacityFindingQuery = - s"SELECT COUNT(event_id) FROM event WHERE status='${EventStatus.TransformingTriples.value}'" } diff --git a/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDbSpec.scala b/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDbSpec.scala index fa1980a7f2..2204246a18 100644 --- a/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDbSpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDbSpec.scala @@ -50,9 +50,8 @@ trait InMemoryEventLogDbSpec } protected def prepareDbForTest(): Unit = execute[Unit] { - Kleisli { session => - val query: Command[Void] = sql"TRUNCATE TABLE #${findAllTables().mkString(", ")} CASCADE".command - session.execute(query).void - } + val tables = findAllTables().mkString(", ") + val query: Command[Void] = sql"TRUNCATE TABLE #$tables CASCADE".command + Kleisli(_.execute(query).void) } } diff --git a/event-log/src/test/scala/io/renku/eventlog/events/producers/CapacityFinderSpec.scala b/event-log/src/test/scala/io/renku/eventlog/events/producers/CapacityFinderSpec.scala index e0360e3760..75f2a3362e 100644 --- a/event-log/src/test/scala/io/renku/eventlog/events/producers/CapacityFinderSpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/events/producers/CapacityFinderSpec.scala @@ -42,8 +42,7 @@ class CapacityFinderSpec extends AnyWordSpec with should.Matchers with CapacityF createEvent(EventStatus.GeneratingTriples) createEvent(Gen.oneOf(EventStatus.all - EventStatus.GeneratingTriples).generateOne) - val finder = CapacityFinder - .queryBased[IO](s"SELECT COUNT(event_id) FROM event WHERE status='${EventStatus.GeneratingTriples.value}'") + val finder = CapacityFinder.ofStatus[IO](EventStatus.GeneratingTriples) finder.findUsedCapacity.unsafeRunSync() shouldBe UsedCapacity(1) diff --git a/event-log/src/test/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategorySpec.scala b/event-log/src/test/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategorySpec.scala index 251b2ffd68..77b18f9d53 100644 --- a/event-log/src/test/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategorySpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/events/producers/awaitinggeneration/SubscriptionCategorySpec.scala @@ -37,7 +37,7 @@ class SubscriptionCategorySpec extends AnyWordSpec with should.Matchers with Cap createEvent(Gen.oneOf(EventStatus.all - EventStatus.GeneratingTriples).generateOne) CapacityFinder - .queryBased[IO](SubscriptionCategory.capacityFindingQuery) + .ofStatus[IO](EventStatus.GeneratingTriples) .findUsedCapacity .unsafeRunSync() shouldBe UsedCapacity(1) } diff --git a/event-log/src/test/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategorySpec.scala b/event-log/src/test/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategorySpec.scala index 2b26f7e4ab..5c1e387fa8 100644 --- a/event-log/src/test/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategorySpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/events/producers/triplesgenerated/SubscriptionCategorySpec.scala @@ -22,7 +22,7 @@ import cats.effect.IO import io.renku.eventlog.events.producers.{CapacityFinder, CapacityFindingQuerySpec, UsedCapacity} import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.events.EventStatus -import io.renku.graph.model.events.EventStatus.GeneratingTriples +import io.renku.graph.model.events.EventStatus.TransformingTriples import org.scalacheck.Gen import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec @@ -31,13 +31,13 @@ class SubscriptionCategorySpec extends AnyWordSpec with should.Matchers with Cap "capacityFindingQuery" should { - s"count events in the $GeneratingTriples status" in { + s"count events in the $TransformingTriples status" in { createEvent(EventStatus.TransformingTriples) - createEvent(Gen.oneOf(EventStatus.all - EventStatus.GeneratingTriples).generateOne) + createEvent(Gen.oneOf(EventStatus.all - EventStatus.TransformingTriples).generateOne) CapacityFinder - .queryBased[IO](SubscriptionCategory.capacityFindingQuery) + .ofStatus[IO](EventStatus.TransformingTriples) .findUsedCapacity .unsafeRunSync() shouldBe UsedCapacity(1) } From 3e3132557658f26dd70f09633a8ab6ada0bb27ab Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Sat, 22 Apr 2023 09:45:44 +0200 Subject: [PATCH 09/28] chore: Update rdf4j-queryparser-sparql from 4.2.3 to 4.2.4 (#1435) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 550324b9a6..66708cc591 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -23,7 +23,7 @@ object Dependencies { val monocle = "2.1.0" val owlapi = "5.5.0" val pureconfig = "0.17.3" - val rdf4jQueryParserSparql = "4.2.3" + val rdf4jQueryParserSparql = "4.2.4" val refined = "0.10.3" val refinedPureconfig = "0.10.3" val scalacheck = "1.17.0" From 209158f8b930dc4b59cceb93cd32b5fc4e854c13 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Mon, 24 Apr 2023 11:04:42 +0200 Subject: [PATCH 10/28] chore: Update log4cats-core from 2.5.0 to 2.6.0 (#1436) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 66708cc591..6e12a7f63e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -16,7 +16,7 @@ object Dependencies { val http4sPrometheus = "0.24.3" val ip4s = "3.3.0" val jsonld4s = "0.10.0" - val log4cats = "2.5.0" + val log4cats = "2.6.0" val log4jCore = "2.20.0" val logback = "1.4.7" val luceneQueryParser = "9.5.0" From b95500f41b3e7d27ae365de45833d570fe30ad6f Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Tue, 25 Apr 2023 08:47:27 +0200 Subject: [PATCH 11/28] chore: Update jsonld4s from 0.10.0 to 0.11.0 (#1442) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6e12a7f63e..3cfc7af4e4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -15,7 +15,7 @@ object Dependencies { val http4sEmber = "0.23.18" val http4sPrometheus = "0.24.3" val ip4s = "3.3.0" - val jsonld4s = "0.10.0" + val jsonld4s = "0.11.0" val log4cats = "2.6.0" val log4jCore = "2.20.0" val logback = "1.4.7" From 6485a12ddf46183f7f51bf6a9214a664218edab3 Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Tue, 25 Apr 2023 09:41:26 +0200 Subject: [PATCH 12/28] feat: status-info inludes webhook installed as activated project (#1434) * Reference a constant for a static value * chore: include webhook case in status info activated The status info "activated" now also includes the case when there are no events, but the webhook does exist. --- .../webhookservice/eventstatus/Endpoint.scala | 8 +- .../eventstatus/StatusInfo.scala | 98 +++++++++++-------- .../eventstatus/StatusInfoFinder.scala | 17 ++-- .../eventstatus/EndpointSpec.scala | 20 ++-- .../eventstatus/StatusInfoFinderSpec.scala | 10 +- .../eventstatus/StatusInfoSpec.scala | 14 +-- 6 files changed, 94 insertions(+), 73 deletions(-) diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala index bc0afd68fb..9655ae188e 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala @@ -76,8 +76,12 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( } private def findStatus(projectId: GitLabId) = - statusInfoFinder - .findStatusInfo(projectId) + EitherT( + statusInfoFinder + .findStatusInfo(projectId) + .map(_.getOrElse(StatusInfo.webhookReady)) + .attempt + ) .biSemiflatMap(internalServerError(projectId), status => Ok(status.asJson)) .merge diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala index 23778f93f2..27d5fa6105 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala @@ -19,80 +19,100 @@ package io.renku.webhookservice.eventstatus import cats.syntax.all._ -import io.circe.Encoder +import io.circe.{Encoder, Json} import io.circe.literal._ import io.renku.graph.model.events.{EventStatus, EventStatusProgress} import EventStatus._ +import io.circe.syntax.EncoderOps +import io.renku.graph.model.events.EventStatusProgress.Stage -private sealed trait StatusInfo { - val activated: Boolean - val progress: Progress +private sealed trait StatusInfo extends Product { + def activated: Boolean + def progress: Progress + + def fold[A](activated: StatusInfo.ActivatedProject => A, whenNotActivated: => A): A } private object StatusInfo { - final case class ActivatedProject(progress: Progress.NonZero, details: Details) extends StatusInfo { + final case class ActivatedProject(progress: Progress, details: Details) extends StatusInfo { override val activated: Boolean = true + def fold[A](whenActivated: StatusInfo.ActivatedProject => A, notActivated: => A): A = whenActivated(this) } def activated(eventStatus: EventStatus): StatusInfo.ActivatedProject = - ActivatedProject(Progress.from(eventStatus), Details(eventStatus)) + ActivatedProject(Progress.from(eventStatus), Details.fromStatus(eventStatus)) + + def webhookReady: StatusInfo.ActivatedProject = + ActivatedProject(Progress.Zero, Details("in-progress", "Webhook has been installed.")) final case object NotActivated extends StatusInfo { override val activated: Boolean = false override val progress: Progress = Progress.Zero + def fold[A](whenActivated: StatusInfo.ActivatedProject => A, whenNotActivated: => A): A = whenNotActivated } - implicit def encoder[PS <: StatusInfo]: Encoder[PS] = { - case info @ ActivatedProject(progress: Progress.NonZero, details) => json"""{ - "activated": ${info.activated}, - "progress": { - "done": ${progress.statusProgress.stage.value}, - "total": ${progress.finalStage.value}, - "percentage": ${progress.statusProgress.completion.value} - }, - "details": { - "status": ${details.status}, - "message": ${details.message} - } - }""" - case info @ NotActivated => json"""{ - "activated": ${info.activated}, - "progress": { - "done": 0, - "total": ${info.progress.finalStage.value}, - "percentage": 0.00 - } - }""" - } + implicit def encoder[PS <: StatusInfo]: Encoder[PS] = + Encoder.instance { statusInfo => + Json + .obj( + "activated" -> statusInfo.activated.asJson, + "progress" -> statusInfo.progress.asJson, + "details" -> statusInfo.fold(_.details.some, None).asJson + ) + .deepDropNullValues + } } -private sealed trait Progress extends Product with Serializable { - lazy val finalStage: EventStatusProgress.Stage = EventStatusProgress.Stage.Final +private sealed trait Progress extends Product { + final val total: Int = Stage.Final.value + def done: Int + def percentage: Float } private object Progress { - final case object Zero extends Progress + final case object Zero extends Progress { + val done = 0 + val percentage = 0f + } final case class NonZero(statusProgress: EventStatusProgress) extends Progress { lazy val currentStage: EventStatusProgress.Stage = statusProgress.stage lazy val completion: EventStatusProgress.Completion = statusProgress.completion + + lazy val done = currentStage.value + lazy val percentage = completion.value } def from(eventStatus: EventStatus): Progress.NonZero = Progress.NonZero(EventStatusProgress(eventStatus)) + + implicit val jsonEncoder: Encoder[Progress] = Encoder.instance { progress => + Json.obj( + "done" -> progress.done.asJson, + "total" -> progress.total.asJson, + "percentage" -> progress.percentage.asJson + ) + } } -private final case class Details(eventStatus: EventStatus) { +private final case class Details(status: String, message: String) + +private object Details { + def fromStatus(eventStatus: EventStatus): Details = { + val status: String = eventStatus match { + case New | GeneratingTriples | GenerationRecoverableFailure | TriplesGenerated | TransformingTriples | + TransformationRecoverableFailure | AwaitingDeletion | Deleting => + "in-progress" + case Skipped | TriplesStore => "success" + case GenerationNonRecoverableFailure | TransformationNonRecoverableFailure => "failure" + } - lazy val status: String = eventStatus match { - case New | GeneratingTriples | GenerationRecoverableFailure | TriplesGenerated | TransformingTriples | - TransformationRecoverableFailure | AwaitingDeletion | Deleting => - "in-progress" - case Skipped | TriplesStore => "success" - case GenerationNonRecoverableFailure | TransformationNonRecoverableFailure => "failure" + val message: String = eventStatus.show.toLowerCase.replace('_', ' ') + Details(status, message) } - lazy val message: String = eventStatus.show.toLowerCase.replace('_', ' ') + implicit val jsonEncoder: Encoder[Details] = + Encoder.instance(d => Json.obj("status" -> d.status.asJson, "message" -> d.message.asJson)) } diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfoFinder.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfoFinder.scala index 2a28de24fd..aefae65020 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfoFinder.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfoFinder.scala @@ -19,7 +19,6 @@ package io.renku.webhookservice.eventstatus import cats.MonadThrow -import cats.data.EitherT import cats.effect.Async import cats.syntax.all._ import io.renku.graph.eventlog.EventLogClient @@ -29,7 +28,7 @@ import io.renku.http.rest.paging.model.PerPage import org.typelevel.log4cats.Logger private trait StatusInfoFinder[F[_]] { - def findStatusInfo(projectId: projects.GitLabId): EitherT[F, Throwable, StatusInfo] + def findStatusInfo(projectId: projects.GitLabId): F[Option[StatusInfo]] } private object StatusInfoFinder { @@ -40,7 +39,7 @@ private object StatusInfoFinder { private class StatusInfoFinderImpl[F[_]: MonadThrow](eventLogClient: EventLogClient[F]) extends StatusInfoFinder[F] { - override def findStatusInfo(projectId: projects.GitLabId): EitherT[F, Throwable, StatusInfo] = EitherT { + override def findStatusInfo(projectId: projects.GitLabId): F[Option[StatusInfo]] = eventLogClient .getEvents( EventLogClient.SearchCriteria @@ -48,12 +47,12 @@ private class StatusInfoFinderImpl[F[_]: MonadThrow](eventLogClient: EventLogCli .withPerPage(PerPage(1)) .sortBy(EventLogClient.SearchCriteria.Sort.EventDateDesc) ) - .map(toStatusInfo) - } + .map(_.toEither.map(toStatusInfo)) + .rethrow - private lazy val toStatusInfo: EventLogClient.Result[List[EventInfo]] => Either[Throwable, StatusInfo] = - _.toEither.map { - case Nil => StatusInfo.NotActivated - case event :: _ => StatusInfo.activated(event.status) + private def toStatusInfo(events: List[EventInfo]): Option[StatusInfo] = + events match { + case Nil => None + case event :: _ => StatusInfo.activated(event.status).some } } diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala index 842f36864b..b5332fe0a1 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala @@ -19,14 +19,8 @@ package io.renku.webhookservice package eventstatus -import Generators._ -import cats.data.EitherT -import cats.data.EitherT.{leftT, right, rightT} import cats.effect.IO import cats.syntax.all._ -import hookvalidation.HookValidator -import hookvalidation.HookValidator.HookValidationResult.{HookExists, HookMissing} -import hookvalidation.HookValidator.NoAccessTokenException import io.circe.Json import io.circe.syntax._ import io.renku.generators.CommonGraphGenerators.authUsers @@ -38,12 +32,16 @@ import io.renku.graph.model.projects.GitLabId import io.renku.http.ErrorMessage._ import io.renku.http.client.AccessToken import io.renku.http.server.EndpointTester._ -import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.http.server.security.model.AuthUser +import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.{Error, Warn} import io.renku.logging.TestExecutionTimeRecorder import io.renku.testtools.IOSpec +import io.renku.webhookservice.eventstatus.Generators._ +import io.renku.webhookservice.hookvalidation.HookValidator +import io.renku.webhookservice.hookvalidation.HookValidator.HookValidationResult.{HookExists, HookMissing} +import io.renku.webhookservice.hookvalidation.HookValidator.NoAccessTokenException import org.http4s.MediaType.application import org.http4s.Status._ import org.http4s.headers.`Content-Type` @@ -60,7 +58,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val statusInfo = statusInfos.generateOne - givenStatusInfoFinding(projectId, returning = rightT(statusInfo)) + givenStatusInfoFinding(projectId, returning = IO.pure(statusInfo.some)) val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() @@ -115,7 +113,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne - givenStatusInfoFinding(projectId, returning = leftT(exception)) + givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() @@ -131,7 +129,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne - givenStatusInfoFinding(projectId, returning = right(exception.raiseError[IO, StatusInfo])) + givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() @@ -165,7 +163,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit .expects(projectId, authUser.accessToken.some) .returning(returning) - def givenStatusInfoFinding(projectId: projects.GitLabId, returning: EitherT[IO, Throwable, StatusInfo]) = + def givenStatusInfoFinding(projectId: projects.GitLabId, returning: IO[Option[StatusInfo]]) = (statusInfoFinder .findStatusInfo(_: GitLabId)) .expects(projectId) diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoFinderSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoFinderSpec.scala index b230eb034d..d6fd4489c5 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoFinderSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoFinderSpec.scala @@ -42,14 +42,14 @@ class StatusInfoFinderSpec extends AnyWordSpec with should.Matchers with MockFac val eventInfo = eventInfos(projectIdGen = fixed(projectId)).generateOne givenGetEvents(projectId, returning = EventLogClient.Result.Success(List(eventInfo)).pure[Try]) - fetcher.findStatusInfo(projectId).value shouldBe StatusInfo.activated(eventInfo.status).asRight.pure[Try] + fetcher.findStatusInfo(projectId) shouldBe StatusInfo.activated(eventInfo.status).some.pure[Try] } - "return non-activated project StatusInfo when no events found for the project" in new TestCase { + "return activated project StatusInfo when no events found for the project" in new TestCase { givenGetEvents(projectId, returning = EventLogClient.Result.Success(List.empty).pure[Try]) - fetcher.findStatusInfo(projectId).value shouldBe StatusInfo.NotActivated.asRight.pure[Try] + fetcher.findStatusInfo(projectId) shouldBe Option.empty[StatusInfo].pure[Try] } "return an Exception if EL responds with a failure" in new TestCase { @@ -57,7 +57,7 @@ class StatusInfoFinderSpec extends AnyWordSpec with should.Matchers with MockFac val message = nonEmptyStrings().generateOne givenGetEvents(projectId, returning = EventLogClient.Result.failure(message).pure[Try]) - val Success(result) = fetcher.findStatusInfo(projectId).value + val Success(result) = fetcher.findStatusInfo(projectId).attempt result shouldBe a[Left[_, _]] result.leftMap(_.getMessage) shouldBe message.asLeft @@ -67,7 +67,7 @@ class StatusInfoFinderSpec extends AnyWordSpec with should.Matchers with MockFac givenGetEvents(projectId, returning = EventLogClient.Result.unavailable.pure[Try]) - val Success(result) = fetcher.findStatusInfo(projectId).value + val Success(result) = fetcher.findStatusInfo(projectId).attempt result shouldBe a[Left[_, _]] result shouldBe EventLogClient.Result.Unavailable.asLeft diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoSpec.scala index 20121f9f29..dbca5c2feb 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/StatusInfoSpec.scala @@ -41,9 +41,9 @@ class StatusInfoSpec extends AnyWordSpec with should.Matchers with ScalaCheckPro info.asJson shouldBe json"""{ "activated": true, "progress": { - "done": ${info.progress.currentStage.value}, - "total": ${info.progress.finalStage.value}, - "percentage": ${info.progress.completion.value} + "done": ${info.progress.done}, + "total": ${info.progress.total}, + "percentage": ${info.progress.percentage} }, "details": { "status": ${info.details.status}, @@ -61,7 +61,7 @@ class StatusInfoSpec extends AnyWordSpec with should.Matchers with ScalaCheckPro "activated": false, "progress": { "done": 0, - "total": ${info.progress.finalStage.value}, + "total": ${info.progress.total}, "percentage": 0.00 } }""" @@ -78,14 +78,14 @@ class ProgressSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope val progressStatus = Progress.from(eventStatus) progressStatus.currentStage shouldBe EventStatusProgress(eventStatus).stage - progressStatus.finalStage shouldBe EventStatusProgress.Stage.Final + progressStatus.total shouldBe EventStatusProgress.Stage.Final.value progressStatus.completion shouldBe EventStatusProgress(eventStatus).completion } } } "Progress.Zero to have final stage set to EventStatusProgress.Stage.Final" in { - Progress.Zero.finalStage shouldBe EventStatusProgress.Stage.Final + Progress.Zero.total shouldBe EventStatusProgress.Stage.Final.value } } @@ -111,7 +111,7 @@ class DetailsSpec extends AnyWordSpec with should.Matchers with TableDrivenPrope ) ) { (eventStatus, status, message) => show"provide '$status' as status and '$message' as message for the '$eventStatus' status" in { - val details = Details(eventStatus) + val details = Details.fromStatus(eventStatus) details.status shouldBe status details.message shouldBe message From c42963a51dba0ca4807df530cc355023b541e29e Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 25 Apr 2023 11:51:33 +0200 Subject: [PATCH 13/28] fix: lineage api not working with implicit params (#1441) --- .../files/lineage/NodeDetailsFinder.scala | 189 +++++++++--------- .../files/lineage/NodeDetailsFinderSpec.scala | 46 ++++- .../io/renku/graph/model/entityModel.scala | 6 + .../testentities/LineageExemplarData.scala | 1 + .../StepPlanCommandParameter.scala | 38 +++- 5 files changed, 181 insertions(+), 99 deletions(-) diff --git a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinder.scala b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinder.scala index 82a314b889..e72dd0661f 100644 --- a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinder.scala +++ b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinder.scala @@ -24,13 +24,14 @@ import cats.syntax.all._ import eu.timepit.refined.auto._ import io.circe.{Decoder, DecodingFailure} import io.renku.graph.config.RenkuUrlLoader +import io.renku.graph.model.{projects, GraphClass, RenkuUrl} import io.renku.graph.model.Schemas._ import io.renku.graph.model.projects.ResourceId -import io.renku.graph.model.views.RdfResource -import io.renku.graph.model.{GraphClass, RenkuUrl, projects} +import io.renku.jsonld.syntax._ import io.renku.tinytypes.json.TinyTypeDecoders -import io.renku.triplesstore.SparqlQuery.Prefixes import io.renku.triplesstore._ +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ import model.{ExecutionInfo, Node} import org.typelevel.log4cats.Logger @@ -102,7 +103,7 @@ private class NodeDetailsFinderImpl[F[_]: Async: Parallel: Logger: SparqlQueryTi case location: Node.Location => new IllegalArgumentException(s"No entity with $location").raiseError[F, Node] case runInfo: ExecutionInfo => - new IllegalArgumentException(s"No plan with ${runInfo.entityId}").raiseError[F, Node] + new IllegalArgumentException(s"No activity with ${runInfo.entityId}").raiseError[F, Node] case other => new IllegalArgumentException(s"Entity $other not recognisable").raiseError[F, Node] } @@ -120,29 +121,29 @@ private object NodeDetailsFinder { SparqlQuery.of( name = "lineage - entity details", Prefixes of (prov -> "prov", schema -> "schema", renku -> "renku"), - s"""|SELECT DISTINCT ?type ?location ?label - |WHERE { - | { - | GRAPH <${GraphClass.Project.id(projectId)}> { - | ?entity prov:atLocation '$location'; - | a prov:Entity; - | a ?type - | } - | } { - | GRAPH <${GraphClass.Project.id(projectId)}> { - | ?activityId a prov:Activity; - | prov:qualifiedUsage / prov:entity ?entity; - | ^renku:hasActivity ${projectId.showAs[RdfResource]} - | } - | } UNION { - | GRAPH <${GraphClass.Project.id(projectId)}> { - | ?entity prov:qualifiedGeneration / prov:activity / ^renku:hasActivity ${projectId.showAs[RdfResource]} - | } - | } - | BIND ('$location' AS ?location) - | BIND ('$location' AS ?label) - |} - |""".stripMargin + sparql"""|SELECT DISTINCT ?type ?location ?label + |WHERE { + | { + | GRAPH ${GraphClass.Project.id(projectId)} { + | ?entity prov:atLocation ${location.asObject}; + | a prov:Entity; + | a ?type + | } + | } { + | GRAPH ${GraphClass.Project.id(projectId)} { + | ?activityId a prov:Activity; + | prov:qualifiedUsage / prov:entity ?entity; + | ^renku:hasActivity ${projectId.asEntityId} + | } + | } UNION { + | GRAPH ${GraphClass.Project.id(projectId)} { + | ?entity prov:qualifiedGeneration / prov:activity / ^renku:hasActivity ${projectId.asEntityId} + | } + | } + | BIND (${location.asObject} AS ?location) + | BIND (${location.asObject} AS ?label) + |} + |""".stripMargin ) implicit val activityIdQuery: (ExecutionInfo, ResourceId) => SparqlQuery = { @@ -150,73 +151,73 @@ private object NodeDetailsFinder { SparqlQuery.of( name = "lineage - plan details", Prefixes of (prov -> "prov", renku -> "renku", schema -> "schema"), - s"""|SELECT DISTINCT ?type (CONCAT(STR(?command), (GROUP_CONCAT(?commandParameter; separator=' '))) AS ?label) ?location - |WHERE { - | { - | SELECT DISTINCT ?command ?type ?location - | WHERE { - | GRAPH <${GraphClass.Project.id(projectId)}> { - | <$activityId> prov:qualifiedAssociation/prov:hadPlan ?planId; - | a ?type. - | OPTIONAL { ?planId renku:command ?maybeCommand. } - | } - | BIND (IF(bound(?maybeCommand), CONCAT(STR(?maybeCommand), STR(' ')), '') AS ?command). - | BIND (<$activityId> AS ?location) - | } - | } { - | SELECT ?position ?commandParameter - | WHERE { - | { # inputs - | GRAPH <${GraphClass.Project.id(projectId)}> { - | <$activityId> prov:qualifiedAssociation/prov:hadPlan ?planId. - | ?planId renku:hasInputs ?input . - | ?paramValue a renku:ParameterValue ; - | schema:valueReference ?input . - | ?paramValue schema:value ?value . - | OPTIONAL { ?input renku:mappedTo/renku:streamType ?maybeStreamType. } - | ?input renku:position ?position. - | OPTIONAL { ?input renku:prefix ?maybePrefix } - | } - | BIND (IF(bound(?maybeStreamType), ?maybeStreamType, '') AS ?streamType). - | BIND (IF(?streamType = 'stdin', '< ', '') AS ?streamOperator). - | BIND (IF(bound(?maybePrefix), STR(?maybePrefix), '') AS ?prefix). - | BIND (CONCAT(?prefix, ?streamOperator, STR(?value)) AS ?commandParameter) . - | } UNION { # outputs - | GRAPH <${GraphClass.Project.id(projectId)}> { - | <$activityId> prov:qualifiedAssociation/prov:hadPlan ?planId. - | ?planId renku:hasOutputs ?output . - | ?paramValue a renku:ParameterValue ; - | schema:valueReference ?output . - | ?paramValue schema:value ?value . - | ?output renku:position ?position. - | OPTIONAL { ?output renku:mappedTo/renku:streamType ?maybeStreamType. } - | OPTIONAL { ?output renku:prefix ?maybePrefix } - | } - | BIND (IF(bound(?maybeStreamType), ?maybeStreamType, '') AS ?streamType). - | BIND (IF(?streamType = 'stdout', '> ', IF(?streamType = 'stderr', '2> ', '')) AS ?streamOperator). - | BIND (IF(bound(?maybePrefix), STR(?maybePrefix), '') AS ?prefix) . - | BIND (CONCAT(?prefix, ?streamOperator, STR(?value)) AS ?commandParameter) . - | } UNION { # parameters - | GRAPH <${GraphClass.Project.id(projectId)}> { - | <$activityId> prov:qualifiedAssociation/prov:hadPlan ?planId. - | ?planId renku:hasArguments ?parameter . - | ?paramValue a renku:ParameterValue; - | schema:valueReference ?parameter . - | ?paramValue schema:value ?value . - | ?parameter renku:position ?position . - | OPTIONAL { ?parameter renku:prefix ?maybePrefix } - | } - | BIND (IF(bound(?maybePrefix), STR(?maybePrefix), '') AS ?prefix) . - | BIND (CONCAT(?prefix, STR(?value)) AS ?commandParameter) . - | } - | } - | GROUP BY ?position ?commandParameter - | HAVING (COUNT(*) > 0) - | ORDER BY ?position - | } - |} - |GROUP BY ?command ?type ?location - |""".stripMargin + sparql"""|SELECT DISTINCT ?type (CONCAT(STR(?command), (GROUP_CONCAT(?commandParameter; separator=' '))) AS ?label) ?location + |WHERE { + | { + | SELECT DISTINCT ?command ?type ?location + | WHERE { + | GRAPH ${GraphClass.Project.id(projectId)} { + | $activityId prov:qualifiedAssociation/prov:hadPlan ?planId; + | a ?type. + | OPTIONAL { ?planId renku:command ?maybeCommand. } + | } + | BIND (IF(BOUND(?maybeCommand), CONCAT(STR(?maybeCommand), STR(' ')), '') AS ?command). + | BIND ($activityId AS ?location) + | } + | } { + | SELECT ?commandParameter + | WHERE { + | { # inputs + | GRAPH ${GraphClass.Project.id(projectId)} { + | $activityId prov:qualifiedAssociation/prov:hadPlan ?planId. + | ?planId renku:hasInputs ?input . + | ?paramValue a renku:ParameterValue ; + | schema:valueReference ?input . + | ?paramValue schema:value ?value . + | OPTIONAL { ?input renku:mappedTo/renku:streamType ?maybeStreamType } + | OPTIONAL { ?input renku:position ?maybePosition } + | OPTIONAL { ?input renku:prefix ?maybePrefix } + | } + | BIND (IF(BOUND(?maybeStreamType), ?maybeStreamType, '') AS ?streamType). + | BIND (IF(?streamType = 'stdin', '< ', '') AS ?streamOperator). + | BIND (IF(BOUND(?maybePrefix), STR(?maybePrefix), '') AS ?prefix). + | BIND (IF(BOUND(?maybePosition), CONCAT(?prefix, ?streamOperator, STR(?value)), '') AS ?commandParameter) . + | } UNION { # outputs + | GRAPH ${GraphClass.Project.id(projectId)} { + | $activityId prov:qualifiedAssociation/prov:hadPlan ?planId. + | ?planId renku:hasOutputs ?output . + | ?paramValue a renku:ParameterValue ; + | schema:valueReference ?output . + | ?paramValue schema:value ?value . + | OPTIONAL { ?output renku:position ?maybePosition } + | OPTIONAL { ?output renku:mappedTo/renku:streamType ?maybeStreamType } + | OPTIONAL { ?output renku:prefix ?maybePrefix } + | } + | BIND (IF(BOUND(?maybeStreamType), ?maybeStreamType, '') AS ?streamType). + | BIND (IF(?streamType = 'stdout', '> ', IF(?streamType = 'stderr', '2> ', '')) AS ?streamOperator). + | BIND (IF(BOUND(?maybePrefix), STR(?maybePrefix), '') AS ?prefix) . + | BIND (IF(BOUND(?maybePosition), CONCAT(?prefix, ?streamOperator, STR(?value)), '') AS ?commandParameter) . + | } UNION { # parameters + | GRAPH ${GraphClass.Project.id(projectId)} { + | $activityId prov:qualifiedAssociation/prov:hadPlan ?planId. + | ?planId renku:hasArguments ?parameter . + | ?paramValue a renku:ParameterValue; + | schema:valueReference ?parameter . + | ?paramValue schema:value ?value . + | OPTIONAL { ?parameter renku:position ?maybePosition } + | OPTIONAL { ?parameter renku:prefix ?maybePrefix } + | } + | BIND (IF(BOUND(?maybePrefix), STR(?maybePrefix), '') AS ?prefix) . + | BIND (IF(BOUND(?maybePosition), CONCAT(?prefix, STR(?value)), '') AS ?commandParameter) . + | } + | } + | GROUP BY ?maybePosition ?commandParameter + | HAVING (COUNT(*) > 0) + | ORDER BY ?maybePosition + | } + |} + |GROUP BY ?command ?type ?location + |""".stripMargin ) } } diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala index 8d4653974b..62492cf4de 100644 --- a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala @@ -21,11 +21,12 @@ package io.renku.knowledgegraph.projects.files.lineage import LineageGenerators._ import cats.effect.IO import cats.syntax.all._ -import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators._ +import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.GraphModelGenerators.projectPaths -import io.renku.graph.model.testentities.StepPlanCommandParameter._ +import io.renku.graph.model.parameterValues.ValueOverride import io.renku.graph.model.testentities._ +import io.renku.graph.model.testentities.StepPlanCommandParameter._ import io.renku.interpreters.TestLogger import io.renku.jsonld.syntax._ import io.renku.logging.TestSparqlQueryTimeRecorder @@ -165,7 +166,46 @@ class NodeDetailsFinderSpec .unsafeRunSync() shouldBe Set(NodeDef(activity1).toNode, NodeDef(activity2).toNode) } + "find details of a Plan with implicit command parameters " + + "- no implicit params, inputs and outputs to be included in the node's label" in new TestCase { + + val parameter = commandParameterDefaultValueGen.generateOne + val input = entityLocations.generateOne + val output = entityLocations.generateOne + + val project = anyRenkuProjectEntities + .addActivity(project => + executionPlanners( + stepPlanEntities( + CommandParameter.implicitFrom(parameter), + CommandInput.implicitFromLocation(input), + CommandOutput.implicitFromLocation(output) + ), + project + ).map( + _.planParameterValues(parameter -> ValueOverride(parameter.value)) + .planInputParameterValuesFromChecksum(input -> entityChecksums.generateOne) + .buildProvenanceUnsafe() + ) + ) + .generateOne + + upload(to = projectsDataset, project) + + val activity :: Nil = project.activities + + nodeDetailsFinder + .findDetails( + Set( + ExecutionInfo(activity.asEntityId.show, activity.startTime.value) + ), + project.path + ) + .unsafeRunSync() shouldBe Set(NodeDef(activity).toNode) + } + "find details of a Plan with mapped command parameters" in new TestCase { + val input +: output +: errOutput +: Nil = entityLocations.generateNonEmptyList(min = 3, max = 3).toList @@ -252,7 +292,7 @@ class NodeDetailsFinderSpec } exception shouldBe an[IllegalArgumentException] - exception.getMessage shouldBe s"No plan with $missingPlan" + exception.getMessage shouldBe s"No activity with $missingPlan" } } diff --git a/renku-model-tiny-types/src/main/scala/io/renku/graph/model/entityModel.scala b/renku-model-tiny-types/src/main/scala/io/renku/graph/model/entityModel.scala index a4a11bb5bf..f5064847a3 100644 --- a/renku-model-tiny-types/src/main/scala/io/renku/graph/model/entityModel.scala +++ b/renku-model-tiny-types/src/main/scala/io/renku/graph/model/entityModel.scala @@ -19,6 +19,7 @@ package io.renku.graph.model import cats.syntax.all._ +import cats.Show import io.renku.graph.model.entityModel.Location.FileOrFolder.from import io.renku.graph.model.views.{EntityIdJsonLDOps, TinyTypeJsonLDOps} import io.renku.jsonld.JsonLDDecoder.decodeString @@ -80,6 +81,11 @@ object entityModel { } implicit lazy val jsonLDEncoder: JsonLDEncoder[Location] = encodeString.contramap(_.value) + + implicit val show: Show[Location] = Show.show { + case Location.File(v) => v + case Location.Folder(v) => v + } } final class Checksum private (val value: String) extends AnyVal with StringTinyType diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/LineageExemplarData.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/LineageExemplarData.scala index 1f6d9eeacc..48ac71f206 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/LineageExemplarData.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/LineageExemplarData.scala @@ -233,6 +233,7 @@ object NodeDef { .sortBy(_._2) .map(_._1.show) .mkString(start = commandComponent, sep = " ", end = "") + .trim } private implicit def parameterValueShow[P <: ParameterValue]: Show[P] = Show.show { diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/StepPlanCommandParameter.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/StepPlanCommandParameter.scala index c5d7e3d999..963d7344e0 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/StepPlanCommandParameter.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/StepPlanCommandParameter.scala @@ -21,11 +21,11 @@ package io.renku.graph.model.testentities import cats.syntax.all._ import eu.timepit.refined.auto._ import io.renku.cli.model.{CliCommandInput, CliCommandOutput} +import io.renku.graph.model._ import io.renku.graph.model.cli.CliConverters -import io.renku.graph.model.commandParameters.IOStream.{StdErr, StdIn, StdOut} import io.renku.graph.model.commandParameters._ +import io.renku.graph.model.commandParameters.IOStream.{StdErr, StdIn, StdOut} import io.renku.graph.model.entityModel.Location -import io.renku.graph.model.{RenkuUrl, commandParameters, entities, plans} import io.renku.jsonld._ import io.renku.jsonld.syntax._ @@ -73,6 +73,17 @@ object StepPlanCommandParameter { plan.id ) + def implicitFrom(value: ParameterDefaultValue): Position => Plan => ImplicitCommandParameter = + _ => + plan => + ImplicitCommandParameter( + Name(value.show), + maybeDescription = None, + maybePrefix = None, + value, + plan.id + ) + implicit def toEntitiesCommandParameter(implicit renkuUrl: RenkuUrl ): CommandParameter => entities.StepPlanCommandParameter.CommandParameter = { @@ -120,6 +131,17 @@ object StepPlanCommandParameter { def streamedFromLocation(defaultValue: Location): Position => Plan => CommandInput = streamedFrom(InputDefaultValue(defaultValue)) + def implicitFromLocation(location: Location): Position => Plan => ImplicitCommandInput = + _ => + plan => + ImplicitCommandInput( + Name(location.show), + maybePrefix = None, + InputDefaultValue(location), + maybeEncodingFormat = None, + plan.id + ) + def from(defaultValue: InputDefaultValue): Position => Plan => LocationCommandInput = position => plan => @@ -249,6 +271,18 @@ object StepPlanCommandParameter { def streamedFromLocation(defaultValue: Location, stream: IOStream.Out): Position => Plan => MappedCommandOutput = streamedFrom(OutputDefaultValue(defaultValue), stream) + def implicitFromLocation(location: Location): Position => Plan => ImplicitCommandOutput = + _ => + plan => + ImplicitCommandOutput( + Name(location.show), + maybePrefix = None, + OutputDefaultValue(location), + FolderCreation.no, + maybeEncodingFormat = None, + plan.id + ) + def stdOut(implicit renkuUrl: RenkuUrl): IOStream.StdOut = IOStream.StdOut(IOStream.ResourceId((renkuUrl / "iostreams" / StdOut.name).value)) From 7d7132849b6f2bfc950bedcec16b93631cbccbae Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 25 Apr 2023 11:52:13 +0200 Subject: [PATCH 14/28] fix: TS clean up process not to read ds date (#1440) --- .../namedgraphs/SameAsHierarchyFixer.scala | 242 ++++++++---------- 1 file changed, 104 insertions(+), 138 deletions(-) diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/namedgraphs/SameAsHierarchyFixer.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/namedgraphs/SameAsHierarchyFixer.scala index fb15578962..48fab7e751 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/namedgraphs/SameAsHierarchyFixer.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/namedgraphs/SameAsHierarchyFixer.scala @@ -23,12 +23,13 @@ import cats.data.Nested import cats.effect.Async import cats.syntax.all._ import eu.timepit.refined.auto._ -import io.circe.{Decoder, DecodingFailure} +import io.circe.Decoder import io.renku.graph.model.{datasets, projects, GraphClass} import io.renku.graph.model.Schemas.{prov, renku, schema} import io.renku.graph.model.datasets._ import io.renku.jsonld.{EntityId, NamedGraph} import io.renku.triplesstore._ +import io.renku.triplesstore.client.syntax._ import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery.Prefixes import org.typelevel.log4cats.Logger @@ -77,17 +78,15 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "find DS projects", Prefixes of (renku -> "renku", schema -> "schema"), - s""" - SELECT DISTINCT ?allProjectIds - WHERE { - GRAPH <$graphId> { - ?datasetId ^renku:hasDataset <$graphId> - } - GRAPH ?projectGraphs { - ?datasetId ^renku:hasDataset ?allProjectIds - } - } - """ + sparql"""|SELECT DISTINCT ?allProjectIds + |WHERE { + | GRAPH $graphId { + | ?datasetId ^renku:hasDataset $graphId + | } + | GRAPH ?projectGraphs { + | ?datasetId ^renku:hasDataset ?allProjectIds + | } + |}""".stripMargin ) } } @@ -105,28 +104,25 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "find proj DS topmostSameAs", Prefixes of (renku -> "renku", schema -> "schema"), - s""" - SELECT ?projectId ?datasetId ?topmostSameAs ?sameAs - WHERE { - GRAPH ?g { - ?projectId renku:projectPath '$path'. - ?datasetId ^renku:hasDataset ?projectId; - a schema:Dataset; - renku:topmostSameAs ?topmostSameAs. - OPTIONAL { ?datasetId schema:sameAs/schema:url ?sameAs } - } - } - """ + sparql"""|SELECT ?projectId ?datasetId ?topmostSameAs ?sameAs + |WHERE { + | GRAPH ?g { + | ?projectId renku:projectPath ${path.asObject}. + | ?datasetId ^renku:hasDataset ?projectId; + | a schema:Dataset; + | renku:topmostSameAs ?topmostSameAs. + | OPTIONAL { ?datasetId schema:sameAs/schema:url ?sameAs } + | } + |}""".stripMargin ) } } - private case class DirectDescendantInfo(graphId: EntityId, - dsId: datasets.ResourceId, - topmostSameAs: TopmostSameAs, - sameAs: SameAs, - createdOrPublished: CreatedOrPublished, - modified: Boolean + private case class DirectDescendantInfo(graphId: EntityId, + dsId: datasets.ResourceId, + topmostSameAs: TopmostSameAs, + sameAs: SameAs, + modified: Boolean ) private def collectDirectDescendants(dsInfo: DSInfo): Nested[F, List, DirectDescendantInfo] = Nested { @@ -136,21 +132,14 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] implicit cursor => for { graphId <- extract[projects.ResourceId]("graphId").map(GraphClass.Project.id) - dsId <- extract[datasets.ResourceId]("descendantId") - topmostSameAs <- extract[TopmostSameAs]("descendantTopmostSameAs") + descId <- extract[datasets.ResourceId]("descendantId") + descTopmostSameAs <- extract[TopmostSameAs]("descendantTopmostSameAs") sameAs <- extract[SameAs]("descendantSameAs") - maybeDatePublished <- extract[Option[DatePublished]]("datePublished") - maybeDateCreated <- extract[Option[DateCreated]]("dateCreated") maybeModificationId <- extract[Option[ResourceId]]("modificationId") - date <- maybeDatePublished - .orElse(maybeDateCreated) - .map(_.asRight) - .getOrElse(DecodingFailure(s"No dates on DS $dsId", Nil).asLeft) } yield DirectDescendantInfo(graphId, - dsId, - topmostSameAs, + descId, + descTopmostSameAs, sameAs, - date, modified = maybeModificationId.nonEmpty ) } @@ -159,20 +148,15 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "find DS descendants info", Prefixes of (prov -> "prov", renku -> "renku", schema -> "schema"), - s""" - SELECT ?graphId ?descendantId ?descendantTopmostSameAs ?descendantSameAs - ?datePublished ?dateCreated ?modificationId - WHERE { - GRAPH ?graphId { - ?descendantId schema:sameAs/schema:url <${dsId.show}>; - renku:topmostSameAs ?descendantTopmostSameAs; - schema:sameAs/schema:url ?descendantSameAs. - OPTIONAL { ?descendantId schema:datePublished ?datePublished } - OPTIONAL { ?descendantId schema:dateCreated ?dateCreated } - OPTIONAL { ?modificationId prov:wasDerivedFrom/schema:url ?descendantId } - } - } - """ + sparql"""|SELECT ?graphId ?descendantId ?descendantTopmostSameAs ?descendantSameAs ?modificationId + |WHERE { + | GRAPH ?graphId { + | ?descendantId schema:sameAs/schema:url ${dsId.asEntityId}; + | renku:topmostSameAs ?descendantTopmostSameAs; + | schema:sameAs/schema:url ?descendantSameAs. + | OPTIONAL { ?modificationId prov:wasDerivedFrom/schema:url ?descendantId } + | } + |}""".stripMargin ) } } @@ -210,36 +194,32 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "clean-up DS descendant SameAs", Prefixes of schema -> "schema", - s""" - DELETE { - GRAPH <${descendant.graphId}> { - ?descendantSameAs ?p ?s. - <${descendant.dsId.show}> schema:sameAs ?descendantSameAs - } - } - WHERE { - GRAPH <${descendant.graphId}> { - <${descendant.dsId.show}> schema:sameAs ?descendantSameAs. - ?descendantSameAs ?p ?s - } - } - """ + sparql"""|DELETE { + | GRAPH ${descendant.graphId} { + | ?descendantSameAs ?p ?s. + | ${descendant.dsId.asEntityId} schema:sameAs ?descendantSameAs + | } + |} + |WHERE { + | GRAPH ${descendant.graphId} { + | ${descendant.dsId.asEntityId} schema:sameAs ?descendantSameAs. + | ?descendantSameAs ?p ?s + | } + |}""".stripMargin ) } private def cleanUpDescendantTopmostSameAs(descendant: DirectDescendantInfo) = updateWithNoResult { - val DirectDescendantInfo(graphId, descendantId, topmostSameAs, _, _, _) = descendant + val DirectDescendantInfo(graphId, descendantId, topmostSameAs, _, _) = descendant SparqlQuery.of( name = "clean-up DS descendant TopmostSameAs", Prefixes of renku -> "renku", - s""" - DELETE DATA { - GRAPH <$graphId> { - <${descendantId.show}> renku:topmostSameAs <${topmostSameAs.show}> - } - } - """ + sparql"""|DELETE DATA { + | GRAPH $graphId { + | ${descendantId.asEntityId} renku:topmostSameAs ${topmostSameAs.asEntityId} + | } + |}""".stripMargin ) } @@ -247,13 +227,11 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "link DS to itself", Prefixes of renku -> "renku", - s""" - INSERT DATA { - GRAPH <${nominated.graphId}> { - <${nominated.dsId.show}> renku:topmostSameAs <${nominated.dsId.show}> - } - } - """ + sparql"""|INSERT DATA { + | GRAPH ${nominated.graphId} { + | ${nominated.dsId.asEntityId} renku:topmostSameAs ${nominated.dsId.asEntityId} + | } + |}""".stripMargin ) } @@ -275,18 +253,16 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] linkDescendant(descendant, newTop) private def linkDescendant(descendantInfo: DirectDescendantInfo, newTop: DirectDescendantInfo) = updateWithNoResult { - val DirectDescendantInfo(_, _, newTopmostSameAs, newSameAs, _, _) = newTop + val DirectDescendantInfo(_, _, newTopmostSameAs, newSameAs, _) = newTop SparqlQuery.of( name = "link DS to new top", Prefixes of (renku -> "renku", schema -> "schema"), - s""" - INSERT DATA { - GRAPH <${descendantInfo.graphId}> { - <${descendantInfo.dsId.show}> renku:topmostSameAs <${newTopmostSameAs.show}>. - <${descendantInfo.dsId.show}> schema:sameAs <${newSameAs.asEntityId}> - } - } - """ + sparql"""|INSERT DATA { + | GRAPH ${descendantInfo.graphId} { + | ${descendantInfo.dsId.asEntityId} renku:topmostSameAs ${newTopmostSameAs.asEntityId}. + | ${descendantInfo.dsId.asEntityId} schema:sameAs ${newSameAs.asEntityId} + | } + |}""".stripMargin ) } @@ -311,15 +287,13 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "find DS descendants info", Prefixes of (renku -> "renku", schema -> "schema"), - s""" - SELECT ?graphId ?descendantId ?descendantSameAs - WHERE { - GRAPH ?graphId { - ?descendantId renku:topmostSameAs <${dsId.show}>; - schema:sameAs/schema:url ?descendantSameAs - } - } - """ + sparql"""|SELECT ?graphId ?descendantId ?descendantSameAs + |WHERE { + | GRAPH ?graphId { + | ?descendantId renku:topmostSameAs ${dsId.asEntityId}; + | schema:sameAs/schema:url ?descendantSameAs + | } + |}""".stripMargin ) } } @@ -330,18 +304,16 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "clean-up DS descendant", Prefixes of renku -> "renku", - s""" - DELETE { - GRAPH <$graphId> { - <${descendantId.show}> renku:topmostSameAs ?topmost - } - } - WHERE { - GRAPH <$graphId> { - <${descendantId.show}> renku:topmostSameAs ?topmost - } - } - """ + sparql"""|DELETE { + | GRAPH $graphId { + | ${descendantId.asEntityId} renku:topmostSameAs ?topmost + | } + |} + |WHERE { + | GRAPH $graphId { + | ${descendantId.asEntityId} renku:topmostSameAs ?topmost + | } + |}""".stripMargin ) } @@ -355,13 +327,11 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "insert new topmostSameAs", Prefixes of renku -> "renku", - s""" - INSERT DATA { - GRAPH <$graphId> { - <${descendantId.show}> renku:topmostSameAs <${newTopmostSameAs.show}> - } - } - """ + sparql"""|INSERT DATA { + | GRAPH $graphId { + | ${descendantId.asEntityId} renku:topmostSameAs ${newTopmostSameAs.asEntityId} + | } + |}""".stripMargin ) } @@ -383,13 +353,11 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "insert new SameAs", Prefixes of schema -> "schema", - s""" - INSERT DATA { - GRAPH <${descendantInfo.graphId}> { - <${descendantInfo.dsId.show}> schema:sameAs <${sameAs.asEntityId}> - } - } - """ + sparql"""|INSERT DATA { + | GRAPH ${descendantInfo.graphId} { + | ${descendantInfo.dsId.asEntityId} schema:sameAs ${sameAs.asEntityId} + | } + |}""".stripMargin ) } @@ -413,16 +381,14 @@ private class SameAsHierarchyFixer[F[_]: Async: Logger: SparqlQueryTimeRecorder] SparqlQuery.of( name = "find TopmostSameAs by SameAs", Prefixes of (renku -> "renku", schema -> "schema"), - s""" - SELECT ?topmostSameAs - WHERE { - GRAPH <$graphId> { - ?dsId schema:sameAs <${sameAs.asEntityId}>; - renku:topmostSameAs ?topmostSameAs - } - } - LIMIT 1 - """ + sparql"""|SELECT ?topmostSameAs + |WHERE { + | GRAPH $graphId { + | ?dsId schema:sameAs ${sameAs.asEntityId}; + | renku:topmostSameAs ?topmostSameAs + | } + |} + |LIMIT 1""".stripMargin ) }(decoder) } From b46470aabcb4fc6ab72bd9800cf694c2283de135 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 25 Apr 2023 12:00:15 +0200 Subject: [PATCH 15/28] chore: jena upgraded to 4.8.0 (renku-jena chart 0.0.20) --- .../src/test/scala/io/renku/triplesstore/InMemoryJena.scala | 4 ++-- helm-chart/renku-graph/requirements.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala b/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala index c589b67ed6..264a219516 100644 --- a/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala +++ b/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala @@ -51,13 +51,13 @@ trait InMemoryJena { lazy val container: SingleContainer[_] = jenaRunMode match { case JenaRunMode.GenericContainer => GenericContainer( - dockerImage = "renku/renku-jena:0.0.19", + dockerImage = "renku/renku-jena:0.0.20", exposedPorts = Seq(3030), waitStrategy = Wait forHttp "/$/ping" ) case JenaRunMode.FixedPortContainer(fixedPort) => FixedHostPortGenericContainer( - imageName = "renku/renku-jena:0.0.19", + imageName = "renku/renku-jena:0.0.20", exposedPorts = Seq(3030), exposedHostPort = fixedPort, exposedContainerPort = fixedPort, diff --git a/helm-chart/renku-graph/requirements.yaml b/helm-chart/renku-graph/requirements.yaml index 458408fd2f..e2e757f718 100644 --- a/helm-chart/renku-graph/requirements.yaml +++ b/helm-chart/renku-graph/requirements.yaml @@ -3,6 +3,6 @@ dependencies: version: "0.0.4" repository: "https://swissdatasciencecenter.github.io/helm-charts/" - name: renku-jena - version: "0.0.19" + version: "0.0.20" repository: "https://swissdatasciencecenter.github.io/helm-charts/" alias: jena From 073372317740d2c7001e6f7b2182bc8cbe47c398 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 25 Apr 2023 14:01:08 +0200 Subject: [PATCH 16/28] fix: flaky NodeDetailsFinderSpec --- .../projects/files/lineage/NodeDetailsFinderSpec.scala | 7 +++---- .../renku/graph/model/testentities/ExecutionPlanner.scala | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala index 62492cf4de..eff86faceb 100644 --- a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/files/lineage/NodeDetailsFinderSpec.scala @@ -183,7 +183,8 @@ class NodeDetailsFinderSpec ), project ).map( - _.planParameterValues(parameter -> ValueOverride(parameter.value)) + _.replaceCommand(planCommands.generateSome) + .planParameterValues(parameter -> ValueOverride(parameter.value)) .planInputParameterValuesFromChecksum(input -> entityChecksums.generateOne) .buildProvenanceUnsafe() ) @@ -196,9 +197,7 @@ class NodeDetailsFinderSpec nodeDetailsFinder .findDetails( - Set( - ExecutionInfo(activity.asEntityId.show, activity.startTime.value) - ), + Set(ExecutionInfo(activity.asEntityId.show, activity.startTime.value)), project.path ) .unsafeRunSync() shouldBe Set(NodeDef(activity).toNode) diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/ExecutionPlanner.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/ExecutionPlanner.scala index 5616fdf3bc..300dfbc9de 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/ExecutionPlanner.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/ExecutionPlanner.scala @@ -42,6 +42,9 @@ final case class ExecutionPlanner(plan: StepPlan, projectDateCreated: projects.DateCreated ) { + def replaceCommand(maybeCommand: Option[plans.Command]): ExecutionPlanner = + this.copy(plan = plan.fold(_.copy(maybeCommand = maybeCommand), _.copy(maybeCommand = maybeCommand))) + def planParameterValues( valuesOverrides: (ParameterDefaultValue, ValueOverride)* ): ExecutionPlanner = this.copy(parametersValueOverrides = parametersValueOverrides ::: valuesOverrides.toList) From a18ed792e008f016c3a388ce424eb5afa2a4a0a0 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 25 Apr 2023 15:30:03 +0200 Subject: [PATCH 17/28] fix: cross-entity search ordering by date issue --- .../main/scala/io/renku/entities/search/EntitiesFinder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entities-search/src/main/scala/io/renku/entities/search/EntitiesFinder.scala b/entities-search/src/main/scala/io/renku/entities/search/EntitiesFinder.scala index 0ccff57f54..320c9e6628 100644 --- a/entities-search/src/main/scala/io/renku/entities/search/EntitiesFinder.scala +++ b/entities-search/src/main/scala/io/renku/entities/search/EntitiesFinder.scala @@ -74,7 +74,7 @@ private class EntitiesFinderImpl[F[_]: Async: NonEmptyParallel: Logger: SparqlQu )(implicit encoder: SparqlEncoder[OrderBy]): String = { def mapPropertyName(property: Criteria.Sort.SortProperty) = property match { case Criteria.Sort.ByName => OrderBy.Property("LCASE(?name)") - case Criteria.Sort.ByDate => OrderBy.Property("?date") + case Criteria.Sort.ByDate => OrderBy.Property("xsd:dateTime(?date)") case Criteria.Sort.ByMatchingScore => OrderBy.Property("?matchingScore") } From 09496ef711092c78b0542238f6d0c44c836ad511 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Fri, 28 Apr 2023 09:28:54 +0200 Subject: [PATCH 18/28] chore: dev cli version to be installed on startup in background (#1445) * chore: dev cli version to be installed on startup in the background * feat: renku to be installed on the container by the tguser * feat: renku cli version check to be lifted if cli dev version configured * fix: tg not to check installed cli version if dev version confgured --- .../scala/io/renku/config/ConfigLoader.scala | 5 +- triples-generator/Dockerfile | 10 ++-- triples-generator/entrypoint.sh | 4 +- .../config/RenkuPythonDevVersionConfig.scala | 41 ++++++++++++---- .../CliVersionCompatibilityVerifier.scala | 44 ++++++++++------- .../init/CliVersionLoader.scala | 14 +++--- .../RenkuPythonDevVersionConfigSpec.scala | 10 +++- ...VersionCompatibilityVerifierImplSpec.scala | 48 ++++++++++++++----- 8 files changed, 119 insertions(+), 57 deletions(-) diff --git a/graph-commons/src/main/scala/io/renku/config/ConfigLoader.scala b/graph-commons/src/main/scala/io/renku/config/ConfigLoader.scala index 3233585e0d..efc974148b 100644 --- a/graph-commons/src/main/scala/io/renku/config/ConfigLoader.scala +++ b/graph-commons/src/main/scala/io/renku/config/ConfigLoader.scala @@ -44,9 +44,7 @@ object ConfigLoader { ConfigSource.fromConfig(config).at(key).load[T] } - private def fromEither[F[_]: MonadThrow, T]( - loadedConfig: ConfigReaderFailures Either T - ): F[T] = + private def fromEither[F[_]: MonadThrow, T](loadedConfig: ConfigReaderFailures Either T): F[T] = MonadThrow[F].fromEither[T] { loadedConfig leftMap (new ConfigLoadingException(_)) } @@ -77,6 +75,5 @@ object ConfigLoader { .leftMap(exception => CannotConvert(stringValue, ttApply.getClass.toString, exception.getMessage)) } .getOrElse(Left(CannotConvert(stringValue, ttApply.getClass.toString, "Not an int value"))) - } } diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile index c719a6ffcf..544a35e455 100644 --- a/triples-generator/Dockerfile +++ b/triples-generator/Dockerfile @@ -26,12 +26,11 @@ COPY --from=builder /work/triples-generator/target/universal/stage . ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV TZ UTC -# Installing Renku and other dependencies +# Installing Renku dependencies other tools RUN apk update && apk add --no-cache tzdata git git-lfs curl bash python3-dev py3-pip py3-wheel openssl-dev libffi-dev linux-headers gcc g++ make libxml2-dev libxslt-dev libc-dev yaml-dev 'rust>1.41.0' cargo tini && \ python3 -m pip install --ignore-installed 'packaging==21.3 '&& \ - python3 -m pip install --upgrade 'pip==23.0.1' && \ + python3 -m pip install --upgrade 'pip==23.1.2' && \ python3 -m pip install jinja2 && \ - python3 -m pip install 'renku==2.3.2' 'sentry-sdk==1.5.11' && \ chown -R daemon:daemon . COPY triples-generator/entrypoint.sh /entrypoint.sh @@ -43,6 +42,11 @@ RUN adduser --disabled-password -g "$GID" -D -u 1000 tguser && \ USER tguser +ENV PATH=$PATH:/home/tguser/.local/bin + +# Installing Renku +RUN python3 -m pip install 'renku==2.3.2' 'sentry-sdk==1.5.11' + RUN git config --global user.name 'renku' && \ git config --global user.email 'renku@renkulab.io' && \ git config --global filter.lfs.smudge "git-lfs smudge --skip %f" && \ diff --git a/triples-generator/entrypoint.sh b/triples-generator/entrypoint.sh index decb5763a3..0b16b6aceb 100755 --- a/triples-generator/entrypoint.sh +++ b/triples-generator/entrypoint.sh @@ -2,8 +2,8 @@ if [ ! -z "$RENKU_PYTHON_DEV_VERSION" ] then - /usr/bin/python3 -m pip uninstall --yes renku - /usr/bin/python3 -m pip install ${RENKU_PYTHON_DEV_VERSION} + /usr/bin/python3 -m pip uninstall --yes renku && \ + /usr/bin/python3 -m pip install renku==${RENKU_PYTHON_DEV_VERSION} & fi # run the command diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfig.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfig.scala index 3478bca0a1..ff2c6f5a86 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfig.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfig.scala @@ -18,22 +18,43 @@ package io.renku.triplesgenerator.config -import cats.MonadThrow +import cats.syntax.all._ +import cats.{MonadThrow, Show} import com.typesafe.config.{Config, ConfigFactory} -import pureconfig.ConfigReader +import pureconfig.ConfigReader.Result +import pureconfig.error.ConfigReaderException +import pureconfig.{ConfigCursor, ConfigReader, ConfigSource, ReadsMissingKeys} -final case class RenkuPythonDevVersion(version: String) extends Product with Serializable +final case class RenkuPythonDevVersion(version: String) +object RenkuPythonDevVersion { + implicit lazy val show: Show[RenkuPythonDevVersion] = Show.show(_.version) +} object RenkuPythonDevVersionConfig { - import io.renku.config.ConfigLoader._ + private val optionalStringReader: ConfigReader[Option[String]] = + new ConfigReader[Option[String]] with ReadsMissingKeys { + override def from(cur: ConfigCursor): Result[Option[String]] = + if (cur.isUndefined) + Right(Option.empty[String]) + else + ConfigReader[Option[String]].from(cur) >>= { + case None => None.asRight + case Some(version) if version.trim.isEmpty => None.asRight + case Some(version) => version.trim.some.asRight + } + } - implicit val reader: ConfigReader[Option[RenkuPythonDevVersion]] = ConfigReader[Option[String]].map { - case Some(version) if version.trim.isEmpty => None - case Some(version) => Some(RenkuPythonDevVersion(version.trim)) - case None => None - } + private implicit val reader: ConfigReader[Option[RenkuPythonDevVersion]] = + ConfigReader.forProduct1[Option[RenkuPythonDevVersion], Option[String]]("renku-python-dev-version") { + _.map(RenkuPythonDevVersion(_)) + }(optionalStringReader) def apply[F[_]: MonadThrow](config: Config = ConfigFactory.load): F[Option[RenkuPythonDevVersion]] = - find[F, Option[RenkuPythonDevVersion]]("renku-python-dev-version", config) + MonadThrow[F].fromEither( + ConfigSource + .fromConfig(config) + .load[Option[RenkuPythonDevVersion]] + .leftMap(ConfigReaderException(_)) + ) } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifier.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifier.scala index 06a5abe064..4aa94d2037 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifier.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifier.scala @@ -18,31 +18,38 @@ package io.renku.triplesgenerator.init -import cats.MonadThrow -import cats.effect.kernel.Async +import cats.effect.Async import cats.syntax.all._ +import cats.{Applicative, MonadThrow} import com.typesafe.config.Config import io.renku.graph.model.versions.CliVersion -import io.renku.triplesgenerator.config.{TriplesGeneration, VersionCompatibilityConfig} import io.renku.triplesgenerator.config.TriplesGeneration.{RemoteTriplesGeneration, RenkuLog} +import io.renku.triplesgenerator.config.{RenkuPythonDevVersion, RenkuPythonDevVersionConfig, TriplesGeneration, VersionCompatibilityConfig} import org.typelevel.log4cats.Logger trait CliVersionCompatibilityVerifier[F[_]] { def run: F[Unit] } -private class CliVersionCompatibilityVerifierImpl[F[_]: MonadThrow]( - cliVersion: CliVersion, - compatibility: VersionCompatibilityConfig +private class CliVersionCompatibilityVerifierImpl[F[_]: MonadThrow: Logger]( + cliVersion: CliVersion, + compatibility: VersionCompatibilityConfig, + maybeRenkuDevVersion: Option[RenkuPythonDevVersion] ) extends CliVersionCompatibilityVerifier[F] { - override def run: F[Unit] = - if (cliVersion != compatibility.cliVersion) - MonadThrow[F].raiseError( + + private val applicative: Applicative[F] = Applicative[F] + import applicative.whenA + + override def run: F[Unit] = maybeRenkuDevVersion match { + case Some(version) => + Logger[F].warn(show"Service running dev version of CLI: $version") + case None => + whenA(cliVersion != compatibility.cliVersion) { new IllegalStateException( - s"Incompatible versions. cliVersion: $cliVersion, configured version: ${compatibility.cliVersion}" - ) - ) - else ().pure[F] + show"Incompatible versions. cliVersion: $cliVersion, configured version: ${compatibility.cliVersion}" + ).raiseError[F, Unit] + } + } } object CliVersionCompatibilityChecker { @@ -50,10 +57,13 @@ object CliVersionCompatibilityChecker { // the concept of TriplesGeneration flag is a temporary solution // to provide acceptance-tests with the expected CLI version def apply[F[_]: Async: Logger](config: Config): F[CliVersionCompatibilityVerifier[F]] = for { - compatConfig <- VersionCompatibilityConfig.fromConfigF(config) + compatConfig <- VersionCompatibilityConfig.fromConfigF(config) + renkuDevVersion <- RenkuPythonDevVersionConfig[F](config) cliVersion <- TriplesGeneration[F](config) >>= { - case RenkuLog => CliVersionLoader[F]() - case RemoteTriplesGeneration => compatConfig.cliVersion.pure[F] + case RemoteTriplesGeneration => + compatConfig.cliVersion.pure[F] + case RenkuLog => + renkuDevVersion.map(v => CliVersion(v.version).pure[F]).getOrElse(CliVersionLoader[F]()) } - } yield new CliVersionCompatibilityVerifierImpl[F](cliVersion, compatConfig) + } yield new CliVersionCompatibilityVerifierImpl[F](cliVersion, compatConfig, renkuDevVersion) } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionLoader.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionLoader.scala index 860c09b193..82a8c63eea 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionLoader.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/init/CliVersionLoader.scala @@ -18,7 +18,7 @@ package io.renku.triplesgenerator.init -import cats.MonadError +import cats.MonadThrow import io.renku.graph.model.versions.CliVersion import pureconfig.ConfigReader @@ -28,19 +28,17 @@ private[init] object CliVersionLoader { private implicit val cliVersionLoader: ConfigReader[CliVersion] = stringTinyTypeReader(CliVersion) - def apply[F[_]]()(implicit ME: MonadError[F, Throwable]): F[CliVersion] = apply(findRenkuVersion) + def apply[F[_]: MonadThrow](): F[CliVersion] = apply(findRenkuVersion) - private[init] def apply[F[_]](renkuVersionFinder: F[CliVersion])(implicit - ME: MonadError[F, Throwable] - ): F[CliVersion] = renkuVersionFinder + private[init] def apply[F[_]: MonadThrow](renkuVersionFinder: F[CliVersion]): F[CliVersion] = renkuVersionFinder - private def findRenkuVersion[F[_]](implicit ME: MonadError[F, Throwable]): F[CliVersion] = { + private def findRenkuVersion[F[_]: MonadThrow]: F[CliVersion] = { import ammonite.ops._ import cats.syntax.all._ for { - versionAsString <- ME.catchNonFatal(%%("renku", "--version")(pwd).out.string.trim) - version <- ME.fromEither(CliVersion.from(versionAsString)) + versionAsString <- MonadThrow[F].catchNonFatal(%%("renku", "--version")(pwd).out.string.trim) + version <- MonadThrow[F].fromEither(CliVersion.from(versionAsString)) } yield version } } diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfigSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfigSpec.scala index 4fd21196fc..17c92d9086 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfigSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/config/RenkuPythonDevVersionConfigSpec.scala @@ -20,7 +20,7 @@ package io.renku.triplesgenerator.config import com.typesafe.config.ConfigFactory import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.nonEmptyStrings +import io.renku.generators.Generators.{blankStrings, nonEmptyStrings} import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec @@ -28,7 +28,9 @@ import scala.jdk.CollectionConverters._ import scala.util.{Success, Try} class RenkuPythonDevVersionConfigSpec extends AnyWordSpec with should.Matchers { + "apply" should { + "return Some(version) if there is a value set" in { val version = nonEmptyStrings().generateOne val config = ConfigFactory.parseMap( @@ -37,6 +39,10 @@ class RenkuPythonDevVersionConfigSpec extends AnyWordSpec with should.Matchers { RenkuPythonDevVersionConfig[Try](config) shouldBe Success(Some(RenkuPythonDevVersion(version))) } + "return None if there is no entry" in { + RenkuPythonDevVersionConfig[Try](ConfigFactory.empty) shouldBe Success(None) + } + "return None if there is no value set" in { val config = ConfigFactory.parseMap( Map("renku-python-dev-version" -> null).asJava @@ -47,7 +53,7 @@ class RenkuPythonDevVersionConfigSpec extends AnyWordSpec with should.Matchers { "return None if there is an empty string" in { val config = ConfigFactory.parseMap( - Map("renku-python-dev-version" -> nonEmptyStrings().generateOne.map(_ => ' ')).asJava + Map("renku-python-dev-version" -> blankStrings().generateOne).asJava ) RenkuPythonDevVersionConfig[Try](config) shouldBe Success(None) } diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifierImplSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifierImplSpec.scala index 20caa97a02..5f1703c5fa 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifierImplSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/init/CliVersionCompatibilityVerifierImplSpec.scala @@ -18,33 +18,59 @@ package io.renku.triplesgenerator.init +import cats.syntax.all._ import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.GraphModelGenerators._ +import io.renku.interpreters.TestLogger +import io.renku.triplesgenerator.config.RenkuPythonDevVersion import io.renku.triplesgenerator.generators.VersionGenerators._ import org.scalamock.scalatest.MockFactory +import org.scalatest.TryValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import scala.util.{Failure, Success, Try} +import scala.util.Try -class CliVersionCompatibilityVerifierImplSpec extends AnyWordSpec with MockFactory with should.Matchers { +class CliVersionCompatibilityVerifierImplSpec extends AnyWordSpec with MockFactory with should.Matchers with TryValues { "run" should { - "return Unit if the cli version matches the cli version from the compatibility config" in { + + "succeed if the cli version matches the cli version from the compatibility config" in { + val cliVersion = cliVersions.generateOne val compatConfig = compatibilityGen.generateOne.copy(configuredCliVersion = cliVersion, renkuDevVersion = None) - val checker = new CliVersionCompatibilityVerifierImpl[Try](cliVersion, compatConfig) - checker.run shouldBe Success(()) + val checker = new CliVersionCompatibilityVerifierImpl[Try](cliVersion, compatConfig, maybeRenkuDevVersion = None) + + checker.run.success.value shouldBe () } "fail if the cli version does not match the cli version from the compatibility config" in { - val cliVersion = cliVersions.generateOne - val compatConfig = compatibilityGen.suchThat(c => c.cliVersion != cliVersion).generateOne - val checker = new CliVersionCompatibilityVerifierImpl[Try](cliVersion, compatConfig) - val Failure(exception) = checker.run - exception shouldBe a[IllegalStateException] - exception.getMessage shouldBe s"Incompatible versions. cliVersion: $cliVersion, configured version: ${compatConfig.cliVersion}" + + val cliVersion = cliVersions.generateOne + val compatConfig = compatibilityGen.suchThat(c => c.cliVersion != cliVersion).generateOne + + val checker = new CliVersionCompatibilityVerifierImpl[Try](cliVersion, compatConfig, maybeRenkuDevVersion = None) + + val failure = checker.run.failure + + failure.exception shouldBe a[IllegalStateException] + failure.exception.getMessage shouldBe show"Incompatible versions. cliVersion: $cliVersion, configured version: ${compatConfig.cliVersion}" + } + + "succeed if there's CliDevVersion configured even if the versions does not match" in { + + val cliVersion = cliVersions.generateOne + val compatConfig = compatibilityGen.generateOne.copy(configuredCliVersion = cliVersion, renkuDevVersion = None) + val renkuDevVersion = RenkuPythonDevVersion(cliVersions.generateOne.value) + + assume(cliVersion.value != renkuDevVersion.version) + + val checker = new CliVersionCompatibilityVerifierImpl[Try](cliVersion, compatConfig, renkuDevVersion.some) + + checker.run.success.value shouldBe () } } + + private implicit lazy val logger: TestLogger[Try] = TestLogger[Try]() } From 9630ff433652d952dc17edf1e382195a5876834a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Fri, 28 Apr 2023 10:37:18 +0200 Subject: [PATCH 19/28] feat: delete project api to work with no data in TS (#1446) * refactor: SecurityRecordsFinder to take an optional AuthUser * feat: GitLab project path Authorizer * feat: KG API to use ProjectPathRecordsFinder when authorizing for a path --- .../stubs/gitlab/GitLabApiStub.scala | 6 + .../stubs/gitlab/Http4sDslUtils.scala | 5 +- .../renku/graph/eventlog/EventLogClient.scala | 1 + .../http/server/security/Authorizer.scala | 18 ++- .../security/DatasetIdRecordsFinder.scala | 5 +- .../security/DatasetSameAsRecordsFinder.scala | 7 +- .../security/GitLabPathRecordsFinder.scala | 136 +++++++++++++++++ .../security/ProjectPathRecordsFinder.scala | 72 +++------ .../server/security/TSPathRecordsFinder.scala | 88 +++++++++++ .../http/server/security/AuthorizerSpec.scala | 118 ++++++++------- .../security/DatasetIdRecordsFinderSpec.scala | 22 ++- .../DatasetSameAsRecordsFinderSpec.scala | 23 ++- .../GitLabPathRecordsFinderSpec.scala | 82 ++++++++++ .../server/security/MembersFinderSpec.scala | 141 ++++++++++++++++++ .../ProjectPathRecordsFinderSpec.scala | 67 +++++---- .../security/TSPathRecordsFinderSpec.scala | 76 ++++++++++ .../security/VisibilityFinderSpec.scala | 120 +++++++++++++++ .../projects/delete/ELProjectFinder.scala | 53 +++++++ .../projects/delete/Endpoint.scala | 35 +++-- ...jectFinder.scala => GLProjectFinder.scala} | 8 +- .../projects/delete/ELProjectFinderSpec.scala | 87 +++++++++++ .../projects/delete/EndpointSpec.scala | 86 ++++++++--- ...erSpec.scala => GLProjectFinderSpec.scala} | 4 +- .../io/renku/graph/model/GraphClass.scala | 2 +- .../api/events/CleanUpEvent.scala | 62 ++++++++ .../triplesgenerator/api/events/Client.scala | 4 + .../api/events/CleanUpEventSpec.scala | 66 ++++++++ .../api/events/Generators.scala | 4 + .../consumers/cleanup/CleanUpEvent.scala | 23 --- .../consumers/cleanup/EventDecoder.scala | 39 ----- .../consumers/cleanup/EventHandler.scala | 16 +- .../projectinfo/ProjectFinder.scala | 13 +- .../projectinfo/ProjectMembersFinder.scala | 3 +- .../consumers/cleanup/EventDecoderSpec.scala | 60 -------- .../consumers/cleanup/EventHandlerSpec.scala | 16 +- 35 files changed, 1215 insertions(+), 353 deletions(-) create mode 100644 graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala create mode 100644 graph-commons/src/main/scala/io/renku/graph/http/server/security/TSPathRecordsFinder.scala create mode 100644 graph-commons/src/test/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinderSpec.scala create mode 100644 graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala create mode 100644 graph-commons/src/test/scala/io/renku/graph/http/server/security/TSPathRecordsFinderSpec.scala create mode 100644 graph-commons/src/test/scala/io/renku/graph/http/server/security/VisibilityFinderSpec.scala create mode 100644 knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinder.scala rename knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/{ProjectFinder.scala => GLProjectFinder.scala} (89%) create mode 100644 knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinderSpec.scala rename knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/{ProjectFinderSpec.scala => GLProjectFinderSpec.scala} (98%) create mode 100644 triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/CleanUpEvent.scala create mode 100644 triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/CleanUpEventSpec.scala delete mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/CleanUpEvent.scala delete mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoder.scala delete mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoderSpec.scala diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala index 85ce18e367..e9eafed308 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala @@ -111,9 +111,15 @@ final class GitLabApiStub[F[_]: Async: Logger](private val stateRef: Ref[F, Stat case GET -> Root / ProjectId(id) => query(findProjectById(id, maybeAuthedReq)).flatMap(OkOrNotFound(_)) + case HEAD -> Root / ProjectId(id) => + query(findProjectById(id, maybeAuthedReq)).flatMap(EmptyOkOrNotFound(_)) + case GET -> Root / ProjectPath(path) => query(findProjectByPath(path, maybeAuthedReq)).flatMap(OkOrNotFound(_)) + case HEAD -> Root / ProjectPath(path) => + query(findProjectByPath(path, maybeAuthedReq)).flatMap(EmptyOkOrNotFound(_)) + case GET -> Root / ProjectPath(path) / ("users" | "members") => query(findProjectByPath(path, maybeAuthedReq)) .map(_.toList.flatMap(_.members.toList)) diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/Http4sDslUtils.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/Http4sDslUtils.scala index fed85758f5..8832f1b6f3 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/Http4sDslUtils.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/Http4sDslUtils.scala @@ -29,7 +29,7 @@ import io.renku.graph.model.{persons, projects} import io.renku.graph.model.events.CommitId import io.renku.http.rest.paging.{PagingRequest, PagingResponse} import io.renku.http.rest.paging.PagingRequest.Decoders.{page, perPage} -import org.http4s.{EntityEncoder, Header, HttpApp, HttpRoutes, QueryParamDecoder, Request, Response} +import org.http4s.{EntityEncoder, Header, HttpApp, HttpRoutes, QueryParamDecoder, Request, Response, Status} import org.http4s.circe.CirceEntityCodec._ import org.http4s.dsl.Http4sDsl import org.http4s.dsl.impl.{OptionalQueryParamDecoderMatcher, QueryParamDecoderMatcher} @@ -76,6 +76,9 @@ private[gitlab] trait Http4sDslUtils { payload.map(Ok(_)).getOrElse(Response.notFound[F].pure[F]) } + def EmptyOkOrNotFound[F[_]: Applicative](payload: Option[Any]): F[Response[F]] = + Response[F](payload.map(_ => Status.Ok).getOrElse(Status.NotFound)).pure[F] + object Membership extends QueryParamDecoderMatcher[Boolean]("membership") implicit val accessLevelDecoder: QueryParamDecoder[AccessLevel] = QueryParamDecoder[Int].map(pv => diff --git a/graph-commons/src/main/scala/io/renku/graph/eventlog/EventLogClient.scala b/graph-commons/src/main/scala/io/renku/graph/eventlog/EventLogClient.scala index 3868e7b1b5..17ccb1bc79 100644 --- a/graph-commons/src/main/scala/io/renku/graph/eventlog/EventLogClient.scala +++ b/graph-commons/src/main/scala/io/renku/graph/eventlog/EventLogClient.scala @@ -125,6 +125,7 @@ object EventLogClient { } object SearchCriteria { + def forStatus(status: EventStatus): SearchCriteria = SearchCriteria(Ior.right(status), None, None) diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/Authorizer.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/Authorizer.scala index 0f52434238..a46567f936 100644 --- a/graph-commons/src/main/scala/io/renku/graph/http/server/security/Authorizer.scala +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/Authorizer.scala @@ -23,8 +23,7 @@ import cats.data.EitherT import cats.effect.Async import cats.syntax.all._ import io.renku.graph.http.server.security.Authorizer.{AuthContext, SecurityRecord, SecurityRecordFinder} -import io.renku.graph.model.persons.GitLabId -import io.renku.graph.model.projects +import io.renku.graph.model.{persons, projects} import io.renku.graph.model.projects.Visibility import io.renku.graph.model.projects.Visibility._ import io.renku.http.server.security.EndpointSecurityException @@ -37,8 +36,11 @@ trait Authorizer[F[_], Key] { } object Authorizer { - type SecurityRecord = (Visibility, projects.Path, Set[GitLabId]) - type SecurityRecordFinder[F[_], Key] = Key => F[List[SecurityRecord]] + final case class SecurityRecord(visibility: Visibility, + projectPath: projects.Path, + allowedPersons: Set[persons.GitLabId] + ) + trait SecurityRecordFinder[F[_], Key] extends ((Key, Option[AuthUser]) => F[List[SecurityRecord]]) final case class AuthContext[Key](maybeAuthUser: Option[AuthUser], key: Key, allowedProjects: Set[projects.Path]) { def addAllowedProject(path: projects.Path): AuthContext[Key] = copy(allowedProjects = allowedProjects + path) @@ -61,7 +63,7 @@ private class AuthorizerImpl[F[_]: MonadThrow, Key](securityRecordsFinder: Secur override def authorize(key: Key, maybeAuthUser: Option[AuthUser] ): EitherT[F, EndpointSecurityException, AuthContext[Key]] = for { - records <- EitherT.right(securityRecordsFinder(key)) + records <- EitherT.right(securityRecordsFinder(key, maybeAuthUser)) authContext <- validate(AuthContext[Key](maybeAuthUser, key, Set.empty), records) } yield authContext @@ -76,9 +78,9 @@ private class AuthorizerImpl[F[_]: MonadThrow, Key](securityRecordsFinder: Secur private def findAllowedProjects(authContext: AuthContext[Key]): List[SecurityRecord] => Set[projects.Path] = _.foldLeft(Set.empty[projects.Path]) { - case (allowed, (Public, path, _)) => allowed + path - case (allowed, (Internal, path, _)) if authContext.maybeAuthUser.isDefined => allowed + path - case (allowed, (Private, path, members)) + case (allowed, SecurityRecord(Public, path, _)) => allowed + path + case (allowed, SecurityRecord(Internal, path, _)) if authContext.maybeAuthUser.isDefined => allowed + path + case (allowed, SecurityRecord(Private, path, members)) if (members intersect authContext.maybeAuthUser.map(_.id).toSet).nonEmpty => allowed + path case (allowed, _) => allowed diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinder.scala index ddc163f549..26d378a284 100644 --- a/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinder.scala +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinder.scala @@ -26,6 +26,7 @@ import io.renku.graph.model.{datasets, projects, GraphClass} import io.renku.graph.model.entities.Person import io.renku.graph.model.persons.GitLabId import io.renku.graph.model.projects.Visibility +import io.renku.http.server.security.model.AuthUser import io.renku.triplesstore._ import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery.Prefixes @@ -41,7 +42,7 @@ private class DatasetIdRecordsFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRec ) extends TSClientImpl(storeConfig) with SecurityRecordFinder[F, datasets.Identifier] { - override def apply(id: datasets.Identifier): F[List[SecurityRecord]] = + override def apply(id: datasets.Identifier, maybeAuthUser: Option[AuthUser]): F[List[SecurityRecord]] = queryExpecting[List[SecurityRecord]](selectQuery = query(id))(recordsDecoder) import eu.timepit.refined.auto._ @@ -86,6 +87,6 @@ private class DatasetIdRecordsFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRec .map(_.map(_.split(rowsSeparator).toList).getOrElse(List.empty)) .flatMap(_.map(GitLabId.parse).sequence.leftMap(ex => DecodingFailure(ex.getMessage, Nil))) .map(_.toSet) - } yield (visibility, path, userIds) + } yield SecurityRecord(visibility, path, userIds) } } diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinder.scala index 4984598c7c..98d27060f7 100644 --- a/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinder.scala +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinder.scala @@ -22,8 +22,9 @@ import cats.effect.Async import cats.syntax.all._ import cats.MonadThrow import io.renku.graph.http.server.security.Authorizer.{SecurityRecord, SecurityRecordFinder} -import io.renku.graph.model.{datasets, GraphClass} +import io.renku.graph.model.{GraphClass, datasets} import io.renku.graph.model.entities.Person +import io.renku.http.server.security.model.AuthUser import io.renku.jsonld.EntityId import io.renku.triplesstore.{ProjectsConnectionConfig, SparqlQueryTimeRecorder, TSClient} import org.typelevel.log4cats.Logger @@ -36,7 +37,7 @@ object DatasetSameAsRecordsFinder { private class DatasetSameAsRecordsFinderImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extends SecurityRecordFinder[F, datasets.SameAs] { - override def apply(sameAs: datasets.SameAs): F[List[SecurityRecord]] = + override def apply(sameAs: datasets.SameAs, maybeAuthUser: Option[AuthUser]): F[List[SecurityRecord]] = tsClient.queryExpecting[List[SecurityRecord]](selectQuery = query(sameAs)) import eu.timepit.refined.auto._ @@ -90,6 +91,6 @@ private class DatasetSameAsRecordsFinderImpl[F[_]: MonadThrow](tsClient: TSClien visibility <- extract[projects.Visibility]("visibility") path <- extract[projects.Path]("path") userIds <- extract[Option[String]]("memberGitLabIds") >>= toSetOfGitLabIds - } yield (visibility, path, userIds) + } yield SecurityRecord(visibility, path, userIds) } } diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala new file mode 100644 index 0000000000..eb83c5ac03 --- /dev/null +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala @@ -0,0 +1,136 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.Parallel +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.circe.Decoder +import io.circe.Decoder.decodeOption +import io.renku.graph.http.server.security.Authorizer.{SecurityRecord, SecurityRecordFinder} +import io.renku.graph.model.{persons, projects} +import io.renku.http.client.{AccessToken, GitLabClient, UserAccessToken} +import io.renku.http.server.security.model.AuthUser +import io.renku.http.tinytypes.TinyTypeURIEncoder._ +import org.http4s.Status.{Forbidden, NotFound, Ok, Unauthorized} +import org.http4s._ +import org.http4s.circe.jsonOf +import org.http4s.implicits._ +import org.typelevel.ci._ +import org.typelevel.log4cats.Logger + +trait GitLabPathRecordsFinder[F[_]] extends SecurityRecordFinder[F, projects.Path] + +object GitLabPathRecordsFinder { + def apply[F[_]: Async: Parallel: Logger: GitLabClient]: F[GitLabPathRecordsFinder[F]] = + new GitLabPathRecordsFinderImpl[F](new VisibilityFinderImpl[F], new MembersFinderImpl[F]).pure[F].widen +} + +private class GitLabPathRecordsFinderImpl[F[_]: Async: Parallel](visibilityFinder: VisibilityFinder[F], + membersFinder: MembersFinder[F] +) extends GitLabPathRecordsFinder[F] { + + import membersFinder.findMembers + import visibilityFinder.findVisibility + + override def apply(path: projects.Path, maybeAuthUser: Option[AuthUser]): F[List[SecurityRecord]] = { + implicit val maybeAccessToken: Option[UserAccessToken] = maybeAuthUser.map(_.accessToken) + + (findVisibility(path) -> findMembers(path)) + .parMapN((maybeVisibility, members) => maybeVisibility.map(vis => SecurityRecord(vis, path, members)).toList) + } +} + +private trait VisibilityFinder[F[_]] { + def findVisibility(path: projects.Path)(implicit mat: Option[AccessToken]): F[Option[projects.Visibility]] +} + +private class VisibilityFinderImpl[F[_]: Async: GitLabClient: Logger] extends VisibilityFinder[F] { + + override def findVisibility(path: projects.Path)(implicit mat: Option[AccessToken]): F[Option[projects.Visibility]] = + GitLabClient[F].get(uri"projects" / path, "single-project")(mapResponse) + + private lazy val mapResponse: PartialFunction[(Status, Request[F], Response[F]), F[Option[projects.Visibility]]] = { + case (Ok, _, response) => response.as[Option[projects.Visibility]] + case (Unauthorized | Forbidden | NotFound, _, _) => Option.empty[projects.Visibility].pure[F] + } + + private implicit lazy val entityDecoder: EntityDecoder[F, Option[projects.Visibility]] = { + + implicit val decoder: Decoder[Option[projects.Visibility]] = { cur => + cur.downField("visibility").as[Option[projects.Visibility]](decodeOption(projects.Visibility.jsonDecoder)) + } + + jsonOf[F, Option[projects.Visibility]] + } +} + +private trait MembersFinder[F[_]] { + def findMembers(path: projects.Path)(implicit mat: Option[AccessToken]): F[Set[persons.GitLabId]] +} + +private class MembersFinderImpl[F[_]: Async: GitLabClient: Logger] extends MembersFinder[F] { + + override def findMembers(path: projects.Path)(implicit mat: Option[AccessToken]): F[Set[persons.GitLabId]] = + fetch(uri"projects" / path / "members") + + private def fetch( + uri: Uri, + maybePage: Option[Int] = None, + allMembers: Set[persons.GitLabId] = Set.empty + )(implicit mat: Option[AccessToken]): F[Set[persons.GitLabId]] = for { + uri <- uriWithPage(uri, maybePage).pure[F] + fetchedUsersAndNextPage <- GitLabClient[F].get(uri, "project-members")(mapResponse) + allMembers <- addNextPage(uri, allMembers, fetchedUsersAndNextPage) + } yield allMembers + + private def uriWithPage(uri: Uri, maybePage: Option[Int]) = maybePage match { + case Some(page) => uri withQueryParam ("page", page) + case None => uri + } + + private lazy val mapResponse + : PartialFunction[(Status, Request[F], Response[F]), F[(Set[persons.GitLabId], Option[Int])]] = { + case (Ok, _, response) => + lazy val maybeNextPage: Option[Int] = response.headers.get(ci"X-Next-Page").flatMap(_.head.value.toIntOption) + response.as[List[persons.GitLabId]].map(_.toSet -> maybeNextPage) + case (Unauthorized | Forbidden | NotFound, _, _) => (Set.empty[persons.GitLabId] -> Option.empty[Int]).pure[F] + } + + private def addNextPage( + url: Uri, + allMembers: Set[persons.GitLabId], + fetchedIdsAndMaybeNextPage: (Set[persons.GitLabId], Option[Int]) + )(implicit maybeAccessToken: Option[AccessToken]): F[Set[persons.GitLabId]] = + fetchedIdsAndMaybeNextPage match { + case (fetchedIds, maybeNextPage @ Some(_)) => fetch(url, maybeNextPage, allMembers ++ fetchedIds) + case (fetchedIds, None) => (allMembers ++ fetchedIds).pure[F] + } + + private implicit lazy val entityDecoder: EntityDecoder[F, List[persons.GitLabId]] = { + + implicit val decoder: Decoder[persons.GitLabId] = { cursor => + import io.renku.tinytypes.json.TinyTypeDecoders.intDecoder + cursor.downField("id").as[persons.GitLabId](intDecoder(persons.GitLabId)) + } + + jsonOf[F, List[persons.GitLabId]] + } +} diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinder.scala index 740b711186..ae10373cac 100644 --- a/graph-commons/src/main/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinder.scala +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinder.scala @@ -20,66 +20,28 @@ package io.renku.graph.http.server.security import cats.effect.Async import cats.syntax.all._ -import io.circe.{Decoder, DecodingFailure} -import io.renku.graph.config.RenkuUrlLoader -import io.renku.graph.http.server.security.Authorizer.{SecurityRecord, SecurityRecordFinder} -import io.renku.graph.model.{projects, GraphClass, RenkuUrl} -import io.renku.graph.model.entities.Person -import io.renku.graph.model.persons.GitLabId -import io.renku.graph.model.projects.{ResourceId, Visibility} -import io.renku.graph.model.projects.Visibility._ -import io.renku.graph.model.views.RdfResource -import io.renku.triplesstore._ -import io.renku.triplesstore.ResultsDecoder._ -import io.renku.triplesstore.SparqlQuery.Prefixes +import cats.{MonadThrow, Parallel} +import io.renku.graph.http.server.security.Authorizer.SecurityRecordFinder +import io.renku.graph.model.projects +import io.renku.http.client.GitLabClient +import io.renku.http.server.security.model.AuthUser +import io.renku.triplesstore.SparqlQueryTimeRecorder import org.typelevel.log4cats.Logger object ProjectPathRecordsFinder { - def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[SecurityRecordFinder[F, projects.Path]] = for { - implicit0(renkuUrl: RenkuUrl) <- RenkuUrlLoader[F]() - storeConfig <- ProjectsConnectionConfig[F]() - } yield new ProjectPathRecordsFinderImpl[F](storeConfig) + def apply[F[_]: Async: Parallel: Logger: SparqlQueryTimeRecorder: GitLabClient] + : F[SecurityRecordFinder[F, projects.Path]] = + (TSPathRecordsFinder[F] -> GitLabPathRecordsFinder[F]) + .mapN(new ProjectPathRecordsFinderImpl[F](_, _)) } -private class ProjectPathRecordsFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder]( - storeConfig: ProjectsConnectionConfig -)(implicit renkuUrl: RenkuUrl) - extends TSClientImpl(storeConfig) - with SecurityRecordFinder[F, projects.Path] { +private class ProjectPathRecordsFinderImpl[F[_]: MonadThrow](tsPathRecordsFinder: TSPathRecordsFinder[F], + glPathRecordsFinder: GitLabPathRecordsFinder[F] +) extends SecurityRecordFinder[F, projects.Path] { - override def apply(path: projects.Path): F[List[SecurityRecord]] = - queryExpecting[List[SecurityRecord]](query(ResourceId(path)))(recordsDecoder(path)) - - import eu.timepit.refined.auto._ - import io.renku.graph.model.Schemas._ - - private def query(resourceId: projects.ResourceId) = SparqlQuery.of( - name = "authorise - project path", - Prefixes of (renku -> "renku", schema -> "schema"), - s"""|SELECT DISTINCT ?projectId ?visibility (GROUP_CONCAT(?maybeMemberGitLabId; separator=',') AS ?memberGitLabIds) - |FROM <${GraphClass.Persons.id}> - |FROM ${resourceId.showAs[RdfResource]} { - | BIND (${resourceId.showAs[RdfResource]} AS ?resourceId) - | ?projectId a schema:Project; - | renku:projectVisibility ?visibility. - | OPTIONAL { - | ?projectId schema:member/schema:sameAs ?sameAsId. - | ?sameAsId schema:additionalType '${Person.gitLabSameAsAdditionalType}'; - | schema:identifier ?maybeMemberGitLabId. - | } - |} - |GROUP BY ?projectId ?visibility - |""".stripMargin - ) - - private def recordsDecoder(path: projects.Path): Decoder[List[SecurityRecord]] = - ResultsDecoder[List, SecurityRecord] { implicit cur => - for { - visibility <- extract[Visibility]("visibility") - maybeUserId <- extract[Option[String]]("memberGitLabIds") - .map(_.map(_.split(",").toList).getOrElse(List.empty)) - .flatMap(_.map(GitLabId.parse).sequence.leftMap(ex => DecodingFailure(ex.getMessage, Nil))) - .map(_.toSet) - } yield (visibility, path, maybeUserId) + override def apply(path: projects.Path, maybeAuthUser: Option[AuthUser]): F[List[Authorizer.SecurityRecord]] = + tsPathRecordsFinder(path, maybeAuthUser) >>= { + case Nil => glPathRecordsFinder(path, maybeAuthUser) + case nonEmpty => nonEmpty.pure[F] } } diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/TSPathRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/TSPathRecordsFinder.scala new file mode 100644 index 0000000000..2e1963198e --- /dev/null +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/TSPathRecordsFinder.scala @@ -0,0 +1,88 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.effect.Async +import cats.syntax.all._ +import io.circe.{Decoder, DecodingFailure} +import io.renku.graph.config.RenkuUrlLoader +import io.renku.graph.http.server.security.Authorizer.{SecurityRecord, SecurityRecordFinder} +import io.renku.graph.model.{GraphClass, RenkuUrl, projects} +import io.renku.graph.model.entities.Person +import io.renku.graph.model.persons.GitLabId +import io.renku.graph.model.projects.{ResourceId, Visibility} +import io.renku.graph.model.projects.Visibility._ +import io.renku.graph.model.views.RdfResource +import io.renku.http.server.security.model.AuthUser +import io.renku.triplesstore._ +import io.renku.triplesstore.ResultsDecoder._ +import io.renku.triplesstore.SparqlQuery.Prefixes +import org.typelevel.log4cats.Logger + +trait TSPathRecordsFinder[F[_]] extends SecurityRecordFinder[F, projects.Path] + +object TSPathRecordsFinder { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[TSPathRecordsFinder[F]] = for { + implicit0(renkuUrl: RenkuUrl) <- RenkuUrlLoader[F]() + storeConfig <- ProjectsConnectionConfig[F]() + } yield new TSPathRecordsFinderImpl[F](storeConfig) +} + +private class TSPathRecordsFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder]( + storeConfig: ProjectsConnectionConfig +)(implicit renkuUrl: RenkuUrl) + extends TSClientImpl(storeConfig) + with TSPathRecordsFinder[F] { + + override def apply(path: projects.Path, maybeAuthUser: Option[AuthUser]): F[List[SecurityRecord]] = + queryExpecting[List[SecurityRecord]](query(ResourceId(path)))(recordsDecoder(path)) + + import eu.timepit.refined.auto._ + import io.renku.graph.model.Schemas._ + + private def query(resourceId: projects.ResourceId) = SparqlQuery.of( + name = "authorise - project path", + Prefixes of (renku -> "renku", schema -> "schema"), + s"""|SELECT DISTINCT ?projectId ?visibility (GROUP_CONCAT(?maybeMemberGitLabId; separator=',') AS ?memberGitLabIds) + |FROM <${GraphClass.Persons.id}> + |FROM ${resourceId.showAs[RdfResource]} { + | BIND (${resourceId.showAs[RdfResource]} AS ?resourceId) + | ?projectId a schema:Project; + | renku:projectVisibility ?visibility. + | OPTIONAL { + | ?projectId schema:member/schema:sameAs ?sameAsId. + | ?sameAsId schema:additionalType '${Person.gitLabSameAsAdditionalType}'; + | schema:identifier ?maybeMemberGitLabId. + | } + |} + |GROUP BY ?projectId ?visibility + |""".stripMargin + ) + + private def recordsDecoder(path: projects.Path): Decoder[List[SecurityRecord]] = + ResultsDecoder[List, SecurityRecord] { implicit cur => + for { + visibility <- extract[Visibility]("visibility") + maybeUserId <- extract[Option[String]]("memberGitLabIds") + .map(_.map(_.split(",").toList).getOrElse(List.empty)) + .flatMap(_.map(GitLabId.parse).sequence.leftMap(ex => DecodingFailure(ex.getMessage, Nil))) + .map(_.toSet) + } yield SecurityRecord(visibility, path, maybeUserId) + } +} diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/AuthorizerSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/AuthorizerSpec.scala index 6c8b748284..dc9e43676f 100644 --- a/graph-commons/src/test/scala/io/renku/graph/http/server/security/AuthorizerSpec.scala +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/AuthorizerSpec.scala @@ -22,7 +22,6 @@ import cats.data.EitherT.{leftT, rightT} import cats.syntax.all._ import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ -import io.renku.graph.http.server.security.Authorizer.{AuthContext, SecurityRecord} import io.renku.graph.model.GraphModelGenerators._ import io.renku.graph.model.persons.GitLabId import io.renku.graph.model.projects.Visibility @@ -32,6 +31,7 @@ import org.scalacheck.Arbitrary import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec +import Authorizer._ import scala.util.Try @@ -40,10 +40,11 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { "authorize - public projects" should { "succeed if found SecurityRecord is for a public project and there's no auth user" in new TestCase { + val projectPath = projectPaths.generateOne - securityRecordsFinder - .expects(key) - .returning(List((Visibility.Public, projectPath, Set.empty[GitLabId])).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, None) + .returning(List(SecurityRecord(Visibility.Public, projectPath, Set.empty[GitLabId])).pure[Try]) authorizer.authorize(key, maybeAuthUser = None) shouldBe rightT[Try, EndpointSecurityException]( AuthContext[Key](None, key, Set(projectPath)) @@ -51,28 +52,32 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { } "succeed if found SecurityRecord is for a public project and the user has rights to the project" in new TestCase { + val projectPath = projectPaths.generateOne val authUser = authUsers.generateOne - securityRecordsFinder - .expects(key) - .returning(List((Visibility.Public, projectPath, personGitLabIds.generateSet() + authUser.id)).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, authUser.some) + .returning( + List(SecurityRecord(Visibility.Public, projectPath, personGitLabIds.generateSet() + authUser.id)).pure[Try] + ) authorizer.authorize(key, authUser.some) shouldBe rightT[Try, EndpointSecurityException]( - AuthContext[Key](Some(authUser), key, Set(projectPath)) + AuthContext[Key](authUser.some, key, Set(projectPath)) ) } "succeed if found SecurityRecord is for a public project and the user has no explicit rights to the project" in new TestCase { + val projectPath = projectPaths.generateOne - val authUser = authUsers.generateOne + val authUser = authUsers.generateSome - securityRecordsFinder - .expects(key) - .returning(List((Visibility.Public, projectPath, personGitLabIds.generateSet())).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, authUser) + .returning(List(SecurityRecord(Visibility.Public, projectPath, personGitLabIds.generateSet())).pure[Try]) - authorizer.authorize(key, authUser.some) shouldBe rightT[Try, EndpointSecurityException]( - AuthContext[Key](Some(authUser), key, Set(projectPath)) + authorizer.authorize(key, authUser) shouldBe rightT[Try, EndpointSecurityException]( + AuthContext[Key](authUser, key, Set(projectPath)) ) } } @@ -81,10 +86,11 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { Visibility.Internal :: Visibility.Private :: Nil foreach { visibility => s"fail if found SecurityRecord is for a $visibility project and there's no auth user" in new TestCase { + val projectPath = projectPaths.generateOne - securityRecordsFinder - .expects(key) - .returning(List((visibility, projectPath, Set.empty[GitLabId])).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, None) + .returning(List(SecurityRecord(visibility, projectPath, Set.empty[GitLabId])).pure[Try]) authorizer.authorize(key, maybeAuthUser = None) shouldBe leftT[Try, Unit](AuthorizationFailure) } @@ -92,49 +98,57 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { Visibility.Internal :: Visibility.Private :: Nil foreach { visibility => s"succeed if found SecurityRecord is for a $visibility project but the user has rights to the project" in new TestCase { + val projectPath = projectPaths.generateOne val authUser = authUsers.generateOne - securityRecordsFinder - .expects(key) - .returning(List((visibility, projectPath, personGitLabIds.generateSet() + authUser.id)).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, authUser.some) + .returning( + List(SecurityRecord(visibility, projectPath, personGitLabIds.generateSet() + authUser.id)).pure[Try] + ) authorizer.authorize(key, authUser.some) shouldBe rightT[Try, EndpointSecurityException]( - AuthContext[Key](Some(authUser), key, Set(projectPath)) + AuthContext[Key](authUser.some, key, Set(projectPath)) ) } } "succeed if found SecurityRecord is for an internal project even if the user has no explicit rights for the project" in new TestCase { + val projectPath = projectPaths.generateOne - val authUser = authUsers.generateOne + val authUser = authUsers.generateSome - securityRecordsFinder - .expects(key) - .returning(List((Visibility.Internal, projectPath, personGitLabIds.generateSet())).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, authUser) + .returning(List(SecurityRecord(Visibility.Internal, projectPath, personGitLabIds.generateSet())).pure[Try]) - authorizer.authorize(key, authUser.some) shouldBe rightT[Try, EndpointSecurityException]( - AuthContext[Key](Some(authUser), key, Set(projectPath)) + authorizer.authorize(key, authUser) shouldBe rightT[Try, EndpointSecurityException]( + AuthContext[Key](authUser, key, Set(projectPath)) ) } "fail if found SecurityRecord is for a private project and the user has no explicit rights for the project" in new TestCase { + val projectPath = projectPaths.generateOne - val authUser = authUsers.generateOne + val authUser = authUsers.generateSome - securityRecordsFinder - .expects(key) - .returning(List((Visibility.Private, projectPath, personGitLabIds.generateSet())).pure[Try]) + (securityRecordsFinder.apply _) + .expects(key, authUser) + .returning(List(SecurityRecord(Visibility.Private, projectPath, personGitLabIds.generateSet())).pure[Try]) - authorizer.authorize(key, authUser.some) shouldBe leftT[Try, Unit](AuthorizationFailure) + authorizer.authorize(key, authUser) shouldBe leftT[Try, Unit](AuthorizationFailure) } "fail if there's no project with the given id" in new TestCase { - val authUser = authUsers.generateOne - securityRecordsFinder.expects(key).returning(Nil.pure[Try]) + val authUser = authUsers.generateSome + + (securityRecordsFinder.apply _) + .expects(key, authUser) + .returning(Nil.pure[Try]) - authorizer.authorize(key, Some(authUser)) shouldBe leftT[Try, Unit](AuthorizationFailure) + authorizer.authorize(key, authUser) shouldBe leftT[Try, Unit](AuthorizationFailure) } } @@ -142,13 +156,13 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { "succeed with public projects only if there's no auth user" in new TestCase { val publicProject = projectPaths.generateOne - securityRecordsFinder - .expects(key) + (securityRecordsFinder.apply _) + .expects(key, None) .returning( List( - (Visibility.Public, publicProject, personGitLabIds.generateSet()), - (Visibility.Internal, projectPaths.generateOne, personGitLabIds.generateSet()), - (Visibility.Private, projectPaths.generateOne, personGitLabIds.generateSet()) + SecurityRecord(Visibility.Public, publicProject, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Internal, projectPaths.generateOne, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Private, projectPaths.generateOne, personGitLabIds.generateSet()) ).pure[Try] ) @@ -158,17 +172,18 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { } "succeed with public and internal projects only if there's an auth user without explicit access for the private projects" in new TestCase { + val authUser = authUsers.generateOne val publicProject = projectPaths.generateOne val internalProject = projectPaths.generateOne - securityRecordsFinder - .expects(key) + (securityRecordsFinder.apply _) + .expects(key, authUser.some) .returning( List( - (Visibility.Public, publicProject, personGitLabIds.generateSet()), - (Visibility.Internal, internalProject, personGitLabIds.generateSet()), - (Visibility.Private, projectPaths.generateOne, personGitLabIds.generateSet()) + SecurityRecord(Visibility.Public, publicProject, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Internal, internalProject, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Private, projectPaths.generateOne, personGitLabIds.generateSet()) ).pure[Try] ) @@ -178,18 +193,19 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { } "succeed with public, internal and private projects if there's an auth user with explicit access for the private projects" in new TestCase { + val authUser = authUsers.generateOne val publicProject = projectPaths.generateOne val internalProject = projectPaths.generateOne val privateProject = projectPaths.generateOne - securityRecordsFinder - .expects(key) + (securityRecordsFinder.apply _) + .expects(key, authUser.some) .returning( List( - (Visibility.Public, publicProject, personGitLabIds.generateSet()), - (Visibility.Internal, internalProject, personGitLabIds.generateSet()), - (Visibility.Private, privateProject, personGitLabIds.generateSet() + authUser.id) + SecurityRecord(Visibility.Public, publicProject, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Internal, internalProject, personGitLabIds.generateSet()), + SecurityRecord(Visibility.Private, privateProject, personGitLabIds.generateSet() + authUser.id) ).pure[Try] ) @@ -202,8 +218,8 @@ class AuthorizerSpec extends AnyWordSpec with MockFactory with should.Matchers { private case class Key(value: Int) private trait TestCase { + val securityRecordsFinder = mock[SecurityRecordFinder[Try, Key]] val key = Arbitrary.arbInt.arbitrary.map(Key).generateOne - val securityRecordsFinder = mockFunction[Key, Try[List[SecurityRecord]]] val authorizer = new AuthorizerImpl[Try, Key](securityRecordsFinder) } } diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinderSpec.scala index 44511fdfed..17f1f96622 100644 --- a/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinderSpec.scala +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetIdRecordsFinderSpec.scala @@ -19,7 +19,9 @@ package io.renku.graph.http.server.security import cats.effect.IO +import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ +import io.renku.graph.http.server.security.Authorizer.SecurityRecord import io.renku.graph.model.testentities._ import io.renku.interpreters.TestLogger import io.renku.logging.TestSparqlQueryTimeRecorder @@ -44,8 +46,8 @@ class DatasetIdRecordsFinderSpec upload(to = projectsDataset, project) - recordsFinder(dataset.identification.identifier).unsafeRunSync() shouldBe List( - (project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) + recordsFinder(dataset.identification.identifier, maybeAuthUser).unsafeRunSync() shouldBe List( + SecurityRecord(project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) ) } @@ -59,8 +61,8 @@ class DatasetIdRecordsFinderSpec upload(to = projectsDataset, project) - recordsFinder(dataset.identification.identifier).unsafeRunSync() shouldBe List( - (project.visibility, project.path, Set.empty) + recordsFinder(dataset.identification.identifier, maybeAuthUser).unsafeRunSync() shouldBe List( + SecurityRecord(project.visibility, project.path, Set.empty) ) } @@ -75,18 +77,22 @@ class DatasetIdRecordsFinderSpec upload(to = projectsDataset, parentProject, project) - recordsFinder(dataset.identification.identifier).unsafeRunSync() should contain theSameElementsAs List( - (parentProject.visibility, parentProject.path, parentProject.members.flatMap(_.maybeGitLabId)), - (project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) + recordsFinder(dataset.identification.identifier, maybeAuthUser) + .unsafeRunSync() should contain theSameElementsAs List( + SecurityRecord(parentProject.visibility, parentProject.path, parentProject.members.flatMap(_.maybeGitLabId)), + SecurityRecord(project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) ) } "nothing if there's no project with the given path" in new TestCase { - recordsFinder(datasetIdentifiers.generateOne).unsafeRunSync() shouldBe Nil + recordsFinder(datasetIdentifiers.generateOne, maybeAuthUser).unsafeRunSync() shouldBe Nil } } private trait TestCase { + + val maybeAuthUser = authUsers.generateOption + private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() val recordsFinder = new DatasetIdRecordsFinderImpl[IO](projectsDSConnectionInfo) diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinderSpec.scala index 0f470eab82..e8f7c71084 100644 --- a/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinderSpec.scala +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/DatasetSameAsRecordsFinderSpec.scala @@ -19,7 +19,9 @@ package io.renku.graph.http.server.security import cats.effect.IO +import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ +import io.renku.graph.http.server.security.Authorizer.SecurityRecord import io.renku.graph.model import io.renku.graph.model.testentities._ import io.renku.interpreters.TestLogger @@ -45,8 +47,10 @@ class DatasetSameAsRecordsFinderSpec upload(to = projectsDataset, project) - recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value)) - .unsafeRunSync() shouldBe List((project.visibility, project.path, project.members.flatMap(_.maybeGitLabId))) + recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value), maybeAuthUser) + .unsafeRunSync() shouldBe List( + SecurityRecord(project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) + ) } "return SecurityRecord with project visibility, path and no member if project is none" in new TestCase { @@ -59,8 +63,8 @@ class DatasetSameAsRecordsFinderSpec upload(to = projectsDataset, project) - recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value)) - .unsafeRunSync() shouldBe List((project.visibility, project.path, Set.empty)) + recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value), maybeAuthUser) + .unsafeRunSync() shouldBe List(SecurityRecord(project.visibility, project.path, Set.empty)) } "return SecurityRecords with projects visibilities, paths and members in case of forks" in new TestCase { @@ -74,19 +78,22 @@ class DatasetSameAsRecordsFinderSpec upload(to = projectsDataset, parentProject, project) - recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value)) + recordsFinder(model.datasets.SameAs.ofUnsafe(dataset.provenance.topmostSameAs.value), maybeAuthUser) .unsafeRunSync() should contain theSameElementsAs List( - (parentProject.visibility, parentProject.path, parentProject.members.flatMap(_.maybeGitLabId)), - (project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) + SecurityRecord(parentProject.visibility, parentProject.path, parentProject.members.flatMap(_.maybeGitLabId)), + SecurityRecord(project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) ) } "nothing if there's no project with the given path" in new TestCase { - recordsFinder(datasetSameAs.generateOne).unsafeRunSync() shouldBe Nil + recordsFinder(datasetSameAs.generateOne, maybeAuthUser).unsafeRunSync() shouldBe Nil } } private trait TestCase { + + val maybeAuthUser = authUsers.generateOption + private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() private val tsClient = TSClient[IO](projectsDSConnectionInfo) diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinderSpec.scala new file mode 100644 index 0000000000..686f1e2c53 --- /dev/null +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinderSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.effect.IO +import cats.syntax.all._ +import io.renku.generators.CommonGraphGenerators.authUsers +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.model.RenkuTinyTypeGenerators._ +import io.renku.graph.model.{persons, projects} +import io.renku.http.client.AccessToken +import io.renku.testtools.IOSpec +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class GitLabPathRecordsFinderSpec extends AnyWordSpec with should.Matchers with IOSpec with MockFactory { + + "apply" should { + + "return SecurityRecord with project visibility and all project members" in new TestCase { + + val visibility = projectVisibilities.generateOne + givenFindingVisibility(returning = visibility.some.pure[IO]) + + val members = personGitLabIds.generateSet() + givenFindingMembers(returning = members.pure[IO]) + + recordsFinder(projectPath, maybeAuthUser).unsafeRunSync() shouldBe List( + Authorizer.SecurityRecord(visibility, projectPath, members) + ) + } + + "return no Records if no visibility found" in new TestCase { + + givenFindingVisibility(returning = None.pure[IO]) + + val members = personGitLabIds.generateSet() + givenFindingMembers(returning = members.pure[IO]) + + recordsFinder(projectPath, maybeAuthUser).unsafeRunSync() shouldBe Nil + } + } + + private trait TestCase { + + val projectPath = projectPaths.generateOne + val maybeAuthUser = authUsers.generateOption + + implicit val visibilityFinder: VisibilityFinder[IO] = mock[VisibilityFinder[IO]] + implicit val membersFinder: MembersFinder[IO] = mock[MembersFinder[IO]] + val recordsFinder = new GitLabPathRecordsFinderImpl[IO](visibilityFinder, membersFinder) + + def givenFindingVisibility(returning: IO[Option[projects.Visibility]]) = + (visibilityFinder + .findVisibility(_: projects.Path)(_: Option[AccessToken])) + .expects(projectPath, maybeAuthUser.map(_.accessToken)) + .returning(returning) + + def givenFindingMembers(returning: IO[Set[persons.GitLabId]]) = + (membersFinder + .findMembers(_: projects.Path)(_: Option[AccessToken])) + .expects(projectPath, maybeAuthUser.map(_.accessToken)) + .returning(returning) + } +} diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala new file mode 100644 index 0000000000..4459f6258e --- /dev/null +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala @@ -0,0 +1,141 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.effect.IO +import cats.syntax.all._ +import eu.timepit.refined.api.Refined +import eu.timepit.refined.auto._ +import eu.timepit.refined.collection.NonEmpty +import io.circe.Encoder +import io.circe.literal._ +import io.circe.syntax._ +import io.renku.generators.CommonGraphGenerators.accessTokens +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.{fixed, positiveInts} +import io.renku.graph.model.RenkuTinyTypeGenerators._ +import io.renku.graph.model.persons +import io.renku.http.client.RestClient.ResponseMappingF +import io.renku.http.client.{AccessToken, GitLabClient} +import io.renku.http.tinytypes.TinyTypeURIEncoder._ +import io.renku.interpreters.TestLogger +import io.renku.testtools.{GitLabClientTools, IOSpec} +import org.http4s.circe.CirceEntityCodec.circeEntityEncoder +import org.http4s.implicits._ +import org.http4s.{Header, Request, Response, Status, Uri} +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec +import org.typelevel.ci._ + +class MembersFinderSpec + extends AnyWordSpec + with should.Matchers + with IOSpec + with MockFactory + with GitLabClientTools[IO] { + + "findMembers" should { + + "return all project members" in new TestCase { + + val members = personGitLabIds.generateSet() + givenFindingMembers(maybePage = None, returning = (members -> Option.empty[Int]).pure[IO]) + + finder.findMembers(projectPath).unsafeRunSync() shouldBe members + } + + "return project members from all pages" in new TestCase { + + val membersPage1 = personGitLabIds.generateSet() + givenFindingMembers(maybePage = None, returning = (membersPage1 -> 2.some).pure[IO]) + val membersPage2 = personGitLabIds.generateSet() + givenFindingMembers(maybePage = 2.some, returning = (membersPage2 -> Option.empty[Int]).pure[IO]) + + finder.findMembers(projectPath).unsafeRunSync() shouldBe membersPage1 ++ membersPage2 + } + + "return members if OK" in new TestCase { + + val members = personGitLabIds.generateSet() + + mapResponse(Status.Ok, Request(), Response().withEntity(members.map(_.asJson(encoder)).toList.asJson)) + .unsafeRunSync() shouldBe (members, None) + } + + "return members and the next page if OK" in new TestCase { + + val members = personGitLabIds.generateSet() + val nextPage = positiveInts().generateOne.value + + mapResponse( + Status.Ok, + Request(), + Response[IO]() + .withEntity(members.map(_.asJson(encoder)).toList.asJson) + .withHeaders(Header.Raw(ci"X-Next-Page", nextPage.toString)) + ).unsafeRunSync() shouldBe (members, nextPage.some) + } + + Status.NotFound :: Status.Unauthorized :: Status.Forbidden :: Nil foreach { status => + s"return an empty Set for $status" in new TestCase { + mapResponse(status, Request(), Response()).unsafeRunSync() shouldBe (Set.empty, None) + } + } + } + + private trait TestCase { + + val projectPath = projectPaths.generateOne + implicit val maybeAccessToken: Option[AccessToken] = accessTokens.generateOption + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + implicit val gitLabClient: GitLabClient[IO] = mock[GitLabClient[IO]] + val finder = new MembersFinderImpl[IO] + + def givenFindingMembers(maybePage: Option[Int], returning: IO[(Set[persons.GitLabId], Option[Int])]) = { + val endpointName: String Refined NonEmpty = "project-members" + + val uri = { + val uri = uri"projects" / projectPath / "members" + maybePage match { + case Some(page) => uri withQueryParam ("page", page.toString) + case None => uri + } + } + + (gitLabClient + .get(_: Uri, _: String Refined NonEmpty)(_: ResponseMappingF[IO, (Set[persons.GitLabId], Option[Int])])( + _: Option[AccessToken] + )) + .expects(uri, endpointName, *, maybeAccessToken) + .returning(returning) + } + + val mapResponse = + captureMapping(gitLabClient)( + finder.findMembers(projectPath)(maybeAccessToken).unsafeRunSync(), + fixed((Set.empty[persons.GitLabId], Option.empty[Int])) + ) + } + + private lazy val encoder: Encoder[persons.GitLabId] = Encoder.instance { id => + json"""{"id": $id}""" + } +} diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinderSpec.scala index 1718949d71..b9cf7882d3 100644 --- a/graph-commons/src/test/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinderSpec.scala +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/ProjectPathRecordsFinderSpec.scala @@ -18,54 +18,59 @@ package io.renku.graph.http.server.security -import cats.effect.IO +import cats.syntax.all._ +import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ -import io.renku.graph.model.testentities.generators.EntitiesGenerators -import io.renku.interpreters.TestLogger -import io.renku.logging.TestSparqlQueryTimeRecorder -import io.renku.testtools.IOSpec -import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset, SparqlQueryTimeRecorder} +import io.renku.graph.http.server.security.Authorizer.SecurityRecord +import io.renku.graph.model.RenkuTinyTypeGenerators.{personGitLabIds, projectPaths, projectVisibilities} +import org.scalacheck.Gen +import org.scalamock.scalatest.MockFactory +import org.scalatest.TryValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -class ProjectPathRecordsFinderSpec - extends AnyWordSpec - with IOSpec - with EntitiesGenerators - with InMemoryJenaForSpec - with ProjectsDataset - with should.Matchers { +import scala.util.Try + +class ProjectPathRecordsFinderSpec extends AnyWordSpec with should.Matchers with MockFactory with TryValues { "apply" should { - "return SecurityRecords with project visibility and all project members" in new TestCase { - val project = anyProjectEntities.generateOne + "return SecurityRecords from the TS when found" in new TestCase { - upload(to = projectsDataset, project) + val securityRecords = securityRecordsGen.generateList(min = 1) + givenTSPathRecordsFinder(returning = securityRecords.pure[Try]) - recordsFinder(project.path).unsafeRunSync() shouldBe List( - (project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) - ) + recordsFinder(projectPath, maybeAuthUser).success.value shouldBe securityRecords } - "return SecurityRecords with project visibility and no member is project has none" in new TestCase { - val project = renkuProjectEntities(anyVisibility).generateOne.copy(members = Set.empty) + "return SecurityRecords from the GL when no records found in TS" in new TestCase { - upload(to = projectsDataset, project) + givenTSPathRecordsFinder(returning = Nil.pure[Try]) - recordsFinder(project.path).unsafeRunSync() shouldBe List( - (project.visibility, project.path, Set.empty) - ) - } + val securityRecords = securityRecordsGen.generateList(min = 1) + givenGLPathRecordsFinder(returning = securityRecords.pure[Try]) - "nothing if there's no project with the given path" in new TestCase { - recordsFinder(projectPaths.generateOne).unsafeRunSync() shouldBe Nil + recordsFinder(projectPath, maybeAuthUser).success.value shouldBe securityRecords } } private trait TestCase { - private implicit val logger: TestLogger[IO] = TestLogger[IO]() - private implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() - val recordsFinder = new ProjectPathRecordsFinderImpl[IO](projectsDSConnectionInfo) + + val projectPath = projectPaths.generateOne + val maybeAuthUser = authUsers.generateOption + + private val tsPathRecordsFinder = mock[TSPathRecordsFinder[Try]] + private val glPathRecordsFinder = mock[GitLabPathRecordsFinder[Try]] + val recordsFinder = new ProjectPathRecordsFinderImpl[Try](tsPathRecordsFinder, glPathRecordsFinder) + + def givenTSPathRecordsFinder(returning: Try[List[SecurityRecord]]) = + (tsPathRecordsFinder.apply _).expects(projectPath, maybeAuthUser).returning(returning) + + def givenGLPathRecordsFinder(returning: Try[List[SecurityRecord]]) = + (glPathRecordsFinder.apply _).expects(projectPath, maybeAuthUser).returning(returning) + + val securityRecordsGen: Gen[SecurityRecord] = + (projectVisibilities, personGitLabIds.toGeneratorOfSet(min = 0)) + .mapN(SecurityRecord(_, projectPath, _)) } } diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/TSPathRecordsFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/TSPathRecordsFinderSpec.scala new file mode 100644 index 0000000000..70b3744ec2 --- /dev/null +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/TSPathRecordsFinderSpec.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.effect.IO +import io.renku.generators.CommonGraphGenerators.authUsers +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.http.server.security.Authorizer.SecurityRecord +import io.renku.graph.model.testentities.generators.EntitiesGenerators +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.testtools.IOSpec +import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset, SparqlQueryTimeRecorder} +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class TSPathRecordsFinderSpec + extends AnyWordSpec + with IOSpec + with EntitiesGenerators + with InMemoryJenaForSpec + with ProjectsDataset + with should.Matchers { + + "apply" should { + + "return SecurityRecords with project visibility and all project members" in new TestCase { + val project = anyProjectEntities.generateOne + + upload(to = projectsDataset, project) + + recordsFinder(project.path, maybeAuthUser).unsafeRunSync() shouldBe List( + SecurityRecord(project.visibility, project.path, project.members.flatMap(_.maybeGitLabId)) + ) + } + + "return SecurityRecords with project visibility and no member is project has none" in new TestCase { + val project = renkuProjectEntities(anyVisibility).generateOne.copy(members = Set.empty) + + upload(to = projectsDataset, project) + + recordsFinder(project.path, maybeAuthUser).unsafeRunSync() shouldBe List( + SecurityRecord(project.visibility, project.path, Set.empty) + ) + } + + "nothing if there's no project with the given path" in new TestCase { + recordsFinder(projectPaths.generateOne, maybeAuthUser).unsafeRunSync() shouldBe Nil + } + } + + private trait TestCase { + + val maybeAuthUser = authUsers.generateOption + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + private implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + val recordsFinder = new TSPathRecordsFinderImpl[IO](projectsDSConnectionInfo) + } +} diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/VisibilityFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/VisibilityFinderSpec.scala new file mode 100644 index 0000000000..64f2a1110e --- /dev/null +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/VisibilityFinderSpec.scala @@ -0,0 +1,120 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.graph.http.server.security + +import cats.effect.IO +import cats.syntax.all._ +import eu.timepit.refined.api.Refined +import eu.timepit.refined.auto._ +import eu.timepit.refined.collection.NonEmpty +import io.circe.literal._ +import io.circe.syntax._ +import io.circe.{Encoder, Json} +import io.renku.generators.CommonGraphGenerators.accessTokens +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.model.RenkuTinyTypeGenerators._ +import io.renku.graph.model.projects +import io.renku.http.client.RestClient.ResponseMappingF +import io.renku.http.client.{AccessToken, GitLabClient} +import io.renku.http.tinytypes.TinyTypeURIEncoder._ +import io.renku.interpreters.TestLogger +import io.renku.testtools.{GitLabClientTools, IOSpec} +import org.http4s.circe.CirceEntityCodec.circeEntityEncoder +import org.http4s.implicits._ +import org.http4s.{Request, Response, Status, Uri} +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class VisibilityFinderSpec + extends AnyWordSpec + with should.Matchers + with IOSpec + with MockFactory + with GitLabClientTools[IO] { + + "findVisibility" should { + + "return project visibility if found" in new TestCase { + + val visibility = projectVisibilities.generateSome + givenFindingVisibility(returning = visibility.pure[IO]) + + finder.findVisibility(projectPath).unsafeRunSync() shouldBe visibility + } + + "return no visibility if not found" in new TestCase { + + givenFindingVisibility(returning = None.pure[IO]) + + finder.findVisibility(projectPath).unsafeRunSync() shouldBe None + } + + "return visibility if OK and relevant property exists in the response" in new TestCase { + + val visibility = projectVisibilities.generateOne + + mapResponse(Status.Ok, Request(), Response().withEntity(visibility.asJson(encoder))) + .unsafeRunSync() shouldBe visibility.some + } + + "return no visibility if OK and relevant property does not exist in the response" in new TestCase { + mapResponse(Status.Ok, Request(), Response().withEntity(Json.obj())) + .unsafeRunSync() shouldBe None + } + + Status.NotFound :: Status.Unauthorized :: Status.Forbidden :: Nil foreach { status => + s"return no visibility for $status" in new TestCase { + mapResponse(status, Request(), Response()).unsafeRunSync() shouldBe None + } + } + } + + private trait TestCase { + + val projectPath = projectPaths.generateOne + implicit val maybeAccessToken: Option[AccessToken] = accessTokens.generateOption + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + implicit val gitLabClient: GitLabClient[IO] = mock[GitLabClient[IO]] + val finder = new VisibilityFinderImpl[IO] + + def givenFindingVisibility(returning: IO[Option[projects.Visibility]]) = { + val endpointName: String Refined NonEmpty = "single-project" + val uri = uri"projects" / projectPath + + (gitLabClient + .get(_: Uri, _: String Refined NonEmpty)(_: ResponseMappingF[IO, Option[projects.Visibility]])( + _: Option[AccessToken] + )) + .expects(uri, endpointName, *, maybeAccessToken) + .returning(returning) + } + + val mapResponse = + captureMapping(gitLabClient)( + finder.findVisibility(projectPath)(maybeAccessToken).unsafeRunSync(), + projectVisibilities.generateOption + ) + } + + private lazy val encoder: Encoder[projects.Visibility] = Encoder.instance { visibility => + json"""{"visibility": $visibility}""" + } +} diff --git a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinder.scala b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinder.scala new file mode 100644 index 0000000000..3ea623e885 --- /dev/null +++ b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinder.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.knowledgegraph.projects.delete + +import cats.MonadThrow +import cats.effect.Async +import cats.syntax.all._ +import io.renku.events.consumers.Project +import io.renku.graph.eventlog.EventLogClient +import io.renku.graph.eventlog.EventLogClient.{Result, SearchCriteria} +import io.renku.graph.model.events.EventInfo +import io.renku.graph.model.projects +import io.renku.http.rest.paging.model.PerPage +import org.typelevel.log4cats.Logger + +private trait ELProjectFinder[F[_]] { + def findProject(path: projects.Path): F[Option[Project]] +} + +private object ELProjectFinder { + def apply[F[_]: Async: Logger]: F[ELProjectFinder[F]] = + EventLogClient[F].map(new ELProjectFinderImpl[F](_)) +} + +private class ELProjectFinderImpl[F[_]: MonadThrow](elClient: EventLogClient[F]) extends ELProjectFinder[F] { + + override def findProject(path: projects.Path): F[Option[Project]] = + elClient + .getEvents(SearchCriteria.forProject(path).withPerPage(PerPage(1))) + .flatMap(toResult) + + private def toResult: Result[List[EventInfo]] => F[Option[Project]] = + _.toEither.fold( + _.raiseError[F, Option[Project]], + _.headOption.map(ei => Project(ei.project.id, ei.project.path)).pure[F] + ) +} diff --git a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/Endpoint.scala b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/Endpoint.scala index e7b6aa014f..95b118b992 100644 --- a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/Endpoint.scala +++ b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/Endpoint.scala @@ -24,11 +24,13 @@ import io.renku.events.consumers.Project import io.renku.graph.eventlog import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.projects -import io.renku.http.{ErrorMessage, InfoMessage} +import io.renku.http.InfoMessage._ import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.http.server.security.model.AuthUser -import io.renku.http.InfoMessage._ +import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.metrics.MetricsRegistry +import io.renku.triplesgenerator +import io.renku.triplesgenerator.api.events.CleanUpEvent import org.http4s.Response import org.http4s.dsl.Http4sDsl import org.typelevel.log4cats.Logger @@ -41,20 +43,20 @@ trait Endpoint[F[_]] { object Endpoint { def apply[F[_]: Async: Logger: MetricsRegistry: GitLabClient]: F[Endpoint[F]] = - eventlog.api.events - .Client[F] - .map(new EndpointImpl(ProjectFinder[F], ProjectRemover[F], _)) + (ELProjectFinder[F], eventlog.api.events.Client[F], triplesgenerator.api.events.Client[F]).mapN( + new EndpointImpl(GLProjectFinder[F], _, ProjectRemover[F], _, _) + ) } -private class EndpointImpl[F[_]: Async: Logger](projectFinder: ProjectFinder[F], +private class EndpointImpl[F[_]: Async: Logger](glProjectFinder: GLProjectFinder[F], + elProjectFinder: ELProjectFinder[F], projectRemover: ProjectRemover[F], elClient: eventlog.api.events.Client[F], + tgClient: triplesgenerator.api.events.Client[F], waitBeforeNextCheck: Duration = 1 second ) extends Http4sDsl[F] with Endpoint[F] { - import elClient.send - import projectFinder.findProject import projectRemover.deleteProject override def `DELETE /projects/:path`(path: projects.Path, authUser: AuthUser): F[Response[F]] = { @@ -65,17 +67,26 @@ private class EndpointImpl[F[_]: Async: Logger](projectFinder: ProjectFinder[F], NotFound(InfoMessage("Project does not exist")) case Some(project) => deleteProject(project.id) >> - Spawn[F].start(waitForDeletion(project) >> send(CommitSyncRequest(project))) >> + Spawn[F].start(waitForDeletion(project.path) >> sendEvents(project)) >> Accepted(InfoMessage("Project deleted")) } }.handleErrorWith(httpResult(path)) - private def waitForDeletion(project: Project)(implicit ac: AccessToken): F[Unit] = - findProject(project.path) >>= { + private def waitForDeletion(path: projects.Path)(implicit ac: AccessToken): F[Unit] = + glProjectFinder.findProject(path) >>= { case None => ().pure[F] - case Some(_) => Temporal[F].delayBy(waitForDeletion(project), waitBeforeNextCheck) + case Some(_) => Temporal[F].delayBy(waitForDeletion(path), waitBeforeNextCheck) } + private def findProject(path: projects.Path)(implicit ac: AccessToken): F[Option[Project]] = + glProjectFinder.findProject(path) >>= { + case None => elProjectFinder.findProject(path) + case someProject => someProject.pure[F] + } + + private def sendEvents(project: Project): F[Unit] = + elClient.send(CommitSyncRequest(project)) >> tgClient.send(CleanUpEvent(project)) + private def httpResult(path: projects.Path): Throwable => F[Response[F]] = { exception => Logger[F].error(exception)(show"Deleting '$path' project failed") >> InternalServerError(ErrorMessage(s"Project deletion failure: ${exception.getMessage}")) diff --git a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ProjectFinder.scala b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinder.scala similarity index 89% rename from knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ProjectFinder.scala rename to knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinder.scala index 1eea5353ae..c77c1cb127 100644 --- a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/ProjectFinder.scala +++ b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinder.scala @@ -24,15 +24,15 @@ import io.renku.events.consumers.Project import io.renku.graph.model.projects import io.renku.http.client.{AccessToken, GitLabClient} -private trait ProjectFinder[F[_]] { +private trait GLProjectFinder[F[_]] { def findProject(path: projects.Path)(implicit at: AccessToken): F[Option[Project]] } -private object ProjectFinder { - def apply[F[_]: Async: GitLabClient]: ProjectFinder[F] = new ProjectFinderImpl[F] +private object GLProjectFinder { + def apply[F[_]: Async: GitLabClient]: GLProjectFinder[F] = new GLProjectFinderImpl[F] } -private class ProjectFinderImpl[F[_]: Async: GitLabClient] extends ProjectFinder[F] { +private class GLProjectFinderImpl[F[_]: Async: GitLabClient] extends GLProjectFinder[F] { import eu.timepit.refined.auto._ import io.circe.Decoder diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinderSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinderSpec.scala new file mode 100644 index 0000000000..06f0eaf03f --- /dev/null +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ELProjectFinderSpec.scala @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.knowledgegraph.projects.delete + +import cats.syntax.all._ +import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.{exceptions, nonEmptyStrings} +import io.renku.graph.eventlog.EventLogClient +import io.renku.graph.eventlog.EventLogClient.Result.{Failure, Success} +import io.renku.graph.eventlog.EventLogClient.{Result, SearchCriteria} +import io.renku.graph.model.EventContentGenerators.eventInfos +import io.renku.graph.model.events.EventInfo +import io.renku.http.rest.paging.model.PerPage +import org.scalamock.scalatest.MockFactory +import org.scalatest.TryValues +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +import scala.util.Try + +class ELProjectFinderSpec extends AnyWordSpec with should.Matchers with MockFactory with TryValues { + + "findProject" should { + + "return a project if found in EL" in new TestCase { + + givenELProjectFinding(returning = + Success(eventInfos(project.path, project.id).generateFixedSizeList(ofSize = 1)).pure[Try] + ) + + finder.findProject(project.path).success.value shouldBe Some(project) + } + + "return no project if not found in EL" in new TestCase { + + givenELProjectFinding(returning = Success(List.empty[EventInfo]).pure[Try]) + + finder.findProject(project.path).success.value shouldBe None + } + + "fail if finding project return an error" in new TestCase { + + val failure = Failure(nonEmptyStrings().generateOne) + givenELProjectFinding(returning = failure.pure[Try]) + + finder.findProject(project.path).failure.exception shouldBe failure + } + + "fail if finding project fails" in new TestCase { + + val failure = exceptions.generateOne + givenELProjectFinding(returning = failure.raiseError[Try, Result[List[EventInfo]]]) + + finder.findProject(project.path).failure.exception shouldBe failure + } + } + + private trait TestCase { + + val project = consumerProjects.generateOne + + private val elClient = mock[EventLogClient[Try]] + val finder = new ELProjectFinderImpl[Try](elClient) + + def givenELProjectFinding(returning: Try[EventLogClient.Result[List[EventInfo]]]) = + (elClient.getEvents _) + .expects(SearchCriteria.forProject(project.path).withPerPage(PerPage(1))) + .returning(returning) + } +} diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/EndpointSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/EndpointSpec.scala index 5e63531bed..29aa1326ef 100644 --- a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/EndpointSpec.scala +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/EndpointSpec.scala @@ -28,17 +28,19 @@ import io.renku.generators.Generators.exceptions import io.renku.graph.eventlog import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.projects -import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.http.ErrorMessage._ import io.renku.http.InfoMessage._ import io.renku.http.client.AccessToken import io.renku.http.server.EndpointTester._ +import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.Error import io.renku.testtools.IOSpec +import io.renku.triplesgenerator +import io.renku.triplesgenerator.api.events.CleanUpEvent +import org.http4s.MediaType.application import org.http4s.Status.{Accepted, InternalServerError, NotFound} import org.http4s.headers.`Content-Type` -import org.http4s.MediaType.application import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec @@ -50,13 +52,36 @@ class EndpointSpec extends AnyWordSpec with should.Matchers with IOSpec with Moc "fetch project details from GL, " + "call DELETE Project API on GL, " + "wait for the projects to be gone from GL, " + - "sent the COMMIT_SYNC_REQUEST event to EL " + + "send a COMMIT_SYNC_REQUEST event to EL and a CLEAN_UP event to TG " + + "and return 202 Accepted" in new TestCase { + + givenProjectFindingInGL(project.path, returning = project.some.pure[IO]) + givenProjectDelete(project.id, returning = ().pure[IO]) + givenProjectFindingInGL(project.path, returning = None.pure[IO]) + givenCommitSyncRequestSent(project, returning = ().pure[IO]) + givenCleanUpRequestSent(project, returning = ().pure[IO]) + + val response = endpoint.`DELETE /projects/:path`(project.path, authUser).unsafeRunSync() + + response.status shouldBe Accepted + response.contentType shouldBe `Content-Type`(application.json).some + response.as[InfoMessage].unsafeRunSync() shouldBe InfoMessage("Project deleted") + + ensureCommitSyncSent.get.unsafeRunSync() shouldBe true + ensureCleanUpSent.get.unsafeRunSync() shouldBe true + } + + "fetch project details from EL if it cannot be found in GL, " + + "call DELETE Project API on GL, " + + "wait for the projects to be gone from GL, " + + "send a COMMIT_SYNC_REQUEST event to EL and a CLEAN_UP event to TG " + "and return 202 Accepted" in new TestCase { - givenProjectFinding(project.path, returning = project.some.pure[IO]) + givenProjectFindingInGL(project.path, returning = None.pure[IO]).atLeastOnce() + givenProjectFindingInEL(project.path, returning = project.some.pure[IO]) givenProjectDelete(project.id, returning = ().pure[IO]) - givenProjectFinding(project.path, returning = None.pure[IO]) givenCommitSyncRequestSent(project, returning = ().pure[IO]) + givenCleanUpRequestSent(project, returning = ().pure[IO]) val response = endpoint.`DELETE /projects/:path`(project.path, authUser).unsafeRunSync() @@ -64,12 +89,14 @@ class EndpointSpec extends AnyWordSpec with should.Matchers with IOSpec with Moc response.contentType shouldBe `Content-Type`(application.json).some response.as[InfoMessage].unsafeRunSync() shouldBe InfoMessage("Project deleted") - ensureEventSent.get.unsafeRunSync() shouldBe true + ensureCommitSyncSent.get.unsafeRunSync() shouldBe true + ensureCleanUpSent.get.unsafeRunSync() shouldBe true } "return 404 Not Found in case the project does not exist in GL" in new TestCase { - givenProjectFinding(project.path, returning = None.pure[IO]) + givenProjectFindingInGL(project.path, returning = None.pure[IO]) + givenProjectFindingInEL(project.path, returning = None.pure[IO]) val response = endpoint.`DELETE /projects/:path`(project.path, authUser).unsafeRunSync() @@ -80,20 +107,22 @@ class EndpointSpec extends AnyWordSpec with should.Matchers with IOSpec with Moc "be sure the project gets deleted from GL before the COMMIT_SYNC_REQUEST event is sent to EL" in new TestCase { - givenProjectFinding(project.path, returning = project.some.pure[IO]) + givenProjectFindingInGL(project.path, returning = project.some.pure[IO]) givenProjectDelete(project.id, returning = ().pure[IO]) - givenProjectFinding(project.path, returning = project.some.pure[IO]) - givenProjectFinding(project.path, returning = None.pure[IO]) + givenProjectFindingInGL(project.path, returning = project.some.pure[IO]) + givenProjectFindingInGL(project.path, returning = None.pure[IO]) givenCommitSyncRequestSent(project, returning = ().pure[IO]) + givenCleanUpRequestSent(project, returning = ().pure[IO]) endpoint.`DELETE /projects/:path`(project.path, authUser).unsafeRunSync().status shouldBe Accepted - ensureEventSent.get.unsafeRunSync() shouldBe true + ensureCommitSyncSent.get.unsafeRunSync() shouldBe true + ensureCleanUpSent.get.unsafeRunSync() shouldBe true } "return 500 Internal Server Error in case any of the operations fails" in new TestCase { - givenProjectFinding(project.path, returning = project.some.pure[IO]) + givenProjectFindingInGL(project.path, returning = project.some.pure[IO]) val exception = exceptions.generateOne givenProjectDelete(project.id, returning = exception.raiseError[IO, Unit]) @@ -114,28 +143,43 @@ class EndpointSpec extends AnyWordSpec with should.Matchers with IOSpec with Moc val project = consumerProjects.generateOne implicit val logger: TestLogger[IO] = TestLogger[IO]() - private val projectFinder = mock[ProjectFinder[IO]] - private val projectRemover = mock[ProjectRemover[IO]] - private val elClient = mock[eventlog.api.events.Client[IO]] - val endpoint = new EndpointImpl[IO](projectFinder, projectRemover, elClient) - - def givenProjectFinding(path: projects.Path, returning: IO[Option[Project]]) = - (projectFinder + private val glProjectFinder = mock[GLProjectFinder[IO]] + private val elProjectFinder = mock[ELProjectFinder[IO]] + private val projectRemover = mock[ProjectRemover[IO]] + private val elClient = mock[eventlog.api.events.Client[IO]] + private val tgClient = mock[triplesgenerator.api.events.Client[IO]] + val endpoint = new EndpointImpl[IO](glProjectFinder, elProjectFinder, projectRemover, elClient, tgClient) + + def givenProjectFindingInGL(path: projects.Path, returning: IO[Option[Project]]) = + (glProjectFinder .findProject(_: projects.Path)(_: AccessToken)) .expects(path, authUser.accessToken) .returning(returning) + def givenProjectFindingInEL(path: projects.Path, returning: IO[Option[Project]]) = + (elProjectFinder + .findProject(_: projects.Path)) + .expects(path) + .returning(returning) + def givenProjectDelete(id: projects.GitLabId, returning: IO[Unit]) = (projectRemover .deleteProject(_: projects.GitLabId)(_: AccessToken)) .expects(id, authUser.accessToken) .returning(returning) - val ensureEventSent = Deferred.unsafe[IO, Boolean] + val ensureCommitSyncSent = Deferred.unsafe[IO, Boolean] def givenCommitSyncRequestSent(project: Project, returning: IO[Unit]) = (elClient .send(_: CommitSyncRequest)) .expects(CommitSyncRequest(project)) - .onCall((_: CommitSyncRequest) => returning >> ensureEventSent.complete(true).void) + .onCall((_: CommitSyncRequest) => returning >> ensureCommitSyncSent.complete(true).void) + + val ensureCleanUpSent = Deferred.unsafe[IO, Boolean] + def givenCleanUpRequestSent(project: Project, returning: IO[Unit]) = + (tgClient + .send(_: CleanUpEvent)) + .expects(CleanUpEvent(project)) + .onCall((_: CleanUpEvent) => returning >> ensureCleanUpSent.complete(true).void) } } diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ProjectFinderSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinderSpec.scala similarity index 98% rename from knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ProjectFinderSpec.scala rename to knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinderSpec.scala index c1092cd109..2150b912c3 100644 --- a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/ProjectFinderSpec.scala +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/projects/delete/GLProjectFinderSpec.scala @@ -42,7 +42,7 @@ import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -class ProjectFinderSpec +class GLProjectFinderSpec extends AnyWordSpec with should.Matchers with MockFactory @@ -86,7 +86,7 @@ class ProjectFinderSpec val project = consumerProjects.generateOne private implicit val glClient: GitLabClient[IO] = mock[GitLabClient[IO]] - val finder = new ProjectFinderImpl[IO] + val finder = new GLProjectFinderImpl[IO] def givenSingleProjectAPICall(path: projects.Path, returning: IO[Option[Project]]) = { val endpointName: String Refined NonEmpty = "single-project" diff --git a/renku-model/src/main/scala/io/renku/graph/model/GraphClass.scala b/renku-model/src/main/scala/io/renku/graph/model/GraphClass.scala index a31765cc9a..cc4f62338d 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/GraphClass.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/GraphClass.scala @@ -28,7 +28,7 @@ object GraphClass { import io.renku.jsonld.syntax._ - lazy val all: Set[GraphClass] = Set(Default, Project, Persons, Datasets, ProjectViewedTimes) + lazy val all: Set[GraphClass] = Set(Default, Project, Persons, Datasets, ProjectViewedTimes, PersonViewings) case object Default extends GraphClass type Default = Default.type diff --git a/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/CleanUpEvent.scala b/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/CleanUpEvent.scala new file mode 100644 index 0000000000..41ceb642d6 --- /dev/null +++ b/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/CleanUpEvent.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.api.events + +import cats.Show +import cats.syntax.all._ +import io.circe.DecodingFailure.Reason.CustomReason +import io.circe.literal._ +import io.circe.{Decoder, DecodingFailure, Encoder} +import io.renku.events.CategoryName +import io.renku.events.consumers.Project +import io.renku.graph.model.projects + +final case class CleanUpEvent(project: Project) + +object CleanUpEvent { + + val categoryName: CategoryName = CategoryName("CLEAN_UP") + + implicit val encoder: Encoder[CleanUpEvent] = Encoder.instance { case CleanUpEvent(Project(id, path)) => + json"""{ + "categoryName": $categoryName, + "project": { + "id": $id, + "path": $path + } + }""" + } + + implicit val decoder: Decoder[CleanUpEvent] = Decoder.instance { cursor => + import io.renku.tinytypes.json.TinyTypeDecoders._ + + lazy val validateCategory = cursor.downField("categoryName").as[CategoryName] >>= { + case `categoryName` => ().asRight + case other => DecodingFailure(CustomReason(s"Expected $categoryName but got $other"), cursor).asLeft + } + + for { + _ <- validateCategory + id <- cursor.downField("project").downField("id").as[projects.GitLabId] + path <- cursor.downField("project").downField("path").as[projects.Path] + } yield CleanUpEvent(Project(id, path)) + } + + implicit val show: Show[CleanUpEvent] = Show[Project].contramap(_.project) +} diff --git a/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/Client.scala b/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/Client.scala index 464d63d807..08243e7240 100644 --- a/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/Client.scala +++ b/triples-generator-api/src/main/scala/io/renku/triplesgenerator/api/events/Client.scala @@ -33,6 +33,7 @@ trait Client[F[_]] { def send(event: ProjectViewedEvent): F[Unit] def send(event: DatasetViewedEvent): F[Unit] def send(event: ProjectViewingDeletion): F[Unit] + def send(event: CleanUpEvent): F[Unit] } object Client { @@ -59,6 +60,9 @@ private class ClientImpl[F[_]](eventSender: EventSender[F]) extends Client[F] { override def send(event: ProjectViewingDeletion): F[Unit] = send(event, ProjectViewingDeletion.categoryName) + override def send(event: CleanUpEvent): F[Unit] = + send(event, CleanUpEvent.categoryName) + private def send[E](event: E, category: CategoryName)(implicit enc: Encoder[E], show: Show[E]): F[Unit] = eventSender.sendEvent( EventRequestContent.NoPayload(event.asJson), diff --git a/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/CleanUpEventSpec.scala b/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/CleanUpEventSpec.scala new file mode 100644 index 0000000000..570cff02a3 --- /dev/null +++ b/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/CleanUpEventSpec.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.api.events + +import io.circe.literal._ +import io.circe.syntax._ +import io.renku.events.consumers.Project +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.nonEmptyStrings +import io.renku.graph.model.projects +import io.renku.triplesgenerator.api.events.Generators.cleanUpEvents +import org.scalatest.EitherValues +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class CleanUpEventSpec extends AnyWordSpec with should.Matchers with EitherValues { + + "json codec" should { + + "encode and decode" in { + + val event = cleanUpEvents.generateOne + + event.asJson.hcursor.as[CleanUpEvent].value shouldBe event + } + + "be able to decode json valid from the contract point of view" in { + + json"""{ + "categoryName": "CLEAN_UP", + "project": { + "id": 123, + "path": "space/name" + } + }""".hcursor.as[CleanUpEvent].value shouldBe CleanUpEvent( + Project(projects.GitLabId(123), projects.Path("space/name")) + ) + } + + "fail if categoryName does not match" in { + + val otherCategory = nonEmptyStrings().generateOne + val result = json"""{ + "categoryName": $otherCategory + }""".hcursor.as[CleanUpEvent] + + result.left.value.getMessage() should include(s"Expected CLEAN_UP but got $otherCategory") + } + } +} diff --git a/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/Generators.scala b/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/Generators.scala index 3611a3f5da..e81ddc0ad0 100644 --- a/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/Generators.scala +++ b/triples-generator-api/src/test/scala/io/renku/triplesgenerator/api/events/Generators.scala @@ -19,6 +19,7 @@ package io.renku.triplesgenerator.api.events import cats.syntax.all._ +import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators.timestampsNotInTheFuture import io.renku.graph.model.RenkuTinyTypeGenerators._ @@ -40,4 +41,7 @@ object Generators { val projectViewingDeletions: Gen[ProjectViewingDeletion] = projectPaths.map(ProjectViewingDeletion.apply) + + val cleanUpEvents: Gen[CleanUpEvent] = + consumerProjects.map(CleanUpEvent.apply) } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/CleanUpEvent.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/CleanUpEvent.scala deleted file mode 100644 index 18ee200f04..0000000000 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/CleanUpEvent.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.triplesgenerator.events.consumers.cleanup - -import io.renku.events.consumers.Project - -case class CleanUpEvent(project: Project) diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoder.scala deleted file mode 100644 index 4b57029f20..0000000000 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoder.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.triplesgenerator.events.consumers.cleanup - -import cats.syntax.all._ -import io.circe.{DecodingFailure, Error, Json} -import io.renku.events.EventRequestContent - -private trait EventDecoder { - val decode: EventRequestContent => Either[Exception, CleanUpEvent] -} - -private object EventDecoder extends EventDecoder { - - import io.renku.events.consumers.EventDecodingTools._ - - lazy val decode: EventRequestContent => Either[Exception, CleanUpEvent] = req => - req.event.getProject.map(CleanUpEvent).leftMap(toMeaningfulError(req.event)) - - private def toMeaningfulError(event: Json): DecodingFailure => Error = { failure => - failure.withMessage(s"CleanUpEvent cannot be decoded: '$event'") - } -} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandler.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandler.scala index 20d0d71a88..79cbae12eb 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandler.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandler.scala @@ -21,10 +21,11 @@ package io.renku.triplesgenerator.events.consumers.cleanup import cats.effect.{Async, MonadCancelThrow} import cats.syntax.all._ import eu.timepit.refined.auto._ -import io.renku.events.{CategoryName, consumers} -import io.renku.events.consumers.subscriptions.SubscriptionMechanism import io.renku.events.consumers.ProcessExecutor +import io.renku.events.consumers.subscriptions.SubscriptionMechanism +import io.renku.events.{CategoryName, consumers} import io.renku.metrics.MetricsRegistry +import io.renku.triplesgenerator.api.events.CleanUpEvent import io.renku.triplesgenerator.events.consumers.TSReadinessForEventsChecker import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations.reprovisioning.ReProvisioningStatus import io.renku.triplesstore.SparqlQueryTimeRecorder @@ -34,7 +35,6 @@ private class EventHandler[F[_]: MonadCancelThrow: Logger]( override val categoryName: CategoryName, tsReadinessChecker: TSReadinessForEventsChecker[F], eventProcessor: EventProcessor[F], - eventDecoder: EventDecoder, subscriptionMechanism: SubscriptionMechanism[F], processExecutor: ProcessExecutor[F] ) extends consumers.EventHandlerWithProcessLimiter[F](processExecutor) { @@ -43,7 +43,7 @@ private class EventHandler[F[_]: MonadCancelThrow: Logger]( override def createHandlingDefinition(): EventHandlingDefinition = EventHandlingDefinition( - eventDecoder.decode, + _.event.as[CleanUpEvent], e => eventProcessor.process(e.project), precondition = tsReadinessChecker.verifyTSReady, onRelease = subscriptionMechanism.renewSubscription().some @@ -58,11 +58,5 @@ private object EventHandler { tsReadinessChecker <- TSReadinessForEventsChecker[F] eventProcessor <- EventProcessor[F] processExecutor <- ProcessExecutor.concurrent(processesCount = 1) - } yield new EventHandler[F](categoryName, - tsReadinessChecker, - eventProcessor, - EventDecoder, - subscriptionMechanism, - processExecutor - ) + } yield new EventHandler[F](categoryName, tsReadinessChecker, eventProcessor, subscriptionMechanism, processExecutor) } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectFinder.scala index f9f3c38d2f..6b1d098c77 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectFinder.scala @@ -23,15 +23,15 @@ import cats.effect.Async import cats.syntax.all._ import eu.timepit.refined.auto._ import io.circe.Decoder -import io.renku.graph.model.{persons, projects} import io.renku.graph.model.entities.Project.{GitLabProjectInfo, ProjectMember} import io.renku.graph.model.images.ImageUri +import io.renku.graph.model.{persons, projects} import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.triplesgenerator.events.consumers.ProcessingRecoverableError import io.renku.triplesgenerator.events.consumers.tsprovisioning.RecoverableErrorsRecovery -import org.http4s.{EntityDecoder, Request, Response, Status} import org.http4s.dsl.io.{NotFound, Ok} import org.http4s.implicits._ +import org.http4s.{EntityDecoder, Request, Response, Status} import org.typelevel.log4cats.Logger private trait ProjectFinder[F[_]] { @@ -55,9 +55,9 @@ private class ProjectFinderImpl[F[_]: Async: GitLabClient: Logger]( private type ProjectAndCreator = (GitLabProjectInfo, Option[persons.GitLabId]) - override def findProject(path: projects.Path)(implicit - maybeAccessToken: Option[AccessToken] - ): EitherT[F, ProcessingRecoverableError, Option[GitLabProjectInfo]] = EitherT { + override def findProject( + path: projects.Path + )(implicit mat: Option[AccessToken]): EitherT[F, ProcessingRecoverableError, Option[GitLabProjectInfo]] = EitherT { { for { (project, maybeCreatorId) <- fetchProject(path) @@ -130,6 +130,5 @@ private class ProjectFinderImpl[F[_]: Async: GitLabClient: Logger]( username <- cursor.downField("username").as[persons.Username] } yield ProjectMember(name, username, gitLabId) - private implicit lazy val memberEntityDecoder: EntityDecoder[F, ProjectMember] = jsonOf[F, ProjectMember] - private implicit lazy val membersDecoder: EntityDecoder[F, List[ProjectMember]] = jsonOf[F, List[ProjectMember]] + private implicit lazy val memberEntityDecoder: EntityDecoder[F, ProjectMember] = jsonOf[F, ProjectMember] } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala index 78af468f9e..5507bd3e9c 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala @@ -73,8 +73,7 @@ private class ProjectMembersFinderImpl[F[_]: Async: NonEmptyParallel: GitLabClie username <- cursor.downField("username").as[persons.Username] } yield ProjectMember(name, username, gitLabId) - private implicit lazy val memberEntityDecoder: EntityDecoder[F, ProjectMember] = jsonOf[F, ProjectMember] - private implicit lazy val membersDecoder: EntityDecoder[F, List[ProjectMember]] = jsonOf[F, List[ProjectMember]] + private implicit lazy val membersDecoder: EntityDecoder[F, List[ProjectMember]] = jsonOf[F, List[ProjectMember]] private def fetch( uri: Uri, diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoderSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoderSpec.scala deleted file mode 100644 index 3e3faa6dac..0000000000 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventDecoderSpec.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2023 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.triplesgenerator.events.consumers.cleanup - -import io.circe._ -import io.circe.literal._ -import io.renku.events.EventRequestContent -import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects -import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.jsons -import org.scalatest.EitherValues -import org.scalatest.matchers.should -import org.scalatest.wordspec.AnyWordSpec - -class EventDecoderSpec extends AnyWordSpec with should.Matchers with EitherValues { - - "decode" should { - - "produce a CleanUpEvent if the Json string can be successfully deserialized" in { - - val project = consumerProjects.generateOne - lazy val event = json"""{ - "categoryName": "CLEAN_UP", - "project": { - "id": ${project.id}, - "path": ${project.path} - } - }""" - - EventDecoder - .decode(EventRequestContent.NoPayload(event)) - .value shouldBe CleanUpEvent(project) - } - - "fail if decoding fails" in { - - val event = jsons.generateOne - val result = EventDecoder.decode(EventRequestContent.NoPayload(event)) - - result.left.value shouldBe a[DecodingFailure] - result.left.value.asInstanceOf[DecodingFailure].message shouldBe s"CleanUpEvent cannot be decoded: '$event'" - } - } -} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandlerSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandlerSpec.scala index 21725295e9..73d727ac8c 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandlerSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/cleanup/EventHandlerSpec.scala @@ -20,23 +20,32 @@ package io.renku.triplesgenerator.events.consumers.cleanup import cats.effect.{IO, Ref} import cats.syntax.all._ +import io.circe.syntax._ +import io.renku.events.EventRequestContent import io.renku.events.consumers.ConsumersModelGenerators.{consumerProjects, eventSchedulingResults} import io.renku.events.consumers.ProcessExecutor import io.renku.events.consumers.subscriptions.SubscriptionMechanism import io.renku.generators.Generators.Implicits._ import io.renku.interpreters.TestLogger import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.CleanUpEvent import io.renku.triplesgenerator.events.consumers.TSReadinessForEventsChecker import org.scalamock.scalatest.MockFactory +import org.scalatest.EitherValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -class EventHandlerSpec extends AnyWordSpec with MockFactory with IOSpec with should.Matchers { +class EventHandlerSpec extends AnyWordSpec with MockFactory with IOSpec with should.Matchers with EitherValues { "handlingDefinition.decode" should { "be the eventDecoder.decode" in new TestCase { - handler.createHandlingDefinition().decode shouldBe EventDecoder.decode + + val event = consumerProjects.map(CleanUpEvent(_)).generateOne + + handler + .createHandlingDefinition() + .decode(EventRequestContent.NoPayload(event.asJson)).value shouldBe event } } @@ -44,7 +53,7 @@ class EventHandlerSpec extends AnyWordSpec with MockFactory with IOSpec with sho "be the EventProcessor.process" in new TestCase { - val event = consumerProjects.map(CleanUpEvent).generateOne + val event = consumerProjects.map(CleanUpEvent(_)).generateOne (eventProcessor.process _).expects(event.project).returns(().pure[IO]) @@ -86,7 +95,6 @@ class EventHandlerSpec extends AnyWordSpec with MockFactory with IOSpec with sho val handler = new EventHandler[IO](categoryName, tsReadinessChecker, eventProcessor, - EventDecoder, subscriptionMechanism, mock[ProcessExecutor[IO]] ) From b2b6e375de4911dcb0b7280e3968f5fb3f61c731 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Fri, 28 Apr 2023 10:48:13 +0200 Subject: [PATCH 20/28] chore: Update sentry-logback from 6.17.0 to 6.18.0 (#1449) Co-authored-by: RenkuBot --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3cfc7af4e4..df25aee17f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -30,7 +30,7 @@ object Dependencies { val scalamock = "5.2.0" val scalatest = "3.2.15" val scalatestScalacheck = "3.2.2.0" - val sentryLogback = "6.17.0" + val sentryLogback = "6.18.0" val skunk = "0.5.1" val swaggerParser = "2.1.13" val testContainersScala = "0.40.15" From eff883a0d9296e10033518bf19ad905bad892c08 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Fri, 28 Apr 2023 11:07:25 +0200 Subject: [PATCH 21/28] chore: Update widoco from 1.4.17 to 1.4.18 (#1448) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index df25aee17f..b0e977fd9f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -34,7 +34,7 @@ object Dependencies { val skunk = "0.5.1" val swaggerParser = "2.1.13" val testContainersScala = "0.40.15" - val widoco = "1.4.17" + val widoco = "1.4.18" val wiremock = "2.35.0" } From 592c84035f88df2d6f3c0d739c7aedb2d04a342a Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Fri, 28 Apr 2023 17:35:20 +0200 Subject: [PATCH 22/28] fix: project members to be found from project/id/members/all (#1450) * fix: project members to be found from project/:id/members/all * fix: timeout extended for project members sparql queries * fix: MEMBER_SYNC handler to take 1 process at a time --- .../stubs/gitlab/GitLabApiStub.scala | 2 +- .../security/GitLabPathRecordsFinder.scala | 2 +- .../server/security/MembersFinderSpec.scala | 2 +- .../consumers/membersync/EventHandler.scala | 2 +- .../GitLabProjectMembersFinder.scala | 40 ++++----- .../namedgraphs/KGPersonFinder.scala | 15 +++- .../namedgraphs/KGProjectMembersFinder.scala | 10 ++- .../namedgraphs/KGSynchronizer.scala | 4 +- .../projectinfo/ProjectMembersFinder.scala | 31 ++++--- .../GitLabProjectMembersFinderSpec.scala | 67 +++++++-------- .../ProjectMembersFinderSpec.scala | 81 ++++++------------- 11 files changed, 106 insertions(+), 150 deletions(-) diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala index e9eafed308..f6a7e32587 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/GitLabApiStub.scala @@ -120,7 +120,7 @@ final class GitLabApiStub[F[_]: Async: Logger](private val stateRef: Ref[F, Stat case HEAD -> Root / ProjectPath(path) => query(findProjectByPath(path, maybeAuthedReq)).flatMap(EmptyOkOrNotFound(_)) - case GET -> Root / ProjectPath(path) / ("users" | "members") => + case GET -> Root / ProjectPath(path) / "members" / "all" => query(findProjectByPath(path, maybeAuthedReq)) .map(_.toList.flatMap(_.members.toList)) .flatMap(Ok(_)) diff --git a/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala b/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala index eb83c5ac03..af3cc4bfdc 100644 --- a/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala +++ b/graph-commons/src/main/scala/io/renku/graph/http/server/security/GitLabPathRecordsFinder.scala @@ -89,7 +89,7 @@ private trait MembersFinder[F[_]] { private class MembersFinderImpl[F[_]: Async: GitLabClient: Logger] extends MembersFinder[F] { override def findMembers(path: projects.Path)(implicit mat: Option[AccessToken]): F[Set[persons.GitLabId]] = - fetch(uri"projects" / path / "members") + fetch(uri"projects" / path / "members" / "all") private def fetch( uri: Uri, diff --git a/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala b/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala index 4459f6258e..a73fe1082d 100644 --- a/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala +++ b/graph-commons/src/test/scala/io/renku/graph/http/server/security/MembersFinderSpec.scala @@ -113,7 +113,7 @@ class MembersFinderSpec val endpointName: String Refined NonEmpty = "project-members" val uri = { - val uri = uri"projects" / projectPath / "members" + val uri = uri"projects" / projectPath / "members" / "all" maybePage match { case Some(page) => uri withQueryParam ("page", page.toString) case None => uri diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/EventHandler.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/EventHandler.scala index 6fe1addff8..535641b716 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/EventHandler.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/EventHandler.scala @@ -63,7 +63,7 @@ private object EventHandler { ): F[consumers.EventHandler[F]] = for { tsReadinessChecker <- TSReadinessForEventsChecker[F] membersSynchronizer <- MembersSynchronizer[F] - processExecutor <- ProcessExecutor.concurrent(processesCount = 10) + processExecutor <- ProcessExecutor.concurrent(processesCount = 1) } yield new EventHandler[F](categoryName, tsReadinessChecker, membersSynchronizer, diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinder.scala index 8a7fb47eb2..d22cc90f66 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinder.scala @@ -28,8 +28,8 @@ import io.renku.graph.model.persons.{GitLabId, Name} import io.renku.graph.model.projects.Path import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.tinytypes.json.TinyTypeDecoders._ -import org.http4s._ import org.http4s.Status.{Forbidden, NotFound, Ok, Unauthorized} +import org.http4s._ import org.http4s.circe.jsonOf import org.http4s.implicits.http4sLiteralsSyntax import org.typelevel.ci._ @@ -41,30 +41,27 @@ private trait GitLabProjectMembersFinder[F[_]] { private class GitLabProjectMembersFinderImpl[F[_]: Async: GitLabClient: Logger] extends GitLabProjectMembersFinder[F] { - override def findProjectMembers( - path: Path - )(implicit maybeAccessToken: Option[AccessToken]): F[Set[GitLabProjectMember]] = for { - users <- fetch(uri"projects" / path.show / "users", "project-users") - members <- fetch(uri"projects" / path.show / "members", "project-members") - } yield users ++ members + override def findProjectMembers(path: Path)(implicit mat: Option[AccessToken]): F[Set[GitLabProjectMember]] = + fetch(uri"projects" / path.show / "members" / "all") + + private val glApiName: String Refined NonEmpty = "project-members" private def fetch( - uri: Uri, - endpointName: NonEmptyString, - maybePage: Option[Int] = None, - allUsers: Set[GitLabProjectMember] = Set.empty + uri: Uri, + maybePage: Option[Int] = None, + allMembers: Set[GitLabProjectMember] = Set.empty )(implicit maybeAccessToken: Option[AccessToken]): F[Set[GitLabProjectMember]] = for { uri <- addPageToUrl(uri, maybePage).pure[F] - fetchedUsersAndNextPage <- GitLabClient[F].get(uri, endpointName)(mapResponse(uri, endpointName)) - allUsers <- addNextPage(uri, endpointName, allUsers, fetchedUsersAndNextPage) - } yield allUsers + fetchedUsersAndNextPage <- GitLabClient[F].get(uri, glApiName)(mapResponse(uri)) + allResults <- addNextPage(uri, allMembers, fetchedUsersAndNextPage) + } yield allResults private def addPageToUrl(uri: Uri, maybePage: Option[Int] = None) = maybePage match { case Some(page) => uri.withQueryParam("page", page.toString) case None => uri } - private def mapResponse(uri: Uri, endpointName: NonEmptyString)(implicit + private def mapResponse(uri: Uri)(implicit maybeAccessToken: Option[AccessToken] ): PartialFunction[(Status, Request[F], Response[F]), F[(Set[GitLabProjectMember], Option[Int])]] = { case (Ok, _, response) => @@ -75,19 +72,18 @@ private class GitLabProjectMembersFinderImpl[F[_]: Async: GitLabClient: Logger] (Set.empty[GitLabProjectMember] -> Option.empty[Int]).pure[F] case (Unauthorized | Forbidden, _, _) => maybeAccessToken match { - case Some(_) => fetch(uri, endpointName)(maybeAccessToken = None).map(_ -> None) + case Some(_) => fetch(uri)(maybeAccessToken = None).map(_ -> None) case None => (Set.empty[GitLabProjectMember] -> Option.empty[Int]).pure[F] } } private def addNextPage( uri: Uri, - endpointName: NonEmptyString, allUsers: Set[GitLabProjectMember], fetchedUsersAndMaybeNextPage: (Set[GitLabProjectMember], Option[Int]) )(implicit maybeAccessToken: Option[AccessToken]): F[Set[GitLabProjectMember]] = fetchedUsersAndMaybeNextPage match { - case (fetchedUsers, maybeNextPage @ Some(_)) => fetch(uri, endpointName, maybeNextPage, allUsers ++ fetchedUsers) + case (fetchedUsers, maybeNextPage @ Some(_)) => fetch(uri, maybeNextPage, allUsers ++ fetchedUsers) case (fetchedUsers, None) => (allUsers ++ fetchedUsers).pure[F] } @@ -98,16 +94,12 @@ private class GitLabProjectMembersFinderImpl[F[_]: Async: GitLabClient: Logger] import io.renku.graph.model.persons implicit val decoder: Decoder[GitLabProjectMember] = { cursor => - for { - id <- cursor.downField("id").as[GitLabId] - name <- cursor.downField("name").as[persons.Name] - } yield GitLabProjectMember(id, name) + (cursor.downField("id").as[GitLabId] -> cursor.downField("name").as[persons.Name]) + .mapN(GitLabProjectMember) } jsonOf[F, List[GitLabProjectMember]] } - - type NonEmptyString = String Refined NonEmpty } private object GitLabProjectMembersFinder { diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGPersonFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGPersonFinder.scala index c08b4648a2..c7b7a8467a 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGPersonFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGPersonFinder.scala @@ -21,22 +21,29 @@ package namedgraphs import cats.effect.Async import cats.syntax.all._ -import io.renku.graph.model.{persons, GraphClass} import io.renku.graph.model.Schemas.schema import io.renku.graph.model.entities.Person import io.renku.graph.model.persons.{GitLabId, ResourceId} -import io.renku.triplesstore._ +import io.renku.graph.model.{GraphClass, persons} import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ import org.typelevel.log4cats.Logger +import scala.concurrent.duration._ + private trait KGPersonFinder[F[_]] { def findPersonIds(membersToAdd: Set[GitLabProjectMember]): F[Set[(GitLabProjectMember, Option[ResourceId])]] } private class KGPersonFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder]( - connectionConfig: ProjectsConnectionConfig -) extends TSClientImpl(connectionConfig) + connectionConfig: ProjectsConnectionConfig, + idleTimeout: Duration = 21 minutes, + requestTimeout: Duration = 20 minutes +) extends TSClientImpl(connectionConfig, + idleTimeoutOverride = idleTimeout.some, + requestTimeoutOverride = requestTimeout.some + ) with KGPersonFinder[F] { import eu.timepit.refined.auto._ diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGProjectMembersFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGProjectMembersFinder.scala index d88638487b..16b4135ea4 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGProjectMembersFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGProjectMembersFinder.scala @@ -32,15 +32,21 @@ import io.renku.triplesstore._ import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery.Prefixes import org.typelevel.log4cats.Logger +import scala.concurrent.duration._ private trait KGProjectMembersFinder[F[_]] { def findProjectMembers(path: projects.Path): F[Set[KGProjectMember]] } private class KGProjectMembersFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder]( - connectionConfig: ProjectsConnectionConfig + connectionConfig: ProjectsConnectionConfig, + idleTimeout: Duration = 21 minutes, + requestTimeout: Duration = 20 minutes )(implicit renkuUrl: RenkuUrl) - extends TSClientImpl(connectionConfig) + extends TSClientImpl(connectionConfig, + idleTimeoutOverride = idleTimeout.some, + requestTimeoutOverride = requestTimeout.some + ) with KGProjectMembersFinder[F] { import eu.timepit.refined.auto._ diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGSynchronizer.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGSynchronizer.scala index b217344cc5..ee6b3bf5bb 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGSynchronizer.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/membersync/namedgraphs/KGSynchronizer.scala @@ -36,8 +36,8 @@ private[membersync] object KGSynchronizer { updatesCreator <- UpdatesCreator[F] connectionConfig <- ProjectsConnectionConfig[F]() tsClient <- TSClient[F](connectionConfig, - idleTimeoutOverride = (11 minutes).some, - requestTimeoutOverride = (10 minutes).some + idleTimeoutOverride = (21 minutes).some, + requestTimeoutOverride = (20 minutes).some ).pure[F] } yield new KGSynchronizerImpl[F](kgProjectMembersFinder, kgPersonFinder, updatesCreator, tsClient) diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala index 5507bd3e9c..223da880fa 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinder.scala @@ -26,8 +26,8 @@ import eu.timepit.refined.api.Refined import eu.timepit.refined.auto._ import eu.timepit.refined.collection.NonEmpty import io.circe.Decoder -import io.renku.graph.model.{persons, projects} import io.renku.graph.model.entities.Project.ProjectMember +import io.renku.graph.model.{persons, projects} import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.triplesgenerator.events.consumers.ProcessingRecoverableError import io.renku.triplesgenerator.events.consumers.tsprovisioning.RecoverableErrorsRecovery @@ -53,15 +53,13 @@ private class ProjectMembersFinderImpl[F[_]: Async: NonEmptyParallel: GitLabClie recoveryStrategy: RecoverableErrorsRecovery = RecoverableErrorsRecovery ) extends ProjectMembersFinder[F] { + import io.renku.http.tinytypes.TinyTypeURIEncoder._ import io.renku.tinytypes.json.TinyTypeDecoders._ - override def findProjectMembers(path: projects.Path)(implicit - maybeAccessToken: Option[AccessToken] - ): EitherT[F, ProcessingRecoverableError, Set[ProjectMember]] = EitherT { - ( - fetch(uri"projects" / path.show / "members", "project-members"), - fetch(uri"projects" / path.show / "users", "project-users") - ).parMapN(_ ++ _) + override def findProjectMembers( + path: projects.Path + )(implicit mat: Option[AccessToken]): EitherT[F, ProcessingRecoverableError, Set[ProjectMember]] = EitherT { + fetch(uri"projects" / path / "members" / "all") .map(_.asRight[ProcessingRecoverableError]) .recoverWith(recoveryStrategy.maybeRecoverableError) } @@ -75,15 +73,16 @@ private class ProjectMembersFinderImpl[F[_]: Async: NonEmptyParallel: GitLabClie private implicit lazy val membersDecoder: EntityDecoder[F, List[ProjectMember]] = jsonOf[F, List[ProjectMember]] + private val endpointName: String Refined NonEmpty = "project-members" + private def fetch( - uri: Uri, - endpointName: String Refined NonEmpty, - maybePage: Option[Int] = None, - allMembers: Set[ProjectMember] = Set.empty + uri: Uri, + maybePage: Option[Int] = None, + allMembers: Set[ProjectMember] = Set.empty )(implicit maybeAccessToken: Option[AccessToken]): F[Set[ProjectMember]] = for { uri <- uriWithPage(uri, maybePage).pure[F] fetchedUsersAndNextPage <- GitLabClient[F].get(uri, endpointName)(mapResponse) - allMembers <- addNextPage(uri, endpointName, allMembers, fetchedUsersAndNextPage) + allMembers <- addNextPage(uri, allMembers, fetchedUsersAndNextPage) } yield allMembers private def uriWithPage(uri: Uri, maybePage: Option[Int]) = maybePage match { @@ -101,13 +100,11 @@ private class ProjectMembersFinderImpl[F[_]: Async: NonEmptyParallel: GitLabClie private def addNextPage( url: Uri, - endpointName: String Refined NonEmpty, allMembers: Set[ProjectMember], fetchedUsersAndMaybeNextPage: (Set[ProjectMember], Option[Int]) )(implicit maybeAccessToken: Option[AccessToken]): F[Set[ProjectMember]] = fetchedUsersAndMaybeNextPage match { - case (fetchedUsers, maybeNextPage @ Some(_)) => - fetch(url, endpointName, maybeNextPage, allMembers ++ fetchedUsers) - case (fetchedUsers, None) => (allMembers ++ fetchedUsers).pure[F] + case (fetchedUsers, maybeNextPage @ Some(_)) => fetch(url, maybeNextPage, allMembers ++ fetchedUsers) + case (fetchedUsers, None) => (allMembers ++ fetchedUsers).pure[F] } } diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinderSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinderSpec.scala index 623f33bb98..e89439379e 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinderSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/membersync/GitLabProjectMembersFinderSpec.scala @@ -51,6 +51,7 @@ import io.renku.graph.model.projects import io.renku.http.client.RestClient.ResponseMappingF import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.http.server.EndpointTester.jsonEntityEncoder +import io.renku.http.tinytypes.TinyTypeURIEncoder._ import io.renku.interpreters.TestLogger import io.renku.stubbing.ExternalServiceStubbing import io.renku.testtools.{GitLabClientTools, IOSpec} @@ -63,7 +64,7 @@ import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import org.typelevel.ci.CIStringSyntax +import org.typelevel.ci._ class GitLabProjectMembersFinderSpec extends AnyWordSpec @@ -76,36 +77,29 @@ class GitLabProjectMembersFinderSpec "findProjectMembers" should { - "return a set of project members and users" in new TestCase { - forAll { (gitLabProjectUsers: Set[GitLabProjectMember], gitLabProjectMembers: Set[GitLabProjectMember]) => - setGitLabClientExpectation(path, "users", "project-users", None, returning = (gitLabProjectUsers, None)) - setGitLabClientExpectation(path, "members", "project-members", None, returning = (gitLabProjectMembers, None)) + "return a set of all project members" in new TestCase { + forAll { gitLabProjectMembers: Set[GitLabProjectMember] => + setGitLabClientExpectation(path, None, returning = (gitLabProjectMembers, None)) - finder.findProjectMembers(path).unsafeRunSync() shouldBe (gitLabProjectUsers ++ gitLabProjectMembers) + finder.findProjectMembers(path).unsafeRunSync() shouldBe gitLabProjectMembers } } - "collect users from paged results" in new TestCase { - val projectUsers = gitLabProjectMembers.generateNonEmptyList(min = 2).toList.toSet + "collect members from all pages of results" in new TestCase { + val projectMembers = gitLabProjectMembers.generateNonEmptyList(min = 2).toList.toSet - setGitLabClientExpectation(path, "users", "project-users", None, returning = (Set(projectUsers.head), 2.some)) - setGitLabClientExpectation(path, "users", "project-users", 2.some, returning = (projectUsers.tail, None)) - setGitLabClientExpectation(path, - "members", - "project-members", - None, - returning = (Set(projectMembers.head), 2.some) - ) - setGitLabClientExpectation(path, "members", "project-members", 2.some, returning = (projectMembers.tail, None)) + setGitLabClientExpectation(path, None, returning = (Set(projectMembers.head), 2.some)) + setGitLabClientExpectation(path, 2.some, returning = (projectMembers.tail, None)) - finder.findProjectMembers(path).unsafeRunSync() shouldBe (projectUsers ++ projectMembers) + finder.findProjectMembers(path).unsafeRunSync() shouldBe projectMembers } // test map response "parse results and request next page" in new TestCase { - val projectUsers = gitLabProjectMembers.generateNonEmptyList(min = 2).toList.toSet + + val members = gitLabProjectMembers.generateNonEmptyList(min = 2).toList.toSet val nextPage = 2 val totalPages = 2 @@ -113,8 +107,8 @@ class GitLabProjectMembersFinderSpec List(Header.Raw(ci"X-Next-Page", nextPage.toString), Header.Raw(ci"X-Total-Pages", totalPages.toString)) ) - mapResponse(Status.Ok, Request(), Response().withEntity(projectUsers.asJson).withHeaders(headers)) - .unsafeRunSync() shouldBe (projectUsers, Some(2)) + mapResponse(Status.Ok, Request(), Response().withEntity(members.asJson).withHeaders(headers)) + .unsafeRunSync() shouldBe (members, Some(2)) } "return an empty set when service responds with NOT_FOUND" in new TestCase { @@ -124,6 +118,7 @@ class GitLabProjectMembersFinderSpec Forbidden +: Unauthorized +: Nil foreach { status => s"try without an access token when service responds with $status" in new TestCase { + val members = gitLabProjectMembers.generateNonEmptyList().toList.toSet implicit override val maybeAccessToken: Option[AccessToken] = accessTokens.generateSome @@ -131,29 +126,22 @@ class GitLabProjectMembersFinderSpec override val mapResponse = captureMapping(gitLabClient)( finder.findProjectMembers(path)(maybeAccessToken).unsafeRunSync(), - Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])), - expectedNumberOfCalls = 2 + Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])) ) - setGitLabClientExpectation(path, - "members", - "project-members", - maybePage = None, - maybeAccessTokenOverride = None, - returning = (members, None) - ) + setGitLabClientExpectation(path, maybePage = None, maybeAccessTokenOverride = None, returning = (members, None)) mapResponse(status, Request(), Response()).unsafeRunSync() shouldBe (members, None) } s"return an empty set when service responds with $status without access token" in new TestCase { + implicit override val maybeAccessToken: Option[AccessToken] = Option.empty[AccessToken] override val mapResponse = captureMapping(gitLabClient)( finder.findProjectMembers(path)(maybeAccessToken).unsafeRunSync(), - Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])), - expectedNumberOfCalls = 2 + Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])) ) val actual = mapResponse(status, Request(), Response()).unsafeRunSync() @@ -173,14 +161,14 @@ class GitLabProjectMembersFinderSpec val finder = new GitLabProjectMembersFinderImpl[IO] def setGitLabClientExpectation(projectPath: projects.Path, - path: String, - endpointName: String Refined NonEmpty, maybePage: Option[Int] = None, maybeAccessTokenOverride: Option[AccessToken] = maybeAccessToken, returning: (Set[GitLabProjectMember], Option[Int]) ) = { + val endpointName: String Refined NonEmpty = "project-members" + val uri = { - val uri = uri"projects" / projectPath.show / path + val uri = uri"projects" / projectPath / "members" / "all" maybePage match { case Some(page) => uri withQueryParam ("page", page.toString) case None => uri @@ -198,17 +186,16 @@ class GitLabProjectMembersFinderSpec val mapResponse = captureMapping(gitLabClient)( finder.findProjectMembers(path)(maybeAccessToken).unsafeRunSync(), - Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])), - expectedNumberOfCalls = 2 + Gen.const((Set.empty[GitLabProjectMember], Option.empty[Int])) ) } private implicit val projectMemberEncoder: Encoder[GitLabProjectMember] = Encoder.instance[GitLabProjectMember] { member => json"""{ - "id": ${member.gitLabId.value}, - "username": ${member.name.value}, - "name": ${member.name.value} + "id": ${member.gitLabId.value}, + "username": ${member.name.value}, + "name": ${member.name.value} }""" } } diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinderSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinderSpec.scala index 6789b74842..a3dc01221b 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinderSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/projectinfo/ProjectMembersFinderSpec.scala @@ -36,6 +36,7 @@ import io.renku.graph.model.testentities.generators.EntitiesGenerators._ import io.renku.http.client.RestClient.ResponseMappingF import io.renku.http.client.RestClientError.UnexpectedResponseException import io.renku.http.client.{AccessToken, GitLabClient} +import io.renku.http.tinytypes.TinyTypeURIEncoder._ import io.renku.interpreters.TestLogger import io.renku.stubbing.ExternalServiceStubbing import io.renku.testtools.{GitLabClientTools, IOSpec} @@ -46,17 +47,17 @@ import org.http4s.implicits.http4sLiteralsSyntax import org.http4s.{Request, Response, Status, Uri} import org.scalacheck.Gen import org.scalamock.scalatest.MockFactory +import org.scalatest.EitherValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import scala.util.Random - class ProjectMembersFinderSpec extends AnyWordSpec with IOSpec with ExternalServiceStubbing with should.Matchers + with EitherValues with GitLabClientTools[IO] with MockFactory with ScalaCheckPropertyChecks { @@ -64,41 +65,28 @@ class ProjectMembersFinderSpec "findProject" should { "fetch and merge project users and members" in new TestCase { - forAll { (members: Set[ProjectMemberNoEmail], users: Set[ProjectMemberNoEmail]) => - setGitLabClientExpectationUsers(projectPath, returning = (users, None).pure[IO]) - setGitLabClientExpectationMembers(projectPath, returning = (members, None).pure[IO]) + forAll { members: Set[ProjectMemberNoEmail] => + setGitLabClientExpectation(projectPath, returning = (members, None).pure[IO]) - finder.findProjectMembers(projectPath).value.unsafeRunSync() shouldBe (members ++ users).asRight + finder.findProjectMembers(projectPath).value.unsafeRunSync().value shouldBe members } } "collect members from all the pages" in new TestCase { - val allMembers = projectMembersNoEmail.generateFixedSizeList(4) - val (users, members) = allMembers.splitAt(allMembers.size / 2) + val members = projectMembersNoEmail.generateFixedSizeSet(ofSize = 4) - setGitLabClientExpectationUsers(projectPath, returning = (Set(users.head), 2.some).pure[IO]) - setGitLabClientExpectationUsers(projectPath, maybePage = 2.some, returning = (users.tail.toSet, None).pure[IO]) - setGitLabClientExpectationMembers(projectPath, returning = (Set(members.head), 2.some).pure[IO]) - setGitLabClientExpectationMembers(projectPath, - maybePage = 2.some, - returning = (members.tail.toSet, None).pure[IO] - ) + setGitLabClientExpectation(projectPath, returning = (Set(members.head), 2.some).pure[IO]) + setGitLabClientExpectation(projectPath, maybePage = 2.some, returning = (members.tail, None).pure[IO]) - finder.findProjectMembers(projectPath).value.unsafeRunSync() shouldBe allMembers.toSet.asRight + finder.findProjectMembers(projectPath).value.unsafeRunSync().value shouldBe members } - "return members even if one of the endpoints responds with NOT_FOUND" in new TestCase { - val members = projectMembersNoEmail.generateSet() - if (Random.nextBoolean()) { - setGitLabClientExpectationUsers(projectPath, returning = (Set.empty[ProjectMemberNoEmail], None).pure[IO]) - setGitLabClientExpectationMembers(projectPath, returning = (members, None).pure[IO]) - } else { - setGitLabClientExpectationUsers(projectPath, returning = (members, None).pure[IO]) - setGitLabClientExpectationMembers(projectPath, returning = (Set.empty[ProjectMemberNoEmail], None).pure[IO]) - } + "return an empty set even if GL endpoint responds with NOT_FOUND" in new TestCase { + + setGitLabClientExpectation(projectPath, returning = (Set.empty[ProjectMemberNoEmail], None).pure[IO]) - finder.findProjectMembers(projectPath).value.unsafeRunSync() shouldBe members.asRight + finder.findProjectMembers(projectPath).value.unsafeRunSync().value shouldBe Set.empty } val errorMessage = nonEmptyStrings().generateOne @@ -108,19 +96,11 @@ class ProjectMembersFinderSpec "Forbidden" -> UnexpectedResponseException(Forbidden, errorMessage), "Unauthorized" -> UnexpectedResponseException(Unauthorized, errorMessage) ) foreach { case (problemName, error) => - s"return a Recoverable Failure for $problemName when fetching project members or users" in new TestCase { - if (Random.nextBoolean()) { - setGitLabClientExpectationUsers(projectPath, returning = (Set.empty[ProjectMemberNoEmail], None).pure[IO]) - .noMoreThanOnce() - setGitLabClientExpectationMembers(projectPath, returning = IO.raiseError(error)) - } else { - setGitLabClientExpectationUsers(projectPath, returning = IO.raiseError(error)) - setGitLabClientExpectationMembers(projectPath, returning = (Set.empty[ProjectMemberNoEmail], None).pure[IO]) - .noMoreThanOnce() - } + s"return a Recoverable Failure for $problemName when fetching project members" in new TestCase { + + setGitLabClientExpectation(projectPath, returning = IO.raiseError(error)) - val Left(failure) = finder.findProjectMembers(projectPath).value.unsafeRunSync() - failure shouldBe a[ProcessingRecoverableError] + finder.findProjectMembers(projectPath).value.unsafeRunSync().left.value shouldBe a[ProcessingRecoverableError] } } @@ -146,24 +126,13 @@ class ProjectMembersFinderSpec implicit val gitLabClient: GitLabClient[IO] = mock[GitLabClient[IO]] val finder = new ProjectMembersFinderImpl[IO] - def setGitLabClientExpectationUsers(projectPath: projects.Path, - maybePage: Option[Int] = None, - returning: IO[(Set[ProjectMemberNoEmail], Option[Int])] - ) = setGitLabClientExpectation(projectPath, "users", "project-users", maybePage, returning) - - def setGitLabClientExpectationMembers(projectPath: projects.Path, - maybePage: Option[Int] = None, - returning: IO[(Set[ProjectMemberNoEmail], Option[Int])] - ) = setGitLabClientExpectation(projectPath, "members", "project-members", maybePage, returning) - - private def setGitLabClientExpectation(projectPath: projects.Path, - endpoint: String, - endpointName: String Refined NonEmpty, - maybePage: Option[Int] = None, - returning: IO[(Set[ProjectMemberNoEmail], Option[Int])] + def setGitLabClientExpectation(projectPath: projects.Path, + maybePage: Option[Int] = None, + returning: IO[(Set[ProjectMemberNoEmail], Option[Int])] ) = { + val endpointName: String Refined NonEmpty = "project-members" val uri = { - val uri = uri"projects" / projectPath.show / endpoint + val uri = uri"projects" / projectPath / "members" / "all" maybePage match { case Some(page) => uri withQueryParam ("page", page.toString) case None => uri @@ -181,8 +150,7 @@ class ProjectMembersFinderSpec val mapResponse = captureMapping(gitLabClient)( finder.findProjectMembers(projectPath)(maybeAccessToken).value.unsafeRunSync(), - Gen.const((Set.empty[ProjectMemberNoEmail], Option.empty[Int])), - expectedNumberOfCalls = 2 + Gen.const((Set.empty[ProjectMemberNoEmail], Option.empty[Int])) ) } @@ -193,5 +161,4 @@ class ProjectMembersFinderSpec "username": ${member.username} }""" } - } From 971a15ffb17fa9b043d683512da2031efd8bfdaf Mon Sep 17 00:00:00 2001 From: eikek <701128+eikek@users.noreply.github.com> Date: Fri, 28 Apr 2023 19:06:54 +0200 Subject: [PATCH 23/28] feat: Don't require authenticated user for event/statuts endpoint (#1451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Introduce MaybeAuthUser, refactoring accordingly This is to better distinguish the case "no token provided" vs "token invalid" when reading code. The `…ifNeeded` auth middleware is fulfilling the route always, either with a `None` or `Some` based on the existence of a token. That means the routes cannot be combined via semigroupK anymore, since there is no "fall through" anymore. This also applies to the mandatory auth middleware. That means both ways cannot be combined via "the usual way". To overcome this, the `MaybeUser` provides some convenient methods to support cases where a user is mandatory and optional, so routes can be combined. --- .../http/server/security/Authentication.scala | 24 ++- .../io/renku/http/server/security/model.scala | 38 ++++- .../io/renku/http/server/EndpointTester.scala | 14 +- .../server/security/AuthenticationSpec.scala | 12 +- .../knowledgegraph/MicroserviceRoutes.scala | 44 +++--- .../MicroserviceRoutesSpec.scala | 149 ++++++++++-------- webhook-service/README.md | 60 +++---- .../webhookservice/MicroserviceRoutes.scala | 31 ++-- .../webhookservice/eventstatus/Endpoint.scala | 11 +- .../eventstatus/StatusInfo.scala | 4 +- .../hookfetcher/ProjectHookFetcher.scala | 11 +- .../hookvalidation/HookValidator.scala | 5 +- .../hookvalidation/ProjectHookVerifier.scala | 2 +- .../MicroserviceRoutesSpec.scala | 39 +++-- .../eventstatus/EndpointSpec.scala | 140 +++++++++------- 15 files changed, 337 insertions(+), 247 deletions(-) diff --git a/graph-commons/src/main/scala/io/renku/http/server/security/Authentication.scala b/graph-commons/src/main/scala/io/renku/http/server/security/Authentication.scala index 6b723ac54d..7d6081fe8e 100644 --- a/graph-commons/src/main/scala/io/renku/http/server/security/Authentication.scala +++ b/graph-commons/src/main/scala/io/renku/http/server/security/Authentication.scala @@ -21,9 +21,8 @@ package io.renku.http.server.security import cats.data.{Kleisli, OptionT} import cats.syntax.all._ import cats.{Applicative, MonadThrow} -import io.renku.http.client.UserAccessToken import io.renku.http.client.AccessToken.{PersonalAccessToken, UserOAuthAccessToken} -import io.renku.http.server.security.EndpointSecurityException.AuthenticationFailure +import io.renku.http.client.UserAccessToken import io.renku.http.server.security.model._ import org.http4s.AuthScheme.Bearer import org.http4s.Credentials.Token @@ -32,7 +31,7 @@ import org.http4s.{AuthedRoutes, Request} import org.typelevel.ci._ private trait Authentication[F[_]] { - def authenticateIfNeeded: Kleisli[F, Request[F], Either[EndpointSecurityException, Option[AuthUser]]] + def authenticateIfNeeded: Kleisli[F, Request[F], Either[EndpointSecurityException, MaybeAuthUser]] def authenticate: Kleisli[F, Request[F], Either[EndpointSecurityException, AuthUser]] } @@ -40,21 +39,16 @@ private class AuthenticationImpl[F[_]: MonadThrow](authenticator: Authenticator[ import org.http4s.{Header, Request} - override val authenticateIfNeeded: Kleisli[F, Request[F], Either[EndpointSecurityException, Option[AuthUser]]] = + override val authenticateIfNeeded: Kleisli[F, Request[F], Either[EndpointSecurityException, MaybeAuthUser]] = Kleisli { request => request.getBearerToken orElse request.getPrivateAccessToken match { - case Some(token) => authenticator.authenticate(token).map(_.map(Option.apply)) - case None => Option.empty[AuthUser].asRight[EndpointSecurityException].pure[F] + case Some(token) => authenticator.authenticate(token).map(_.map(MaybeAuthUser.apply)) + case None => MaybeAuthUser.noUser.asRight[EndpointSecurityException].pure[F] } } override val authenticate: Kleisli[F, Request[F], Either[EndpointSecurityException, AuthUser]] = - Kleisli { request => - request.getBearerToken orElse request.getPrivateAccessToken match { - case Some(token) => authenticator.authenticate(token) - case None => (AuthenticationFailure: EndpointSecurityException).asLeft[AuthUser].pure[F] - } - } + authenticateIfNeeded.map(_.flatMap(_.required)) private implicit class RequestOps(request: Request[F]) { @@ -80,14 +74,14 @@ object Authentication { def middlewareAuthenticatingIfNeeded[F[_]: MonadThrow]( authenticator: Authenticator[F] - ): F[AuthMiddleware[F, Option[AuthUser]]] = MonadThrow[F].catchNonFatal { + ): F[AuthMiddleware[F, MaybeAuthUser]] = MonadThrow[F].catchNonFatal { middlewareAuthenticatingIfNeeded[F](new AuthenticationImpl[F](authenticator)) } private[security] def middlewareAuthenticatingIfNeeded[F[_]: MonadThrow]( authentication: Authentication[F] - ): AuthMiddleware[F, Option[AuthUser]] = - AuthMiddleware[F, EndpointSecurityException, Option[AuthUser]](authentication.authenticateIfNeeded, onFailure) + ): AuthMiddleware[F, MaybeAuthUser] = + AuthMiddleware[F, EndpointSecurityException, MaybeAuthUser](authentication.authenticateIfNeeded, onFailure) def middleware[F[_]: MonadThrow]( authenticator: Authenticator[F] diff --git a/graph-commons/src/main/scala/io/renku/http/server/security/model.scala b/graph-commons/src/main/scala/io/renku/http/server/security/model.scala index 1f7b735342..f71afabf4a 100644 --- a/graph-commons/src/main/scala/io/renku/http/server/security/model.scala +++ b/graph-commons/src/main/scala/io/renku/http/server/security/model.scala @@ -18,12 +18,44 @@ package io.renku.http.server.security +import cats.Applicative +import cats.syntax.all._ import io.renku.graph.model.persons +import io.renku.http.InfoMessage.messageJsonEntityEncoder import io.renku.http.client.UserAccessToken -import org.http4s.Response +import io.renku.http.server.security.EndpointSecurityException.AuthenticationFailure +import io.renku.http.{ErrorMessage, InfoMessage} +import org.http4s.{Response, Status} + +import java.util.Objects object model { final case class AuthUser(id: persons.GitLabId, accessToken: UserAccessToken) + + final class MaybeAuthUser(private val user: Option[AuthUser]) { + val option: Option[AuthUser] = user + val required: Either[EndpointSecurityException, AuthUser] = + user.toRight(AuthenticationFailure: EndpointSecurityException) + + def withAuthenticatedUser[F[_]: Applicative](code: AuthUser => F[Response[F]]): F[Response[F]] = + required.fold(_.toHttpResponse[F].pure[F], code) + + def withUserOrNotFound[F[_]: Applicative](code: AuthUser => F[Response[F]]): F[Response[F]] = + required.fold(_ => Response.notFound[F].withEntity(InfoMessage("Resource not found")).pure[F], code) + + override def equals(obj: Any): Boolean = obj match { + case o: MaybeAuthUser => o.user == user + case _ => false + } + + override def hashCode(): Int = Objects.hashCode(user, "MaybeUser") + } + + object MaybeAuthUser { + val noUser: MaybeAuthUser = new MaybeAuthUser(None) + def apply(user: Option[AuthUser]): MaybeAuthUser = new MaybeAuthUser(user) + def apply(user: AuthUser): MaybeAuthUser = apply(Some(user)) + } } sealed trait EndpointSecurityException extends Exception with Product with Serializable { @@ -32,10 +64,6 @@ sealed trait EndpointSecurityException extends Exception with Product with Seria object EndpointSecurityException { - import io.renku.http.ErrorMessage - import io.renku.http.ErrorMessage._ - import org.http4s.{Response, Status} - final case object AuthenticationFailure extends EndpointSecurityException { override lazy val getMessage: String = "User authentication failure" diff --git a/graph-commons/src/test/scala/io/renku/http/server/EndpointTester.scala b/graph-commons/src/test/scala/io/renku/http/server/EndpointTester.scala index 4fdb188de0..11baf08949 100644 --- a/graph-commons/src/test/scala/io/renku/http/server/EndpointTester.scala +++ b/graph-commons/src/test/scala/io/renku/http/server/EndpointTester.scala @@ -27,7 +27,7 @@ import io.circe._ import io.renku.http.ErrorMessage.ErrorMessage import io.renku.http.rest.Links import io.renku.http.rest.Links.{Href, Rel} -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.{AuthUser, MaybeAuthUser} import io.renku.json.JsonOps.JsonExt import org.http4s._ import org.http4s.circe.{jsonEncoderOf, jsonOf} @@ -96,17 +96,23 @@ object EndpointTester { } yield link.href } - def givenAuthIfNeededMiddleware(returning: OptionT[IO, Option[AuthUser]]): AuthMiddleware[IO, Option[AuthUser]] = + def givenAuthIfNeededMiddleware(returning: IO[MaybeAuthUser]): AuthMiddleware[IO, MaybeAuthUser] = AuthMiddleware { - Kleisli liftF returning + Kleisli.liftF(OptionT.liftF(returning)) } + def givenAuthAsUnauthorized: AuthMiddleware[IO, MaybeAuthUser] = + AuthMiddleware.noSpider[IO, MaybeAuthUser]( + Kleisli.liftF(OptionT.none[IO, MaybeAuthUser]), + AuthMiddleware.defaultAuthFailure[IO] + ) + def givenAuthMiddleware(returning: OptionT[IO, AuthUser]): AuthMiddleware[IO, AuthUser] = AuthMiddleware { Kleisli liftF returning } - def givenAuthFailing(): AuthMiddleware[IO, Option[AuthUser]] = AuthMiddleware { + def givenAuthFailing(): AuthMiddleware[IO, MaybeAuthUser] = AuthMiddleware { Kleisli(_ => OptionT.none) } diff --git a/graph-commons/src/test/scala/io/renku/http/server/security/AuthenticationSpec.scala b/graph-commons/src/test/scala/io/renku/http/server/security/AuthenticationSpec.scala index e8abf79624..2f7d84be10 100644 --- a/graph-commons/src/test/scala/io/renku/http/server/security/AuthenticationSpec.scala +++ b/graph-commons/src/test/scala/io/renku/http/server/security/AuthenticationSpec.scala @@ -27,7 +27,7 @@ import io.renku.http.ErrorMessage._ import io.renku.http.client.AccessToken.UserOAuthAccessToken import io.renku.http.server.EndpointTester._ import io.renku.http.server.security.EndpointSecurityException.AuthenticationFailure -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.{AuthUser, MaybeAuthUser} import io.renku.testtools.IOSpec import org.http4s.dsl.Http4sDsl import org.http4s.{AuthedRoutes, Request, Response} @@ -59,14 +59,16 @@ class AuthenticationSpec authentication.authenticateIfNeeded( request.withHeaders(accessToken.toHeader) - ) shouldBe authUser.some.asRight[EndpointSecurityException].pure[Try] + ) shouldBe MaybeAuthUser(authUser.some).asRight[EndpointSecurityException].pure[Try] } } "return a function which succeeds authenticating given request and return no user " + "if the request does not contain an Authorization token" in new TestCase { authentication - .authenticateIfNeeded(request) shouldBe Option.empty[AuthUser].asRight[EndpointSecurityException].pure[Try] + .authenticateIfNeeded(request) shouldBe MaybeAuthUser.noUser + .asRight[EndpointSecurityException] + .pure[Try] } "return a function which fails authenticating the given request " + @@ -127,8 +129,8 @@ class AuthenticationSpec val authentication = mock[Authentication[IO]] val exception = securityExceptions.generateOne - val authenticate: Kleisli[IO, Request[IO], Either[EndpointSecurityException, Option[AuthUser]]] = - Kleisli.liftF(exception.asLeft[Option[AuthUser]].pure[IO]) + val authenticate: Kleisli[IO, Request[IO], Either[EndpointSecurityException, MaybeAuthUser]] = + Kleisli.liftF(exception.asLeft[MaybeAuthUser].pure[IO]) (() => authentication.authenticateIfNeeded) .expects() diff --git a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/MicroserviceRoutes.scala b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/MicroserviceRoutes.scala index e10861e1b3..96607b3c81 100644 --- a/knowledge-graph/src/main/scala/io/renku/knowledgegraph/MicroserviceRoutes.scala +++ b/knowledge-graph/src/main/scala/io/renku/knowledgegraph/MicroserviceRoutes.scala @@ -27,7 +27,7 @@ import io.renku.entities.search.{Criteria => EntitiesSearchCriteria} import io.renku.graph.config.RenkuUrlLoader import io.renku.graph.http.server.security._ import io.renku.graph.model -import io.renku.graph.model.{persons, RenkuUrl} +import io.renku.graph.model.{RenkuUrl, persons} import io.renku.graph.tokenrepository.AccessTokenFinder import io.renku.http.InfoMessage import io.renku.http.InfoMessage._ @@ -38,7 +38,7 @@ import io.renku.http.rest.paging.PagingRequest.Decoders._ import io.renku.http.rest.paging.model.{Page, PerPage} import io.renku.http.server.QueryParameterTools._ import io.renku.http.server.security.Authentication -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.{AuthUser, MaybeAuthUser} import io.renku.http.server.version import io.renku.knowledgegraph.datasets.details.RequestedDataset import io.renku.metrics.{MetricsRegistry, RoutesMetrics} @@ -60,7 +60,7 @@ private class MicroserviceRoutes[F[_]: Async]( lineageEndpoint: projects.files.lineage.Endpoint[F], docsEndpoint: docs.Endpoint[F], usersProjectsEndpoint: users.projects.Endpoint[F], - authMiddleware: AuthMiddleware[F, Option[AuthUser]], + authMiddleware: AuthMiddleware[F, MaybeAuthUser], projectPathAuthorizer: Authorizer[F, model.projects.Path], datasetIdAuthorizer: Authorizer[F, model.datasets.Identifier], datasetSameAsAuthorizer: Authorizer[F, model.datasets.SameAs], @@ -90,21 +90,21 @@ private class MicroserviceRoutes[F[_]: Async]( `GET /datasets/*` <+> `GET /entities/*` <+> `GET /projects/*` <+> `GET /users/*` } - private lazy val `GET /datasets/*` : AuthedRoutes[Option[AuthUser], F] = { + private lazy val `GET /datasets/*` : AuthedRoutes[MaybeAuthUser, F] = { import datasets.Endpoint.Query.query import datasets.Endpoint.Sort.sort AuthedRoutes.of { case GET -> Root / "knowledge-graph" / "datasets" :? query(maybePhrase) +& sort(sortBy) +& page(page) +& perPage(perPage) as maybeUser => - searchForDatasets(maybePhrase, sortBy, page, perPage, maybeUser) + searchForDatasets(maybePhrase, sortBy, page, perPage, maybeUser.option) case GET -> Root / "knowledge-graph" / "datasets" / RequestedDataset(dsId) as maybeUser => - fetchDataset(dsId, maybeUser) + fetchDataset(dsId, maybeUser.option) } } // format: off - private lazy val `GET /entities/*`: AuthedRoutes[Option[AuthUser], F] = { + private lazy val `GET /entities/*`: AuthedRoutes[MaybeAuthUser, F] = { import io.renku.entities.search.Criteria._ import Sort.sort import entities.QueryParamDecoders._ @@ -116,12 +116,12 @@ private class MicroserviceRoutes[F[_]: Async]( +& since(maybeSince) +& until(maybeUntil) +& sort(maybeSort) +& page(maybePage) +& perPage(maybePerPage) as maybeUser => searchForEntities(maybeQuery, maybeTypes, maybeCreators, maybeVisibilities, maybeNamespaces, - maybeSince, maybeUntil, maybeSort, maybePage, maybePerPage, maybeUser, req.req) + maybeSince, maybeUntil, maybeSort, maybePage, maybePerPage, maybeUser.option, req.req) } } // format: on - private lazy val `GET /users/*` : AuthedRoutes[Option[AuthUser], F] = { + private lazy val `GET /users/*` : AuthedRoutes[MaybeAuthUser, F] = { import users.binders._ import users.projects.Endpoint._ import users.projects.Endpoint.Criteria.Filters @@ -134,26 +134,28 @@ private class MicroserviceRoutes[F[_]: Async]( :? activationState(maybeState) +& page(maybePage) +& perPage(maybePerPage) as maybeUser => (maybeState getOrElse ActivationState.All.validNel, PagingRequest(maybePage, maybePerPage)) .mapN { (activationState, paging) => - `GET /users/:id/projects`(Criteria(userId = id, Filters(activationState), paging, maybeUser), req.req) + `GET /users/:id/projects`(Criteria(userId = id, Filters(activationState), paging, maybeUser.option), + req.req + ) } .fold(toBadRequest, identity) } } - private lazy val `GET /projects/*` : AuthedRoutes[Option[AuthUser], F] = AuthedRoutes.of { + private lazy val `GET /projects/*` : AuthedRoutes[MaybeAuthUser, F] = AuthedRoutes.of { case authReq @ GET -> "knowledge-graph" /: "projects" /: path as maybeUser => - routeToProjectsEndpoints(path, maybeUser)(authReq.req) - - case DELETE -> "knowledge-graph" /: "projects" /: path as Some(user) => - path.segments.toList - .map(_.toString) - .toProjectPath - .flatTap(authorizePath(_, user.some).leftMap(_.toHttpResponse)) - .semiflatMap(`DELETE /projects/:path`(_, user)) - .merge + routeToProjectsEndpoints(path, maybeUser.option)(authReq.req) - case DELETE -> "knowledge-graph" /: "projects" /: _ as None => resourceNotFound[F] + case DELETE -> "knowledge-graph" /: "projects" /: path as maybeUser => + maybeUser.withUserOrNotFound { user => + path.segments.toList + .map(_.toString) + .toProjectPath + .flatTap(authorizePath(_, user.some).leftMap(_.toHttpResponse)) + .semiflatMap(`DELETE /projects/:path`(_, user)) + .merge + } } private lazy val nonAuthorizedRoutes: HttpRoutes[F] = HttpRoutes.of[F] { diff --git a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/MicroserviceRoutesSpec.scala b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/MicroserviceRoutesSpec.scala index cd50965f10..1ae985e489 100644 --- a/knowledge-graph/src/test/scala/io/renku/knowledgegraph/MicroserviceRoutesSpec.scala +++ b/knowledge-graph/src/test/scala/io/renku/knowledgegraph/MicroserviceRoutesSpec.scala @@ -18,7 +18,7 @@ package io.renku.knowledgegraph -import cats.data.{EitherT, Kleisli, OptionT} +import cats.data.{EitherT, Kleisli} import cats.data.EitherT.{leftT, rightT} import cats.effect.{IO, Resource} import cats.syntax.all._ @@ -42,7 +42,7 @@ import io.renku.http.rest.paging.model.{Page, PerPage} import io.renku.http.server.EndpointTester._ import io.renku.http.server.security.EndpointSecurityException import io.renku.http.server.security.EndpointSecurityException.AuthorizationFailure -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.{AuthUser, MaybeAuthUser} import io.renku.http.server.version import io.renku.interpreters.TestRoutesMetrics import io.renku.knowledgegraph.datasets.details.RequestedDataset @@ -80,14 +80,14 @@ class MicroserviceRoutesSpec s"return $Ok when a valid 'query' and no 'sort', `page` and `per_page` parameters given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) val phrase = nonEmptyStrings().generateOne (datasetsSearchEndpoint .searchForDatasets(_: Option[Phrase], _: Sorting[Sort.type], _: PagingRequest, _: Option[AuthUser])) .expects(Phrase(phrase).some, Sorting(Sort.By(TitleProperty, Direction.Asc)), PagingRequest(Page.first, PerPage.default), - maybeAuthUser + maybeAuthUser.option ) .returning(IO.pure(Response[IO](Ok))) @@ -98,14 +98,14 @@ class MicroserviceRoutesSpec s"return $Ok when no ${query.parameterName} parameter given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) (datasetsSearchEndpoint .searchForDatasets(_: Option[Phrase], _: Sorting[Sort.type], _: PagingRequest, _: Option[AuthUser])) .expects(Option.empty[Phrase], Sorting(Sort.By(TitleProperty, Direction.Asc)), PagingRequest(Page.first, PerPage.default), - maybeAuthUser + maybeAuthUser.option ) .returning(IO.pure(Response[IO](Ok))) @@ -122,7 +122,7 @@ class MicroserviceRoutesSpec val sortBy = Sort.By(sortProperty, Gen.oneOf(SortBy.Direction.Asc, SortBy.Direction.Desc).generateOne) s"return $Ok when '${query.parameterName}' and 'sort=${sortBy.property}:${sortBy.direction}' parameters given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) val phrase = phrases.generateOne val request = Request[IO]( @@ -133,7 +133,7 @@ class MicroserviceRoutesSpec ) (datasetsSearchEndpoint .searchForDatasets(_: Option[Phrase], _: Sorting[Sort.type], _: PagingRequest, _: Option[AuthUser])) - .expects(phrase.some, Sorting(sortBy), PagingRequest.default, maybeAuthUser) + .expects(phrase.some, Sorting(sortBy), PagingRequest.default, maybeAuthUser.option) .returning(IO.pure(Response[IO](Ok))) val response = routes(maybeAuthUser).call(request) @@ -146,7 +146,7 @@ class MicroserviceRoutesSpec s"return $Ok when query, ${PagingRequest.Decoders.page.parameterName} and ${PagingRequest.Decoders.perPage.parameterName} parameters given" in new TestCase { forAll(phrases, pages, perPages) { (phrase, page, perPage) => - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) val request = Request[IO]( GET, @@ -160,7 +160,7 @@ class MicroserviceRoutesSpec .expects(phrase.some, Sorting(Sort.By(TitleProperty, Direction.Asc)), PagingRequest(page, perPage), - maybeAuthUser + maybeAuthUser.option ) .returning(IO.pure(Response[IO](Ok))) @@ -214,10 +214,14 @@ class MicroserviceRoutesSpec } { (dsIdType, requestedDS) => s"return $Ok when a valid $dsIdType given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) - val authContext = AuthContext(maybeAuthUser, requestedDS, projectPaths.generateSet()) - givenDSAuthorizer(requestedDS, maybeAuthUser, returning = rightT[IO, EndpointSecurityException](authContext)) + val authContext = AuthContext(maybeAuthUser.option, requestedDS, projectPaths.generateSet()) + givenDSAuthorizer( + requestedDS, + maybeAuthUser.option, + returning = rightT[IO, EndpointSecurityException](authContext) + ) (datasetDetailsEndpoint.`GET /datasets/:id` _) .expects(requestedDS, authContext) @@ -236,7 +240,7 @@ class MicroserviceRoutesSpec } s"return $Unauthorized when user authentication failed" in new TestCase { - routes(givenAuthIfNeededMiddleware(returning = OptionT.none[IO, Option[AuthUser]])) + routes(givenAuthAsUnauthorized) .call(Request(GET, uri"/knowledge-graph/datasets" / RequestedDataset(datasetIdentifiers.generateOne))) .status shouldBe Unauthorized } @@ -244,11 +248,12 @@ class MicroserviceRoutesSpec s"return $NotFound when user has no rights for the project the dataset belongs to" in new TestCase { val id = datasetIdentifiers.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) - givenDSIdAuthorizer(id, - maybeAuthUser, - returning = leftT[IO, AuthContext[model.datasets.Identifier]](AuthorizationFailure) + givenDSIdAuthorizer( + id, + maybeAuthUser.option, + returning = leftT[IO, AuthContext[model.datasets.Identifier]](AuthorizationFailure) ) val response = routes(maybeAuthUser).call(Request(GET, uri"/knowledge-graph/datasets" / RequestedDataset(id))) @@ -414,12 +419,12 @@ class MicroserviceRoutesSpec } "authenticate user from the request if given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser.apply(authUsers.generateOption) val request = Request[IO](GET, uri"/knowledge-graph/entities") val responseBody = jsons.generateOne (entitiesEndpoint.`GET /entities` _) - .expects(Criteria(maybeUser = maybeAuthUser), request) + .expects(Criteria(maybeUser = maybeAuthUser.option), request) .returning(Response[IO](Ok).withEntity(responseBody).pure[IO]) routes(maybeAuthUser).call(request).status shouldBe Ok @@ -454,41 +459,41 @@ class MicroserviceRoutesSpec s"return $Accepted for valid path parameters and user" in new TestCase { - val authUser = authUsers.generateOne + val authUser = MaybeAuthUser.apply(authUsers.generateOne) (projectPathAuthorizer.authorize _) - .expects(projectPath, authUser.some) - .returning(rightT[IO, EndpointSecurityException](AuthContext(authUser.some, projectPath, Set(projectPath)))) + .expects(projectPath, authUser.option) + .returning(rightT[IO, EndpointSecurityException](AuthContext(authUser.option, projectPath, Set(projectPath)))) (projectDeleteEndpoint .`DELETE /projects/:path`(_: model.projects.Path, _: AuthUser)) - .expects(projectPath, authUser) + .expects(projectPath, authUser.option.get) .returning(Response[IO](Accepted).pure[IO]) - routes(authUser.some).call(request).status shouldBe Accepted + routes(authUser).call(request).status shouldBe Accepted routesMetrics.clearRegistry() } s"return $Unauthorized when authentication fails" in new TestCase { - routes(givenAuthIfNeededMiddleware(returning = OptionT.none[IO, Option[AuthUser]])) + routes(givenAuthAsUnauthorized) .call(request) .status shouldBe Unauthorized } s"return $NotFound when no auth header" in new TestCase { - routes(maybeAuthUser = None).call(request).status shouldBe NotFound + routes(maybeAuthUser = MaybeAuthUser.noUser).call(request).status shouldBe NotFound } s"return $NotFound when the user has no rights to the project" in new TestCase { - val authUser = authUsers.generateOne + val authUser = MaybeAuthUser(authUsers.generateOne) (projectPathAuthorizer.authorize _) - .expects(projectPath, authUser.some) + .expects(projectPath, authUser.option) .returning(leftT[IO, AuthContext[model.projects.Path]](AuthorizationFailure)) - val response = routes(authUser.some).call(request) + val response = routes(authUser).call(request) response.status shouldBe NotFound response.contentType shouldBe Some(`Content-Type`(application.json)) @@ -500,17 +505,19 @@ class MicroserviceRoutesSpec s"return $Ok for valid path parameters" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val projectPath = projectPaths.generateOne (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) val request = Request[IO](GET, Uri.unsafeFromString(s"knowledge-graph/projects/$projectPath")) (projectDetailsEndpoint .`GET /projects/:path`(_: model.projects.Path, _: Option[AuthUser])(_: Request[IO])) - .expects(projectPath, maybeAuthUser, request) + .expects(projectPath, maybeAuthUser.option, request) .returning(Response[IO](Ok).pure[IO]) routes(maybeAuthUser).call(request).status shouldBe Ok @@ -520,7 +527,7 @@ class MicroserviceRoutesSpec s"return $NotFound for invalid project paths" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val namespace = nonBlankStrings().generateOne.value val response = routes(maybeAuthUser).call( @@ -533,18 +540,18 @@ class MicroserviceRoutesSpec } s"return $Unauthorized when user authentication fails" in new TestCase { - routes(givenAuthIfNeededMiddleware(returning = OptionT.none[IO, Option[AuthUser]])) + routes(givenAuthAsUnauthorized) .call(Request(GET, Uri.unsafeFromString(s"knowledge-graph/projects/${projectPaths.generateOne}"))) .status shouldBe Unauthorized } s"return $NotFound when auth user has no rights for the project" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val projectPath = projectPaths.generateOne (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) + .expects(projectPath, maybeAuthUser.option) .returning(leftT[IO, AuthContext[model.projects.Path]](AuthorizationFailure)) val response = routes(maybeAuthUser).call( @@ -561,11 +568,13 @@ class MicroserviceRoutesSpec s"return $Ok for valid path parameters" in new TestCase { val projectPath = projectPaths.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) (projectDatasetsEndpoint.getProjectDatasets _).expects(projectPath).returning(IO.pure(Response[IO](Ok))) @@ -577,7 +586,7 @@ class MicroserviceRoutesSpec } s"return $Unauthorized when user authentication fails" in new TestCase { - routes(givenAuthIfNeededMiddleware(returning = OptionT.none[IO, Option[AuthUser]])) + routes(givenAuthAsUnauthorized) .call( Request(GET, Uri.unsafeFromString(s"knowledge-graph/projects/${projectPaths.generateOne}/datasets")) ) @@ -586,10 +595,10 @@ class MicroserviceRoutesSpec s"return $NotFound when auth user has no rights for the project" in new TestCase { val projectPath = projectPaths.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) + .expects(projectPath, maybeAuthUser.option) .returning(leftT[IO, AuthContext[model.projects.Path]](AuthorizationFailure)) val response = routes(maybeAuthUser) @@ -668,17 +677,19 @@ class MicroserviceRoutesSpec "authenticate user from the request if given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val request = Request[IO](GET, projectDsTagsUri) (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) val responseBody = jsons.generateOne (projectDatasetTagsEndpoint .`GET /projects/:path/datasets/:name/tags`(_: Criteria)(_: Request[IO])) - .expects(Criteria(projectPath, datasetName, maybeUser = maybeAuthUser), request) + .expects(Criteria(projectPath, datasetName, maybeUser = maybeAuthUser.option), request) .returning(Response[IO](Ok).withEntity(responseBody).pure[IO]) routes(maybeAuthUser).call(request).status shouldBe Ok @@ -696,18 +707,20 @@ class MicroserviceRoutesSpec s"return $Ok when the lineage is found" in new TestCase { val projectPath = projectPaths.generateOne val location = nodeLocations.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val uri = lineageUri(projectPath, location) val request = Request[IO](GET, uri) val responseBody = jsons.generateOne (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) (lineageEndpoint.`GET /lineage` _) - .expects(projectPath, location, maybeAuthUser) + .expects(projectPath, location, maybeAuthUser.option) .returning(Response[IO](Ok).withEntity(responseBody).pure[IO]) val response = routes(maybeAuthUser).call(request) @@ -720,15 +733,17 @@ class MicroserviceRoutesSpec s"return $NotFound for a lineage which isn't found" in new TestCase { val projectPath = projectPaths.generateOne val location = nodeLocations.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val uri = lineageUri(projectPath, location) (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) (lineageEndpoint.`GET /lineage` _) - .expects(projectPath, location, maybeAuthUser) + .expects(projectPath, location, maybeAuthUser.option) .returning(Response[IO](NotFound).pure[IO]) val response = routes(maybeAuthUser).call(Request[IO](GET, uri)) @@ -740,17 +755,19 @@ class MicroserviceRoutesSpec val projectPath = projectPaths.generateOne val location = nodeLocations.generateOne - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val request = Request[IO](GET, lineageUri(projectPath, location)) val responseBody = jsons.generateOne (projectPathAuthorizer.authorize _) - .expects(projectPath, maybeAuthUser) - .returning(rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser, projectPath, Set(projectPath)))) + .expects(projectPath, maybeAuthUser.option) + .returning( + rightT[IO, EndpointSecurityException](AuthContext(maybeAuthUser.option, projectPath, Set(projectPath))) + ) (lineageEndpoint.`GET /lineage` _) - .expects(projectPath, location, maybeAuthUser) + .expects(projectPath, location, maybeAuthUser.option) .returning(Response[IO](Ok).withEntity(responseBody).pure[IO]) routes(maybeAuthUser).call(request).status shouldBe Ok @@ -836,12 +853,12 @@ class MicroserviceRoutesSpec "authenticate user from the request if given" in new TestCase { - val maybeAuthUser = authUsers.generateOption + val maybeAuthUser = MaybeAuthUser(authUsers.generateOption) val request = Request[IO](GET, uri"/knowledge-graph/users" / userId.value / "projects") val responseBody = jsons.generateOne (usersProjectsEndpoint.`GET /users/:id/projects` _) - .expects(Criteria(userId, maybeUser = maybeAuthUser), request) + .expects(Criteria(userId, maybeUser = maybeAuthUser.option), request) .returning(Response[IO](Ok).withEntity(responseBody).pure[IO]) routes(maybeAuthUser).call(request).status shouldBe Ok @@ -900,11 +917,13 @@ class MicroserviceRoutesSpec val routesMetrics = TestRoutesMetrics() private val versionRoutes = mock[version.Routes[IO]] - def routes(maybeAuthUser: Option[AuthUser] = None): Resource[IO, Kleisli[IO, Request[IO], Response[IO]]] = routes( - givenAuthIfNeededMiddleware(returning = OptionT.some[IO](maybeAuthUser)) + def routes( + maybeAuthUser: MaybeAuthUser = MaybeAuthUser.noUser + ): Resource[IO, Kleisli[IO, Request[IO], Response[IO]]] = routes( + givenAuthIfNeededMiddleware(returning = IO.pure(maybeAuthUser)) ) - def routes(middleware: AuthMiddleware[IO, Option[AuthUser]]): Resource[IO, Kleisli[IO, Request[IO], Response[IO]]] = + def routes(middleware: AuthMiddleware[IO, MaybeAuthUser]): Resource[IO, Kleisli[IO, Request[IO], Response[IO]]] = new MicroserviceRoutes[IO]( datasetsSearchEndpoint, datasetDetailsEndpoint, diff --git a/webhook-service/README.md b/webhook-service/README.md index e254ced5ff..0fc15f61b2 100644 --- a/webhook-service/README.md +++ b/webhook-service/README.md @@ -44,6 +44,13 @@ Verifies service health. Returns information about activation and processing progress of project events. +**request format** + +The endpoint accepts an authorization headers passed in the request: +- `Authorization: Bearer ` with oauth token obtained from gitlab +- `PRIVATE-TOKEN: ` with user's personal access token in gitlab +The headers are not required. + **Response** | Status | Description | @@ -86,18 +93,18 @@ creates a webhook for a project with the given `project id`. **request format** -the endpoint requires an authorization token passed in the request header as: -- `authorization: bearer ` with oauth token obtained from gitlab -- `private-token: ` with user's personal access token in gitlab +The endpoint requires an authorization token passed in the request header as: +- `Authorization: Bearer ` with oauth token obtained from gitlab +- `PRIVATE-TOKEN: ` with user's personal access token in gitlab **response** -| status | description | -|----------------------------|-------------------------------------------------------------------------------------------------| -| ok (200) | when hook already exists for the project | -| created (201) | when a new hook was created | -| unauthorized (401) | when there is neither `private-token` nor `authorization: bearer` in the header or it's invalid | -| internal server error (500)| when there are problems with webhook creation | +| status | description | +|-----------------------------|-------------------------------------------------------------------------------------------------| +| OK (200) | when hook already exists for the project | +| CREATED (201) | when a new hook was created | +| UNAUTHORIZED (401) | when there is neither `private-token` nor `authorization: bearer` in the header or it's invalid | +| INTERNAL_SERVER_ERROR (500) | when there are problems with webhook creation | #### DELETE /projects/:id/webhooks @@ -105,26 +112,25 @@ deletes a webhook for a project with the given `project id`. **request format** -the endpoint requires an authorization token passed in the request header as: -- `authorization: bearer ` with oauth token obtained from gitlab -- `private-token: ` with user's personal access token in gitlab +The endpoint requires an authorization token passed in the request header as: +- `Authorization: Bearer ` with oauth token obtained from gitlab +- `PRIVATE-TOKEN: ` with user's personal access token in gitlab **response** -| status | description | -|----------------------------|-------------------------------------------------------------------------------------------------| -| ok (200) | when hook is successfully deleted | -| not found (404) | when the project does not exists | -| unauthorized (401) | when there is neither `private-token` nor `authorization: bearer` in the header or it's invalid | -| internal server error (500)| when there are problems with webhook creation | +| status | description | +|-----------------------------|-------------------------------------------------------------------------------------------------| +| OK (200) | when hook is successfully deleted | +| NOT_FOUND (404) | when the project does not exists | +| UNAUTHORIZED (401) | when there is neither `private-token` nor `authorization: bearer` in the header or it's invalid | +| INTERNAL_SERVER_ERROR (500) | when there are problems with webhook creation | #### POST /projects/:id/webhooks/validation **Notice** -This endpoint is under development and works just for public projects. For non-public projects it responds with INTERNAL SERVER ERROR (500). - -Validates the webhook for the project with the given `project id`. It succeeds (OK) if either the project is public and there's a hook for it or it's private, there's a hook for it and a Personal Access Token (PAT). If either there's no webhook or there's no PAT in case of a private project, the call results with NOT_FOUND. In case of private projects, if there's a hook created for a project but no PAT available (or the PAT doesn't work), the hook will be removed as part of the validation process. +This API validates the renku webhook for the project with the given `id`. +It succeeds (OK) if the hook exists. If there's no webhook the call responds with NOT_FOUND. **Request format** @@ -134,12 +140,12 @@ The endpoint requires an authorization token passed in the request header as: **Response** -| Status | Description | -|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| OK (200) | When the hook exists for the project and the project is either public or there's a Personal Access Token available for it | -| NOT_FOUND (404) | When the hook either does not exists or there's no Personal Access Token available for it. If the hook exists but there's no PAT for it, the hook will be removed | -| UNAUTHORIZED (401) | When there is neither `PRIVATE-TOKEN` nor `AUTHORIZATION: BEARER` in the header or it's invalid | -| INTERNAL SERVER ERROR (500)| When there are problems with validating the hook presence | +| Status | Description | +|----------------------------|-------------------------------------------------------------------------------------------------| +| OK (200) | When the hook exists for the project | +| NOT_FOUND (404) | When there's no hook for the project | +| UNAUTHORIZED (401) | When there is neither `PRIVATE-TOKEN` nor `AUTHORIZATION: BEARER` in the header or it's invalid | +| INTERNAL SERVER ERROR (500)| When there are problems with validating the hook presence | #### GET /version diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala b/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala index 59da710ae1..c6d9684a1c 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/MicroserviceRoutes.scala @@ -25,7 +25,7 @@ import io.renku.graph.http.server.binders.ProjectId import io.renku.graph.http.server.security.GitLabAuthenticator import io.renku.http.client.GitLabClient import io.renku.http.server.security.Authentication -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.MaybeAuthUser import io.renku.http.server.version import io.renku.logging.ExecutionTimeRecorder import io.renku.metrics.{MetricsRegistry, RoutesMetrics} @@ -45,7 +45,7 @@ private class MicroserviceRoutes[F[_]: MonadThrow]( hookValidationEndpoint: HookValidationEndpoint[F], hookDeletionEndpoint: HookDeletionEndpoint[F], eventStatusEndpoint: eventstatus.Endpoint[F], - authMiddleware: AuthMiddleware[F, AuthUser], + optAuthMiddleware: AuthMiddleware[F, MaybeAuthUser], routesMetrics: RoutesMetrics[F], versionRoutes: version.Routes[F] ) extends Http4sDsl[F] { @@ -58,24 +58,29 @@ private class MicroserviceRoutes[F[_]: MonadThrow]( import eventStatusEndpoint._ import routesMetrics._ - // format: off - private lazy val authorizedRoutes: HttpRoutes[F] = authMiddleware { + private lazy val optionalAuthorizedRoutes: HttpRoutes[F] = optAuthMiddleware { AuthedRoutes.of { - case GET -> Root / "projects" / ProjectId(projectId) / "events" / "status" as authUser => fetchProcessingStatus(projectId, authUser) - case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => createHook(projectId, authUser) - case DELETE -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => deleteHook(projectId, authUser) - case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" / "validation" as authUser => validateHook(projectId, authUser) + case GET -> Root / "projects" / ProjectId(projectId) / "events" / "status" as authUser => + fetchProcessingStatus(projectId, authUser.option) + + case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => + authUser.withAuthenticatedUser(createHook(projectId, _)) + + case DELETE -> Root / "projects" / ProjectId(projectId) / "webhooks" as authUser => + authUser.withAuthenticatedUser(deleteHook(projectId, _)) + + case POST -> Root / "projects" / ProjectId(projectId) / "webhooks" / "validation" as authUser => + authUser.withAuthenticatedUser(validateHook(projectId, _)) } } private lazy val nonAuthorizedRoutes: HttpRoutes[F] = HttpRoutes.of[F] { - case GET -> Root / "ping" => Ok("pong") + case GET -> Root / "ping" => Ok("pong") case request @ POST -> Root / "webhooks" / "events" => processPushEvent(request) } - // format: on lazy val routes: Resource[F, HttpRoutes[F]] = - (versionRoutes() <+> nonAuthorizedRoutes <+> authorizedRoutes).withMetrics + (versionRoutes() <+> nonAuthorizedRoutes <+> optionalAuthorizedRoutes).withMetrics } private object MicroserviceRoutes { @@ -88,7 +93,7 @@ private object MicroserviceRoutes { hookValidationEndpoint <- HookValidationEndpoint(projectHookUrl) hookDeletionEndpoint <- HookDeletionEndpoint(projectHookUrl) authenticator <- GitLabAuthenticator[F] - authMiddleware <- Authentication.middleware(authenticator) + optAuthMiddleware <- Authentication.middlewareAuthenticatingIfNeeded(authenticator) versionRoutes <- version.Routes[F] } yield new MicroserviceRoutes[F]( webhookEventsEndpoint, @@ -96,7 +101,7 @@ private object MicroserviceRoutes { hookValidationEndpoint, hookDeletionEndpoint, eventStatusEndpoint, - authMiddleware, + optAuthMiddleware, new RoutesMetrics[F], versionRoutes ) diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala index 9655ae188e..8cdeeeaf5c 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala @@ -40,7 +40,7 @@ import org.http4s.dsl.Http4sDsl import org.typelevel.log4cats.Logger trait Endpoint[F[_]] { - def fetchProcessingStatus(projectId: GitLabId, authUser: AuthUser): F[Response[F]] + def fetchProcessingStatus(projectId: GitLabId, authUser: Option[AuthUser]): F[Response[F]] } private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( @@ -53,7 +53,7 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( private val executionTimeRecorder = ExecutionTimeRecorder[F] import executionTimeRecorder._ - def fetchProcessingStatus(projectId: GitLabId, authUser: AuthUser): F[Response[F]] = measureExecutionTime { + def fetchProcessingStatus(projectId: GitLabId, authUser: Option[AuthUser]): F[Response[F]] = measureExecutionTime { validateHook(projectId, authUser) .semiflatMap { case HookExists => findStatus(projectId) @@ -63,10 +63,13 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( .recoverWith(internalServerError(projectId)) } map logExecutionTime(withMessage = show"Finding status info for project '$projectId' finished") - private def validateHook(projectId: GitLabId, authUser: AuthUser): EitherT[F, Response[F], HookValidationResult] = + private def validateHook( + projectId: GitLabId, + authUser: Option[AuthUser] + ): EitherT[F, Response[F], HookValidationResult] = EitherT { hookValidator - .validateHook(projectId, authUser.accessToken.some) + .validateHook(projectId, authUser.map(_.accessToken)) .map(_.asRight[Response[F]]) .recoverWith(noAccessTokenToNotFound) } diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala index 27d5fa6105..5dd73e066a 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala @@ -81,8 +81,8 @@ private object Progress { lazy val currentStage: EventStatusProgress.Stage = statusProgress.stage lazy val completion: EventStatusProgress.Completion = statusProgress.completion - lazy val done = currentStage.value - lazy val percentage = completion.value + lazy val done: Int = currentStage.value + lazy val percentage: Float = completion.value } def from(eventStatus: EventStatus): Progress.NonZero = diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcher.scala b/webhook-service/src/main/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcher.scala index 287d806859..106cd88823 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcher.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcher.scala @@ -18,7 +18,8 @@ package io.renku.webhookservice.hookfetcher -import cats.effect.{Async, MonadCancelThrow} +import cats.MonadThrow +import cats.effect.Async import cats.syntax.all._ import eu.timepit.refined.auto._ import io.circe.Decoder.decodeList @@ -46,19 +47,19 @@ private[webhookservice] class ProjectHookFetcherImpl[F[_]: Async: GitLabClient: import io.circe._ import io.renku.http.client.RestClientError.UnauthorizedException - import org.http4s.Status.Unauthorized + import io.renku.http.tinytypes.TinyTypeURIEncoder._ + import org.http4s.Status.{NotFound, Ok, Unauthorized} import org.http4s._ import org.http4s.circe._ - import org.http4s.dsl.io._ private lazy val mapResponse: PartialFunction[(Status, Request[F], Response[F]), F[List[HookIdAndUrl]]] = { case (Ok, _, response) => response.as[List[HookIdAndUrl]] case (NotFound, _, _) => List.empty[HookIdAndUrl].pure[F] - case (Unauthorized, _, _) => MonadCancelThrow[F].raiseError(UnauthorizedException) + case (Unauthorized, _, _) => MonadThrow[F].raiseError(UnauthorizedException) } override def fetchProjectHooks(projectId: projects.GitLabId, accessToken: AccessToken): F[List[HookIdAndUrl]] = - GitLabClient[F].get(uri"projects" / projectId.show / "hooks", "project-hooks")(mapResponse)(accessToken.some) + GitLabClient[F].get(uri"projects" / projectId / "hooks", "project-hooks")(mapResponse)(accessToken.some) private implicit lazy val hooksIdsAndUrlsDecoder: EntityDecoder[F, List[HookIdAndUrl]] = { implicit val hookIdAndUrlDecoder: Decoder[List[HookIdAndUrl]] = decodeList { cursor => diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/HookValidator.scala b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/HookValidator.scala index 2d343b2da7..54cb7501c7 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/HookValidator.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/HookValidator.scala @@ -73,10 +73,7 @@ class HookValidatorImpl[F[_]: MonadThrow: Logger]( } .onError(logError(projectId)) - private def findToken( - projectId: GitLabId, - maybeAccessToken: Option[AccessToken] - ): F[Token] = + private def findToken(projectId: GitLabId, maybeAccessToken: Option[AccessToken]): F[Token] = maybeAccessToken .map(GivenToken(_).widen.pure[F]) .getOrElse(findStoredToken(projectId)) diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala index 458fc32b4c..d9cb48c251 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/hookvalidation/ProjectHookVerifier.scala @@ -35,7 +35,7 @@ private trait ProjectHookVerifier[F[_]] { private object ProjectHookVerifier { - def apply[F[_]: Async: GitLabClient: Logger] = + def apply[F[_]: Async: GitLabClient: Logger]: F[ProjectHookVerifierImpl[F]] = ProjectHookFetcher[F] map (new ProjectHookVerifierImpl[F](_)) } diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala index e79783bb9a..794be04057 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/MicroserviceRoutesSpec.scala @@ -18,7 +18,6 @@ package io.renku.webhookservice -import cats.data.OptionT import cats.effect.IO import cats.syntax.all._ import io.renku.generators.CommonGraphGenerators.{authUsers, httpStatuses} @@ -26,7 +25,7 @@ import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.GraphModelGenerators._ import io.renku.graph.model.projects import io.renku.http.server.EndpointTester._ -import io.renku.http.server.security.model.AuthUser +import io.renku.http.server.security.model.{AuthUser, MaybeAuthUser} import io.renku.http.server.version import io.renku.http.tinytypes.TinyTypeURIEncoder._ import io.renku.interpreters.TestRoutesMetrics @@ -118,24 +117,29 @@ class MicroserviceRoutesSpec val request = Request[IO](Method.GET, uri"/projects" / projectId / "events" / "status") val responseStatus = Gen.oneOf(Ok, BadRequest).generateOne (eventStatusEndpoint - .fetchProcessingStatus(_: projects.GitLabId, _: AuthUser)) - .expects(projectId, authUser) + .fetchProcessingStatus(_: projects.GitLabId, _: Option[AuthUser])) + .expects(projectId, authUser.some) .returning(IO.pure(Response[IO](responseStatus))) routes.call(request).status shouldBe responseStatus } - "return NotFound when no :id path parameter given" in new TestCase { - routes.call(Request(Method.GET, uri"/projects/")).status shouldBe NotFound - } + "return Ok response from the endpoint without a user" in new TestCase { + override val authenticationResponse = IO.pure(MaybeAuthUser.noUser) - "return Unauthorized when user is not authorized" in new TestCase { - - override val authenticationResponse = OptionT.none[IO, AuthUser] + val projectId = projectIds.generateOne + val request = Request[IO](Method.GET, uri"/projects" / projectId / "events" / "status") + val responseStatus = Gen.oneOf(Ok, BadRequest).generateOne + (eventStatusEndpoint + .fetchProcessingStatus(_: projects.GitLabId, _: Option[AuthUser])) + .expects(projectId, None) + .returning(IO.pure(Response[IO](responseStatus))) - val request = Request[IO](Method.GET, uri"/projects" / projectIds.generateOne / "events" / "status") + routes.call(request).status shouldBe responseStatus + } - routes.call(request).status shouldBe Unauthorized + "return NotFound when no :id path parameter given" in new TestCase { + routes.call(Request(Method.GET, uri"/projects/")).status shouldBe NotFound } "return response from the endpoint" in new TestCase { @@ -156,7 +160,7 @@ class MicroserviceRoutesSpec "return Unauthorized when the user is not authorized" in new TestCase { - override val authenticationResponse = OptionT.none[IO, AuthUser] + override val authenticationResponse = IO.pure(MaybeAuthUser.noUser) val projectId = projectIds.generateOne val request = Request[IO](Method.POST, uri"/projects" / projectId / "webhooks") @@ -182,7 +186,7 @@ class MicroserviceRoutesSpec "return Unauthorized when user is not authorized" in new TestCase { - override val authenticationResponse = OptionT.none[IO, AuthUser] + override val authenticationResponse = IO.pure(MaybeAuthUser.noUser) val projectId = projectIds.generateOne val request = Request[IO](Method.POST, uri"/projects" / projectId / "webhooks" / "validation") @@ -199,8 +203,9 @@ class MicroserviceRoutesSpec private trait TestCase { - val authUser = authUsers.generateOne - val authenticationResponse = OptionT.some[IO](authUser) + val authUser = authUsers.generateOne + + val authenticationResponse = IO.pure(MaybeAuthUser(authUser)) val webhookEventsEndpoint = mock[webhookevents.Endpoint[IO]] val hookCreationEndpoint = mock[HookCreationEndpoint[IO]] val hookDeletionEndpoint = mock[HookDeletionEndpoint[IO]] @@ -214,7 +219,7 @@ class MicroserviceRoutesSpec hookValidationEndpoint, hookDeletionEndpoint, eventStatusEndpoint, - givenAuthMiddleware(returning = authenticationResponse), + givenAuthIfNeededMiddleware(returning = authenticationResponse), routesMetrics, versionRoutes ).routes.map(_.or(notAvailableResponse)) diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala index b5332fe0a1..6cbea02ae9 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala @@ -45,105 +45,127 @@ import io.renku.webhookservice.hookvalidation.HookValidator.NoAccessTokenExcepti import org.http4s.MediaType.application import org.http4s.Status._ import org.http4s.headers.`Content-Type` +import org.scalacheck.Gen import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers with IOSpec { +class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers with IOSpec with ScalaCheckPropertyChecks { "fetchProcessingStatus" should { "return OK with the status info if webhook for the project exists" in new TestCase { + forAll(authUserGen) { authUser => + logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) - givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) + val statusInfo = statusInfos.generateOne + givenStatusInfoFinding(projectId, returning = IO.pure(statusInfo.some)) - val statusInfo = statusInfos.generateOne - givenStatusInfoFinding(projectId, returning = IO.pure(statusInfo.some)) + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + response.status shouldBe Ok + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe statusInfo.asJson - response.status shouldBe Ok - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe statusInfo.asJson - - logger.loggedOnly( - Warn(s"Finding status info for project '$projectId' finished${executionTimeRecorder.executionTimeInfo}") - ) + logger.loggedOnly( + Warn(s"Finding status info for project '$projectId' finished${executionTimeRecorder.executionTimeInfo}") + ) + } } "return OK with activated = false if the webhook does not exist" in new TestCase { + forAll(authUserGen) { authUser => + logger.reset() + givenHookValidation(projectId, authUser, returning = HookMissing.pure[IO]) - givenHookValidation(projectId, authUser, returning = HookMissing.pure[IO]) - - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - response.status shouldBe Ok - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe StatusInfo.NotActivated.asJson + response.status shouldBe Ok + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe StatusInfo.NotActivated.asJson + } } "return NOT_FOUND if no Access Token found for the project" in new TestCase { - - val exception = NoAccessTokenException("error") - givenHookValidation(projectId, authUser, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) - - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - - response.status shouldBe NotFound - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe InfoMessage("Info about project cannot be found").asJson + forAll(authUserGen) { authUser => + logger.reset() + val exception = NoAccessTokenException("error") + givenHookValidation(projectId, + authUser, + returning = exception.raiseError[IO, HookValidator.HookValidationResult] + ) + + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + + response.status shouldBe NotFound + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe InfoMessage("Info about project cannot be found").asJson + } } "return INTERNAL_SERVER_ERROR when checking if the webhook exists fails" in new TestCase { - - val exception = exceptions.generateOne - givenHookValidation(projectId, authUser, returning = exception.raiseError[IO, HookValidator.HookValidationResult]) - - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - - response.status shouldBe InternalServerError - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson - - logger.logged(Error(statusInfoFindingErrorMessage, exception)) + forAll(authUserGen) { authUser => + logger.reset() + val exception = exceptions.generateOne + givenHookValidation(projectId, + authUser, + returning = exception.raiseError[IO, HookValidator.HookValidationResult] + ) + + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + + response.status shouldBe InternalServerError + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson + + logger.logged(Error(statusInfoFindingErrorMessage, exception)) + } } "return INTERNAL_SERVER_ERROR when finding status returns a failure" in new TestCase { + forAll(authUserGen) { authUser => + logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) - givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) + val exception = exceptions.generateOne + givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) - val exception = exceptions.generateOne - givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + response.status shouldBe InternalServerError + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson - response.status shouldBe InternalServerError - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson - - logger.logged(Error(statusInfoFindingErrorMessage, exception)) + logger.logged(Error(statusInfoFindingErrorMessage, exception)) + } } "return INTERNAL_SERVER_ERROR when finding status info fails" in new TestCase { + forAll(authUserGen) { authUser => + logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) - givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) - - val exception = exceptions.generateOne - givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) + val exception = exceptions.generateOne + givenStatusInfoFinding(projectId, returning = IO.raiseError(exception)) - val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() - response.status shouldBe InternalServerError - response.contentType shouldBe Some(`Content-Type`(application.json)) - response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson + response.status shouldBe InternalServerError + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe ErrorMessage(statusInfoFindingErrorMessage).asJson - logger.logged(Error(statusInfoFindingErrorMessage, exception)) + logger.logged(Error(statusInfoFindingErrorMessage, exception)) + } } } private trait TestCase { - val authUser = authUsers.generateOne + val authUserGen: Gen[Option[AuthUser]] = + Gen.option(authUsers) + val projectId = projectIds.generateOne private val hookValidator = mock[HookValidator[IO]] @@ -155,12 +177,12 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit lazy val statusInfoFindingErrorMessage = show"Finding status info for project '$projectId' failed" def givenHookValidation(projectId: projects.GitLabId, - authUser: AuthUser, + authUser: Option[AuthUser], returning: IO[HookValidator.HookValidationResult] ) = (hookValidator .validateHook(_: GitLabId, _: Option[AccessToken])) - .expects(projectId, authUser.accessToken.some) + .expects(projectId, authUser.map(_.accessToken)) .returning(returning) def givenStatusInfoFinding(projectId: projects.GitLabId, returning: IO[Option[StatusInfo]]) = From da8a909fa959ba2ca04e9823a1f1dd9f4050acdf Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Sat, 29 Apr 2023 16:08:11 +0200 Subject: [PATCH 24/28] chore: COMMIT_SYNC_REQUEST to be sent when no status info is found (#1452) --- .../CommitSyncRequestSender.scala | 69 ------------- .../ProjectInfoFinder.scala | 16 ++-- .../webhookservice/eventstatus/Endpoint.scala | 55 ++++++----- .../eventstatus/StatusInfo.scala | 2 + .../hookcreation/HookCreator.scala | 37 +++---- .../scala/io/renku/webhookservice/model.scala | 12 +-- .../webhookevents/Endpoint.scala | 28 +++--- .../CommitSyncRequestSenderSpec.scala | 96 ------------------- .../ProjectInfoFinderSpec.scala | 23 +++-- .../WebhookServiceGenerators.scala | 9 -- .../eventstatus/EndpointSpec.scala | 59 +++++++++++- .../hookcreation/HookCreatorSpec.scala | 21 ++-- .../webhookevents/EndpointSpec.scala | 30 +++--- 13 files changed, 167 insertions(+), 290 deletions(-) delete mode 100644 webhook-service/src/main/scala/io/renku/webhookservice/CommitSyncRequestSender.scala rename webhook-service/src/main/scala/io/renku/webhookservice/{hookcreation => }/ProjectInfoFinder.scala (85%) delete mode 100644 webhook-service/src/test/scala/io/renku/webhookservice/CommitSyncRequestSenderSpec.scala rename webhook-service/src/test/scala/io/renku/webhookservice/{hookcreation => }/ProjectInfoFinderSpec.scala (91%) diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/CommitSyncRequestSender.scala b/webhook-service/src/main/scala/io/renku/webhookservice/CommitSyncRequestSender.scala deleted file mode 100644 index 464139e158..0000000000 --- a/webhook-service/src/main/scala/io/renku/webhookservice/CommitSyncRequestSender.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2023 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.webhookservice - -import cats.MonadThrow -import cats.effect.Async -import cats.syntax.all._ -import io.renku.events.{CategoryName, EventRequestContent} -import io.renku.events.producers.EventSender -import io.renku.graph.config.EventLogUrl -import io.renku.metrics.MetricsRegistry -import io.renku.webhookservice.model.CommitSyncRequest -import org.typelevel.log4cats.Logger - -private trait CommitSyncRequestSender[F[_]] { - def sendCommitSyncRequest(commitSyncRequest: CommitSyncRequest, processName: String): F[Unit] -} - -private class CommitSyncRequestSenderImpl[F[_]: MonadThrow: Logger](eventSender: EventSender[F]) - extends CommitSyncRequestSender[F] { - - import eventSender._ - import io.circe.Encoder - import io.circe.literal._ - import io.circe.syntax._ - - def sendCommitSyncRequest(syncRequest: CommitSyncRequest, processName: String): F[Unit] = - sendEvent( - EventRequestContent.NoPayload(syncRequest.asJson), - EventSender.EventContext(CategoryName("COMMIT_SYNC_REQUEST"), - show"$processName - sending COMMIT_SYNC_REQUEST for ${syncRequest.project} failed" - ) - ) >> logInfo(processName)(syncRequest) - - private implicit lazy val entityEncoder: Encoder[CommitSyncRequest] = Encoder.instance[CommitSyncRequest] { event => - json"""{ - "categoryName": "COMMIT_SYNC_REQUEST", - "project": { - "id": ${event.project.id.value}, - "path": ${event.project.path.value} - } - }""" - } - - private def logInfo(processName: String): CommitSyncRequest => F[Unit] = { case CommitSyncRequest(project) => - Logger[F].info(show"$processName - COMMIT_SYNC_REQUEST sent for $project") - } -} - -private object CommitSyncRequestSender { - def apply[F[_]: Async: Logger: MetricsRegistry]: F[CommitSyncRequestSender[F]] = - EventSender[F](EventLogUrl).map(new CommitSyncRequestSenderImpl(_)) -} diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinder.scala b/webhook-service/src/main/scala/io/renku/webhookservice/ProjectInfoFinder.scala similarity index 85% rename from webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinder.scala rename to webhook-service/src/main/scala/io/renku/webhookservice/ProjectInfoFinder.scala index 8795ff30cc..957c88e7b8 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinder.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/ProjectInfoFinder.scala @@ -16,15 +16,17 @@ * limitations under the License. */ -package io.renku.webhookservice.hookcreation +package io.renku.webhookservice -import cats.effect.{Async, MonadCancelThrow} +import cats.MonadThrow +import cats.effect.Async import cats.syntax.all._ import eu.timepit.refined.auto._ +import io.renku.events.consumers.Project import io.renku.graph.model.projects import io.renku.http.client.{AccessToken, GitLabClient} -import io.renku.webhookservice.model.Project -import org.http4s.implicits.http4sLiteralsSyntax +import io.renku.http.tinytypes.TinyTypeURIEncoder._ +import org.http4s.implicits._ import org.typelevel.log4cats.Logger private trait ProjectInfoFinder[F[_]] { @@ -36,16 +38,16 @@ private class ProjectInfoFinderImpl[F[_]: Async: GitLabClient: Logger] extends P import io.circe._ import io.renku.http.client.RestClientError.UnauthorizedException import io.renku.tinytypes.json.TinyTypeDecoders._ - import org.http4s._ import org.http4s.Status.{Ok, Unauthorized} + import org.http4s._ import org.http4s.circe._ def findProjectInfo(projectId: projects.GitLabId)(implicit maybeAccessToken: Option[AccessToken]): F[Project] = - GitLabClient[F].get(uri"projects" / projectId.show, "single-project")(mapResponse) + GitLabClient[F].get(uri"projects" / projectId, "single-project")(mapResponse) private lazy val mapResponse: PartialFunction[(Status, Request[F], Response[F]), F[Project]] = { case (Ok, _, response) => response.as[Project] - case (Unauthorized, _, _) => MonadCancelThrow[F].raiseError(UnauthorizedException) + case (Unauthorized, _, _) => MonadThrow[F].raiseError(UnauthorizedException) } private implicit lazy val projectEntityDecoder: EntityDecoder[F, Project] = { diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala index 8cdeeeaf5c..33a012d2a9 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/Endpoint.scala @@ -16,20 +16,24 @@ * limitations under the License. */ -package io.renku.webhookservice.eventstatus +package io.renku.webhookservice +package eventstatus import cats.MonadThrow import cats.data.EitherT import cats.effect._ import cats.syntax.all._ import io.circe.syntax._ +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.projects import io.renku.graph.model.projects.GitLabId -import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.http.ErrorMessage._ import io.renku.http.client.GitLabClient import io.renku.http.server.security.model.AuthUser +import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.logging.ExecutionTimeRecorder +import io.renku.metrics.MetricsRegistry import io.renku.webhookservice.hookvalidation import io.renku.webhookservice.hookvalidation.HookValidator import io.renku.webhookservice.hookvalidation.HookValidator.{HookValidationResult, NoAccessTokenException} @@ -44,29 +48,35 @@ trait Endpoint[F[_]] { } private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( - hookValidator: HookValidator[F], - statusInfoFinder: StatusInfoFinder[F] + hookValidator: HookValidator[F], + statusInfoFinder: StatusInfoFinder[F], + projectInfoFinder: ProjectInfoFinder[F], + elClient: eventlog.api.events.Client[F] ) extends Http4sDsl[F] with Endpoint[F] { import HookValidationResult._ + import projectInfoFinder.findProjectInfo + import statusInfoFinder.findStatusInfo private val executionTimeRecorder = ExecutionTimeRecorder[F] import executionTimeRecorder._ def fetchProcessingStatus(projectId: GitLabId, authUser: Option[AuthUser]): F[Response[F]] = measureExecutionTime { validateHook(projectId, authUser) .semiflatMap { - case HookExists => findStatus(projectId) - case HookMissing => Ok(StatusInfo.NotActivated.asJson) + case HookMissing => + Ok(StatusInfo.NotActivated.asJson) + case HookExists => + findStatusInfo(projectId) + .flatTap(sendCommitSyncIfNone(projectId, authUser)) + .map(_.getOrElse(StatusInfo.webhookReady.widen)) + .flatMap(si => Ok(si.asJson)) } .merge .recoverWith(internalServerError(projectId)) } map logExecutionTime(withMessage = show"Finding status info for project '$projectId' finished") - private def validateHook( - projectId: GitLabId, - authUser: Option[AuthUser] - ): EitherT[F, Response[F], HookValidationResult] = + private def validateHook(projectId: GitLabId, authUser: Option[AuthUser]) = EitherT { hookValidator .validateHook(projectId, authUser.map(_.accessToken)) @@ -78,15 +88,12 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( case _: NoAccessTokenException => NotFound(InfoMessage("Info about project cannot be found")).map(_.asLeft) } - private def findStatus(projectId: GitLabId) = - EitherT( - statusInfoFinder - .findStatusInfo(projectId) - .map(_.getOrElse(StatusInfo.webhookReady)) - .attempt - ) - .biSemiflatMap(internalServerError(projectId), status => Ok(status.asJson)) - .merge + private def sendCommitSyncIfNone(projectId: GitLabId, authUser: Option[AuthUser]): Option[StatusInfo] => F[Unit] = { + case Some(_) => ().pure[F] + case None => + findProjectInfo(projectId)(authUser.map(_.accessToken)).map(CommitSyncRequest(_)) >>= + elClient.send + } private def internalServerError(projectId: projects.GitLabId): PartialFunction[Throwable, F[Response[F]]] = { case exception => @@ -96,10 +103,12 @@ private class EndpointImpl[F[_]: MonadThrow: Logger: ExecutionTimeRecorder]( } object Endpoint { - def apply[F[_]: Async: GitLabClient: ExecutionTimeRecorder: Logger]( + def apply[F[_]: Async: GitLabClient: ExecutionTimeRecorder: Logger: MetricsRegistry]( projectHookUrl: ProjectHookUrl ): F[Endpoint[F]] = for { - finder <- StatusInfoFinder[F] - hookValidator <- hookvalidation.HookValidator(projectHookUrl) - } yield new EndpointImpl[F](hookValidator, finder) + hookValidator <- hookvalidation.HookValidator(projectHookUrl) + statusInfoFinder <- StatusInfoFinder[F] + projectInfoFinder <- ProjectInfoFinder[F] + elClient <- eventlog.api.events.Client[F] + } yield new EndpointImpl[F](hookValidator, statusInfoFinder, projectInfoFinder, elClient) } diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala index 5dd73e066a..b99e90873a 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/eventstatus/StatusInfo.scala @@ -31,6 +31,8 @@ private sealed trait StatusInfo extends Product { def progress: Progress def fold[A](activated: StatusInfo.ActivatedProject => A, whenNotActivated: => A): A + + lazy val widen: StatusInfo = this } private object StatusInfo { diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/HookCreator.scala b/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/HookCreator.scala index a8fa90cccc..5e7edbe68f 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/HookCreator.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/hookcreation/HookCreator.scala @@ -18,14 +18,15 @@ package io.renku.webhookservice.hookcreation +import cats.Show import cats.effect._ import cats.syntax.all._ -import cats.Show +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.projects import io.renku.graph.model.projects.GitLabId import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.metrics.MetricsRegistry -import io.renku.webhookservice.{hookvalidation, CommitSyncRequestSender} import io.renku.webhookservice.crypto.HookTokenCrypto import io.renku.webhookservice.hookcreation.HookCreator.CreationResult import io.renku.webhookservice.hookcreation.ProjectHookCreator.ProjectHook @@ -33,6 +34,7 @@ import io.renku.webhookservice.hookvalidation.HookValidator import io.renku.webhookservice.hookvalidation.HookValidator.HookValidationResult import io.renku.webhookservice.model._ import io.renku.webhookservice.tokenrepository.AccessTokenAssociator +import io.renku.webhookservice.{ProjectInfoFinder, hookvalidation} import org.typelevel.log4cats.Logger private trait HookCreator[F[_]] { @@ -40,18 +42,17 @@ private trait HookCreator[F[_]] { } private class HookCreatorImpl[F[_]: Spawn: Logger]( - projectHookUrl: ProjectHookUrl, - projectHookValidator: HookValidator[F], - projectInfoFinder: ProjectInfoFinder[F], - hookTokenCrypto: HookTokenCrypto[F], - projectHookCreator: ProjectHookCreator[F], - accessTokenAssociator: AccessTokenAssociator[F], - commitSyncRequestSender: CommitSyncRequestSender[F] + projectHookUrl: ProjectHookUrl, + projectHookValidator: HookValidator[F], + projectInfoFinder: ProjectInfoFinder[F], + hookTokenCrypto: HookTokenCrypto[F], + projectHookCreator: ProjectHookCreator[F], + accessTokenAssociator: AccessTokenAssociator[F], + elClient: eventlog.api.events.Client[F] ) extends HookCreator[F] { import HookCreator.CreationResult._ import accessTokenAssociator._ - import commitSyncRequestSender._ import hookTokenCrypto._ import projectHookCreator.create import projectHookValidator._ @@ -88,8 +89,8 @@ private class HookCreatorImpl[F[_]: Spawn: Logger]( s"Hook creation - sending COMMIT_SYNC_REQUEST failure; finding project info for projectId $projectId failed" ) ) - .map(CommitSyncRequest) - .flatMap(sendCommitSyncRequest(_, "HookCreation")) + .map(CommitSyncRequest(_)) + .flatMap(elClient.send) private def loggingError(projectId: GitLabId): PartialFunction[Throwable, F[Unit]] = { case exception => Logger[F].error(exception)(s"Hook creation failed for project with id $projectId") @@ -114,11 +115,11 @@ private object HookCreator { def apply[F[_]: Async: GitLabClient: Logger: MetricsRegistry](projectHookUrl: ProjectHookUrl, hookTokenCrypto: HookTokenCrypto[F] ): F[HookCreator[F]] = for { - commitSyncRequestSender <- CommitSyncRequestSender[F] - hookValidator <- hookvalidation.HookValidator(projectHookUrl) - projectInfoFinder <- ProjectInfoFinder[F] - hookCreator <- ProjectHookCreator[F] - tokenAssociator <- AccessTokenAssociator[F] + hookValidator <- hookvalidation.HookValidator(projectHookUrl) + projectInfoFinder <- ProjectInfoFinder[F] + hookCreator <- ProjectHookCreator[F] + tokenAssociator <- AccessTokenAssociator[F] + elClient <- eventlog.api.events.Client[F] } yield new HookCreatorImpl[F]( projectHookUrl, hookValidator, @@ -126,6 +127,6 @@ private object HookCreator { hookTokenCrypto, hookCreator, tokenAssociator, - commitSyncRequestSender + elClient ) } diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/model.scala b/webhook-service/src/main/scala/io/renku/webhookservice/model.scala index a925361026..6df7914836 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/model.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/model.scala @@ -18,11 +18,10 @@ package io.renku.webhookservice +import cats.MonadThrow import cats.syntax.all._ -import cats.{MonadThrow, Show} import com.typesafe.config.{Config, ConfigFactory} import io.renku.config.ConfigLoader.{find, urlTinyTypeReader} -import io.renku.graph.model.projects import io.renku.graph.model.projects.GitLabId import io.renku.tinytypes.constraints.{Url, UrlOps} import io.renku.tinytypes.{StringTinyType, TinyTypeFactory, UrlTinyType} @@ -31,15 +30,6 @@ import pureconfig.ConfigReader object model { final case class HookToken(projectId: GitLabId) - final case class CommitSyncRequest(project: Project) - - final case class Project(id: projects.GitLabId, path: projects.Path) - object Project { - implicit lazy val show: Show[Project] = Show.show { case Project(id, path) => - s"projectId = $id, projectPath = $path" - } - } - final class ProjectHookUrl private (val value: String) extends AnyVal with StringTinyType object ProjectHookUrl { diff --git a/webhook-service/src/main/scala/io/renku/webhookservice/webhookevents/Endpoint.scala b/webhook-service/src/main/scala/io/renku/webhookservice/webhookevents/Endpoint.scala index 9883421b45..83a6b086e8 100644 --- a/webhook-service/src/main/scala/io/renku/webhookservice/webhookevents/Endpoint.scala +++ b/webhook-service/src/main/scala/io/renku/webhookservice/webhookevents/Endpoint.scala @@ -22,16 +22,18 @@ import cats.data.NonEmptyList import cats.effect._ import cats.syntax.all._ import io.circe.Decoder +import io.renku.events.consumers.Project +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.events.CommitId import io.renku.graph.model.projects.{GitLabId, Path} import io.renku.http.ErrorMessage._ import io.renku.http.client.RestClientError.UnauthorizedException import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.metrics.MetricsRegistry -import io.renku.webhookservice.CommitSyncRequestSender import io.renku.webhookservice.crypto.HookTokenCrypto import io.renku.webhookservice.crypto.HookTokenCrypto.SerializedHookToken -import io.renku.webhookservice.model.{CommitSyncRequest, HookToken, Project} +import io.renku.webhookservice.model.HookToken import org.http4s._ import org.http4s.circe._ import org.http4s.dsl.Http4sDsl @@ -45,13 +47,12 @@ trait Endpoint[F[_]] { } class EndpointImpl[F[_]: Concurrent: Logger]( - hookTokenCrypto: HookTokenCrypto[F], - commitSyncRequestSender: CommitSyncRequestSender[F] + hookTokenCrypto: HookTokenCrypto[F], + elClient: eventlog.api.events.Client[F] ) extends Http4sDsl[F] with Endpoint[F] { import Endpoint._ - import commitSyncRequestSender._ import hookTokenCrypto._ def processPushEvent(request: Request[F]): F[Response[F]] = { @@ -60,7 +61,7 @@ class EndpointImpl[F[_]: Concurrent: Logger]( authToken <- findHookToken(request) hookToken <- decrypt(authToken) recoverWith unauthorizedException _ <- validate(hookToken, commitSyncRequest) - _ <- Spawn[F].start(sendCommitSyncRequest(commitSyncRequest, "HookEvent")) + _ <- Spawn[F].start(elClient.send(commitSyncRequest)) _ <- logInfo(pushEvent) response <- Accepted(InfoMessage("Event accepted")) } yield response @@ -116,18 +117,15 @@ class EndpointImpl[F[_]: Concurrent: Logger]( object Endpoint { - def apply[F[_]: Async: Logger: MetricsRegistry]( - hookTokenCrypto: HookTokenCrypto[F] - ): F[Endpoint[F]] = for { - commitSyncRequestSender <- CommitSyncRequestSender[F] - } yield new EndpointImpl[F](hookTokenCrypto, commitSyncRequestSender) + def apply[F[_]: Async: Logger: MetricsRegistry](hookTokenCrypto: HookTokenCrypto[F]): F[Endpoint[F]] = + eventlog.api.events + .Client[F] + .map(new EndpointImpl[F](hookTokenCrypto, _)) private implicit val projectDecoder: Decoder[Project] = cursor => { import io.renku.tinytypes.json.TinyTypeDecoders._ - for { - id <- cursor.downField("id").as[GitLabId] - path <- cursor.downField("path_with_namespace").as[Path] - } yield Project(id, path) + (cursor.downField("id").as[GitLabId], cursor.downField("path_with_namespace").as[Path]) + .mapN(Project(_, _)) } implicit val pushEventDecoder: Decoder[(CommitId, CommitSyncRequest)] = cursor => { diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/CommitSyncRequestSenderSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/CommitSyncRequestSenderSpec.scala deleted file mode 100644 index c3bb8e972e..0000000000 --- a/webhook-service/src/test/scala/io/renku/webhookservice/CommitSyncRequestSenderSpec.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.webhookservice - -import cats.syntax.all._ -import io.circe.Encoder -import io.circe.literal._ -import io.circe.syntax._ -import io.renku.events.producers.EventSender -import io.renku.events.{CategoryName, EventRequestContent} -import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.{exceptions, nonEmptyStrings} -import io.renku.interpreters.TestLogger -import io.renku.interpreters.TestLogger.Level.Info -import io.renku.webhookservice.WebhookServiceGenerators._ -import io.renku.webhookservice.model.CommitSyncRequest -import org.http4s.Status._ -import org.scalamock.scalatest.MockFactory -import org.scalatest.matchers.should -import org.scalatest.wordspec.AnyWordSpec - -import scala.util.Try - -class CommitSyncRequestSenderSpec extends AnyWordSpec with MockFactory with should.Matchers { - - "send" should { - - s"succeed when delivering the event to the Event Log got $Accepted" in new TestCase { - - (eventSender - .sendEvent(_: EventRequestContent.NoPayload, _: EventSender.EventContext)) - .expects( - EventRequestContent.NoPayload(syncRequest.asJson), - EventSender.EventContext(CategoryName("COMMIT_SYNC_REQUEST"), - show"$processName - sending COMMIT_SYNC_REQUEST for ${syncRequest.project} failed" - ) - ) - .returning(().pure[Try]) - - requestSender.sendCommitSyncRequest(syncRequest, processName) shouldBe ().pure[Try] - - logger.loggedOnly(Info(show"$processName - COMMIT_SYNC_REQUEST sent for ${syncRequest.project}")) - } - - "fail when sending the event fails" in new TestCase { - val exception = exceptions.generateOne - (eventSender - .sendEvent(_: EventRequestContent.NoPayload, _: EventSender.EventContext)) - .expects( - EventRequestContent.NoPayload(syncRequest.asJson), - EventSender.EventContext(CategoryName("COMMIT_SYNC_REQUEST"), - show"$processName - sending COMMIT_SYNC_REQUEST for ${syncRequest.project} failed" - ) - ) - .returning(exception.raiseError[Try, Unit]) - - requestSender.sendCommitSyncRequest(syncRequest, processName) shouldBe exception.raiseError[Try, Unit] - } - } - - private trait TestCase { - - val processName = nonEmptyStrings().generateOne - val syncRequest = commitSyncRequests.generateOne - - val eventSender = mock[EventSender[Try]] - implicit val logger: TestLogger[Try] = TestLogger[Try]() - val requestSender = new CommitSyncRequestSenderImpl[Try](eventSender) - } - - private implicit lazy val eventEncoder: Encoder[CommitSyncRequest] = Encoder.instance[CommitSyncRequest] { event => - json"""{ - "categoryName": "COMMIT_SYNC_REQUEST", - "project": { - "id": ${event.project.id.value}, - "path": ${event.project.path.value} - } - }""" - } -} diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinderSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/ProjectInfoFinderSpec.scala similarity index 91% rename from webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinderSpec.scala rename to webhook-service/src/test/scala/io/renku/webhookservice/ProjectInfoFinderSpec.scala index f734e4d418..ea5587b517 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/ProjectInfoFinderSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/ProjectInfoFinderSpec.scala @@ -16,7 +16,7 @@ * limitations under the License. */ -package io.renku.webhookservice.hookcreation +package io.renku.webhookservice import cats.effect.IO import cats.syntax.all._ @@ -24,19 +24,20 @@ import eu.timepit.refined.api.Refined import eu.timepit.refined.auto._ import eu.timepit.refined.collection.NonEmpty import io.circe.literal._ +import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects +import io.renku.events.consumers.Project import io.renku.generators.CommonGraphGenerators._ import io.renku.generators.Generators.Implicits._ -import io.renku.http.client.{AccessToken, GitLabClient} import io.renku.http.client.RestClient.ResponseMappingF import io.renku.http.client.RestClientError.UnauthorizedException +import io.renku.http.client.{AccessToken, GitLabClient} +import io.renku.http.tinytypes.TinyTypeURIEncoder._ import io.renku.interpreters.TestLogger import io.renku.stubbing.ExternalServiceStubbing import io.renku.testtools.{GitLabClientTools, IOSpec} -import io.renku.webhookservice.WebhookServiceGenerators.projects -import io.renku.webhookservice.model.Project -import org.http4s.{Request, Response, Status, Uri} import org.http4s.circe.jsonEncoder -import org.http4s.implicits.http4sLiteralsSyntax +import org.http4s.implicits._ +import org.http4s.{Request, Response, Status, Uri} import org.scalacheck.Gen import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should @@ -89,11 +90,9 @@ class ProjectInfoFinderSpec } private trait TestCase { - val project = projects.generateOne - val projectId = project.id - val projectPath = project.path - - val uri: Uri = uri"projects" / projectId.show + val project = consumerProjects.generateOne + val projectId = project.id + val uri: Uri = uri"projects" / projectId val endpointName: String Refined NonEmpty = "single-project" implicit val maybeAccessToken: Option[AccessToken] = accessTokens.generateOption @@ -104,7 +103,7 @@ class ProjectInfoFinderSpec lazy val projectJson: String = json"""{ "id": $projectId, - "path_with_namespace": $projectPath + "path_with_namespace": ${project.path} }""".noSpaces lazy val mapResponse = captureMapping(gitLabClient)( diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/WebhookServiceGenerators.scala b/webhook-service/src/test/scala/io/renku/webhookservice/WebhookServiceGenerators.scala index 7aaa1e6812..5ac7d65f8a 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/WebhookServiceGenerators.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/WebhookServiceGenerators.scala @@ -28,15 +28,6 @@ import org.scalacheck.Gen object WebhookServiceGenerators { - implicit val commitSyncRequests: Gen[CommitSyncRequest] = for { - project <- projects - } yield CommitSyncRequest(project) - - implicit lazy val projects: Gen[Project] = for { - projectId <- projectIds - path <- projectPaths - } yield Project(projectId, path) - implicit val serializedHookTokens: Gen[SerializedHookToken] = nonEmptyStrings() map Refined.unsafeApply implicit val hookTokens: Gen[HookToken] = for { diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala index 6cbea02ae9..425a80ce93 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/eventstatus/EndpointSpec.scala @@ -23,9 +23,13 @@ import cats.effect.IO import cats.syntax.all._ import io.circe.Json import io.circe.syntax._ +import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects +import io.renku.events.consumers.Project import io.renku.generators.CommonGraphGenerators.authUsers import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators.exceptions +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.GraphModelGenerators.projectIds import io.renku.graph.model.projects import io.renku.graph.model.projects.GitLabId @@ -58,10 +62,11 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return OK with the status info if webhook for the project exists" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val statusInfo = statusInfos.generateOne - givenStatusInfoFinding(projectId, returning = IO.pure(statusInfo.some)) + givenStatusInfoFinding(projectId, returning = statusInfo.some.pure[IO]) val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() @@ -75,9 +80,35 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit } } + "send COMMIT_SYNC_REQUEST message and return OK with the status info " + + "if webhook for the project exists but no status info can be found" in new TestCase { + + val authUser = authUserGen.generateOne + + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) + + givenStatusInfoFinding(projectId, returning = None.pure[IO]) + + val project = consumerProjects.generateOne.copy(id = projectId) + givenProjectInfoFinding(projectId, authUser, returning = project.pure[IO]) + + givenCommitSyncRequestSend(project, returning = ().pure[IO]) + + val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() + + response.status shouldBe Ok + response.contentType shouldBe Some(`Content-Type`(application.json)) + response.as[Json].unsafeRunSync() shouldBe StatusInfo.webhookReady.asJson + + logger.loggedOnly( + Warn(s"Finding status info for project '$projectId' finished${executionTimeRecorder.executionTimeInfo}") + ) + } + "return OK with activated = false if the webhook does not exist" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + givenHookValidation(projectId, authUser, returning = HookMissing.pure[IO]) val response = endpoint.fetchProcessingStatus(projectId, authUser).unsafeRunSync() @@ -91,6 +122,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return NOT_FOUND if no Access Token found for the project" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + val exception = NoAccessTokenException("error") givenHookValidation(projectId, authUser, @@ -108,6 +140,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return INTERNAL_SERVER_ERROR when checking if the webhook exists fails" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + val exception = exceptions.generateOne givenHookValidation(projectId, authUser, @@ -124,9 +157,10 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit } } - "return INTERNAL_SERVER_ERROR when finding status returns a failure" in new TestCase { + "return INTERNAL_SERVER_ERROR when finding status info returns a failure" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne @@ -145,6 +179,7 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return INTERNAL_SERVER_ERROR when finding status info fails" in new TestCase { forAll(authUserGen) { authUser => logger.reset() + givenHookValidation(projectId, authUser, returning = HookExists.pure[IO]) val exception = exceptions.generateOne @@ -168,11 +203,13 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit val projectId = projectIds.generateOne - private val hookValidator = mock[HookValidator[IO]] - private val statusInfoFinder = mock[StatusInfoFinder[IO]] + private val hookValidator = mock[HookValidator[IO]] + private val statusInfoFinder = mock[StatusInfoFinder[IO]] + private val projectInfoFinder = mock[ProjectInfoFinder[IO]] + private val elClient = mock[eventlog.api.events.Client[IO]] implicit val logger: TestLogger[IO] = TestLogger[IO]() implicit val executionTimeRecorder: TestExecutionTimeRecorder[IO] = TestExecutionTimeRecorder[IO]() - val endpoint = new EndpointImpl[IO](hookValidator, statusInfoFinder) + val endpoint = new EndpointImpl[IO](hookValidator, statusInfoFinder, projectInfoFinder, elClient) lazy val statusInfoFindingErrorMessage = show"Finding status info for project '$projectId' failed" @@ -190,5 +227,17 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit .findStatusInfo(_: GitLabId)) .expects(projectId) .returning(returning) + + def givenProjectInfoFinding(projectId: projects.GitLabId, authUser: Option[AuthUser], returning: IO[Project]) = + (projectInfoFinder + .findProjectInfo(_: projects.GitLabId)(_: Option[AccessToken])) + .expects(projectId, authUser.map(_.accessToken)) + .returning(returning) + + def givenCommitSyncRequestSend(project: Project, returning: IO[Unit]) = + (elClient + .send(_: CommitSyncRequest)) + .expects(CommitSyncRequest(project)) + .returning(returning) } } diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/HookCreatorSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/HookCreatorSpec.scala index 14f74e5dee..e8d0b35ca7 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/HookCreatorSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/hookcreation/HookCreatorSpec.scala @@ -21,24 +21,28 @@ package io.renku.webhookservice.hookcreation import cats.effect.IO import cats.effect.std.Queue import cats.syntax.all._ +import io.renku.events.consumers.ConsumersModelGenerators.consumerProjects +import io.renku.events.consumers.Project import io.renku.generators.CommonGraphGenerators._ -import io.renku.generators.Generators._ import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators._ +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest import io.renku.graph.model.projects.GitLabId import io.renku.http.client.AccessToken import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.{Error, Info} import io.renku.testtools.IOSpec -import io.renku.webhookservice.CommitSyncRequestSender +import io.renku.webhookservice.ProjectInfoFinder import io.renku.webhookservice.WebhookServiceGenerators._ import io.renku.webhookservice.crypto.HookTokenCrypto import io.renku.webhookservice.crypto.HookTokenCrypto.SerializedHookToken import io.renku.webhookservice.hookcreation.HookCreator.CreationResult.{HookCreated, HookExisted} import io.renku.webhookservice.hookcreation.ProjectHookCreator.ProjectHook import io.renku.webhookservice.hookvalidation.HookValidator -import io.renku.webhookservice.hookvalidation.HookValidator.HookValidationResult.{HookExists, HookMissing} import io.renku.webhookservice.hookvalidation.HookValidator.HookValidationResult -import io.renku.webhookservice.model.{CommitSyncRequest, HookToken, Project} +import io.renku.webhookservice.hookvalidation.HookValidator.HookValidationResult.{HookExists, HookMissing} +import io.renku.webhookservice.model.HookToken import io.renku.webhookservice.tokenrepository.AccessTokenAssociator import org.scalamock.scalatest.MockFactory import org.scalatest.concurrent.Eventually @@ -171,7 +175,7 @@ class HookCreatorSpec extends AnyWordSpec with MockFactory with should.Matchers } private trait TestCase { - val projectInfo = projects.generateOne + val projectInfo = consumerProjects.generateOne val projectId = projectInfo.id val serializedHookToken = serializedHookTokens.generateOne val accessToken = accessTokens.generateOne @@ -188,9 +192,8 @@ class HookCreatorSpec extends AnyWordSpec with MockFactory with should.Matchers projectInfoFinderResponse.take.flatten } private val commitSyncRequestSenderResponse = Queue.bounded[IO, IO[Unit]](1).unsafeRunSync() - private val commitSyncRequestSender = new CommitSyncRequestSender[IO] { - override def sendCommitSyncRequest(commitSyncRequest: CommitSyncRequest, processName: String): IO[Unit] = - commitSyncRequestSenderResponse.take.flatten + private val elClient = new eventlog.api.events.Client[IO] { + override def send(event: CommitSyncRequest): IO[Unit] = commitSyncRequestSenderResponse.take.flatten } val hookCreation = new HookCreatorImpl[IO]( @@ -200,7 +203,7 @@ class HookCreatorSpec extends AnyWordSpec with MockFactory with should.Matchers hookTokenCrypto, projectHookCreator, accessTokenAssociator, - commitSyncRequestSender + elClient ) def givenHookValidation(returning: IO[HookValidationResult]) = diff --git a/webhook-service/src/test/scala/io/renku/webhookservice/webhookevents/EndpointSpec.scala b/webhook-service/src/test/scala/io/renku/webhookservice/webhookevents/EndpointSpec.scala index 9b64e1dce8..1b58a802ab 100644 --- a/webhook-service/src/test/scala/io/renku/webhookservice/webhookevents/EndpointSpec.scala +++ b/webhook-service/src/test/scala/io/renku/webhookservice/webhookevents/EndpointSpec.scala @@ -25,6 +25,9 @@ import io.circe.literal._ import io.circe.syntax._ import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators._ +import io.renku.graph.eventlog +import io.renku.graph.eventlog.api.events.CommitSyncRequest +import io.renku.graph.eventlog.api.events.Generators.commitSyncRequests import io.renku.graph.model.EventsGenerators.commitIds import io.renku.graph.model.GraphModelGenerators.projectIds import io.renku.graph.model.events.CommitId @@ -35,11 +38,9 @@ import io.renku.http.{ErrorMessage, InfoMessage} import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.Info import io.renku.testtools.IOSpec -import io.renku.webhookservice.CommitSyncRequestSender -import io.renku.webhookservice.WebhookServiceGenerators._ import io.renku.webhookservice.crypto.HookTokenCrypto import io.renku.webhookservice.crypto.HookTokenCrypto.SerializedHookToken -import io.renku.webhookservice.model.{CommitSyncRequest, HookToken} +import io.renku.webhookservice.model.HookToken import org.http4s.Status._ import org.http4s._ import org.http4s.headers.`Content-Type` @@ -54,9 +55,9 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit "return ACCEPTED for valid push event payload which are accepted" in new TestCase { - (commitSyncRequestSender - .sendCommitSyncRequest(_: CommitSyncRequest, _: String)) - .expects(syncRequest, "HookEvent") + (elClient + .send(_: CommitSyncRequest)) + .expects(syncRequest) .returning(().pure[IO]) expectDecryptionOf(serializedHookToken, returning = HookToken(syncRequest.project.id)) @@ -150,16 +151,13 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit val serializedHookToken = nonEmptyStrings().map { SerializedHookToken .from(_) - .fold( - exception => throw exception, - identity - ) + .fold(exception => throw exception, identity) }.generateOne implicit val logger: TestLogger[IO] = TestLogger[IO]() - val commitSyncRequestSender = mock[CommitSyncRequestSender[IO]] - val hookTokenCrypto = mock[HookTokenCrypto[IO]] - val endpoint = new EndpointImpl[IO](hookTokenCrypto, commitSyncRequestSender) + val elClient = mock[eventlog.api.events.Client[IO]] + val hookTokenCrypto = mock[HookTokenCrypto[IO]] + val endpoint = new EndpointImpl[IO](hookTokenCrypto, elClient) def expectDecryptionOf(hookAuthToken: SerializedHookToken, returning: HookToken) = (hookTokenCrypto @@ -170,10 +168,10 @@ class EndpointSpec extends AnyWordSpec with MockFactory with should.Matchers wit private def pushEventPayloadFrom(commitId: CommitId, syncRequest: CommitSyncRequest) = json"""{ - "after": ${commitId.value}, + "after": $commitId, "project": { - "id": ${syncRequest.project.id.value}, - "path_with_namespace": ${syncRequest.project.path.value} + "id": ${syncRequest.project.id}, + "path_with_namespace": ${syncRequest.project.path} } }""" } From de133b4918fc0bee7437548e8720e97088aa185c Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Sat, 29 Apr 2023 16:08:37 +0200 Subject: [PATCH 25/28] chore: Update sentry-logback from 6.18.0 to 6.18.1 (#1453) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b0e977fd9f..d581152cf9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -30,7 +30,7 @@ object Dependencies { val scalamock = "5.2.0" val scalatest = "3.2.15" val scalatestScalacheck = "3.2.2.0" - val sentryLogback = "6.18.0" + val sentryLogback = "6.18.1" val skunk = "0.5.1" val swaggerParser = "2.1.13" val testContainersScala = "0.40.15" From 866ce7b91bf0b70961ede20eff35b64328ea2f9b Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Sat, 29 Apr 2023 16:09:22 +0200 Subject: [PATCH 26/28] chore: Update cats-effect from 3.4.9 to 3.4.10 (#1454) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d581152cf9..a268e127a4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ object Dependencies { object V { val ammonite = "2.4.1" val catsCore = "2.9.0" - val catsEffect = "3.4.9" + val catsEffect = "3.4.10" val circeCore = "0.14.5" val circeGenericExtras = "0.14.3" val circeOptics = "0.14.1" From 7148e78302d243b26b9f17bfe7ae84e56c0561d7 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 1 May 2023 20:07:14 +0200 Subject: [PATCH 27/28] chore: cli upgraded to 2.4.0rc2 --- .../io/renku/graph/model/infraSpec.scala | 53 +++++++++++++++---- triples-generator/Dockerfile | 2 +- .../src/main/resources/application.conf | 2 +- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/renku-model/src/test/scala/io/renku/graph/model/infraSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/infraSpec.scala index 902dcba181..1fbacaaadc 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/infraSpec.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/infraSpec.scala @@ -18,55 +18,62 @@ package io.renku.graph.model -import GraphModelGenerators.cliVersions import cats.syntax.all._ import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators._ +import io.renku.graph.model.GraphModelGenerators.cliVersions import io.renku.graph.model.versions.CliVersion import org.scalacheck.Gen +import org.scalatest.EitherValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -class GitLabUrlSpec extends AnyWordSpec with ScalaCheckPropertyChecks with should.Matchers { +class GitLabUrlSpec extends AnyWordSpec with ScalaCheckPropertyChecks with should.Matchers with EitherValues { "from" should { "instantiate GitLabUrl when valid url is given" in { forAll(httpUrls()) { path => - GitLabUrl.from(path).map(_.value) shouldBe path.asRight + GitLabUrl.from(path).map(_.value).value shouldBe path } } "instantiate GitLabUrl when valid url is given with a slash at the end" in { val path = httpUrls().generateOne - GitLabUrl.from(s"$path/").map(_.value) shouldBe path.asRight + GitLabUrl.from(s"$path/").map(_.value).value shouldBe path } "fail instantiation for invalid url" in { val value = nonEmptyStrings().generateOne - val Left(ex) = GitLabUrl.from(value).map(_.value) + val res = GitLabUrl.from(value).map(_.value) - ex shouldBe an[IllegalArgumentException] - ex.getMessage should include(s"Cannot instantiate ${GitLabUrl.typeName} with '$value'") + res.left.value shouldBe an[IllegalArgumentException] + res.left.value.getMessage should include(s"Cannot instantiate ${GitLabUrl.typeName} with '$value'") } } } -class CliVersionSpec extends AnyWordSpec with ScalaCheckPropertyChecks with should.Matchers { +class CliVersionSpec extends AnyWordSpec with ScalaCheckPropertyChecks with should.Matchers with EitherValues { "from" should { "return Right for a valid released version" in { forAll(semanticVersions) { version => - CliVersion.from(version).map(_.show) shouldBe version.asRight + CliVersion.from(version).map(_.show).value shouldBe version + } + } + + "return Right for a valid rc version" in { + forAll(rcVersions()) { version => + CliVersion.from(version).map(_.show).value shouldBe version } } "return Right for a valid dev version" in { forAll(devVersions()) { version => - CliVersion.from(version).map(_.show) shouldBe version.asRight + CliVersion.from(version).map(_.show).value shouldBe version } } } @@ -83,6 +90,16 @@ class CliVersionSpec extends AnyWordSpec with ScalaCheckPropertyChecks with shou } } + "extract the relevant version parts from an rc version" in { + forAll(rcVersions()) { version => + val cli = CliVersion(version) + val s"$major.$minor.$bugfix" = version + cli.major shouldBe major + cli.minor shouldBe minor + cli.bugfix shouldBe bugfix + } + } + "extract the relevant version parts from a dev version" in { forAll(devVersions()) { version => val cli = CliVersion(version) @@ -136,6 +153,18 @@ class CliVersionSpec extends AnyWordSpec with ScalaCheckPropertyChecks with shou } } + "consider the rc part if all majors, minors and bugfix are the same" in { + val semanticVersion = semanticVersions.generateOne + forAll(rcVersions(fixed(semanticVersion)), rcVersions(fixed(semanticVersion))) { (version1, version2) => + whenever(version1 != version2) { + val list = List(version1, version2) + + if ((version1.show compareTo version2.show) < 0) list.sorted shouldBe list + else list.sorted shouldBe list.reverse + } + } + } + "consider the dev part if all majors, minors and bugfix are the same" in { val semanticVersion = semanticVersions.generateOne forAll(devVersions(fixed(semanticVersion)), devVersions(fixed(semanticVersion))) { (version1, version2) => @@ -149,6 +178,10 @@ class CliVersionSpec extends AnyWordSpec with ScalaCheckPropertyChecks with shou } } + private def rcVersions(semanticVersionsGen: Gen[String] = semanticVersions) = + (semanticVersionsGen, positiveInts(999)) + .mapN((version, rcNumber) => s"${version}rc$rcNumber") + private def devVersions(semanticVersionsGen: Gen[String] = semanticVersions) = (semanticVersionsGen, positiveInts(999), shas.map(_.take(7))) .mapN((version, commitsNumber, sha) => s"$version.dev$commitsNumber+g$sha") diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile index 544a35e455..42d0b8ccca 100644 --- a/triples-generator/Dockerfile +++ b/triples-generator/Dockerfile @@ -45,7 +45,7 @@ USER tguser ENV PATH=$PATH:/home/tguser/.local/bin # Installing Renku -RUN python3 -m pip install 'renku==2.3.2' 'sentry-sdk==1.5.11' +RUN python3 -m pip install 'renku==2.4.0rc2' 'sentry-sdk==1.5.11' RUN git config --global user.name 'renku' && \ git config --global user.email 'renku@renkulab.io' && \ diff --git a/triples-generator/src/main/resources/application.conf b/triples-generator/src/main/resources/application.conf index 05d5713bd7..9e88e8c39c 100644 --- a/triples-generator/src/main/resources/application.conf +++ b/triples-generator/src/main/resources/application.conf @@ -19,7 +19,7 @@ triples-generation = "renku-log" # Defines expected version of the renku cli and the schema that is generated by the CLI. compatibility { # The expected version of CLI used by TS. - cli-version = "2.3.2" + cli-version = "2.4.0rc2" # The expected version of the schema as returned by CLI. schema-version = "10" From ec608e233010c4584c4771e4932d27644695c9f6 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 2 May 2023 10:27:36 +0200 Subject: [PATCH 28/28] feat: viewing dates to be deduplicated on persist (#1455) --- build.sbt | 2 +- .../collector/datasets/DSInfoFinder.scala | 6 +- .../collector/datasets/EventUploader.scala | 17 +- .../viewings/collector/persons/Encoder.scala | 4 +- .../PersonViewedDatasetDeduplicator.scala | 73 ++++++++ .../PersonViewedDatasetPersister.scala | 28 +++- .../PersonViewedProjectDeduplicator.scala | 73 ++++++++ .../PersonViewedProjectPersister.scala | 24 ++- .../projects/EventDeduplicator.scala | 72 ++++++++ .../projects/activated/EventPersister.scala | 50 +++--- .../projects/viewed/EventPersister.scala | 27 +-- .../entities/viewings/EntityViewings.scala | 44 +++++ .../collector/persons/Generators.scala | 53 ++++++ .../PersonViewedDatasetDeduplicatorSpec.scala | 144 ++++++++++++++++ .../PersonViewedDatasetPersisterSpec.scala | 117 +++++-------- .../PersonViewedDatasetSpecTools.scala | 86 ++++++++++ .../PersonViewedProjectDeduplicatorSpec.scala | 144 ++++++++++++++++ .../PersonViewedProjectPersisterSpec.scala | 99 ++++------- .../PersonViewedProjectSpecTools.scala | 86 ++++++++++ .../projects/EventDeduplicatorSpec.scala | 125 ++++++++++++++ .../projects/EventPersisterSpecTools.scala | 58 +++++++ .../activated/EventPersisterSpec.scala | 45 +++-- .../projects/viewed/EventPersisterSpec.scala | 49 +++--- .../projects/ViewingRemoverSpec.scala | 15 +- .../migrations/Migrations.scala | 6 +- .../PersonViewedEntityDeduplicator.scala | 65 +++++++ .../ProjectDateViewedDeduplicator.scala | 65 +++++++ .../PersonViewedEntityDeduplicatorSpec.scala | 158 ++++++++++++++++++ .../ProjectDateViewedDeduplicatorSpec.scala | 75 +++++++++ 29 files changed, 1552 insertions(+), 258 deletions(-) create mode 100644 entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicator.scala create mode 100644 entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicator.scala create mode 100644 entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/EventDeduplicator.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/EntityViewings.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/Generators.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicatorSpec.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetSpecTools.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicatorSpec.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectSpecTools.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventDeduplicatorSpec.scala create mode 100644 entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventPersisterSpecTools.scala create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicator.scala create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicator.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicatorSpec.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicatorSpec.scala diff --git a/build.sbt b/build.sbt index 6667c4630e..03c0501eec 100644 --- a/build.sbt +++ b/build.sbt @@ -168,7 +168,7 @@ lazy val triplesGenerator = project .dependsOn( triplesGeneratorApi % "compile->compile; test->test", entitiesSearch, - entitiesViewingsCollector + entitiesViewingsCollector % "compile->compile; test->test" ) .enablePlugins( JavaAppPackaging, diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/DSInfoFinder.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/DSInfoFinder.scala index dd71bc965b..38c98f730d 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/DSInfoFinder.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/DSInfoFinder.scala @@ -35,8 +35,12 @@ private trait DSInfoFinder[F[_]] { private final case class DSInfo(projectPath: projects.Path, dataset: Dataset) private object DSInfoFinder { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[DSInfoFinder[F]] = - ProjectsConnectionConfig[F]().map(TSClient[F](_)).map(new DSInfoFinderImpl[F](_)) + ProjectsConnectionConfig[F]().map(TSClient[F](_)).map(apply(_)) + + def apply[F[_]: MonadThrow](tsClient: TSClient[F]): DSInfoFinder[F] = + new DSInfoFinderImpl[F](tsClient) } private class DSInfoFinderImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extends DSInfoFinder[F] { diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/EventUploader.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/EventUploader.scala index 5ced87af65..7a840c3d43 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/EventUploader.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/datasets/EventUploader.scala @@ -19,25 +19,32 @@ package io.renku.entities.viewings.collector package datasets -import cats.effect.Async -import cats.syntax.all._ import cats.MonadThrow import cats.data.OptionT +import cats.effect.Async +import cats.syntax.all._ import io.renku.entities.viewings.collector.persons.{GLUserViewedDataset, PersonViewedDatasetPersister} import io.renku.entities.viewings.collector.projects.viewed.EventPersister import io.renku.graph.model.projects import io.renku.triplesgenerator.api.events.{DatasetViewedEvent, ProjectViewedEvent, UserId} -import io.renku.triplesstore.SparqlQueryTimeRecorder +import io.renku.triplesstore.{SparqlQueryTimeRecorder, TSClient} import org.typelevel.log4cats.Logger -private trait EventUploader[F[_]] { +private[viewings] trait EventUploader[F[_]] { def upload(event: DatasetViewedEvent): F[Unit] } -private object EventUploader { +private[viewings] object EventUploader { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[EventUploader[F]] = (DSInfoFinder[F], EventPersister[F], PersonViewedDatasetPersister[F]) .mapN(new EventUploaderImpl[F](_, _, _)) + + def apply[F[_]: MonadThrow](tsClient: TSClient[F]): EventUploader[F] = + new EventUploaderImpl[F](DSInfoFinder[F](tsClient), + EventPersister[F](tsClient), + PersonViewedDatasetPersister[F](tsClient) + ) } private class EventUploaderImpl[F[_]: MonadThrow]( diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/Encoder.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/Encoder.scala index ec38f9c484..fc56dbad9f 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/Encoder.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/Encoder.scala @@ -49,7 +49,7 @@ private object Encoder { private lazy val viewedProjectEncoder: JsonLDEncoder[PersonViewedProject] = JsonLDEncoder.instance { case PersonViewedProject(userId, Project(id, path), date) => JsonLD.entity( - EntityId.of(s"$userId/$path"), + EntityId of s"$userId/$path", EntityTypes of PersonViewedProjectOntology.classType, PersonViewedProjectOntology.projectProperty -> id.asJsonLD, PersonViewedProjectOntology.dateViewedProperty.id -> date.asJsonLD @@ -68,7 +68,7 @@ private object Encoder { private lazy val viewedDatasetEncoder: JsonLDEncoder[PersonViewedDataset] = JsonLDEncoder.instance { case PersonViewedDataset(userId, Dataset(id, identifier), date) => JsonLD.entity( - EntityId.of(s"$userId/datasets/$identifier"), + EntityId of s"$userId/datasets/$identifier", EntityTypes of PersonViewedProjectOntology.classType, PersonViewedDatasetOntology.datasetProperty -> id.asJsonLD, PersonViewedDatasetOntology.dateViewedProperty.id -> date.asJsonLD diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicator.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicator.scala new file mode 100644 index 0000000000..d2ce2f367d --- /dev/null +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicator.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import cats.syntax.all._ +import io.renku.graph.model.{datasets, persons} +import io.renku.triplesstore.TSClient + +private trait PersonViewedDatasetDeduplicator[F[_]] { + def deduplicate(personId: persons.ResourceId, datasetId: datasets.ResourceId): F[Unit] +} + +private object PersonViewedDatasetDeduplicator { + def apply[F[_]](tsClient: TSClient[F]): PersonViewedDatasetDeduplicator[F] = + new PersonViewedDatasetDeduplicatorImpl[F](tsClient) +} + +private class PersonViewedDatasetDeduplicatorImpl[F[_]](tsClient: TSClient[F]) + extends PersonViewedDatasetDeduplicator[F] { + + import eu.timepit.refined.auto._ + import io.renku.graph.model.GraphClass + import io.renku.graph.model.Schemas._ + import io.renku.jsonld.syntax._ + import io.renku.triplesstore.SparqlQuery + import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore.client.syntax._ + import tsClient.updateWithNoResult + + override def deduplicate(personId: persons.ResourceId, datasetId: datasets.ResourceId): F[Unit] = updateWithNoResult( + SparqlQuery.ofUnsafe( + show"${GraphClass.PersonViewings}: deduplicate dataset viewings", + Prefixes of renku -> "renku", + sparql"""|DELETE { + | GRAPH ${GraphClass.PersonViewings.id} { ?viewingId renku:dateViewed ?date } + |} + |WHERE { + | GRAPH ${GraphClass.PersonViewings.id} { + | BIND (${personId.asEntityId} AS ?personId) + | { + | SELECT ?viewingId (MAX(?date) AS ?maxDate) + | WHERE { + | ?personId renku:viewedDataset ?viewingId. + | ?viewingId renku:dataset ${datasetId.asEntityId}; + | renku:dateViewed ?date. + | } + | GROUP BY ?viewingId + | HAVING (COUNT(?date) > 1) + | } + | ?viewingId renku:dateViewed ?date. + | FILTER (?date != ?maxDate) + | } + |} + |""".stripMargin + ) + ) +} diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersister.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersister.scala index 90c7b1bd0e..37352f26bb 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersister.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersister.scala @@ -37,25 +37,31 @@ private[viewings] object PersonViewedDatasetPersister { .map(apply[F](_)) def apply[F[_]: MonadThrow](tsClient: TSClient[F]): PersonViewedDatasetPersister[F] = - new PersonViewedDatasetPersisterImpl[F](tsClient, PersonFinder(tsClient)) + new PersonViewedDatasetPersisterImpl[F](tsClient, + PersonFinder(tsClient), + PersonViewedDatasetDeduplicator[F](tsClient) + ) } -private class PersonViewedDatasetPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F], personFinder: PersonFinder[F]) - extends PersonViewedDatasetPersister[F] { +private class PersonViewedDatasetPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F], + personFinder: PersonFinder[F], + deduplicator: PersonViewedDatasetDeduplicator[F] +) extends PersonViewedDatasetPersister[F] { + import Encoder._ import cats.syntax.all._ + import deduplicator._ import eu.timepit.refined.auto._ import io.circe.Decoder import io.renku.graph.model.GraphClass import io.renku.graph.model.Schemas._ import io.renku.jsonld.syntax._ - import io.renku.triplesstore.{ResultsDecoder, SparqlQuery} import io.renku.triplesstore.ResultsDecoder._ - import io.renku.triplesstore.client.syntax._ import io.renku.triplesstore.SparqlQuery.Prefixes - import tsClient._ - import Encoder._ + import io.renku.triplesstore.client.syntax._ + import io.renku.triplesstore.{ResultsDecoder, SparqlQuery} import personFinder._ + import tsClient._ override def persist(event: GLUserViewedDataset): F[Unit] = findPersonId(event.userId) >>= { @@ -65,9 +71,13 @@ private class PersonViewedDatasetPersisterImpl[F[_]: MonadThrow](tsClient: TSCli private def persistIfOlderOrNone(personId: persons.ResourceId, event: GLUserViewedDataset) = findStoredDate(personId, event.dataset.id) >>= { - case None => insert(personId, event) + case None => + insert(personId, event) >> + deduplicate(personId, event.dataset.id) case Some(date) if date < event.date => - deleteOldViewedDate(personId, event.dataset.id) >> insert(personId, event) + deleteOldViewedDate(personId, event.dataset.id) >> + insert(personId, event) >> + deduplicate(personId, event.dataset.id) case _ => ().pure[F] } diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicator.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicator.scala new file mode 100644 index 0000000000..4303a78f84 --- /dev/null +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicator.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import cats.syntax.all._ +import io.renku.graph.model.{persons, projects} +import io.renku.triplesstore.TSClient + +private trait PersonViewedProjectDeduplicator[F[_]] { + def deduplicate(personId: persons.ResourceId, projectId: projects.ResourceId): F[Unit] +} + +private object PersonViewedProjectDeduplicator { + def apply[F[_]](tsClient: TSClient[F]): PersonViewedProjectDeduplicator[F] = + new PersonViewedProjectDeduplicatorImpl[F](tsClient) +} + +private class PersonViewedProjectDeduplicatorImpl[F[_]](tsClient: TSClient[F]) + extends PersonViewedProjectDeduplicator[F] { + + import eu.timepit.refined.auto._ + import io.renku.graph.model.GraphClass + import io.renku.graph.model.Schemas._ + import io.renku.jsonld.syntax._ + import io.renku.triplesstore.SparqlQuery + import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore.client.syntax._ + import tsClient.updateWithNoResult + + override def deduplicate(personId: persons.ResourceId, projectId: projects.ResourceId): F[Unit] = updateWithNoResult( + SparqlQuery.ofUnsafe( + show"${GraphClass.PersonViewings}: deduplicate project viewings", + Prefixes of renku -> "renku", + sparql"""|DELETE { + | GRAPH ${GraphClass.PersonViewings.id} { ?viewingId renku:dateViewed ?date } + |} + |WHERE { + | GRAPH ${GraphClass.PersonViewings.id} { + | BIND (${personId.asEntityId} AS ?personId) + | { + | SELECT ?viewingId (MAX(?date) AS ?maxDate) + | WHERE { + | ?personId renku:viewedProject ?viewingId. + | ?viewingId renku:project ${projectId.asEntityId}; + | renku:dateViewed ?date. + | } + | GROUP BY ?viewingId + | HAVING (COUNT(?date) > 1) + | } + | ?viewingId renku:dateViewed ?date. + | FILTER (?date != ?maxDate) + | } + |} + |""".stripMargin + ) + ) +} diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersister.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersister.scala index 888fd78c7c..36d1c38363 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersister.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersister.scala @@ -28,25 +28,28 @@ private[viewings] trait PersonViewedProjectPersister[F[_]] { private[viewings] object PersonViewedProjectPersister { def apply[F[_]: MonadThrow](tsClient: TSClient[F]): PersonViewedProjectPersister[F] = - new PersonViewedProjectPersisterImpl[F](tsClient, PersonFinder(tsClient)) + new PersonViewedProjectPersisterImpl[F](tsClient, PersonFinder(tsClient), PersonViewedProjectDeduplicator(tsClient)) } -private class PersonViewedProjectPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F], personFinder: PersonFinder[F]) - extends PersonViewedProjectPersister[F] { +private class PersonViewedProjectPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F], + personFinder: PersonFinder[F], + deduplicator: PersonViewedProjectDeduplicator[F] +) extends PersonViewedProjectPersister[F] { + import Encoder._ import cats.syntax.all._ + import deduplicator._ import eu.timepit.refined.auto._ import io.circe.Decoder import io.renku.graph.model.GraphClass import io.renku.graph.model.Schemas._ import io.renku.jsonld.syntax._ - import io.renku.triplesstore.{ResultsDecoder, SparqlQuery} import io.renku.triplesstore.ResultsDecoder._ - import io.renku.triplesstore.client.syntax._ import io.renku.triplesstore.SparqlQuery.Prefixes - import tsClient._ - import Encoder._ + import io.renku.triplesstore.client.syntax._ + import io.renku.triplesstore.{ResultsDecoder, SparqlQuery} import personFinder._ + import tsClient._ override def persist(event: GLUserViewedProject): F[Unit] = findPersonId(event.userId) >>= { @@ -56,9 +59,12 @@ private class PersonViewedProjectPersisterImpl[F[_]: MonadThrow](tsClient: TSCli private def persistIfOlderOrNone(personId: persons.ResourceId, event: GLUserViewedProject) = findStoredDate(personId, event.project.id) >>= { - case None => insert(personId, event) + case None => + insert(personId, event) >> deduplicate(personId, event.project.id) case Some(date) if date < event.date => - deleteOldViewedDate(personId, event.project.id) >> insert(personId, event) + deleteOldViewedDate(personId, event.project.id) >> + insert(personId, event) >> + deduplicate(personId, event.project.id) case _ => ().pure[F] } diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/EventDeduplicator.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/EventDeduplicator.scala new file mode 100644 index 0000000000..0df2bce898 --- /dev/null +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/EventDeduplicator.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.projects + +import cats.syntax.all._ +import io.renku.events.CategoryName +import io.renku.graph.model.projects +import io.renku.triplesstore.TSClient + +private trait EventDeduplicator[F[_]] { + def deduplicate(projectId: projects.ResourceId): F[Unit] +} + +private object EventDeduplicator { + def apply[F[_]](tsClient: TSClient[F], categoryName: CategoryName): EventDeduplicator[F] = + new EventDeduplicatorImpl[F](tsClient, categoryName) +} + +private class EventDeduplicatorImpl[F[_]](tsClient: TSClient[F], categoryName: CategoryName) + extends EventDeduplicator[F] { + + import eu.timepit.refined.auto._ + import io.renku.graph.model.GraphClass + import io.renku.graph.model.Schemas._ + import io.renku.jsonld.syntax._ + import io.renku.triplesstore.SparqlQuery + import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore.client.syntax._ + import tsClient.updateWithNoResult + + override def deduplicate(projectId: projects.ResourceId): F[Unit] = updateWithNoResult( + SparqlQuery.ofUnsafe( + show"${categoryName.show.toLowerCase}: deduplicate", + Prefixes of renku -> "renku", + sparql"""|DELETE { + | GRAPH ${GraphClass.ProjectViewedTimes.id} { ?id renku:dateViewed ?date } + |} + |WHERE { + | GRAPH ${GraphClass.ProjectViewedTimes.id} { + | BIND (${projectId.asEntityId} AS ?id) + | { + | SELECT ?id (MAX(?date) AS ?maxDate) + | WHERE { + | ?id renku:dateViewed ?date + | } + | GROUP BY ?id + | HAVING (COUNT(?date) > 1) + | } + | ?id renku:dateViewed ?date. + | FILTER (?date != ?maxDate) + | } + |} + |""".stripMargin + ) + ) +} diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/activated/EventPersister.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/activated/EventPersister.scala index b5782939f6..7f9614ba92 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/activated/EventPersister.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/activated/EventPersister.scala @@ -19,9 +19,9 @@ package io.renku.entities.viewings.collector.projects package activated +import cats.MonadThrow import cats.effect.Async import cats.syntax.all._ -import cats.MonadThrow import io.renku.triplesgenerator.api.events.ProjectActivated import io.renku.triplesstore._ import org.typelevel.log4cats.Logger @@ -32,21 +32,25 @@ private trait EventPersister[F[_]] { private object EventPersister { def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[EventPersister[F]] = - ProjectsConnectionConfig[F]().map(TSClient[F](_)).map(new EventPersisterImpl[F](_)) + ProjectsConnectionConfig[F]() + .map(TSClient[F](_)) + .map(tsClient => new EventPersisterImpl[F](tsClient, EventDeduplicator(tsClient, categoryName))) } -private class EventPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extends EventPersister[F] { +private class EventPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F], deduplicator: EventDeduplicator[F]) + extends EventPersister[F] { + import Encoder._ + import deduplicator.deduplicate import eu.timepit.refined.auto._ import io.circe.Decoder - import io.renku.graph.model.{projects, GraphClass} import io.renku.graph.model.Schemas._ + import io.renku.graph.model.{GraphClass, projects} import io.renku.jsonld.syntax._ - import Encoder._ import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery - import io.renku.triplesstore.client.syntax._ import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore.client.syntax._ import tsClient.{queryExpecting, upload} override def persist(event: ProjectActivated): F[Unit] = @@ -54,7 +58,7 @@ private class EventPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extend case None => ().pure[F] case Some(projectId) => eventExists(projectId) >>= { - case false => insert(projectId, event) + case false => insert(projectId, event) >> deduplicate(projectId) case true => ().pure[F] } } @@ -63,14 +67,14 @@ private class EventPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extend SparqlQuery.ofUnsafe( show"${categoryName.show.toLowerCase}: find id", Prefixes of (renku -> "renku", schema -> "schema"), - s"""|SELECT DISTINCT ?id - |WHERE { - | GRAPH ?id { - | ?id a schema:Project; - | renku:projectPath ${event.path.asObject.asSparql.sparql} - | } - |} - |""".stripMargin + sparql"""|SELECT DISTINCT ?id + |WHERE { + | GRAPH ?id { + | ?id a schema:Project; + | renku:projectPath ${event.path.asObject} + | } + |} + |""".stripMargin ) }(idDecoder) @@ -85,14 +89,14 @@ private class EventPersisterImpl[F[_]: MonadThrow](tsClient: TSClient[F]) extend SparqlQuery.ofUnsafe( show"${categoryName.show.toLowerCase}: check exists", Prefixes of renku -> "renku", - s"""|SELECT ?date - |WHERE { - | GRAPH ${GraphClass.ProjectViewedTimes.id.sparql} { - | ${projectId.asEntityId.sparql} renku:dateViewed ?date - | } - |} - |LIMIT 1 - |""".stripMargin + sparql"""|SELECT ?date + |WHERE { + | GRAPH ${GraphClass.ProjectViewedTimes.id} { + | ${projectId.asEntityId} renku:dateViewed ?date + | } + |} + |LIMIT 1 + |""".stripMargin ) }(dateDecoder) .map(_.isDefined) diff --git a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersister.scala b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersister.scala index 9af4a5c314..a113ec4dad 100644 --- a/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersister.scala +++ b/entities-viewings-collector/src/main/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersister.scala @@ -20,41 +20,48 @@ package io.renku.entities.viewings.collector package projects package viewed +import cats.MonadThrow import cats.effect.Async import cats.syntax.all._ -import cats.MonadThrow +import io.renku.entities.viewings.collector.persons.{GLUserViewedProject, PersonViewedProjectPersister, Project} import io.renku.triplesgenerator.api.events.ProjectViewedEvent import io.renku.triplesstore._ import org.typelevel.log4cats.Logger -import persons.{GLUserViewedProject, PersonViewedProjectPersister, Project} private[viewings] trait EventPersister[F[_]] { def persist(event: ProjectViewedEvent): F[Unit] } private[viewings] object EventPersister { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[EventPersister[F]] = - ProjectsConnectionConfig[F]() - .map(TSClient[F](_)) - .map(tsClient => new EventPersisterImpl[F](tsClient, PersonViewedProjectPersister(tsClient))) + ProjectsConnectionConfig[F]().map(TSClient[F](_)).map(apply(_)) + + def apply[F[_]: MonadThrow](tsClient: TSClient[F]): EventPersister[F] = + new EventPersisterImpl[F](tsClient, + EventDeduplicator[F](tsClient, categoryName), + PersonViewedProjectPersister(tsClient) + ) } private[viewings] class EventPersisterImpl[F[_]: MonadThrow]( tsClient: TSClient[F], + eventDeduplicator: EventDeduplicator[F], personViewedProjectPersister: PersonViewedProjectPersister[F] ) extends EventPersister[F] { + import Encoder._ import eu.timepit.refined.auto._ + import eventDeduplicator.deduplicate import io.circe.Decoder - import io.renku.graph.model.{projects, GraphClass} import io.renku.graph.model.Schemas._ + import io.renku.graph.model.{GraphClass, projects} import io.renku.jsonld.syntax._ import io.renku.triplesstore.ResultsDecoder._ import io.renku.triplesstore.SparqlQuery - import io.renku.triplesstore.client.syntax._ import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore.client.syntax._ import tsClient.{queryExpecting, updateWithNoResult, upload} - import Encoder._ override def persist(event: ProjectViewedEvent): F[Unit] = findProjectId(event) >>= { @@ -64,9 +71,9 @@ private[viewings] class EventPersisterImpl[F[_]: MonadThrow]( private def persistIfOlderOrNone(event: ProjectViewedEvent, projectId: projects.ResourceId) = findStoredDate(projectId) >>= { - case None => insert(projectId, event) + case None => insert(projectId, event) >> deduplicate(projectId) case Some(date) if date < event.dateViewed => - deleteOldViewedDate(projectId) >> insert(projectId, event) + deleteOldViewedDate(projectId) >> insert(projectId, event) >> deduplicate(projectId) case _ => ().pure[F] } diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/EntityViewings.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/EntityViewings.scala new file mode 100644 index 0000000000..8b1420d089 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/EntityViewings.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings + +import cats.effect.IO +import cats.syntax.all._ +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.triplesgenerator.api.events.{DatasetViewedEvent, ProjectViewedEvent} +import io.renku.triplesstore.{InMemoryJena, ProjectsDataset, TSClient} + +trait EntityViewings { + self: ProjectsDataset with InMemoryJena => + + def provision(event: ProjectViewedEvent): IO[Unit] = + projectViewedEventPersister >>= { _.persist(event) } + + def provision(event: DatasetViewedEvent): IO[Unit] = + datasetViewedEventUploader >>= { _.upload(event) } + + private def tsClient = { + implicit val logger: TestLogger[IO] = TestLogger[IO]() + TestSparqlQueryTimeRecorder[IO].map(implicit sqrt => TSClient[IO](projectsDSConnectionInfo)) + } + + private def projectViewedEventPersister = tsClient.map(collector.projects.viewed.EventPersister[IO](_)) + private def datasetViewedEventUploader = tsClient.map(collector.datasets.EventUploader[IO](_)) +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/Generators.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/Generators.scala new file mode 100644 index 0000000000..da591c9fc7 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/Generators.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import cats.syntax.all._ +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.fixed +import io.renku.graph.model.entities +import io.renku.graph.model.testentities._ +import io.renku.triplesgenerator.api.events.UserId + +object Generators { + + def generateProjectWithCreatorAndDataset(userId: UserId) = + anyRenkuProjectEntities + .map(replaceProjectCreator(generateSomeCreator(userId))) + .addDataset(datasetEntities(provenanceInternal)) + .generateOne + .bimap( + _.to[entities.Dataset[entities.Dataset.Provenance.Internal]], + _.to[entities.Project] + ) + + def generateProjectWithCreator(userId: UserId) = + anyProjectEntities + .map(replaceProjectCreator(generateSomeCreator(userId))) + .generateOne + .to[entities.Project] + + private def generateSomeCreator(userId: UserId) = + userId + .fold( + glId => personEntities(maybeGitLabIds = fixed(glId.some)).map(removeOrcidId), + email => personEntities(withoutGitLabId, maybeEmails = fixed(email.some)).map(removeOrcidId) + ) + .generateSome +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicatorSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicatorSpec.scala new file mode 100644 index 0000000000..c007e2823a --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetDeduplicatorSpec.scala @@ -0,0 +1,144 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import cats.effect.IO +import io.renku.entities.viewings.collector.persons.Generators._ +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestamps +import io.renku.graph.model._ +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.UserId +import io.renku.triplesstore._ +import org.scalamock.scalatest.MockFactory +import org.scalatest.OptionValues +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class PersonViewedDatasetDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with OptionValues + with PersonViewedDatasetSpecTools + with IOSpec + with InMemoryJenaForSpec + with ProjectsDataset + with MockFactory { + + "deduplicate" should { + + "do nothing if there's only one date for the user and dataset" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) + upload(to = projectsDataset, project) + + val event = GLUserViewedDataset(userId, + toCollectorDataset(dataset), + datasetViewedDates(dataset.provenance.date.instant).generateOne + ) + persister.persist(event).unsafeRunSync() shouldBe () + + val userResourceId = project.maybeCreator.value.resourceId + deduplicator.deduplicate(userResourceId, dataset.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set(ViewingRecord(userResourceId, dataset.resourceId, event.date)) + } + + "leave only the latest date if there are many" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) + upload(to = projectsDataset, project) + + val event = GLUserViewedDataset(userId, + toCollectorDataset(dataset), + datasetViewedDates(dataset.provenance.date.instant).generateOne + ) + persister.persist(event).unsafeRunSync() shouldBe () + + val olderDateViewed1 = timestamps(max = event.date.value).generateAs(datasets.DateViewed) + insertOtherDate(dataset.resourceId, olderDateViewed1) + val olderDateViewed2 = timestamps(max = event.date.value).generateAs(datasets.DateViewed) + insertOtherDate(dataset.resourceId, olderDateViewed2) + + val userResourceId = project.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, dataset.resourceId, event.date), + ViewingRecord(userResourceId, dataset.resourceId, olderDateViewed1), + ViewingRecord(userResourceId, dataset.resourceId, olderDateViewed2) + ) + + deduplicator.deduplicate(userResourceId, dataset.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set(ViewingRecord(userResourceId, dataset.resourceId, event.date)) + } + + "do not remove dates for other projects" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + + val dataset1 -> project1 = generateProjectWithCreatorAndDataset(userId) + upload(to = projectsDataset, project1) + + val dataset2 -> project2 = generateProjectWithCreatorAndDataset(userId) + upload(to = projectsDataset, project2) + + val event1 = GLUserViewedDataset(userId, + toCollectorDataset(dataset1), + datasetViewedDates(dataset1.provenance.date.instant).generateOne + ) + persister.persist(event1).unsafeRunSync() shouldBe () + + val event2 = GLUserViewedDataset(userId, + toCollectorDataset(dataset2), + datasetViewedDates(dataset2.provenance.date.instant).generateOne + ) + persister.persist(event2).unsafeRunSync() shouldBe () + + val userResourceId = project1.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, dataset1.resourceId, event1.date), + ViewingRecord(userResourceId, dataset2.resourceId, event2.date) + ) + + deduplicator.deduplicate(userResourceId, dataset1.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, dataset1.resourceId, event1.date), + ViewingRecord(userResourceId, dataset2.resourceId, event2.date) + ) + } + } + + private trait TestCase { + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + private val tsClient = TSClient[IO](projectsDSConnectionInfo) + val deduplicator = new PersonViewedDatasetDeduplicatorImpl[IO](tsClient) + + val persister = PersonViewedDatasetPersister[IO](tsClient) + } +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersisterSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersisterSpec.scala index 9086f92314..27137b6ed6 100644 --- a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersisterSpec.scala +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetPersisterSpec.scala @@ -20,48 +20,49 @@ package io.renku.entities.viewings.collector.persons import cats.effect.IO import cats.syntax.all._ -import eu.timepit.refined.auto._ import io.renku.entities.viewings.collector -import io.renku.generators.Generators.{fixed, timestamps, timestampsNotInTheFuture} +import io.renku.entities.viewings.collector.persons.Generators._ import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.{timestamps, timestampsNotInTheFuture} import io.renku.graph.model._ import io.renku.graph.model.testentities._ -import io.renku.graph.model.Schemas.renku import io.renku.interpreters.TestLogger import io.renku.logging.TestSparqlQueryTimeRecorder import io.renku.testtools.IOSpec import io.renku.triplesgenerator.api.events.Generators.userIds import io.renku.triplesgenerator.api.events.UserId import io.renku.triplesstore._ -import io.renku.triplesstore.client.syntax._ -import io.renku.triplesstore.SparqlQuery.Prefixes +import org.scalamock.scalatest.MockFactory import org.scalatest.OptionValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import java.time.Instant - class PersonViewedDatasetPersisterSpec extends AnyWordSpec with should.Matchers with OptionValues + with PersonViewedDatasetSpecTools with IOSpec with InMemoryJenaForSpec - with ProjectsDataset { + with ProjectsDataset + with MockFactory { "persist" should { - "insert the given GLUserViewedDataset to the TS if it doesn't exist yet " + - "case with a user identified with GitLab id" in new TestCase { + "insert the given GLUserViewedDataset to the TS and run the deduplicate query " + + "if the viewing doesn't exist yet " + + "- case with a user identified with GitLab id" in new TestCase { val userId = UserId(personGitLabIds.generateOne) - val dataset -> project = generateProjectWithCreator(userId) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) upload(to = projectsDataset, project) val dateViewed = datasetViewedDates(dataset.provenance.date.instant).generateOne val event = GLUserViewedDataset(userId, toCollectorDataset(dataset), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, dataset.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -69,17 +70,20 @@ class PersonViewedDatasetPersisterSpec ) } - "insert the given GLUserViewedDataset to the TS if it doesn't exist yet " + - "case with a user identified with email" in new TestCase { + "insert the given GLUserViewedDataset to the TS and run the deduplicate query " + + "if the viewing doesn't exist yet " + + "- case with a user identified with email" in new TestCase { val userId = UserId(personEmails.generateOne) - val dataset -> project = generateProjectWithCreator(userId) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) upload(to = projectsDataset, project) val dateViewed = datasetViewedDates(dataset.provenance.date.instant).generateOne val event = GLUserViewedDataset(userId, toCollectorDataset(dataset), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, dataset.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -87,18 +91,20 @@ class PersonViewedDatasetPersisterSpec ) } - "update the date for the user and ds from the GLUserViewedDataset " + + "update the date for the user and ds from the GLUserViewedDataset and run the deduplicate query " + "if an event for the ds already exists in the TS " + "and the date from the new event is newer than this in the TS" in new TestCase { val userId = userIds.generateOne - val dataset -> project = generateProjectWithCreator(userId) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) upload(to = projectsDataset, project) val dateViewed = datasetViewedDates(dataset.provenance.date.instant).generateOne val event = GLUserViewedDataset(userId, toCollectorDataset(dataset), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, dataset.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -107,6 +113,8 @@ class PersonViewedDatasetPersisterSpec val newDate = timestampsNotInTheFuture(butYoungerThan = event.date.value).generateAs(datasets.DateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, dataset.resourceId, returning = ().pure[IO]) + persister.persist(event.copy(date = newDate)).unsafeRunSync() shouldBe () findAllViewings shouldBe Set(ViewingRecord(project.maybeCreator.value.resourceId, dataset.resourceId, newDate)) @@ -115,13 +123,15 @@ class PersonViewedDatasetPersisterSpec "do nothing if the event date is older than the date in the TS" in new TestCase { val userId = userIds.generateOne - val dataset -> project = generateProjectWithCreator(userId) + val dataset -> project = generateProjectWithCreatorAndDataset(userId) upload(to = projectsDataset, project) val dateViewed = datasetViewedDates(dataset.provenance.date.instant).generateOne val event = GLUserViewedDataset(userId, toCollectorDataset(dataset), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, dataset.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -135,23 +145,25 @@ class PersonViewedDatasetPersisterSpec findAllViewings shouldBe Set(ViewingRecord(project.maybeCreator.value.resourceId, dataset.resourceId, dateViewed)) } - "update the date for the user and project from the GLUserViewedProject " + + "update the date for the user and project from the GLUserViewedProject, run the deduplicate query" + "and leave other user viewings if they exist" in new TestCase { val userId = userIds.generateOne - val dataset1 -> project1 = generateProjectWithCreator(userId) - val dataset2 -> project2 = generateProjectWithCreator(userId) + val dataset1 -> project1 = generateProjectWithCreatorAndDataset(userId) + val dataset2 -> project2 = generateProjectWithCreatorAndDataset(userId) upload(to = projectsDataset, project1, project2) val dataset1DateViewed = datasetViewedDates(dataset1.provenance.date.instant).generateOne val dataset1Event = collector.persons.GLUserViewedDataset(userId, toCollectorDataset(dataset1), dataset1DateViewed) + givenEventDeduplication(project1.maybeCreator.value.resourceId, dataset1.resourceId, returning = ().pure[IO]) persister.persist(dataset1Event).unsafeRunSync() shouldBe () val dataset2DateViewed = datasetViewedDates(dataset2.provenance.date.instant).generateOne val dataset2Event = collector.persons.GLUserViewedDataset(userId, toCollectorDataset(dataset2), dataset2DateViewed) + givenEventDeduplication(project2.maybeCreator.value.resourceId, dataset2.resourceId, returning = ().pure[IO]) persister.persist(dataset2Event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -162,6 +174,7 @@ class PersonViewedDatasetPersisterSpec val newDate = timestampsNotInTheFuture(butYoungerThan = dataset1Event.date.value).generateAs(datasets.DateViewed) + givenEventDeduplication(project1.maybeCreator.value.resourceId, dataset1.resourceId, returning = ().pure[IO]) persister.persist(dataset1Event.copy(date = newDate)).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -172,7 +185,7 @@ class PersonViewedDatasetPersisterSpec "do nothing if the given event is for a non-existing user" in new TestCase { - val dataset -> _ = generateProjectWithCreator(userIds.generateOne) + val dataset -> _ = generateProjectWithCreatorAndDataset(userIds.generateOne) val event = collector.persons.GLUserViewedDataset(userIds.generateOne, toCollectorDataset(dataset), @@ -188,57 +201,15 @@ class PersonViewedDatasetPersisterSpec private trait TestCase { private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() - private val tsClient = TSClient[IO](projectsDSConnectionInfo) - val persister = new PersonViewedDatasetPersisterImpl[IO](tsClient, PersonFinder(tsClient)) + private val tsClient = TSClient[IO](projectsDSConnectionInfo) + private val eventDeduplicator = mock[PersonViewedDatasetDeduplicator[IO]] + val persister = new PersonViewedDatasetPersisterImpl[IO](tsClient, PersonFinder(tsClient), eventDeduplicator) + + def givenEventDeduplication(personResourceId: persons.ResourceId, + datasetResourceId: datasets.ResourceId, + returning: IO[Unit] + ) = (eventDeduplicator.deduplicate _) + .expects(personResourceId, datasetResourceId) + .returning(returning) } - - private def generateProjectWithCreator(userId: UserId) = { - - val creator = userId - .fold( - glId => personEntities(maybeGitLabIds = fixed(glId.some)).map(removeOrcidId), - email => personEntities(withoutGitLabId, maybeEmails = fixed(email.some)).map(removeOrcidId) - ) - .generateSome - - anyRenkuProjectEntities - .map(replaceProjectCreator(creator)) - .addDataset(datasetEntities(provenanceInternal)) - .generateOne - .bimap( - _.to[entities.Dataset[entities.Dataset.Provenance.Internal]], - _.to[entities.Project] - ) - } - - private def findAllViewings = - runSelect( - on = projectsDataset, - SparqlQuery.of( - "test find user project viewings", - Prefixes of renku -> "renku", - sparql"""|SELECT ?id ?datasetId ?date - |FROM ${GraphClass.PersonViewings.id} { - | ?id renku:viewedDataset ?viewingId. - | ?viewingId renku:dataset ?datasetId; - | renku:dateViewed ?date. - |} - |""".stripMargin - ) - ).unsafeRunSync() - .map(row => - ViewingRecord(persons.ResourceId(row("id")), - datasets.ResourceId(row("datasetId")), - datasets.DateViewed(Instant.parse(row("date"))) - ) - ) - .toSet - - private case class ViewingRecord(userId: persons.ResourceId, - datasetId: datasets.ResourceId, - date: datasets.DateViewed - ) - - private def toCollectorDataset(ds: entities.Dataset[entities.Dataset.Provenance]) = - collector.persons.Dataset(ds.resourceId, ds.identification.identifier) } diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetSpecTools.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetSpecTools.scala new file mode 100644 index 0000000000..5c71735ad9 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedDatasetSpecTools.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import eu.timepit.refined.auto._ +import io.renku.entities.viewings.collector +import io.renku.graph.model.Schemas.renku +import io.renku.graph.model.{GraphClass, datasets, entities, persons} +import io.renku.jsonld.syntax._ +import io.renku.testtools.IOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import io.renku.triplesstore.client.syntax._ + +import java.time.Instant + +trait PersonViewedDatasetSpecTools { + self: InMemoryJenaForSpec with ProjectsDataset with IOSpec => + + protected def findAllViewings = + runSelect( + on = projectsDataset, + SparqlQuery.of( + "test find user dataset viewings", + Prefixes of renku -> "renku", + sparql"""|SELECT ?id ?datasetId ?date + |FROM ${GraphClass.PersonViewings.id} { + | ?id renku:viewedDataset ?viewingId. + | ?viewingId renku:dataset ?datasetId; + | renku:dateViewed ?date. + |} + |""".stripMargin + ) + ).unsafeRunSync() + .map(row => + ViewingRecord(persons.ResourceId(row("id")), + datasets.ResourceId(row("datasetId")), + datasets.DateViewed(Instant.parse(row("date"))) + ) + ) + .toSet + + protected case class ViewingRecord(userId: persons.ResourceId, + datasetId: datasets.ResourceId, + date: datasets.DateViewed + ) + + protected def toCollectorDataset(ds: entities.Dataset[entities.Dataset.Provenance]) = + collector.persons.Dataset(ds.resourceId, ds.identification.identifier) + + protected def insertOtherDate(datasetId: datasets.ResourceId, dateViewed: datasets.DateViewed) = + runUpdate( + on = projectsDataset, + SparqlQuery.of( + "test add another user dataset dateViewed", + Prefixes of renku -> "renku", + sparql"""|INSERT { + | GRAPH ${GraphClass.PersonViewings.id} { + | ?viewingId renku:dateViewed ${dateViewed.asObject} + | } + |} + |WHERE { + | GRAPH ${GraphClass.PersonViewings.id} { + | ?viewingId renku:dataset ${datasetId.asEntityId} + | } + |} + |""".stripMargin + ) + ).unsafeRunSync() +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicatorSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicatorSpec.scala new file mode 100644 index 0000000000..2b1fa94028 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectDeduplicatorSpec.scala @@ -0,0 +1,144 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import cats.effect.IO +import io.renku.entities.viewings.collector.persons.Generators._ +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestamps +import io.renku.graph.model._ +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.UserId +import io.renku.triplesstore._ +import org.scalamock.scalatest.MockFactory +import org.scalatest.OptionValues +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class PersonViewedProjectDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with OptionValues + with PersonViewedProjectSpecTools + with IOSpec + with InMemoryJenaForSpec + with ProjectsDataset + with MockFactory { + + "deduplicate" should { + + "do nothing if there's only one date for the user and project" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + val project = generateProjectWithCreator(userId) + upload(to = projectsDataset, project) + + val dateViewed = projectViewedDates(project.dateCreated.value).generateOne + val event = GLUserViewedProject(userId, toCollectorProject(project), dateViewed) + persister.persist(event).unsafeRunSync() shouldBe () + + val userResourceId = project.maybeCreator.value.resourceId + deduplicator.deduplicate(userResourceId, project.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set( + ViewingRecord(project.maybeCreator.value.resourceId, project.resourceId, dateViewed) + ) + } + + "leave only the latest date if there are many" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + val project = generateProjectWithCreator(userId) + upload(to = projectsDataset, project) + + val event = GLUserViewedProject(userId, + toCollectorProject(project), + projectViewedDates(project.dateCreated.value).generateOne + ) + persister.persist(event).unsafeRunSync() shouldBe () + + val olderDateViewed1 = timestamps(max = event.date.value).generateAs(projects.DateViewed) + insertOtherDate(project.resourceId, olderDateViewed1) + val olderDateViewed2 = timestamps(max = event.date.value).generateAs(projects.DateViewed) + insertOtherDate(project.resourceId, olderDateViewed2) + + val userResourceId = project.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, project.resourceId, event.date), + ViewingRecord(userResourceId, project.resourceId, olderDateViewed1), + ViewingRecord(userResourceId, project.resourceId, olderDateViewed2) + ) + + deduplicator.deduplicate(userResourceId, project.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set(ViewingRecord(userResourceId, project.resourceId, event.date)) + } + + "do not remove dates for other projects" in new TestCase { + + val userId = UserId(personGitLabIds.generateOne) + + val project1 = generateProjectWithCreator(userId) + upload(to = projectsDataset, project1) + + val project2 = generateProjectWithCreator(userId) + upload(to = projectsDataset, project2) + + val event1 = GLUserViewedProject(userId, + toCollectorProject(project1), + projectViewedDates(project1.dateCreated.value).generateOne + ) + persister.persist(event1).unsafeRunSync() shouldBe () + + val event2 = GLUserViewedProject(userId, + toCollectorProject(project2), + projectViewedDates(project2.dateCreated.value).generateOne + ) + persister.persist(event2).unsafeRunSync() shouldBe () + + val userResourceId = project1.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, project1.resourceId, event1.date), + ViewingRecord(userResourceId, project2.resourceId, event2.date) + ) + + deduplicator.deduplicate(userResourceId, project1.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set( + ViewingRecord(userResourceId, project1.resourceId, event1.date), + ViewingRecord(userResourceId, project2.resourceId, event2.date) + ) + } + } + + private trait TestCase { + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + private val tsClient = TSClient[IO](projectsDSConnectionInfo) + val deduplicator = new PersonViewedProjectDeduplicatorImpl[IO](tsClient) + + val persister = PersonViewedProjectPersister[IO](tsClient) + } +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersisterSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersisterSpec.scala index 0e33d84c22..80ffc8c12e 100644 --- a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersisterSpec.scala +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectPersisterSpec.scala @@ -20,39 +20,38 @@ package io.renku.entities.viewings.collector.persons import cats.effect.IO import cats.syntax.all._ -import eu.timepit.refined.auto._ import io.renku.entities.viewings.collector -import io.renku.generators.Generators.{fixed, timestamps, timestampsNotInTheFuture} +import io.renku.entities.viewings.collector.persons.Generators._ import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.{timestamps, timestampsNotInTheFuture} import io.renku.graph.model._ import io.renku.graph.model.testentities._ -import io.renku.graph.model.Schemas.renku import io.renku.interpreters.TestLogger import io.renku.logging.TestSparqlQueryTimeRecorder import io.renku.testtools.IOSpec import io.renku.triplesgenerator.api.events.Generators.userIds import io.renku.triplesgenerator.api.events.UserId import io.renku.triplesstore._ -import io.renku.triplesstore.client.syntax._ -import io.renku.triplesstore.SparqlQuery.Prefixes +import org.scalamock.scalatest.MockFactory import org.scalatest.OptionValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import java.time.Instant - class PersonViewedProjectPersisterSpec extends AnyWordSpec with should.Matchers with OptionValues + with PersonViewedProjectSpecTools with IOSpec with InMemoryJenaForSpec - with ProjectsDataset { + with ProjectsDataset + with MockFactory { "persist" should { - "insert the given GLUserViewedProject to the TS if it doesn't exist yet " + - "case with a user identified with GitLab id" in new TestCase { + "insert the given GLUserViewedProject to the TS and run the deduplication query " + + "if it doesn't exist yet " + + "- case with a user identified with GitLab id" in new TestCase { val userId = UserId(personGitLabIds.generateOne) val project = generateProjectWithCreator(userId) @@ -62,6 +61,8 @@ class PersonViewedProjectPersisterSpec val dateViewed = projectViewedDates(project.dateCreated.value).generateOne val event = GLUserViewedProject(userId, toCollectorProject(project), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, project.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -69,8 +70,9 @@ class PersonViewedProjectPersisterSpec ) } - "insert the given GLUserViewedProject to the TS if it doesn't exist yet " + - "case with a user identified with email" in new TestCase { + "insert the given GLUserViewedProject to the TS and run the deduplication query " + + "if it doesn't exist yet " + + "- case with a user identified with email" in new TestCase { val userId = UserId(personEmails.generateOne) val project = generateProjectWithCreator(userId) @@ -80,6 +82,8 @@ class PersonViewedProjectPersisterSpec val dateViewed = projectViewedDates(project.dateCreated.value).generateOne val event = collector.persons.GLUserViewedProject(userId, toCollectorProject(project), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, project.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -87,7 +91,7 @@ class PersonViewedProjectPersisterSpec ) } - "update the date for the user and project from the GLUserViewedProject " + + "update the date for the user and project from the GLUserViewedProject and run the deduplicate query " + "if an event for the project already exists in the TS " + "and the date from the new event is newer than this in the TS" in new TestCase { @@ -99,6 +103,8 @@ class PersonViewedProjectPersisterSpec val dateViewed = projectViewedDates(project.dateCreated.value).generateOne val event = collector.persons.GLUserViewedProject(userId, toCollectorProject(project), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, project.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -107,6 +113,8 @@ class PersonViewedProjectPersisterSpec val newDate = timestampsNotInTheFuture(butYoungerThan = event.date.value).generateAs(projects.DateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, project.resourceId, returning = ().pure[IO]) + persister.persist(event.copy(date = newDate)).unsafeRunSync() shouldBe () findAllViewings shouldBe Set(ViewingRecord(project.maybeCreator.value.resourceId, project.resourceId, newDate)) @@ -122,6 +130,8 @@ class PersonViewedProjectPersisterSpec val dateViewed = projectViewedDates(project.dateCreated.value).generateOne val event = collector.persons.GLUserViewedProject(userId, toCollectorProject(project), dateViewed) + givenEventDeduplication(project.maybeCreator.value.resourceId, project.resourceId, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -135,7 +145,7 @@ class PersonViewedProjectPersisterSpec findAllViewings shouldBe Set(ViewingRecord(project.maybeCreator.value.resourceId, project.resourceId, dateViewed)) } - "update the date for the user and project from the GLUserViewedProject " + + "update the date for the user and project from the GLUserViewedProject, run the deduplicate query " + "and leave other user viewings if they exist" in new TestCase { val userId = userIds.generateOne @@ -147,11 +157,13 @@ class PersonViewedProjectPersisterSpec val project1DateViewed = projectViewedDates(project1.dateCreated.value).generateOne val project1Event = collector.persons.GLUserViewedProject(userId, toCollectorProject(project1), project1DateViewed) + givenEventDeduplication(project1.maybeCreator.value.resourceId, project1.resourceId, returning = ().pure[IO]) persister.persist(project1Event).unsafeRunSync() shouldBe () val project2DateViewed = projectViewedDates(project2.dateCreated.value).generateOne val project2Event = collector.persons.GLUserViewedProject(userId, toCollectorProject(project2), project2DateViewed) + givenEventDeduplication(project2.maybeCreator.value.resourceId, project2.resourceId, returning = ().pure[IO]) persister.persist(project2Event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -162,6 +174,7 @@ class PersonViewedProjectPersisterSpec val newDate = timestampsNotInTheFuture(butYoungerThan = project1Event.date.value).generateAs(projects.DateViewed) + givenEventDeduplication(project1.maybeCreator.value.resourceId, project1.resourceId, returning = ().pure[IO]) persister.persist(project1Event.copy(date = newDate)).unsafeRunSync() shouldBe () findAllViewings shouldBe Set( @@ -188,53 +201,15 @@ class PersonViewedProjectPersisterSpec private trait TestCase { private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() - private val tsClient = TSClient[IO](projectsDSConnectionInfo) - val persister = new PersonViewedProjectPersisterImpl[IO](tsClient, PersonFinder(tsClient)) + private val tsClient = TSClient[IO](projectsDSConnectionInfo) + private val eventDeduplicator = mock[PersonViewedProjectDeduplicator[IO]] + val persister = new PersonViewedProjectPersisterImpl[IO](tsClient, PersonFinder(tsClient), eventDeduplicator) + + def givenEventDeduplication(personResourceId: persons.ResourceId, + projectResourceId: projects.ResourceId, + returning: IO[Unit] + ) = (eventDeduplicator.deduplicate _) + .expects(personResourceId, projectResourceId) + .returning(returning) } - - private def generateProjectWithCreator(userId: UserId) = { - - val creator = userId - .fold( - glId => personEntities(maybeGitLabIds = fixed(glId.some)).map(removeOrcidId), - email => personEntities(withoutGitLabId, maybeEmails = fixed(email.some)).map(removeOrcidId) - ) - .generateSome - - anyProjectEntities - .map(replaceProjectCreator(creator)) - .generateOne - .to[entities.Project] - } - - private def findAllViewings = - runSelect( - on = projectsDataset, - SparqlQuery.of( - "test find user project viewings", - Prefixes of renku -> "renku", - sparql"""|SELECT ?id ?projectId ?date - |FROM ${GraphClass.PersonViewings.id} { - | ?id renku:viewedProject ?viewingId. - | ?viewingId renku:project ?projectId; - | renku:dateViewed ?date. - |} - |""".stripMargin - ) - ).unsafeRunSync() - .map(row => - ViewingRecord(persons.ResourceId(row("id")), - projects.ResourceId(row("projectId")), - projects.DateViewed(Instant.parse(row("date"))) - ) - ) - .toSet - - private case class ViewingRecord(userId: persons.ResourceId, - projectId: projects.ResourceId, - date: projects.DateViewed - ) - - private def toCollectorProject(project: entities.Project) = - collector.persons.Project(project.resourceId, project.path) } diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectSpecTools.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectSpecTools.scala new file mode 100644 index 0000000000..a371c71ff1 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/persons/PersonViewedProjectSpecTools.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.persons + +import eu.timepit.refined.auto._ +import io.renku.entities.viewings.collector +import io.renku.graph.model.Schemas.renku +import io.renku.graph.model.{GraphClass, entities, persons, projects} +import io.renku.jsonld.syntax._ +import io.renku.testtools.IOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import io.renku.triplesstore.client.syntax._ + +import java.time.Instant + +trait PersonViewedProjectSpecTools { + self: InMemoryJenaForSpec with ProjectsDataset with IOSpec => + + protected def findAllViewings = + runSelect( + on = projectsDataset, + SparqlQuery.of( + "test find user project viewings", + Prefixes of renku -> "renku", + sparql"""|SELECT ?id ?projectId ?date + |FROM ${GraphClass.PersonViewings.id} { + | ?id renku:viewedProject ?viewingId. + | ?viewingId renku:project ?projectId; + | renku:dateViewed ?date. + |} + |""".stripMargin + ) + ).unsafeRunSync() + .map(row => + ViewingRecord(persons.ResourceId(row("id")), + projects.ResourceId(row("projectId")), + projects.DateViewed(Instant.parse(row("date"))) + ) + ) + .toSet + + protected case class ViewingRecord(userId: persons.ResourceId, + projectId: projects.ResourceId, + date: projects.DateViewed + ) + + protected def toCollectorProject(project: entities.Project) = + collector.persons.Project(project.resourceId, project.path) + + protected def insertOtherDate(projectId: projects.ResourceId, dateViewed: projects.DateViewed) = + runUpdate( + on = projectsDataset, + SparqlQuery.of( + "test add another user project dateViewed", + Prefixes of renku -> "renku", + sparql"""|INSERT { + | GRAPH ${GraphClass.PersonViewings.id} { + | ?viewingId renku:dateViewed ${dateViewed.asObject} + | } + |} + |WHERE { + | GRAPH ${GraphClass.PersonViewings.id} { + | ?viewingId renku:project ${projectId.asEntityId} + | } + |} + |""".stripMargin + ) + ).unsafeRunSync() +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventDeduplicatorSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventDeduplicatorSpec.scala new file mode 100644 index 0000000000..a32e5088df --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventDeduplicatorSpec.scala @@ -0,0 +1,125 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.projects + +import cats.effect.IO +import cats.syntax.all._ +import io.renku.entities.viewings.collector.persons.PersonViewedProjectPersister +import io.renku.entities.viewings.collector.projects.viewed.EventPersisterImpl +import io.renku.events.Generators.categoryNames +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestamps +import io.renku.graph.model.testentities._ +import io.renku.graph.model.{entities, projects} +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.Generators._ +import io.renku.triplesstore._ +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class EventDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with EventPersisterSpecTools + with IOSpec + with InMemoryJenaForSpec + with ProjectsDataset + with MockFactory { + + "deduplicate" should { + + "do nothing if there's only one date for the project" in new TestCase { + + val project = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project) + + val event = projectViewedEvents.generateOne.copy(path = project.path) + + persister.persist(event).unsafeRunSync() shouldBe () + + deduplicator.deduplicate(project.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set(project.resourceId -> event.dateViewed) + } + + "leave only the latest date if there are many" in new TestCase { + + val project = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project) + + val event = projectViewedEvents.generateOne.copy(path = project.path) + persister.persist(event).unsafeRunSync() shouldBe () + + val olderDateViewed1 = timestamps(max = event.dateViewed.value).generateAs(projects.DateViewed) + insertOtherDate(project, olderDateViewed1) + val olderDateViewed2 = timestamps(max = event.dateViewed.value).generateAs(projects.DateViewed) + insertOtherDate(project, olderDateViewed2) + + findAllViewings shouldBe Set( + project.resourceId -> event.dateViewed, + project.resourceId -> olderDateViewed1, + project.resourceId -> olderDateViewed2 + ) + + deduplicator.deduplicate(project.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set(project.resourceId -> event.dateViewed) + } + + "do not remove dates for other projects" in new TestCase { + + val project1 = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project1) + val project2 = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project2) + + val event1 = projectViewedEvents.generateOne.copy(path = project1.path) + persister.persist(event1).unsafeRunSync() shouldBe () + val event2 = projectViewedEvents.generateOne.copy(path = project2.path) + persister.persist(event2).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set( + project1.resourceId -> event1.dateViewed, + project2.resourceId -> event2.dateViewed + ) + + deduplicator.deduplicate(project1.resourceId).unsafeRunSync() shouldBe () + + findAllViewings shouldBe Set( + project1.resourceId -> event1.dateViewed, + project2.resourceId -> event2.dateViewed + ) + } + } + + private trait TestCase { + + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + private val tsClient = TSClient[IO](projectsDSConnectionInfo) + val deduplicator = new EventDeduplicatorImpl[IO](tsClient, categoryNames.generateOne) + + private val personViewingPersister = mock[PersonViewedProjectPersister[IO]] + (personViewingPersister.persist _).expects(*).returning(().pure[IO]).anyNumberOfTimes() + val persister = new EventPersisterImpl[IO](tsClient, deduplicator, personViewingPersister) + } +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventPersisterSpecTools.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventPersisterSpecTools.scala new file mode 100644 index 0000000000..eb65d77159 --- /dev/null +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/EventPersisterSpecTools.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.entities.viewings.collector.projects + +import eu.timepit.refined.auto._ +import io.renku.entities.viewings.collector.ProjectViewedTimeOntology.dataViewedProperty +import io.renku.graph.model.Schemas.renku +import io.renku.graph.model.{GraphClass, entities, projects} +import io.renku.jsonld.syntax._ +import io.renku.testtools.IOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import io.renku.triplesstore.client.model.Quad +import io.renku.triplesstore.client.syntax._ + +import java.time.Instant + +trait EventPersisterSpecTools { + self: InMemoryJenaForSpec with ProjectsDataset with IOSpec => + + protected def findAllViewings = + runSelect( + on = projectsDataset, + SparqlQuery.of( + "test find project viewing", + Prefixes of renku -> "renku", + sparql"""|SELECT DISTINCT ?id ?date + |FROM ${GraphClass.ProjectViewedTimes.id} { + | ?id renku:dateViewed ?date. + |} + |""".stripMargin + ) + ).unsafeRunSync() + .map(row => projects.ResourceId(row("id")) -> projects.DateViewed(Instant.parse(row("date")))) + .toSet + + protected def insertOtherDate(project: entities.Project, dateViewed: projects.DateViewed) = + insert( + to = projectsDataset, + Quad(GraphClass.ProjectViewedTimes.id, project.resourceId.asEntityId, dataViewedProperty.id, dateViewed.asObject) + ) +} diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/activated/EventPersisterSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/activated/EventPersisterSpec.scala index 424cff37ca..003ddabc53 100644 --- a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/activated/EventPersisterSpec.scala +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/activated/EventPersisterSpec.scala @@ -16,38 +16,37 @@ * limitations under the License. */ -package io.renku.entities.viewings.collector.projects.activated +package io.renku.entities.viewings.collector.projects +package activated import cats.effect.IO -import eu.timepit.refined.auto._ +import cats.syntax.all._ import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators.timestampsNotInTheFuture -import io.renku.graph.model.{projects, GraphClass} +import io.renku.graph.model.projects import io.renku.graph.model.testentities._ -import io.renku.graph.model.Schemas.renku import io.renku.interpreters.TestLogger import io.renku.logging.TestSparqlQueryTimeRecorder import io.renku.testtools.IOSpec import io.renku.triplesgenerator.api.events.Generators._ import io.renku.triplesgenerator.api.events.ProjectActivated import io.renku.triplesstore._ -import io.renku.triplesstore.client.syntax._ -import io.renku.triplesstore.SparqlQuery.Prefixes +import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import java.time.Instant - class EventPersisterSpec extends AnyWordSpec with should.Matchers + with EventPersisterSpecTools + with MockFactory with IOSpec with InMemoryJenaForSpec with ProjectsDataset { "persist" should { - "insert the given ProjectActivated event into the TS " + + "insert the given ProjectActivated event into the TS and run the deduplication query " + "if there's no event for the project yet" in new TestCase { val project = anyProjectEntities.generateOne @@ -55,6 +54,8 @@ class EventPersisterSpec val event = projectActivatedEvents.generateOne.copy(path = project.path) + givenEventDeduplication(project, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set(project.resourceId -> projects.DateViewed(event.dateActivated.value)) @@ -67,6 +68,8 @@ class EventPersisterSpec val event = projectActivatedEvents.generateOne.copy(path = project.path) + givenEventDeduplication(project, returning = ().pure[IO]) + persister.persist(event).unsafeRunSync() shouldBe () findAllViewings shouldBe Set(project.resourceId -> projects.DateViewed(event.dateActivated.value)) @@ -82,22 +85,12 @@ class EventPersisterSpec private trait TestCase { private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() - val persister = new EventPersisterImpl[IO](TSClient[IO](projectsDSConnectionInfo)) - } + private val eventDeduplicator = mock[EventDeduplicator[IO]] + val persister = new EventPersisterImpl[IO](TSClient[IO](projectsDSConnectionInfo), eventDeduplicator) - private def findAllViewings = - runSelect( - on = projectsDataset, - SparqlQuery.of( - "test find project viewing", - Prefixes of renku -> "renku", - s"""|SELECT ?id ?date - |FROM ${GraphClass.ProjectViewedTimes.id.asSparql.sparql} { - | ?id renku:dateViewed ?date. - |} - |""".stripMargin - ) - ).unsafeRunSync() - .map(row => projects.ResourceId(row("id")) -> projects.DateViewed(Instant.parse(row("date")))) - .toSet + def givenEventDeduplication(project: Project, returning: IO[Unit]) = + (eventDeduplicator.deduplicate _) + .expects(project.resourceId) + .returning(returning) + } } diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersisterSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersisterSpec.scala index 266f428183..a6e2c70c65 100644 --- a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersisterSpec.scala +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/collector/projects/viewed/EventPersisterSpec.scala @@ -21,31 +21,26 @@ package viewed import cats.effect.IO import cats.syntax.all._ -import eu.timepit.refined.auto._ import io.renku.entities.viewings.collector import io.renku.entities.viewings.collector.persons.{GLUserViewedProject, PersonViewedProjectPersister} -import io.renku.generators.Generators.{timestamps, timestampsNotInTheFuture} import io.renku.generators.Generators.Implicits._ -import io.renku.graph.model.{projects, GraphClass} +import io.renku.generators.Generators.{timestamps, timestampsNotInTheFuture} +import io.renku.graph.model.projects import io.renku.graph.model.testentities._ -import io.renku.graph.model.Schemas.renku import io.renku.interpreters.TestLogger import io.renku.logging.TestSparqlQueryTimeRecorder import io.renku.testtools.IOSpec import io.renku.triplesgenerator.api.events.Generators._ import io.renku.triplesgenerator.api.events.ProjectViewedEvent import io.renku.triplesstore._ -import io.renku.triplesstore.client.syntax._ -import io.renku.triplesstore.SparqlQuery.Prefixes import org.scalamock.scalatest.MockFactory import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import java.time.Instant - class EventPersisterSpec extends AnyWordSpec with should.Matchers + with EventPersisterSpecTools with IOSpec with InMemoryJenaForSpec with ProjectsDataset @@ -53,7 +48,9 @@ class EventPersisterSpec "persist" should { - "insert the given ProjectViewedEvent to the TS and persist it in the PersonViewing " + + "insert the given ProjectViewedEvent to the TS, " + + "run the deduplication query and " + + "persist a PersonViewing event " + "if there's no event for the project yet" in new TestCase { val project = anyProjectEntities.generateOne @@ -61,6 +58,8 @@ class EventPersisterSpec val event = projectViewedEvents.generateOne.copy(path = project.path) + givenEventDeduplication(project, returning = ().pure[IO]) + givenPersonViewingPersisting(event, project, returning = ().pure[IO]) persister.persist(event).unsafeRunSync() shouldBe () @@ -68,7 +67,8 @@ class EventPersisterSpec findAllViewings shouldBe Set(project.resourceId -> event.dateViewed) } - "update the date for the project from the ProjectViewedEvent " + + "update the date for the project from the ProjectViewedEvent, " + + "run the deduplication query " + "if an event for the project already exists in the TS " + "and the date from the event is newer than this in the TS" in new TestCase { @@ -76,7 +76,7 @@ class EventPersisterSpec upload(to = projectsDataset, project) val event = projectViewedEvents.generateOne.copy(path = project.path) - + givenEventDeduplication(project, returning = ().pure[IO]) givenPersonViewingPersisting(event, project, returning = ().pure[IO]) persister.persist(event).unsafeRunSync() shouldBe () @@ -86,6 +86,7 @@ class EventPersisterSpec val newDate = timestampsNotInTheFuture(butYoungerThan = event.dateViewed.value).generateAs(projects.DateViewed) val newEvent = event.copy(dateViewed = newDate) + givenEventDeduplication(project, returning = ().pure[IO]) givenPersonViewingPersisting(newEvent, project, returning = ().pure[IO]) persister.persist(newEvent).unsafeRunSync() shouldBe () @@ -100,6 +101,7 @@ class EventPersisterSpec val event = projectViewedEvents.generateOne.copy(path = project.path) + givenEventDeduplication(project, returning = ().pure[IO]) givenPersonViewingPersisting(event, project, returning = ().pure[IO]) persister.persist(event).unsafeRunSync() shouldBe () @@ -129,8 +131,15 @@ class EventPersisterSpec private trait TestCase { private implicit val logger: TestLogger[IO] = TestLogger[IO]() private implicit val sqtr: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + private val eventDeduplicator = mock[EventDeduplicator[IO]] private val personViewingPersister = mock[PersonViewedProjectPersister[IO]] - val persister = new EventPersisterImpl[IO](TSClient[IO](projectsDSConnectionInfo), personViewingPersister) + val persister = + new EventPersisterImpl[IO](TSClient[IO](projectsDSConnectionInfo), eventDeduplicator, personViewingPersister) + + def givenEventDeduplication(project: Project, returning: IO[Unit]) = + (eventDeduplicator.deduplicate _) + .expects(project.resourceId) + .returning(returning) def givenPersonViewingPersisting(event: ProjectViewedEvent, project: Project, returning: IO[Unit]) = event.maybeUserId.map(userId => @@ -141,20 +150,4 @@ class EventPersisterSpec .returning(returning) ) } - - private def findAllViewings = - runSelect( - on = projectsDataset, - SparqlQuery.of( - "test find project viewing", - Prefixes of renku -> "renku", - s"""|SELECT ?id ?date - |FROM ${GraphClass.ProjectViewedTimes.id.asSparql.sparql} { - | ?id renku:dateViewed ?date. - |} - |""".stripMargin - ) - ).unsafeRunSync() - .map(row => projects.ResourceId(row("id")) -> projects.DateViewed(Instant.parse(row("date")))) - .toSet } diff --git a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/deletion/projects/ViewingRemoverSpec.scala b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/deletion/projects/ViewingRemoverSpec.scala index 6c8d3e27fc..d661cb155e 100644 --- a/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/deletion/projects/ViewingRemoverSpec.scala +++ b/entities-viewings-collector/src/test/scala/io/renku/entities/viewings/deletion/projects/ViewingRemoverSpec.scala @@ -21,26 +21,25 @@ package deletion.projects import cats.effect.IO import cats.syntax.all._ -import collector.persons.PersonViewedProjectPersister -import collector.projects.viewed.EventPersisterImpl import eu.timepit.refined.auto._ +import io.renku.entities.viewings.collector.projects.viewed.EventPersister import io.renku.generators.Generators.Implicits._ import io.renku.generators.Generators.fixed -import io.renku.graph.model.{persons, projects, GraphClass} -import io.renku.graph.model.testentities._ import io.renku.graph.model.Schemas.renku +import io.renku.graph.model.testentities._ +import io.renku.graph.model.{GraphClass, persons, projects} import io.renku.interpreters.TestLogger import io.renku.jsonld.syntax._ import io.renku.logging.TestSparqlQueryTimeRecorder import io.renku.testtools.IOSpec -import io.renku.triplesgenerator.api.events.{ProjectViewingDeletion, UserId} import io.renku.triplesgenerator.api.events.Generators._ +import io.renku.triplesgenerator.api.events.{ProjectViewingDeletion, UserId} +import io.renku.triplesstore.SparqlQuery.Prefixes import io.renku.triplesstore._ import io.renku.triplesstore.client.syntax._ -import io.renku.triplesstore.SparqlQuery.Prefixes +import org.scalatest.OptionValues import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec -import org.scalatest.OptionValues class ViewingRemoverSpec extends AnyWordSpec @@ -139,7 +138,7 @@ class ViewingRemoverSpec val remover = new ViewingRemoverImpl[IO](TSClient[IO](projectsDSConnectionInfo)) private val tsClient = TSClient[IO](projectsDSConnectionInfo) - val eventPersister = new EventPersisterImpl[IO](tsClient, PersonViewedProjectPersister[IO](tsClient)) + val eventPersister = EventPersister[IO](tsClient) def insertViewing(project: Project, userId: UserId): Unit = { diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala index 8b04c6b3d2..f94b1668a0 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala @@ -42,6 +42,8 @@ private[tsmigrationrequest] object Migrations { migrationToV10 <- v10migration.MigrationToV10[F] v10VersionSetter <- V10VersionUpdater[F] projectsDateViewedCreator <- ProjectsDateViewedCreator[F] + projectDateViewedDeduplicator <- ProjectDateViewedDeduplicator[F] + personViewedEntityDeduplicator <- PersonViewedEntityDeduplicator[F] migrations <- validateNames( datasetsCreator, datasetsRemover, @@ -51,7 +53,9 @@ private[tsmigrationrequest] object Migrations { addRenkuPlanWhereMissing, migrationToV10, v10VersionSetter, - projectsDateViewedCreator + projectsDateViewedCreator, + projectDateViewedDeduplicator, + personViewedEntityDeduplicator ) } yield migrations diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicator.scala new file mode 100644 index 0000000000..f13213602e --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicator.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.renku +import io.renku.metrics.MetricsRegistry +import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations.tooling.UpdateQueryMigration +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import org.typelevel.log4cats.Logger + +private object PersonViewedEntityDeduplicator { + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + UpdateQueryMigration[F](name, query).widen + + private lazy val name = Migration.Name("Remove Person Viewed Entity duplicates") + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of renku -> "renku", + sparql"""|DELETE { + | GRAPH ${GraphClass.PersonViewings.id} { + | ?viewingId renku:dateViewed ?date + | } + |} + |WHERE { + | GRAPH ${GraphClass.PersonViewings.id} { + | { + | SELECT ?viewingId (MAX(?date) AS ?maxDate) + | WHERE { + | ?viewingId renku:dateViewed ?date. + | } + | GROUP BY ?viewingId + | HAVING (COUNT(?date) > 1) + | } + | ?viewingId renku:dateViewed ?date. + | FILTER (?date != ?maxDate) + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicator.scala new file mode 100644 index 0000000000..2f5eae8f92 --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicator.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.renku +import io.renku.metrics.MetricsRegistry +import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations.tooling.UpdateQueryMigration +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import org.typelevel.log4cats.Logger + +private object ProjectDateViewedDeduplicator { + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + UpdateQueryMigration[F](name, query).widen + + private lazy val name = Migration.Name("Remove Project DateViewed duplicates") + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of renku -> "renku", + sparql"""|DELETE { + | GRAPH ${GraphClass.ProjectViewedTimes.id} { + | ?id renku:dateViewed ?date + | } + |} + |WHERE { + | GRAPH ${GraphClass.ProjectViewedTimes.id} { + | { + | SELECT ?id (MAX(?date) AS ?maxDate) + | WHERE { + | ?id renku:dateViewed ?date + | } + | GROUP BY ?id + | HAVING (COUNT(?date) > 1) + | } + | ?id renku:dateViewed ?date. + | FILTER (?date != ?maxDate) + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicatorSpec.scala new file mode 100644 index 0000000000..f7984a115a --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/PersonViewedEntityDeduplicatorSpec.scala @@ -0,0 +1,158 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations + +import cats.syntax.all._ +import io.renku.entities.viewings.EntityViewings +import io.renku.entities.viewings.collector.persons.Generators.{generateProjectWithCreator, generateProjectWithCreatorAndDataset} +import io.renku.entities.viewings.collector.persons.{PersonViewedDatasetSpecTools, PersonViewedProjectSpecTools} +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestamps +import io.renku.graph.model.testentities._ +import io.renku.graph.model.{datasets, projects} +import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.{DatasetViewedEvent, ProjectViewedEvent, UserId} +import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset} +import org.scalatest.OptionValues +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class PersonViewedProjectDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with OptionValues + with IOSpec + with InMemoryJenaForSpec + with PersonViewedProjectSpecTools + with ProjectsDataset + with EntityViewings { + + "run" should { + + "remove obsolete project viewing dates when multiple exist for a single user and project" in { + + // project1 + val project1UserId = UserId(personGitLabIds.generateOne) + val project1 = generateProjectWithCreator(project1UserId) + upload(to = projectsDataset, project1) + + val eventProject1 = ProjectViewedEvent( + project1.path, + projectViewedDates(project1.dateCreated.value).generateOne, + project1UserId.some + ) + provision(eventProject1).unsafeRunSync() + + val olderDateDataset1 = timestamps(max = eventProject1.dateViewed.value).generateAs(projects.DateViewed) + insertOtherDate(project1.resourceId, olderDateDataset1) + + // project2 + val project2UserId = UserId(personGitLabIds.generateOne) + val project2 = generateProjectWithCreator(project2UserId) + upload(to = projectsDataset, project2) + + val eventProject2 = ProjectViewedEvent( + project2.path, + projectViewedDates(project2.dateCreated.value).generateOne, + project2UserId.some + ) + provision(eventProject2).unsafeRunSync() + + // assumptions check + val project1UserResourceId = project1.maybeCreator.value.resourceId + val project2UserResourceId = project2.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(project1UserResourceId, project1.resourceId, eventProject1.dateViewed), + ViewingRecord(project1UserResourceId, project1.resourceId, olderDateDataset1), + ViewingRecord(project2UserResourceId, project2.resourceId, eventProject2.dateViewed) + ) + + runUpdate(projectsDataset, PersonViewedEntityDeduplicator.query).unsafeRunSync() + + // verification + findAllViewings shouldBe Set( + ViewingRecord(project1UserResourceId, project1.resourceId, eventProject1.dateViewed), + ViewingRecord(project2UserResourceId, project2.resourceId, eventProject2.dateViewed) + ) + } + } +} + +class PersonViewedDatasetDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with OptionValues + with IOSpec + with InMemoryJenaForSpec + with PersonViewedDatasetSpecTools + with ProjectsDataset + with EntityViewings { + + "run" should { + + "remove obsolete dataset viewing dates when multiple exist for a single user and dataset on a project" in { + + // project1 + val project1UserId = personGitLabIds.generateOne + val datasetProject1 -> project1 = generateProjectWithCreatorAndDataset(UserId(project1UserId)) + upload(to = projectsDataset, project1) + + val eventDatasetProject1 = DatasetViewedEvent( + datasetProject1.identification.identifier, + datasetViewedDates(datasetProject1.provenance.date.instant).generateOne, + project1UserId.some + ) + provision(eventDatasetProject1).unsafeRunSync() + + val olderDateDataset1 = timestamps(max = eventDatasetProject1.dateViewed.value).generateAs(datasets.DateViewed) + insertOtherDate(datasetProject1.resourceId, olderDateDataset1) + + // project2 + val project2UserId = personGitLabIds.generateOne + val datasetProject2 -> project2 = generateProjectWithCreatorAndDataset(UserId(project2UserId)) + upload(to = projectsDataset, project2) + + val eventDatasetProject2 = DatasetViewedEvent( + datasetProject2.identification.identifier, + datasetViewedDates(datasetProject2.provenance.date.instant).generateOne, + project2UserId.some + ) + provision(eventDatasetProject2).unsafeRunSync() + + // assumptions check + val project1UserResourceId = project1.maybeCreator.value.resourceId + val project2UserResourceId = project2.maybeCreator.value.resourceId + + findAllViewings shouldBe Set( + ViewingRecord(project1UserResourceId, datasetProject1.resourceId, eventDatasetProject1.dateViewed), + ViewingRecord(project1UserResourceId, datasetProject1.resourceId, olderDateDataset1), + ViewingRecord(project2UserResourceId, datasetProject2.resourceId, eventDatasetProject2.dateViewed) + ) + + runUpdate(projectsDataset, PersonViewedEntityDeduplicator.query).unsafeRunSync() + + // verification + findAllViewings shouldBe Set( + ViewingRecord(project1UserResourceId, datasetProject1.resourceId, eventDatasetProject1.dateViewed), + ViewingRecord(project2UserResourceId, datasetProject2.resourceId, eventDatasetProject2.dateViewed) + ) + } + } +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicatorSpec.scala new file mode 100644 index 0000000000..fcbd72685b --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectDateViewedDeduplicatorSpec.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations + +import io.renku.entities.viewings.EntityViewings +import io.renku.entities.viewings.collector.projects.EventPersisterSpecTools +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestamps +import io.renku.graph.model.testentities._ +import io.renku.graph.model.{entities, projects} +import io.renku.testtools.IOSpec +import io.renku.triplesgenerator.api.events.Generators.projectViewedEvents +import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset} +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class ProjectDateViewedDeduplicatorSpec + extends AnyWordSpec + with should.Matchers + with IOSpec + with InMemoryJenaForSpec + with EventPersisterSpecTools + with ProjectsDataset + with EntityViewings { + + "run" should { + + "remove obsolete project viewing dates when multiple exist for a single project" in { + + val project1 = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project1) + + val eventProject1 = projectViewedEvents.generateOne.copy(path = project1.path) + provision(eventProject1).unsafeRunSync() + + val otherProject1Date = timestamps(max = eventProject1.dateViewed.value).generateAs(projects.DateViewed) + insertOtherDate(project1, otherProject1Date) + + val project2 = anyProjectEntities.generateOne.to[entities.Project] + upload(to = projectsDataset, project2) + + val eventProject2 = projectViewedEvents.generateOne.copy(path = project2.path) + provision(eventProject2).unsafeRunSync() + + findAllViewings shouldBe Set( + project1.resourceId -> eventProject1.dateViewed, + project1.resourceId -> otherProject1Date, + project2.resourceId -> eventProject2.dateViewed + ) + + runUpdate(projectsDataset, ProjectDateViewedDeduplicator.query).unsafeRunSync() + + findAllViewings shouldBe Set( + project1.resourceId -> eventProject1.dateViewed, + project2.resourceId -> eventProject2.dateViewed + ) + } + } +}