From 90f68807db60dd0c343bdc2ac4a26fee9f14bbfb Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 15 Nov 2022 18:50:31 +0100 Subject: [PATCH] chore: prepare 2.22.2 (#1189) * chore: CLI upgraded to 1.9.2 * feat: TS migration fixing creation dates on Plans (#1183) --- .../renku/eventlog/InMemoryEventLogDb.scala | 1 - .../eventlog/init/BatchDateAdderSpec.scala | 1 - .../init/ProjectPathRemoverSpec.scala | 1 - generators/build.sbt | 4 +- graph-commons/build.sbt | 2 +- project/plugins.sbt | 2 +- .../renku/graph/model/entities/Activity.scala | 14 +- .../graph/model/entities/ActivityLens.scala | 4 + .../model/entities/AssociationLens.scala | 12 ++ .../renku/graph/model/entities/PlanLens.scala | 8 ++ .../renku/graph/model/entities/Project.scala | 72 ++++++---- .../graph/model/entities/ProjectLens.scala | 8 ++ .../scala/io/renku/graph/model/plans.scala | 4 +- .../graph/model/entities/ActivitySpec.scala | 3 +- .../graph/model/entities/ProjectSpec.scala | 33 +++++ tiny-types/build.sbt | 4 +- .../repository/InMemoryProjectsTokensDb.scala | 1 - triples-generator/Dockerfile | 2 +- .../src/main/resources/application.conf | 2 +- .../FixPlansYoungerThanActivities.scala | 54 ++++++++ .../migrations/Migrations.scala | 12 +- .../TransformationStepsCreator.scala | 4 + .../namedgraphs/plans/KGInfoFinder.scala | 69 ++++++++++ .../namedgraphs/plans/PlanTransformer.scala | 73 ++++++++++ .../namedgraphs/plans/UpdatesCreator.scala | 53 ++++++++ .../FixPlansYoungerThanActivitiesSpec.scala | 123 +++++++++++++++++ .../TransformationStepsCreatorSpec.scala | 7 +- .../namedgraphs/plans/KGInfoFinderSpec.scala | 69 ++++++++++ .../plans/PlanTransformerSpec.scala | 126 ++++++++++++++++++ .../plans/UpdatesCreatorSpec.scala | 111 +++++++++++++++ 30 files changed, 827 insertions(+), 52 deletions(-) create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivities.scala create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinder.scala create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformer.scala create mode 100644 triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreator.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivitiesSpec.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinderSpec.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformerSpec.scala create mode 100644 triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreatorSpec.scala diff --git a/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala b/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala index e7deae0dc4..7a18a2da56 100644 --- a/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala +++ b/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala @@ -20,7 +20,6 @@ package io.renku.eventlog import cats.data.Kleisli import cats.effect.IO -import cats.syntax.all._ import com.dimafeng.testcontainers._ import io.renku.db.PostgresContainer import io.renku.eventlog.EventLogDB.SessionResource diff --git a/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala b/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala index 3119f98866..3e222f85bb 100644 --- a/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala @@ -20,7 +20,6 @@ package io.renku.eventlog.init import cats.data.Kleisli import cats.effect.IO -import cats.syntax.all._ import io.circe.literal._ import io.renku.eventlog.EventContentGenerators._ import io.renku.eventlog.init.Generators._ diff --git a/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala b/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala index 955afa365f..7df8d1ec5a 100644 --- a/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala +++ b/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala @@ -20,7 +20,6 @@ package io.renku.eventlog.init import cats.data.Kleisli import cats.effect.IO -import cats.syntax.all._ import io.renku.graph.model.projects import io.renku.interpreters.TestLogger import io.renku.interpreters.TestLogger.Level.Info diff --git a/generators/build.sbt b/generators/build.sbt index df2cb06c39..925b775b23 100644 --- a/generators/build.sbt +++ b/generators/build.sbt @@ -21,6 +21,6 @@ name := "generators" libraryDependencies += "eu.timepit" %% "refined" % "0.10.1" libraryDependencies += "io.circe" %% "circe-core" % "0.14.3" -libraryDependencies += "io.renku" %% "jsonld4s" % "0.6.0" -libraryDependencies += "org.typelevel" %% "cats-core" % "2.8.0" +libraryDependencies += "io.renku" %% "jsonld4s" % "0.7.0" +libraryDependencies += "org.typelevel" %% "cats-core" % "2.9.0" libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.17.0" diff --git a/graph-commons/build.sbt b/graph-commons/build.sbt index 8ec845f5b8..839aba96d5 100644 --- a/graph-commons/build.sbt +++ b/graph-commons/build.sbt @@ -38,7 +38,7 @@ libraryDependencies += "org.http4s" %% "http4s-dsl" % http4sVersi libraryDependencies += "org.http4s" %% "http4s-prometheus-metrics" % http4sPrometheusVersion libraryDependencies += "org.http4s" %% "http4s-server" % http4sVersion -libraryDependencies += "org.typelevel" %% "cats-effect" % "3.3.14" +libraryDependencies += "org.typelevel" %% "cats-effect" % "3.4.0" libraryDependencies += "org.typelevel" %% "log4cats-core" % "2.5.0" // Test dependencies diff --git a/project/plugins.sbt b/project/plugins.sbt index 254ecee172..f6ac3b6d32 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,7 +16,7 @@ * limitations under the License. */ -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.11") diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala index 467869348b..f5fe7ad773 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala @@ -133,12 +133,14 @@ object Activity { import io.renku.jsonld.JsonLDDecoder.decodeList def checkValid(association: Association)(implicit id: ResourceId): StartTime => JsonLDDecoder.Result[Unit] = - startTime => - dependencyLinks.findStepPlan(association.planId) match { - case Some(plan) if (startTime.value compareTo plan.dateCreated.value) < 0 => - DecodingFailure(show"Activity $id date $startTime is older than plan ${plan.dateCreated}", Nil).asLeft - case _ => ().asRight - } + _ => association.asRight.map(_ => ()) +// This code has been temporarily disabled; see https://github.com/SwissDataScienceCenter/renku-graph/issues/1187 +// startTime => +// dependencyLinks.findStepPlan(association.planId) match { +// case Some(plan) if (startTime.value compareTo plan.dateCreated.value) < 0 => +// DecodingFailure(show"Activity $id date $startTime is older than plan ${plan.dateCreated}", Nil).asLeft +// case _ => ().asRight +// } def checkSingle[T](prop: String)(implicit id: ResourceId): List[T] => JsonLDDecoder.Result[T] = { case prop :: Nil => Right(prop) diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala index ec55c024cc..2b0eb6dbfc 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala @@ -19,6 +19,7 @@ package io.renku.graph.model.entities import cats.syntax.all._ +import io.renku.graph.model.entities.ProjectLens.collectStepPlans import monocle.Lens object ActivityLens { @@ -31,4 +32,7 @@ object ActivityLens { val activityAssociationAgent: Lens[Activity, Either[Agent, Person]] = activityAssociation >>> AssociationLens.associationAgent + + def activityStepPlan(plans: List[Plan]): Lens[Activity, StepPlan] = + ActivityLens.activityAssociation >>> AssociationLens.associationStepPlan(collectStepPlans(plans)) } diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala index 3d9e0ae345..22ec2a9cfa 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala @@ -38,4 +38,16 @@ object AssociationLens { assoc.copy(agent = agent) } } + + def associationStepPlan(stepPlans: List[StepPlan]): Lens[Association, StepPlan] = + Lens[Association, StepPlan](a => + stepPlans + .find(_.resourceId == a.planId) + .getOrElse( + throw new IllegalStateException(s"Association ${a.resourceId} pointing to non-existing plan ${a.planId}") + ) + )(p => { + case a: Association.WithPersonAgent => a.copy(planId = p.resourceId) + case a: Association.WithRenkuAgent => a.copy(planId = p.resourceId) + }) } diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala index 0a44c6cc70..3d1939d34c 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala @@ -18,6 +18,7 @@ package io.renku.graph.model.entities +import io.renku.graph.model.plans.DateCreated import monocle.Lens object PlanLens { @@ -32,4 +33,11 @@ object PlanLens { val planCreators: Lens[Plan, List[Person]] = Lens[Plan, List[Person]](_.creators) { persons => { case plan: StepPlan => stepPlanCreators.modify(_ => persons)(plan) } } + + val planDateCreated: Lens[Plan, DateCreated] = Lens[Plan, DateCreated](_.dateCreated) { date => + { + case plan: StepPlan.NonModified => plan.copy(dateCreated = date) + case plan: StepPlan.Modified => plan.copy(dateCreated = date) + } + } } diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala index 25c79b6705..0a6a7dbd41 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala @@ -18,6 +18,7 @@ package io.renku.graph.model.entities +import PlanLens.planDateCreated import cats.Show import cats.data.{NonEmptyList, Validated, ValidatedNel} import cats.syntax.all._ @@ -130,27 +131,26 @@ object RenkuProject { validateDates(dateCreated, activities, datasets, plans), validatePlansDates(plans), validateDatasets(datasets), - updatePlansOriginalId(plans) - ) - .mapN { (_, _, _, updatedPlans) => - val (syncedActivities, syncedDatasets, syncedPlans) = - syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans) - RenkuProject.WithoutParent(resourceId, - path, - name, - maybeDescription, - agent, - dateCreated, - maybeCreator, - visibility, - keywords, - members, - version, - syncedActivities, - syncedDatasets, - syncedPlans - ) - } + updatePlansOriginalId(updatePlansDateCreated(plans, activities)) + ).mapN { (_, _, _, updatedPlans) => + val (syncedActivities, syncedDatasets, syncedPlans) = + syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans) + RenkuProject.WithoutParent(resourceId, + path, + name, + maybeDescription, + agent, + dateCreated, + maybeCreator, + visibility, + keywords, + members, + version, + syncedActivities, + syncedDatasets, + syncedPlans + ) + } private def validateDates(dateCreated: DateCreated, activities: List[Activity], @@ -230,9 +230,9 @@ object RenkuProject { parentResourceId: ResourceId ): ValidatedNel[String, RenkuProject.WithParent] = ( validateDatasets(datasets), - updatePlansOriginalId(plans), - validatePlansDates(plans) - ) mapN { (_, updatedPlans, _) => + validatePlansDates(plans), + updatePlansOriginalId(updatePlansDateCreated(plans, activities)) + ) mapN { (_, _, updatedPlans) => val (syncedActivities, syncedDatasets, syncedPlans) = syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans) RenkuProject.WithParent( @@ -301,6 +301,30 @@ object RenkuProject { .map(_.reverse) } + // The Plan dateCreated is updated only because of a bug on CLI which can produce Activities with dates before the Plan + // Though CLI fixed the issue for new projects, there still might be old ones affected with the issue. + // CLI is going to add a migration which will fix the old projects so this update won't be needed. + // See https://github.com/SwissDataScienceCenter/renku-graph/issues/1187 + protected def updatePlansDateCreated(plans: List[Plan], activities: List[Activity]): List[Plan] = { + + def findMinActivityDate(planId: model.plans.ResourceId): Option[model.activities.StartTime] = + activities.collect { + case a if a.association.planId == planId => a.startTime + } match { + case Nil => None + case dates => dates.min.some + } + + plans + .map(p => + findMinActivityDate(p.resourceId) match { + case None => p + case Some(minActivityDate) if (p.dateCreated.value compareTo minActivityDate.value) <= 0 => p + case Some(minActivityDate) => planDateCreated.set(model.plans.DateCreated(minActivityDate.value))(p) + } + ) + } + private def findParentPlan(derivedFrom: model.plans.DerivedFrom, plans: List[Plan]) = Validated.fromOption(plans.find(_.resourceId.value == derivedFrom.value), NonEmptyList.one(show"Cannot find parent plan $derivedFrom") diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala index 1b421f5601..41f3a243b7 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala @@ -18,7 +18,15 @@ package io.renku.graph.model.entities +import monocle.Lens + object ProjectLens { val collectStepPlans: List[Plan] => List[StepPlan] = _.collect { case p: StepPlan => p } + + def plansLens[P <: Project]: Lens[P, List[Plan]] = Lens[P, List[Plan]](_.plans)(plans => { + case p: RenkuProject.WithParent => p.copy(plans = plans).asInstanceOf[P] + case p: RenkuProject.WithoutParent => p.copy(plans = plans).asInstanceOf[P] + case p => p + }) } diff --git a/renku-model/src/main/scala/io/renku/graph/model/plans.scala b/renku-model/src/main/scala/io/renku/graph/model/plans.scala index 2fd5e678d2..c88b0ed3fe 100644 --- a/renku-model/src/main/scala/io/renku/graph/model/plans.scala +++ b/renku-model/src/main/scala/io/renku/graph/model/plans.scala @@ -29,11 +29,11 @@ import java.time.Instant object plans { class ResourceId private (val value: String) extends AnyVal with StringTinyType - implicit object ResourceId extends TinyTypeFactory[ResourceId](new ResourceId(_)) with Url[ResourceId] - with EntityIdJsonLDOps[ResourceId] { + with EntityIdJsonLDOps[ResourceId] + with AnyResourceRenderer[ResourceId] { def apply(identifier: Identifier)(implicit renkuUrl: RenkuUrl): ResourceId = ResourceId((renkuUrl / "plans" / identifier).value) diff --git a/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala b/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala index c585ae7e73..f09d1e4eaa 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala @@ -186,7 +186,8 @@ class ActivitySpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope error.message should endWith(s"Activity ${entitiesActivity.resourceId} without or with multiple authors") } - "fail if Activity startTime is older than Plan creation date" in { + // This test needed to be temporarily disabled; see https://github.com/SwissDataScienceCenter/renku-graph/issues/1187 + "fail if Activity startTime is older than Plan creation date" ignore { val activity = { val a = activityEntities(stepPlanEntities())(projectCreatedDates().generateOne).generateOne diff --git a/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala index 886727dc08..ad2c627627 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala @@ -18,6 +18,7 @@ package io.renku.graph.model.entities +import PlanLens._ import cats.data.NonEmptyList import cats.syntax.all._ import io.circe.DecodingFailure @@ -316,6 +317,38 @@ class ProjectSpec extends AnyWordSpec with should.Matchers with ScalaCheckProper actualPlan3 shouldBe (modifiedPlanDerivation >>> planDerivationOriginalId) .set(entitiesPlan.resourceId)(entitiesPlanModification2) } + + s"update Plans' dateCreated if there are Activities created before the Plan for project $projectType" in new TestCase { + + val resourceId = projects.ResourceId(info.path) + val activity = { + val a = activityEntities(stepPlanEntities())(info.dateCreated).generateOne + a.replaceStartTime( + timestamps(min = info.dateCreated.value, max = a.plan.dateCreated.value.minusSeconds(1)) + .generateAs(activities.StartTime) + ) + } + val entitiesActivity = activity.to[entities.Activity] + val plan = activity.plan + val entitiesPlan = plan.to[entities.Plan] + + val jsonLD = cliLikeJsonLD( + resourceId, + cliVersion, + schemaVersion, + info.maybeDescription, + info.keywords, + maybeCreator = None, + info.dateCreated, + activities = entitiesActivity :: Nil, + plans = entitiesPlan :: Nil + ) + + val Right(actual :: Nil) = jsonLD.cursor.as(decodeList(entities.Project.decoder(info))) + + actual.plans shouldBe List(planDateCreated.set(plans.DateCreated(entitiesActivity.startTime.value))(entitiesPlan)) + actual.activities shouldBe List(entitiesActivity) + } } "return a DecodingFailure when there's a Person entity that cannot be decoded" in new TestCase { diff --git a/tiny-types/build.sbt b/tiny-types/build.sbt index 218368ae3b..c276b8d07a 100644 --- a/tiny-types/build.sbt +++ b/tiny-types/build.sbt @@ -29,9 +29,9 @@ libraryDependencies += "io.circe" %% "circe-generic" % circeVersion libraryDependencies += "io.circe" %% "circe-optics" % circeOpticsVersion libraryDependencies += "io.circe" %% "circe-parser" % circeVersion -libraryDependencies += "io.renku" %% "jsonld4s" % "0.6.0" +libraryDependencies += "io.renku" %% "jsonld4s" % "0.7.0" -val catsVersion = "2.8.0" +val catsVersion = "2.9.0" libraryDependencies += "org.typelevel" %% "cats-core" % catsVersion libraryDependencies += "org.typelevel" %% "cats-free" % catsVersion diff --git a/token-repository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala b/token-repository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala index 6bb58e4382..985d801496 100644 --- a/token-repository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala +++ b/token-repository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala @@ -20,7 +20,6 @@ package io.renku.tokenrepository.repository import cats.data.Kleisli import cats.effect.IO -import cats.syntax.all._ import com.dimafeng.testcontainers._ import io.renku.db.{PostgresContainer, SessionResource} import io.renku.testtools.IOSpec diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile index 3031effa94..b3496ed840 100644 --- a/triples-generator/Dockerfile +++ b/triples-generator/Dockerfile @@ -31,7 +31,7 @@ RUN apk update && apk add --no-cache tzdata git git-lfs curl bash python3-dev py python3 -m pip install --ignore-installed packaging && \ python3 -m pip install --upgrade 'pip==22.2.2' && \ python3 -m pip install jinja2 && \ - python3 -m pip install 'renku==1.9.1' 'sentry-sdk==1.5.11' && \ + python3 -m pip install 'renku==1.9.2' 'sentry-sdk==1.5.11' && \ chown -R daemon:daemon . COPY triples-generator/entrypoint.sh /entrypoint.sh diff --git a/triples-generator/src/main/resources/application.conf b/triples-generator/src/main/resources/application.conf index e292f860bd..ae52ff28e5 100644 --- a/triples-generator/src/main/resources/application.conf +++ b/triples-generator/src/main/resources/application.conf @@ -23,7 +23,7 @@ triples-generation = "renku-log" # * ["0.16.1 -> 9", ...] as above # * ["0.16.1 -> 9", "0.16.0 -> 9", ...] then the decision about re-provisioning is taken using the 9 schema version and 0.16.1 CLI version compatibility-matrix = [ - "1.9.1 -> 9", + "1.9.2 -> 9", "0.16.2 -> 8" ] diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivities.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivities.scala new file mode 100644 index 0000000000..16d7884b93 --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivities.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.Schemas.{prov, schema, xsd} +import io.renku.metrics.MetricsRegistry +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import org.typelevel.log4cats.Logger +import tooling.UpdateQueryMigration + +private object FixPlansYoungerThanActivities { + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + UpdateQueryMigration[F](name, query).widen + + private lazy val name = Migration.Name("Fix Activities older than Plan") + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of (prov -> "prov", schema -> "schema", xsd -> "xsd"), + s"""|DELETE { GRAPH ?projectId { ?planId schema:dateCreated ?dateCreated } } + |INSERT { GRAPH ?projectId { ?planId schema:dateCreated ?startTime } } + |WHERE { + | GRAPH ?projectId { + | ?activityId prov:startedAtTime ?startTime; + | prov:qualifiedAssociation/prov:hadPlan ?planId. + | ?planId schema:dateCreated ?dateCreated + | } + | FILTER (xsd:dateTime(?startTime) < xsd:dateTime(?dateCreated)) + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala index 4e5ffed45a..67c9135a4e 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala @@ -33,15 +33,17 @@ private[tsmigrationrequest] object Migrations { def apply[F[_]: Async: ReProvisioningStatus: Logger: MetricsRegistry: SparqlQueryTimeRecorder]( config: Config ): F[List[Migration[F]]] = for { - datasetsCreator <- DatasetsCreator[F] - datasetsRemover <- DatasetsRemover[F] - reProvisioning <- ReProvisioning[F](config) - removeNotLinkedPersons <- RemoveNotLinkedPersons[F] + datasetsCreator <- DatasetsCreator[F] + datasetsRemover <- DatasetsRemover[F] + reProvisioning <- ReProvisioning[F](config) + removeNotLinkedPersons <- RemoveNotLinkedPersons[F] + fixPlansYoungerThanActivities <- FixPlansYoungerThanActivities[F] migrations <- validateNames( datasetsCreator, datasetsRemover, reProvisioning, - removeNotLinkedPersons + removeNotLinkedPersons, + fixPlansYoungerThanActivities ) } yield migrations diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreator.scala index fb25e7eee5..855b605505 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreator.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreator.scala @@ -34,6 +34,7 @@ private[tsprovisioning] class TransformationStepsCreatorImpl[F[_]: MonadThrow]( personTransformer: namedgraphs.persons.PersonTransformer[F], projectTransformer: namedgraphs.projects.ProjectTransformer[F], datasetTransformer: namedgraphs.datasets.DatasetTransformer[F], + planTransformer: namedgraphs.plans.PlanTransformer[F], activityTransformer: namedgraphs.activities.ActivityTransformer[F] ) extends TransformationStepsCreator[F] { @@ -41,6 +42,7 @@ private[tsprovisioning] class TransformationStepsCreatorImpl[F[_]: MonadThrow]( personTransformer.createTransformationStep, projectTransformer.createTransformationStep, datasetTransformer.createTransformationStep, + planTransformer.createTransformationStep, activityTransformer.createTransformationStep ) } @@ -51,10 +53,12 @@ private[consumers] object TransformationStepsCreator { personTransformer <- namedgraphs.persons.PersonTransformer[F] projectTransformer <- namedgraphs.projects.ProjectTransformer[F] datasetTransformer <- namedgraphs.datasets.DatasetTransformer[F] + planTransformer <- namedgraphs.plans.PlanTransformer[F] activityTransformer <- namedgraphs.activities.ActivityTransformer[F] } yield new TransformationStepsCreatorImpl[F](personTransformer, projectTransformer, datasetTransformer, + planTransformer, activityTransformer ) } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinder.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinder.scala new file mode 100644 index 0000000000..6bacbbd5db --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinder.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning.transformation.namedgraphs.plans + +import cats.effect.Async +import cats.syntax.all._ +import io.renku.graph.model.{plans, projects} +import io.renku.triplesstore.{ProjectsConnectionConfig, SparqlQueryTimeRecorder, TSClient} +import org.typelevel.log4cats.Logger + +private trait KGInfoFinder[F[_]] { + def findDateCreated(projectId: projects.ResourceId, planId: plans.ResourceId): F[Option[plans.DateCreated]] +} + +private object KGInfoFinder { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[KGInfoFinder[F]] = + ProjectsConnectionConfig[F]().map(new KGInfoFinderImpl(_)) +} + +private class KGInfoFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder](storeConfig: ProjectsConnectionConfig) + extends TSClient(storeConfig) + with KGInfoFinder[F] { + + import eu.timepit.refined.auto._ + import io.circe.Decoder + import io.renku.graph.model.Schemas.{prov, schema} + import io.renku.graph.model._ + import io.renku.graph.model.views.RdfResource + import io.renku.tinytypes.json.TinyTypeDecoders._ + import io.renku.triplesstore.SparqlQuery.Prefixes + import io.renku.triplesstore._ + + override def findDateCreated(projectId: projects.ResourceId, + planId: plans.ResourceId + ): F[Option[plans.DateCreated]] = { + implicit val decoder: Decoder[Option[plans.DateCreated]] = ResultsDecoder[Option, plans.DateCreated] { + implicit cur => extract[plans.DateCreated]("dateCreated") + } + + queryExpecting[Option[plans.DateCreated]]( + SparqlQuery.of( + name = "transformation - find plan dateCreated", + Prefixes of (schema -> "schema", prov -> "prov"), + s"""|SELECT DISTINCT ?dateCreated + |FROM <${GraphClass.Project.id(projectId)}> { + | ${planId.showAs[RdfResource]} a prov:Plan; + | schema:dateCreated ?dateCreated. + |} + |""".stripMargin + ) + ) + } +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformer.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformer.scala new file mode 100644 index 0000000000..d38a4961b6 --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformer.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning +package transformation.namedgraphs.plans + +import TransformationStep._ +import cats.MonadThrow +import cats.data.EitherT +import cats.effect.Async +import cats.syntax.all._ +import io.renku.graph.model.entities.Project +import io.renku.triplesgenerator.events.consumers.ProcessingRecoverableError +import io.renku.triplesstore.SparqlQueryTimeRecorder +import org.typelevel.log4cats.Logger + +private[transformation] trait PlanTransformer[F[_]] { + def createTransformationStep: TransformationStep[F] +} + +private[transformation] object PlanTransformer { + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[PlanTransformer[F]] = + KGInfoFinder[F].map(new PlanTransformerImpl[F](_, UpdatesCreator)) +} + +private class PlanTransformerImpl[F[_]: MonadThrow](kgInfoFinder: KGInfoFinder[F], + updatesCreator: UpdatesCreator, + recoverableErrorsRecovery: RecoverableErrorsRecovery = + RecoverableErrorsRecovery +) extends PlanTransformer[F] { + + import eu.timepit.refined.auto._ + import io.renku.graph.model.entities.ProjectLens._ + import kgInfoFinder._ + import recoverableErrorsRecovery._ + import updatesCreator._ + + override def createTransformationStep: TransformationStep[F] = + TransformationStep("Plan Updates", createTransformation) + + private def createTransformation: Transformation[F] = project => + EitherT { + updateDateCreated(project -> Queries.empty) + .map(_.asRight[ProcessingRecoverableError]) + .recoverWith(maybeRecoverableError("Problem finding activity details in KG")) + } + + private lazy val updateDateCreated: ((Project, Queries)) => F[(Project, Queries)] = { case (project, queries) => + collectStepPlans(project.plans) + .map(plan => + findDateCreated(project.resourceId, plan.resourceId) + .map(queriesDeletingDate(project.resourceId, plan, _)) + ) + .sequence + .map(_.flatten) + .map(quers => project -> (queries |+| Queries.preDataQueriesOnly(quers))) + } +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreator.scala new file mode 100644 index 0000000000..727d69d26a --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreator.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning.transformation.namedgraphs.plans + +import eu.timepit.refined.auto._ +import io.renku.graph.model.Schemas.{prov, schema} +import io.renku.graph.model.entities.StepPlan +import io.renku.graph.model.views.RdfResource +import io.renku.graph.model.{GraphClass, plans, projects} +import io.renku.triplesstore.SparqlQuery +import io.renku.triplesstore.SparqlQuery.Prefixes + +private object UpdatesCreator extends UpdatesCreator + +private trait UpdatesCreator { + + def queriesDeletingDate(projectId: projects.ResourceId, + stepPlan: StepPlan, + maybeDateCreated: Option[plans.DateCreated] + ): List[SparqlQuery] = Option + .when(!maybeDateCreated.forall(_ == stepPlan.dateCreated)) { + SparqlQuery.of( + name = "transformation - delete activity author link", + Prefixes of (schema -> "schema", prov -> "prov"), + s"""|DELETE { GRAPH <${GraphClass.Project.id(projectId)}> { ?planId schema:dateCreated ?dateCreated } } + |WHERE { + | BIND (${stepPlan.resourceId.showAs[RdfResource]} AS ?planId) + | GRAPH <${GraphClass.Project.id(projectId)}> { + | ?planId a prov:Plan; + | schema:dateCreated ?dateCreated + | } + |} + |""".stripMargin + ) + } + .toList +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivitiesSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivitiesSpec.scala new file mode 100644 index 0000000000..30f22e8ce7 --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/FixPlansYoungerThanActivitiesSpec.scala @@ -0,0 +1,123 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations + +import cats.effect.IO +import eu.timepit.refined.auto._ +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestampsNotInTheFuture +import io.renku.graph.model._ +import io.renku.graph.model.entities.ProjectLens._ +import io.renku.graph.model.entities._ +import ActivityLens.activityStepPlan +import io.renku.graph.model.plans.DateCreated +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.metrics.MetricsRegistry +import io.renku.testtools.IOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec +import tooling._ + +import java.time.Instant + +class FixPlansYoungerThanActivitiesSpec + extends AnyWordSpec + with should.Matchers + with IOSpec + with InMemoryJenaForSpec + with ProjectsDataset + with MockFactory { + + "run" should { + + "fix all Plans that are associated with Activities having startTime before Plan creation" in { + + val brokenProject = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .map(_.to[entities.RenkuProject]) + .map(movePlanDateBeforeActivity) + .generateOne + val validProject = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .map(_.to[entities.RenkuProject]) + .generateOne + + upload(to = projectsDataset, brokenProject, validProject) + + findAllDates shouldBe Set(brokenProject, validProject).flatMap(proj => + proj.activities.map(a => a.startTime -> activityStepPlan(proj.plans).get(a).dateCreated) + ) + + runUpdate(on = projectsDataset, FixPlansYoungerThanActivities.query).unsafeRunSync() shouldBe () + + findAllDates shouldBe Set( + validProject.activities + .map(a => a.startTime -> activityStepPlan(validProject.plans).get(a).dateCreated), + brokenProject.activities + .map(a => a.startTime -> plans.DateCreated(a.startTime.value)) + ).flatten + } + } + + "apply" should { + "return an QueryBasedMigration" in { + implicit val logger: TestLogger[IO] = TestLogger[IO]() + implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO] + implicit val metricsRegistry: MetricsRegistry[IO] = new MetricsRegistry.DisabledMetricsRegistry[IO]() + FixPlansYoungerThanActivities[IO].unsafeRunSync().getClass shouldBe classOf[UpdateQueryMigration[IO]] + } + } + + private def findAllDates: Set[(activities.StartTime, plans.DateCreated)] = runSelect( + on = projectsDataset, + SparqlQuery.of( + "fetch activity and plan dates", + Prefixes of (prov -> "prov", schema -> "schema"), + s"""|SELECT ?startTime ?dateCreated + |WHERE { + | GRAPH ?projectId { + | ?activityId prov:startedAtTime ?startTime; + | prov:qualifiedAssociation/prov:hadPlan/schema:dateCreated ?dateCreated + | } + |} + |""".stripMargin + ) + ).unsafeRunSync() + .map(row => + activities.StartTime(Instant.parse(row("startTime"))) -> plans.DateCreated(Instant.parse(row("dateCreated"))) + ) + .toSet + + private def movePlanDateBeforeActivity(project: entities.RenkuProject): entities.RenkuProject = { + val tweakedPlans = project.activities + .map(a => a.startTime -> activityStepPlan(project.plans).get(a)) + .map { case (startTime, plan) => + PlanLens.planDateCreated.set( + timestampsNotInTheFuture(butYoungerThan = startTime.value.minusSeconds(1)).generateAs(DateCreated) + )(plan) + } + + plansLens.set(tweakedPlans)(project) + } +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreatorSpec.scala index fcd5b3c521..2d403c3259 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreatorSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/TransformationStepsCreatorSpec.scala @@ -31,12 +31,13 @@ class TransformationStepsCreatorSpec extends AnyWordSpec with MockFactory with s "createSteps" should { "combine steps from person/project/dataset/activity transformers" in new TestCase { - val steps @ step1 :: step2 :: step3 :: step4 :: Nil = transformationSteps[Try].generateFixedSizeList(4) + val steps @ step1 :: step2 :: step3 :: step4 :: step5 :: Nil = transformationSteps[Try].generateFixedSizeList(5) (() => personTransformer.createTransformationStep).expects().returning(step1) (() => projectTransformer.createTransformationStep).expects().returning(step2) (() => datasetTransformer.createTransformationStep).expects().returning(step3) - (() => activityTransformer.createTransformationStep).expects().returning(step4) + (() => planTransformer.createTransformationStep).expects().returning(step4) + (() => activityTransformer.createTransformationStep).expects().returning(step5) stepsCreator.createSteps shouldBe steps } @@ -46,10 +47,12 @@ class TransformationStepsCreatorSpec extends AnyWordSpec with MockFactory with s val personTransformer = mock[namedgraphs.persons.PersonTransformer[Try]] val projectTransformer = mock[namedgraphs.projects.ProjectTransformer[Try]] val datasetTransformer = mock[namedgraphs.datasets.DatasetTransformer[Try]] + val planTransformer = mock[namedgraphs.plans.PlanTransformer[Try]] val activityTransformer = mock[namedgraphs.activities.ActivityTransformer[Try]] val stepsCreator = new TransformationStepsCreatorImpl[Try](personTransformer, projectTransformer, datasetTransformer, + planTransformer, activityTransformer ) } diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinderSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinderSpec.scala new file mode 100644 index 0000000000..7b0d937c4f --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/KGInfoFinderSpec.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning.transformation.namedgraphs.plans + +import cats.effect.IO +import cats.syntax.all._ +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.model.GraphModelGenerators.projectResourceIds +import io.renku.graph.model.entities +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.testtools.IOSpec +import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset, SparqlQueryTimeRecorder} +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +class KGInfoFinderSpec + extends AnyWordSpec + with IOSpec + with should.Matchers + with InMemoryJenaForSpec + with ProjectsDataset { + + "findDateCreated" should { + + "return plan's dateCreated" in new TestCase { + + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .generateOne + .to[entities.RenkuProject] + + val plan = project.plans.headOption.getOrElse(fail("Plan expected")) + + upload(to = projectsDataset, project) + + kgInfoFinder.findDateCreated(project.resourceId, plan.resourceId).unsafeRunSync() shouldBe plan.dateCreated.some + } + + "return no dateCreated if there's no Plan with the given id" in new TestCase { + kgInfoFinder + .findDateCreated(projectResourceIds.generateOne, planResourceIds.generateOne) + .unsafeRunSync() shouldBe Option.empty + } + } + + private trait TestCase { + private implicit val logger: TestLogger[IO] = TestLogger[IO]() + private implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO] + val kgInfoFinder = new KGInfoFinderImpl[IO](projectsDSConnectionInfo) + } +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformerSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformerSpec.scala new file mode 100644 index 0000000000..ff5904ee03 --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/PlanTransformerSpec.scala @@ -0,0 +1,126 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning +package transformation +package namedgraphs.plans + +import Generators.recoverableClientErrors +import TransformationStep.Queries +import cats.syntax.all._ +import io.renku.generators.CommonGraphGenerators.sparqlQueries +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.{exceptions, timestampsNotInTheFuture} +import io.renku.graph.model._ +import io.renku.graph.model.entities.ProjectLens._ +import io.renku.graph.model.testentities._ +import io.renku.triplesgenerator.events.consumers.ProcessingRecoverableError +import io.renku.triplesstore.SparqlQuery +import org.scalamock.scalatest.MockFactory +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +import scala.util.{Success, Try} + +class PlanTransformerSpec extends AnyWordSpec with should.Matchers with MockFactory { + + "createTransformationStep" should { + + "create update queries for changed plans' creation dates" in new TestCase { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities()).multiple: _*) + .generateOne + .to[entities.RenkuProject] + + val updateQueries = collectStepPlans(project.plans) >>= givenDateUpdates(project.resourceId) + + val step = transformer.createTransformationStep + + (step run project).value shouldBe (project -> Queries.preDataQueriesOnly(updateQueries)).asRight.pure[Try] + } + + "return the ProcessingRecoverableFailure if calls to KG fails with a network or HTTP error" in new TestCase { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .generateOne + .to[entities.RenkuProject] + + val exception = recoverableClientErrors.generateOne + findingPlanDateCreated(project.resourceId, + project.plans.head.resourceId, + returning = exception.raiseError[Try, Option[plans.DateCreated]] + ) + + val step = transformer.createTransformationStep + + val Success(Left(recoverableError)) = step.run(project).value + + recoverableError shouldBe a[ProcessingRecoverableError] + recoverableError.getMessage should startWith("Problem finding activity details in KG") + } + + "fail with NonRecoverableFailure if calls to KG fails with an unknown exception" in new TestCase { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .generateOne + .to[entities.RenkuProject] + + val exception = exceptions.generateOne + findingPlanDateCreated(project.resourceId, + project.plans.head.resourceId, + returning = exception.raiseError[Try, Option[plans.DateCreated]] + ) + + transformer.createTransformationStep.run(project).value shouldBe + exception.raiseError[Try, Either[ProcessingRecoverableError, (RenkuProject, Queries)]] + } + } + + private trait TestCase { + val kgInfoFinder = mock[KGInfoFinder[Try]] + val updatesCreator = mock[UpdatesCreator] + val transformer = new PlanTransformerImpl[Try](kgInfoFinder, updatesCreator) + + def givenDateUpdates(projectId: projects.ResourceId)(plan: entities.StepPlan): List[SparqlQuery] = { + val maybeDateCreatedInKG = timestampsNotInTheFuture.toGeneratorOf(plans.DateCreated).generateOption + findingPlanDateCreated(projectId, plan.resourceId, returning = maybeDateCreatedInKG.pure[Try]) + + val updateQueries = sparqlQueries.generateList() + prepareQueriesUpdatingDates(projectId, plan, maybeDateCreatedInKG, returning = updateQueries) + + updateQueries + } + + def findingPlanDateCreated(projectId: projects.ResourceId, + resourceId: plans.ResourceId, + returning: Try[Option[plans.DateCreated]] + ) = (kgInfoFinder + .findDateCreated(_: projects.ResourceId, _: plans.ResourceId)) + .expects(projectId, resourceId) + .returning(returning) + + def prepareQueriesUpdatingDates(projectId: projects.ResourceId, + plan: entities.StepPlan, + maybeKGDate: Option[plans.DateCreated], + returning: List[SparqlQuery] + ) = (updatesCreator + .queriesDeletingDate(_: projects.ResourceId, _: entities.StepPlan, _: Option[plans.DateCreated])) + .expects(projectId, plan, maybeKGDate) + .returning(returning) + } +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreatorSpec.scala new file mode 100644 index 0000000000..9a7972f5ec --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsprovisioning/transformation/namedgraphs/plans/UpdatesCreatorSpec.scala @@ -0,0 +1,111 @@ +/* + * Copyright 2022 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsprovisioning.transformation.namedgraphs.plans + +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.generators.Generators.Implicits._ +import io.renku.generators.Generators.timestampsNotInTheFuture +import io.renku.graph.model._ +import io.renku.graph.model.entities.ProjectLens._ +import io.renku.graph.model.testentities._ +import io.renku.graph.model.views.RdfResource +import io.renku.testtools.IOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.{InMemoryJenaForSpec, ProjectsDataset, SparqlQuery} +import org.scalatest.matchers.should +import org.scalatest.wordspec.AnyWordSpec + +import java.time.Instant + +class UpdatesCreatorSpec + extends AnyWordSpec + with IOSpec + with should.Matchers + with InMemoryJenaForSpec + with ProjectsDataset { + + "queriesDeletingDate" should { + + "prepare delete query if new Plan has different dateCreated that it's set in the TS" in { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .map(_.to[entities.RenkuProject]) + .generateOne + + upload(to = projectsDataset, project) + + val plan = collectStepPlans(project.plans).headOption.getOrElse(fail("Expected plan")) + + findPlanDateCreated(project.resourceId, plan.resourceId) shouldBe List(plan.dateCreated) + + UpdatesCreator + .queriesDeletingDate(project.resourceId, + plan, + timestampsNotInTheFuture.toGeneratorOf(plans.DateCreated).generateSome + ) + .runAll(on = projectsDataset) + .unsafeRunSync() + + findPlanDateCreated(project.resourceId, plan.resourceId) shouldBe List.empty + } + + "do nothing if there's no date set for the Plan in the TS" in { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .map(_.to[entities.RenkuProject]) + .generateOne + + val plan = collectStepPlans(project.plans).headOption.getOrElse(fail("Expected plan")) + + UpdatesCreator + .queriesDeletingDate(project.resourceId, plan, plan.dateCreated.some) shouldBe Nil + } + + "prepare no queries if there's no change in Plan dateCreated" in { + val project = anyRenkuProjectEntities + .withActivities(activityEntities(stepPlanEntities())) + .map(_.to[entities.RenkuProject]) + .generateOne + + val plan = collectStepPlans(project.plans).headOption.getOrElse(fail("Expected plan")) + + UpdatesCreator + .queriesDeletingDate(project.resourceId, plan, plan.dateCreated.some) shouldBe Nil + } + } + + private def findPlanDateCreated(projectId: projects.ResourceId, planId: plans.ResourceId): List[plans.DateCreated] = + runSelect( + on = projectsDataset, + SparqlQuery.of( + "fetch agent", + Prefixes.of(prov -> "prov", schema -> "schema"), + s"""|SELECT ?dateCreated + |FROM <${GraphClass.Project.id(projectId)}> { + | ${planId.showAs[RdfResource]} a prov:Plan; + | schema:dateCreated ?dateCreated. + |} + |""".stripMargin + ) + ).unsafeRunSync() + .flatMap(row => + row.get("dateCreated").map(Instant.parse).map(plans.DateCreated.from).map(_.fold(throw _, identity)) + ) +}