chore: prepare 2.22.2 (#1189)

* chore: CLI upgraded to 1.9.2 * feat: TS migration fixing creation dates on Plans (#1183)
SwissDataScienceCenter · Nov 15, 2022 · 90f6880 · 90f6880
1 parent 717217d
commit 90f6880
Show file tree

Hide file tree

Showing 30 changed files with 827 additions and 52 deletions.
diff --git a/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala b/event-log/src/test/scala/io/renku/eventlog/InMemoryEventLogDb.scala
@@ -20,7 +20,6 @@ package io.renku.eventlog
 
 import cats.data.Kleisli
 import cats.effect.IO
-import cats.syntax.all._
 import com.dimafeng.testcontainers._
 import io.renku.db.PostgresContainer
 import io.renku.eventlog.EventLogDB.SessionResource

diff --git a/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala b/event-log/src/test/scala/io/renku/eventlog/init/BatchDateAdderSpec.scala
@@ -20,7 +20,6 @@ package io.renku.eventlog.init
 
 import cats.data.Kleisli
 import cats.effect.IO
-import cats.syntax.all._
 import io.circe.literal._
 import io.renku.eventlog.EventContentGenerators._
 import io.renku.eventlog.init.Generators._

diff --git a/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala b/event-log/src/test/scala/io/renku/eventlog/init/ProjectPathRemoverSpec.scala
@@ -20,7 +20,6 @@ package io.renku.eventlog.init
 
 import cats.data.Kleisli
 import cats.effect.IO
-import cats.syntax.all._
 import io.renku.graph.model.projects
 import io.renku.interpreters.TestLogger
 import io.renku.interpreters.TestLogger.Level.Info

diff --git a/generators/build.sbt b/generators/build.sbt
@@ -21,6 +21,6 @@ name := "generators"
 
 libraryDependencies += "eu.timepit"     %% "refined"    % "0.10.1"
 libraryDependencies += "io.circe"       %% "circe-core" % "0.14.3"
-libraryDependencies += "io.renku"       %% "jsonld4s"   % "0.6.0"
-libraryDependencies += "org.typelevel"  %% "cats-core"  % "2.8.0"
+libraryDependencies += "io.renku"       %% "jsonld4s"   % "0.7.0"
+libraryDependencies += "org.typelevel"  %% "cats-core"  % "2.9.0"
 libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.17.0"
diff --git a/graph-commons/build.sbt b/graph-commons/build.sbt
@@ -38,7 +38,7 @@ libraryDependencies += "org.http4s" %% "http4s-dsl"                % http4sVersi
 libraryDependencies += "org.http4s" %% "http4s-prometheus-metrics" % http4sPrometheusVersion
 libraryDependencies += "org.http4s" %% "http4s-server"             % http4sVersion
 
-libraryDependencies += "org.typelevel" %% "cats-effect"   % "3.3.14"
+libraryDependencies += "org.typelevel" %% "cats-effect"   % "3.4.0"
 libraryDependencies += "org.typelevel" %% "log4cats-core" % "2.5.0"
 
 // Test dependencies

diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0")
 
 addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.11")
 

diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/Activity.scala
@@ -133,12 +133,14 @@ object Activity {
       import io.renku.jsonld.JsonLDDecoder.decodeList
 
       def checkValid(association: Association)(implicit id: ResourceId): StartTime => JsonLDDecoder.Result[Unit] =
-        startTime =>
-          dependencyLinks.findStepPlan(association.planId) match {
-            case Some(plan) if (startTime.value compareTo plan.dateCreated.value) < 0 =>
-              DecodingFailure(show"Activity $id date $startTime is older than plan ${plan.dateCreated}", Nil).asLeft
-            case _ => ().asRight
-          }
+        _ => association.asRight.map(_ => ())
+//    This code has been temporarily disabled; see https://github.com/SwissDataScienceCenter/renku-graph/issues/1187
+//        startTime =>
+//          dependencyLinks.findStepPlan(association.planId) match {
+//            case Some(plan) if (startTime.value compareTo plan.dateCreated.value) < 0 =>
+//              DecodingFailure(show"Activity $id date $startTime is older than plan ${plan.dateCreated}", Nil).asLeft
+//            case _ => ().asRight
+//          }
 
       def checkSingle[T](prop: String)(implicit id: ResourceId): List[T] => JsonLDDecoder.Result[T] = {
         case prop :: Nil => Right(prop)

diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/ActivityLens.scala
@@ -19,6 +19,7 @@
 package io.renku.graph.model.entities
 
 import cats.syntax.all._
+import io.renku.graph.model.entities.ProjectLens.collectStepPlans
 import monocle.Lens
 
 object ActivityLens {
@@ -31,4 +32,7 @@ object ActivityLens {
 
   val activityAssociationAgent: Lens[Activity, Either[Agent, Person]] =
     activityAssociation >>> AssociationLens.associationAgent
+
+  def activityStepPlan(plans: List[Plan]): Lens[Activity, StepPlan] =
+    ActivityLens.activityAssociation >>> AssociationLens.associationStepPlan(collectStepPlans(plans))
 }
diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/AssociationLens.scala
@@ -38,4 +38,16 @@ object AssociationLens {
           assoc.copy(agent = agent)
       }
     }
+
+  def associationStepPlan(stepPlans: List[StepPlan]): Lens[Association, StepPlan] =
+    Lens[Association, StepPlan](a =>
+      stepPlans
+        .find(_.resourceId == a.planId)
+        .getOrElse(
+          throw new IllegalStateException(s"Association ${a.resourceId} pointing to non-existing plan ${a.planId}")
+        )
+    )(p => {
+      case a: Association.WithPersonAgent => a.copy(planId = p.resourceId)
+      case a: Association.WithRenkuAgent  => a.copy(planId = p.resourceId)
+    })
 }
diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/PlanLens.scala
@@ -18,6 +18,7 @@
 
 package io.renku.graph.model.entities
 
+import io.renku.graph.model.plans.DateCreated
 import monocle.Lens
 
 object PlanLens {
@@ -32,4 +33,11 @@ object PlanLens {
   val planCreators: Lens[Plan, List[Person]] = Lens[Plan, List[Person]](_.creators) { persons =>
     { case plan: StepPlan => stepPlanCreators.modify(_ => persons)(plan) }
   }
+
+  val planDateCreated: Lens[Plan, DateCreated] = Lens[Plan, DateCreated](_.dateCreated) { date =>
+    {
+      case plan: StepPlan.NonModified => plan.copy(dateCreated = date)
+      case plan: StepPlan.Modified    => plan.copy(dateCreated = date)
+    }
+  }
 }
diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/Project.scala
@@ -18,6 +18,7 @@
 
 package io.renku.graph.model.entities
 
+import PlanLens.planDateCreated
 import cats.Show
 import cats.data.{NonEmptyList, Validated, ValidatedNel}
 import cats.syntax.all._
@@ -130,27 +131,26 @@ object RenkuProject {
       validateDates(dateCreated, activities, datasets, plans),
       validatePlansDates(plans),
       validateDatasets(datasets),
-      updatePlansOriginalId(plans)
-    )
-      .mapN { (_, _, _, updatedPlans) =>
-        val (syncedActivities, syncedDatasets, syncedPlans) =
-          syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans)
-        RenkuProject.WithoutParent(resourceId,
-                                   path,
-                                   name,
-                                   maybeDescription,
-                                   agent,
-                                   dateCreated,
-                                   maybeCreator,
-                                   visibility,
-                                   keywords,
-                                   members,
-                                   version,
-                                   syncedActivities,
-                                   syncedDatasets,
-                                   syncedPlans
-        )
-      }
+      updatePlansOriginalId(updatePlansDateCreated(plans, activities))
+    ).mapN { (_, _, _, updatedPlans) =>
+      val (syncedActivities, syncedDatasets, syncedPlans) =
+        syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans)
+      RenkuProject.WithoutParent(resourceId,
+                                 path,
+                                 name,
+                                 maybeDescription,
+                                 agent,
+                                 dateCreated,
+                                 maybeCreator,
+                                 visibility,
+                                 keywords,
+                                 members,
+                                 version,
+                                 syncedActivities,
+                                 syncedDatasets,
+                                 syncedPlans
+      )
+    }
 
     private def validateDates(dateCreated: DateCreated,
                               activities:  List[Activity],
@@ -230,9 +230,9 @@ object RenkuProject {
              parentResourceId: ResourceId
     ): ValidatedNel[String, RenkuProject.WithParent] = (
       validateDatasets(datasets),
-      updatePlansOriginalId(plans),
-      validatePlansDates(plans)
-    ) mapN { (_, updatedPlans, _) =>
+      validatePlansDates(plans),
+      updatePlansOriginalId(updatePlansDateCreated(plans, activities))
+    ) mapN { (_, _, updatedPlans) =>
       val (syncedActivities, syncedDatasets, syncedPlans) =
         syncPersons(projectPersons = members ++ maybeCreator, activities, datasets, updatedPlans)
       RenkuProject.WithParent(
@@ -301,6 +301,30 @@ object RenkuProject {
         .map(_.reverse)
     }
 
+    // The Plan dateCreated is updated only because of a bug on CLI which can produce Activities with dates before the Plan
+    // Though CLI fixed the issue for new projects, there still might be old ones affected with the issue.
+    // CLI is going to add a migration which will fix the old projects so this update won't be needed.
+    // See https://github.com/SwissDataScienceCenter/renku-graph/issues/1187
+    protected def updatePlansDateCreated(plans: List[Plan], activities: List[Activity]): List[Plan] = {
+
+      def findMinActivityDate(planId: model.plans.ResourceId): Option[model.activities.StartTime] =
+        activities.collect {
+          case a if a.association.planId == planId => a.startTime
+        } match {
+          case Nil   => None
+          case dates => dates.min.some
+        }
+
+      plans
+        .map(p =>
+          findMinActivityDate(p.resourceId) match {
+            case None                                                                                => p
+            case Some(minActivityDate) if (p.dateCreated.value compareTo minActivityDate.value) <= 0 => p
+            case Some(minActivityDate) => planDateCreated.set(model.plans.DateCreated(minActivityDate.value))(p)
+          }
+        )
+    }
+
     private def findParentPlan(derivedFrom: model.plans.DerivedFrom, plans: List[Plan]) =
       Validated.fromOption(plans.find(_.resourceId.value == derivedFrom.value),
                            NonEmptyList.one(show"Cannot find parent plan $derivedFrom")

diff --git a/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala b/renku-model/src/main/scala/io/renku/graph/model/entities/ProjectLens.scala
@@ -18,7 +18,15 @@
 
 package io.renku.graph.model.entities
 
+import monocle.Lens
+
 object ProjectLens {
 
   val collectStepPlans: List[Plan] => List[StepPlan] = _.collect { case p: StepPlan => p }
+
+  def plansLens[P <: Project]: Lens[P, List[Plan]] = Lens[P, List[Plan]](_.plans)(plans => {
+    case p: RenkuProject.WithParent    => p.copy(plans = plans).asInstanceOf[P]
+    case p: RenkuProject.WithoutParent => p.copy(plans = plans).asInstanceOf[P]
+    case p => p
+  })
 }
diff --git a/renku-model/src/main/scala/io/renku/graph/model/plans.scala b/renku-model/src/main/scala/io/renku/graph/model/plans.scala
@@ -29,11 +29,11 @@ import java.time.Instant
 object plans {
 
   class ResourceId private (val value: String) extends AnyVal with StringTinyType
-
   implicit object ResourceId
       extends TinyTypeFactory[ResourceId](new ResourceId(_))
       with Url[ResourceId]
-      with EntityIdJsonLDOps[ResourceId] {
+      with EntityIdJsonLDOps[ResourceId]
+      with AnyResourceRenderer[ResourceId] {
 
     def apply(identifier: Identifier)(implicit renkuUrl: RenkuUrl): ResourceId =
       ResourceId((renkuUrl / "plans" / identifier).value)

diff --git a/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala b/renku-model/src/test/scala/io/renku/graph/model/entities/ActivitySpec.scala
@@ -186,7 +186,8 @@ class ActivitySpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope
       error.message should endWith(s"Activity ${entitiesActivity.resourceId} without or with multiple authors")
     }
 
-    "fail if Activity startTime is older than Plan creation date" in {
+    // This test needed to be temporarily disabled; see https://github.com/SwissDataScienceCenter/renku-graph/issues/1187
+    "fail if Activity startTime is older than Plan creation date" ignore {
 
       val activity = {
         val a = activityEntities(stepPlanEntities())(projectCreatedDates().generateOne).generateOne

diff --git a/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/entities/ProjectSpec.scala
@@ -18,6 +18,7 @@
 
 package io.renku.graph.model.entities
 
+import PlanLens._
 import cats.data.NonEmptyList
 import cats.syntax.all._
 import io.circe.DecodingFailure
@@ -316,6 +317,38 @@ class ProjectSpec extends AnyWordSpec with should.Matchers with ScalaCheckProper
         actualPlan3 shouldBe (modifiedPlanDerivation >>> planDerivationOriginalId)
           .set(entitiesPlan.resourceId)(entitiesPlanModification2)
       }
+
+      s"update Plans' dateCreated if there are Activities created before the Plan for project $projectType" in new TestCase {
+
+        val resourceId = projects.ResourceId(info.path)
+        val activity = {
+          val a = activityEntities(stepPlanEntities())(info.dateCreated).generateOne
+          a.replaceStartTime(
+            timestamps(min = info.dateCreated.value, max = a.plan.dateCreated.value.minusSeconds(1))
+              .generateAs(activities.StartTime)
+          )
+        }
+        val entitiesActivity = activity.to[entities.Activity]
+        val plan             = activity.plan
+        val entitiesPlan     = plan.to[entities.Plan]
+
+        val jsonLD = cliLikeJsonLD(
+          resourceId,
+          cliVersion,
+          schemaVersion,
+          info.maybeDescription,
+          info.keywords,
+          maybeCreator = None,
+          info.dateCreated,
+          activities = entitiesActivity :: Nil,
+          plans = entitiesPlan :: Nil
+        )
+
+        val Right(actual :: Nil) = jsonLD.cursor.as(decodeList(entities.Project.decoder(info)))
+
+        actual.plans      shouldBe List(planDateCreated.set(plans.DateCreated(entitiesActivity.startTime.value))(entitiesPlan))
+        actual.activities shouldBe List(entitiesActivity)
+      }
     }
 
     "return a DecodingFailure when there's a Person entity that cannot be decoded" in new TestCase {

diff --git a/tiny-types/build.sbt b/tiny-types/build.sbt
@@ -29,9 +29,9 @@ libraryDependencies += "io.circe" %% "circe-generic" % circeVersion
 libraryDependencies += "io.circe" %% "circe-optics"  % circeOpticsVersion
 libraryDependencies += "io.circe" %% "circe-parser"  % circeVersion
 
-libraryDependencies += "io.renku" %% "jsonld4s" % "0.6.0"
+libraryDependencies += "io.renku" %% "jsonld4s" % "0.7.0"
 
-val catsVersion = "2.8.0"
+val catsVersion = "2.9.0"
 libraryDependencies += "org.typelevel" %% "cats-core" % catsVersion
 libraryDependencies += "org.typelevel" %% "cats-free" % catsVersion
 

diff --git a/...ository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala b/...ository/src/test/scala/io/renku/tokenrepository/repository/InMemoryProjectsTokensDb.scala
@@ -20,7 +20,6 @@ package io.renku.tokenrepository.repository
 
 import cats.data.Kleisli
 import cats.effect.IO
-import cats.syntax.all._
 import com.dimafeng.testcontainers._
 import io.renku.db.{PostgresContainer, SessionResource}
 import io.renku.testtools.IOSpec

diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile
@@ -31,7 +31,7 @@ RUN apk update && apk add --no-cache tzdata git git-lfs curl bash python3-dev py
     python3 -m pip install --ignore-installed packaging && \
     python3 -m pip install --upgrade 'pip==22.2.2' && \
     python3 -m pip install jinja2 && \
-    python3 -m pip install 'renku==1.9.1' 'sentry-sdk==1.5.11'  && \
+    python3 -m pip install 'renku==1.9.2' 'sentry-sdk==1.5.11'  && \
     chown -R daemon:daemon .
 
 COPY triples-generator/entrypoint.sh /entrypoint.sh

diff --git a/triples-generator/src/main/resources/application.conf b/triples-generator/src/main/resources/application.conf
@@ -23,7 +23,7 @@ triples-generation = "renku-log"
 # *  ["0.16.1 -> 9", ...] as above
 # *  ["0.16.1 -> 9", "0.16.0 -> 9", ...] then the decision about re-provisioning is taken using the 9 schema version and 0.16.1 CLI version
 compatibility-matrix = [
-  "1.9.1  -> 9",
+  "1.9.2  -> 9",
   "0.16.2 -> 8"
 ]