chore: prepare 2.47.0 (#1794)

.git-blame-ignore-revs

@@ -24,3 +24,6 @@ b20f0ba57795f003fe4d90817422e14b880d6c8c
 
 # Scala Steward: Reformat with scalafmt 3.7.15
 506650a1df56b0088fd732560ad1385670395f98
+
+# Scala Steward: Reformat with scalafmt 3.7.17
+36c18f272a4da509eb66c4202092999ea16729c9

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.7.15"
+version = "3.7.17"
 
 runner.dialect = "scala213"
 

entities-search/src/test/scala/io/renku/entities/search/Generators.scala

@@ -30,11 +30,11 @@ import testentities._
 
 object Generators {
 
-  val queryParams: Gen[Filters.Query]      = nonBlankStrings(minLength = 5).map(v => Filters.Query(v.value))
-  val typeParams:  Gen[Filters.EntityType] = Gen.oneOf(Filters.EntityType.all)
-  val sinceParams:    Gen[Filters.Since] = localDatesNotInTheFuture.toGeneratorOf(Filters.Since)
-  val untilParams:    Gen[Filters.Until] = localDatesNotInTheFuture.toGeneratorOf(Filters.Until)
-  val matchingScores: Gen[MatchingScore] = choose(MatchingScore.min.value, 10f).toGeneratorOf(MatchingScore)
+  val queryParams:    Gen[Filters.Query]      = nonBlankStrings(minLength = 5).map(v => Filters.Query(v.value))
+  val typeParams:     Gen[Filters.EntityType] = Gen.oneOf(Filters.EntityType.all)
+  val sinceParams:    Gen[Filters.Since]      = localDatesNotInTheFuture.toGeneratorOf(Filters.Since)
+  val untilParams:    Gen[Filters.Until]      = localDatesNotInTheFuture.toGeneratorOf(Filters.Until)
+  val matchingScores: Gen[MatchingScore]      = choose(MatchingScore.min.value, 10f).toGeneratorOf(MatchingScore)
 
   val modelProjects: Gen[model.Entity.Project] = anyProjectEntities.map(_.to[model.Entity.Project])
   val modelDatasets: Gen[model.Entity.Dataset] =

entities-search/src/test/scala/io/renku/entities/searchgraphs/datasets/Generators.scala

@@ -105,7 +105,7 @@ private object Generators {
 
   implicit lazy val tsDatasetSearchInfoObjects: Gen[TSDatasetSearchInfo] = for {
     topmostSameAs <- datasetTopmostSameAs
-    links <- linkObjectsGen(topmostSameAs).toGeneratorOfList(max = 2)
+    links         <- linkObjectsGen(topmostSameAs).toGeneratorOfList(max = 2)
   } yield TSDatasetSearchInfo(topmostSameAs, links)
 
   def tsDatasetSearchInfoObjects(withLinkTo: projects.ResourceId, and: projects.ResourceId*): Gen[TSDatasetSearchInfo] =
@@ -115,9 +115,9 @@ private object Generators {
       }(i)
     }
 
-  def tsDatasetSearchInfoObjects(withLinkTo: entities.Project,
+  def tsDatasetSearchInfoObjects(withLinkTo:   entities.Project,
                                  topSameAsGen: Gen[datasets.TopmostSameAs] = datasetTopmostSameAs
-                                ): Gen[TSDatasetSearchInfo] =
+  ): Gen[TSDatasetSearchInfo] =
     tsDatasetSearchInfoObjects.map(_.copy(topmostSameAs = topSameAsGen.generateOne)).flatMap { si =>
       linkObjectsGen(si.topmostSameAs)
         .map(linkProjectId replace withLinkTo.resourceId)

knowledge-graph/src/main/scala/io/renku/knowledgegraph/MicroserviceRoutes.scala

@@ -312,7 +312,11 @@ private class MicroserviceRoutes[F[_]: Async](
 
       (
         sort.find(request.uri.query).sequence,
-        PagingRequest(page.find(request.uri.query), perPage.find(request.uri.query))
+        PagingRequest(
+          page.find(request.uri.query),
+          // defaulting to PerPage.max is a temporary fix as CLI does not read DS from other pages yet
+          perPage.find(request.uri.query).fold(ifEmpty = PerPage.max.validNel[ParseFailure])(identity).some
+        )
       ).mapN { (maybeSorts, paging) =>
         val sorting: Sorting[Criteria.Sort.type] = Sorting.fromOptionalListOrDefault(maybeSorts, Sort.default)
         projectSlugParts.toProjectSlug

knowledge-graph/src/test/scala/io/renku/knowledgegraph/MicroserviceRoutesSpec.scala

@@ -703,27 +703,35 @@ class MicroserviceRoutesSpec
     val projectSlug = projectSlugs.generateOne
     val projectDsUri = projectSlug.toNamespaces
       .foldLeft(uri"/knowledge-graph/projects")(_ / _.show) / projectSlug.toPath / "datasets"
+    val defaultPerPage = PerPage.max
+    val defaultPaging  = PagingRequest.default.copy(perPage = defaultPerPage)
 
     forAll {
       Table(
         "uri"        -> "criteria",
-        projectDsUri -> Criteria(projectSlug),
+        projectDsUri -> Criteria(projectSlug, paging = defaultPaging),
         sortingDirections
           .map(dir =>
-            projectDsUri +? ("sort" -> s"name:$dir") -> Criteria(projectSlug, sorting = Sorting(Sort.By(ByName, dir)))
+            projectDsUri +? ("sort" -> s"name:$dir") -> Criteria(projectSlug,
+                                                                 sorting = Sorting(Sort.By(ByName, dir)),
+                                                                 paging = defaultPaging
+            )
           )
           .generateOne,
         sortingDirections
           .map(dir =>
             projectDsUri +? ("sort" -> s"dateModified:$dir") -> Criteria(projectSlug,
-                                                                         sorting = Sorting(Sort.By(ByDateModified, dir))
+                                                                         sorting =
+                                                                           Sorting(Sort.By(ByDateModified, dir)),
+                                                                         paging = defaultPaging
             )
           )
           .generateOne,
         pages
           .map(page =>
-            projectDsUri +? ("page" -> page.show) -> Criteria(projectSlug,
-                                                              paging = PagingRequest.default.copy(page = page)
+            projectDsUri +? ("page" -> page.show) -> Criteria(
+              projectSlug,
+              paging = PagingRequest.default.copy(page = page, perPage = defaultPerPage)
             )
           )
           .generateOne,

triples-generator/Dockerfile

@@ -45,7 +45,7 @@ USER tguser
 ENV PATH=$PATH:/home/tguser/.local/bin
 
 # Installing Renku
-RUN python3 -m pip install 'renku==2.7.0' 'sentry-sdk==1.5.11'
+RUN python3 -m pip install 'renku==2.8.0rc1' 'sentry-sdk==1.5.11'
 
 RUN git config --global user.name 'renku'  && \
     git config --global user.email 'renku@renkulab.io'  && \

triples-generator/src/main/resources/application.conf

@@ -19,7 +19,7 @@ triples-generation = "renku-log"
 # Defines expected version of the renku cli and the schema that is generated by the CLI.
 compatibility {
   # The expected version of CLI used by TS.
-  cli-version = "2.7.0"
+  cli-version = "2.8.0rc1"
 
   # The expected version of the schema as returned by CLI.
   schema-version = "10"

webhook-service/src/main/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcher.scala

@@ -24,10 +24,12 @@ import eu.timepit.refined.auto._
 import io.circe.Decoder.decodeList
 import io.renku.graph.model.projects
 import io.renku.http.client.{AccessToken, GitLabClient}
+import io.renku.http.rest.paging.model.Page
 import io.renku.webhookservice.hookfetcher.ProjectHookFetcher.HookIdAndUrl
 import io.renku.webhookservice.model.ProjectHookUrl
 import org.http4s.implicits.http4sLiteralsSyntax
 import org.typelevel.log4cats.Logger
+import org.typelevel.ci._
 
 private[webhookservice] trait ProjectHookFetcher[F[_]] {
   def fetchProjectHooks(projectId: projects.GitLabId, accessToken: AccessToken): F[Option[List[HookIdAndUrl]]]
@@ -43,30 +45,52 @@ private[webhookservice] object ProjectHookFetcher {
 private[webhookservice] class ProjectHookFetcherImpl[F[_]: Async: GitLabClient: Logger] extends ProjectHookFetcher[F] {
 
   import io.circe._
+  import fs2.Stream
   import io.renku.http.tinytypes.TinyTypeURIEncoder._
   import org.http4s.Status.{Forbidden, NotFound, Ok, Unauthorized}
   import org.http4s._
-  import org.http4s.circe._
+  import org.http4s.circe.CirceEntityDecoder._
 
   override def fetchProjectHooks(projectId:   projects.GitLabId,
                                  accessToken: AccessToken
-  ): F[Option[List[HookIdAndUrl]]] =
-    GitLabClient[F].get(uri"projects" / projectId / "hooks", "project-hooks")(mapResponse)(accessToken.some)
+  ): F[Option[List[HookIdAndUrl]]] = {
+    def fetchHooks(page: Page) =
+      GitLabClient[F].get(uri"projects" / projectId / "hooks" withQueryParam ("page", page), "project-hooks")(
+        mapResponse
+      )(accessToken.some)
 
-  private lazy val mapResponse: PartialFunction[(Status, Request[F], Response[F]), F[Option[List[HookIdAndUrl]]]] = {
-    case (Ok, _, response)                => response.as[List[HookIdAndUrl]].map(_.some)
-    case (NotFound, _, _)                 => List.empty[HookIdAndUrl].some.pure[F]
-    case (Unauthorized | Forbidden, _, _) => Option.empty[List[HookIdAndUrl]].pure[F]
+    def readNextPage
+        : ((Option[List[HookIdAndUrl]], Option[Page])) => Stream[F, (Option[List[HookIdAndUrl]], Option[Page])] = {
+      case (previousHooks, Some(nextPage)) =>
+        Stream
+          .eval(fetchHooks(nextPage))
+          .map { case (currentHooks, maybeNextPage) => (previousHooks |+| currentHooks) -> maybeNextPage }
+          .flatMap(readNextPage)
+      case (previousHooks, None) =>
+        Stream.emit(previousHooks -> None)
+    }
+
+    readNextPage(List.empty[HookIdAndUrl].some -> Page.first.some)
+      .map { case (maybeHooks, _) => maybeHooks }
+      .compile
+      .toList
+      .map(_.sequence.map(_.flatten))
   }
 
-  private implicit lazy val hooksIdsAndUrlsDecoder: EntityDecoder[F, List[HookIdAndUrl]] = {
-    implicit val decoder: Decoder[List[HookIdAndUrl]] = decodeList { cursor =>
-      for {
-        url <- cursor.downField("url").as[String].map(ProjectHookUrl.fromGitlab)
-        id  <- cursor.downField("id").as[Int]
-      } yield HookIdAndUrl(id, url)
-    }
+  private lazy val mapResponse
+      : PartialFunction[(Status, Request[F], Response[F]), F[(Option[List[HookIdAndUrl]], Option[Page])]] = {
+    case (Ok, _, response) =>
+      val maybeNextPage: Option[Page] =
+        response.headers.get(ci"X-Next-Page").flatMap(_.head.value.toIntOption.map(Page(_)))
+      response.as[List[HookIdAndUrl]].map(_.some -> maybeNextPage)
+    case (NotFound, _, _)                 => (List.empty[HookIdAndUrl].some -> Option.empty[Page]).pure[F]
+    case (Unauthorized | Forbidden, _, _) => (Option.empty[List[HookIdAndUrl]] -> Option.empty[Page]).pure[F]
+  }
 
-    jsonOf[F, List[HookIdAndUrl]]
+  private implicit val decoder: Decoder[List[HookIdAndUrl]] = decodeList { cursor =>
+    for {
+      url <- cursor.downField("url").as[String].map(ProjectHookUrl.fromGitlab)
+      id  <- cursor.downField("id").as[Int]
+    } yield HookIdAndUrl(id, url)
   }
 }

webhook-service/src/test/scala/io/renku/webhookservice/hookfetcher/ProjectHookFetcherSpec.scala

@@ -19,6 +19,7 @@
 package io.renku.webhookservice.hookfetcher
 
 import cats.effect.IO
+import cats.effect.testing.scalatest.AsyncIOSpec
 import cats.syntax.all._
 import eu.timepit.refined.api.Refined
 import eu.timepit.refined.auto._
@@ -30,93 +31,98 @@ import io.renku.generators.Generators._
 import io.renku.graph.model.GraphModelGenerators.projectIds
 import io.renku.http.client.RestClient.ResponseMappingF
 import io.renku.http.client.{AccessToken, GitLabClient}
+import io.renku.http.rest.paging.model.Page
 import io.renku.http.server.EndpointTester.jsonEntityEncoder
 import io.renku.http.tinytypes.TinyTypeURIEncoder._
 import io.renku.interpreters.TestLogger
-import io.renku.stubbing.ExternalServiceStubbing
-import io.renku.testtools.{GitLabClientTools, IOSpec}
+import io.renku.testtools.GitLabClientTools
 import io.renku.webhookservice.WebhookServiceGenerators.{hookIdAndUrls, projectHookUrls}
 import io.renku.webhookservice.hookfetcher.ProjectHookFetcher.HookIdAndUrl
 import org.http4s.implicits.http4sLiteralsSyntax
 import org.http4s.{Request, Response, Status, Uri}
 import org.scalacheck.Gen
-import org.scalamock.scalatest.MockFactory
+import org.scalamock.scalatest.AsyncMockFactory
 import org.scalatest.OptionValues
+import org.scalatest.flatspec.AsyncFlatSpec
 import org.scalatest.matchers.should
-import org.scalatest.wordspec.AnyWordSpec
 
 class ProjectHookFetcherSpec
-    extends AnyWordSpec
-    with MockFactory
-    with ExternalServiceStubbing
+    extends AsyncFlatSpec
+    with AsyncIOSpec
+    with AsyncMockFactory
     with GitLabClientTools[IO]
     with should.Matchers
-    with OptionValues
-    with IOSpec {
+    with OptionValues {
 
-  "fetchProjectHooks" should {
+  it should "return list of project hooks from all pages" in {
 
-    "return list of project hooks" in new TestCase {
+    val projectId    = projectIds.generateOne
+    val uri          = uri"projects" / projectId / "hooks"
+    val accessToken  = accessTokens.generateOne
+    val resultsPage1 = hookIdAndUrls.toGeneratorOfNonEmptyList(2).generateOne.toList
+    val resultsPage2 = hookIdAndUrls.toGeneratorOfNonEmptyList(2).generateOne.toList
 
-      val idAndUrls = hookIdAndUrls.toGeneratorOfNonEmptyList(2).generateOne.toList
+    getProjectHooks(uri.withQueryParam("page", 1), accessToken, returning = resultsPage1, maybeNextPage = Page(2).some)
+    getProjectHooks(uri.withQueryParam("page", 2), accessToken, returning = resultsPage2, maybeNextPage = None)
 
-      (gitLabClient
-        .get(_: Uri, _: String Refined NonEmpty)(_: ResponseMappingF[IO, Option[List[HookIdAndUrl]]])(
-          _: Option[AccessToken]
-        ))
-        .expects(uri, endpointName, *, accessToken.some)
-        .returning(idAndUrls.some.pure[IO])
-
-      fetcher
-        .fetchProjectHooks(projectId, accessToken)
-        .unsafeRunSync()
-        .value should contain theSameElementsAs idAndUrls
-    }
+    fetcher
+      .fetchProjectHooks(projectId, accessToken)
+      .asserting(_.value should contain theSameElementsAs (resultsPage1 ::: resultsPage2))
+  }
 
-    "return the list of hooks if the response is Ok" in new TestCase {
+  it should "return the list of hooks if the response is Ok" in {
 
-      val id  = positiveInts().generateOne.value
-      val url = projectHookUrls.generateOne
-      mapResponse((Status.Ok, Request(), Response().withEntity(json"""[{"id":$id, "url":${url.value}}]""")))
-        .unsafeRunSync() shouldBe List(HookIdAndUrl(id, url)).some
-    }
+    val id  = positiveInts().generateOne.value
+    val url = projectHookUrls.generateOne
+    mapResponse((Status.Ok, Request(), Response().withEntity(json"""[{"id":$id, "url":${url.value}}]""")))
+      .asserting(_ shouldBe List(HookIdAndUrl(id, url)).some -> None)
+  }
 
-    "return an empty list of hooks if the project does not exists" in new TestCase {
-      mapResponse(Status.NotFound, Request(), Response()).unsafeRunSync() shouldBe List.empty[HookIdAndUrl].some
-    }
+  it should "return an empty list of hooks if the project does not exists" in {
+    mapResponse(Status.NotFound, Request(), Response())
+      .asserting(_ shouldBe List.empty[HookIdAndUrl].some -> None)
+  }
 
-    Status.Unauthorized :: Status.Forbidden :: Nil foreach { status =>
-      show"return None if remote client responds with $status" in new TestCase {
-        mapResponse(status, Request(), Response()).unsafeRunSync() shouldBe None
-      }
+  Status.Unauthorized :: Status.Forbidden :: Nil foreach { status =>
+    it should show"return None if remote client responds with $status" in {
+      mapResponse(status, Request(), Response())
+        .asserting(_ shouldBe None -> None)
     }
+  }
 
-    "return an Exception if remote client responds with status any of OK , NOT_FOUND, UNAUTHORIZED or FORBIDDEN" in new TestCase {
-      intercept[Exception] {
-        mapResponse(Status.ServiceUnavailable, Request(), Response()).unsafeRunSync()
-      }
-    }
+  it should "return an Exception if remote client responds with status any of OK , NOT_FOUND, UNAUTHORIZED or FORBIDDEN" in {
+    intercept[Exception] {
+      mapResponse(Status.ServiceUnavailable, Request(), Response()).assertNoException
+    } shouldBe a[MatchError]
+  }
 
-    "return a RuntimeException if remote client responds with unexpected body" in new TestCase {
-      intercept[RuntimeException] {
-        mapResponse((Status.Ok, Request(), Response().withEntity("""{}"""))).unsafeRunSync()
-      }.getMessage should include("Could not decode JSON")
-    }
+  it should "return a RuntimeException if remote client responds with unexpected body" in {
+    mapResponse((Status.Ok, Request(), Response().withEntity("""{}""")))
+      .assertThrowsError[Exception](_.getMessage should include("Could not decode JSON"))
   }
 
-  private trait TestCase {
-    val projectId = projectIds.generateOne
-    val uri       = uri"projects" / projectId / "hooks"
-    val endpointName: String Refined NonEmpty = "project-hooks"
-    val accessToken = accessTokens.generateOne
+  private implicit val logger:   TestLogger[IO]   = TestLogger[IO]()
+  private implicit val glClient: GitLabClient[IO] = mock[GitLabClient[IO]]
+  private lazy val fetcher = new ProjectHookFetcherImpl[IO]
 
-    implicit val logger:       TestLogger[IO]   = TestLogger[IO]()
-    implicit val gitLabClient: GitLabClient[IO] = mock[GitLabClient[IO]]
-    val fetcher = new ProjectHookFetcherImpl[IO]
+  private def getProjectHooks(uri:           Uri,
+                              accessToken:   AccessToken,
+                              returning:     List[HookIdAndUrl],
+                              maybeNextPage: Option[Page]
+  ) = {
+    val endpointName: String Refined NonEmpty = "project-hooks"
 
-    val mapResponse = captureMapping(gitLabClient)(fetcher.fetchProjectHooks(projectId, accessToken).unsafeRunSync(),
-                                                   Gen.const(Option.empty[List[HookIdAndUrl]]),
-                                                   underlyingMethod = Get
-    )
+    (glClient
+      .get(_: Uri, _: String Refined NonEmpty)(_: ResponseMappingF[IO, (Option[List[HookIdAndUrl]], Option[Page])])(
+        _: Option[AccessToken]
+      ))
+      .expects(uri, endpointName, *, accessToken.some)
+      .returning((returning.some -> maybeNextPage).pure[IO])
   }
+
+  private lazy val mapResponse = captureMapping(glClient)(
+    fetcher.fetchProjectHooks(projectIds.generateOne, accessTokens.generateOne).unsafeRunSync(),
+    Gen.const(Option.empty[List[HookIdAndUrl]] -> Option.empty[Page]),
+    underlyingMethod = Get
+  )
 }