From 83b132dc50280e764cb72ee21875d6b4e3e64c54 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Sat, 6 Jan 2024 06:02:27 -0500 Subject: [PATCH] CI/CD Cleanup (#156) * CI/CD Cleanup * Restore creation of publish artifacts as part of matrix build as it seems faster --- .github/dependabot.yml | 6 + .github/release-drafter.yml | 36 ++ .github/workflows/ci-cd.yml | 12 +- .github/workflows/ci.yml | 83 ---- .github/workflows/github-dependency-graph.yml | 29 ++ .../workflows/github-dependency-review.yml | 14 + build-alt.sc | 394 ------------------ build.sc | 25 +- 8 files changed, 105 insertions(+), 494 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/release-drafter.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/github-dependency-graph.yml create mode 100644 .github/workflows/github-dependency-review.yml delete mode 100644 build-alt.sc diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5ace4600 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..f613ef99 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,36 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +template: | + # What's Changed + $CHANGES +categories: + - title: 'Breaking' + label: 'type: breaking' + - title: 'New' + label: 'type: feature' + - title: 'Bug Fixes' + label: 'type: bug' + - title: 'Maintenance' + label: 'type: maintenance' + - title: 'Documentation' + label: 'type: docs' + - title: 'Dependency Updates' + label: 'type: dependencies' + +version-resolver: + major: + labels: + - 'type: breaking' + minor: + labels: + - 'type: feature' + patch: + labels: + - 'type: bug' + - 'type: maintenance' + - 'type: docs' + - 'type: dependencies' + - 'type: security' + +exclude-labels: + - 'skip-changelog' diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index f506c551..d69bbbd6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -149,16 +149,17 @@ jobs: out/morphir/${{matrix.scala}}/**/native/ key: ${{ runner.os }}-mill-native-${{matrix.java}}-${{ matrix.scala }}-${{ github.sha }}-${{ hashFiles('out') }} - publish-sonatype: - # when in master repo: all commits to main branch and all additional tags - if: github.repository == 'finos/morphir-jvm' && ( github.ref == 'refs/heads/main' || (github.ref != 'refs/heads/main' && startsWith( github.ref, 'refs/tags/') ) ) + cd: needs: [ci] runs-on: ubuntu-latest + # when in primary repo: all commits to main branch and all additional tags + if: github.repository == 'finos/morphir-jvm' && ( github.ref == 'refs/heads/main' || (github.ref != 'refs/heads/main' && startsWith( github.ref, 'refs/tags/') ) ) # only run one publish job for the same sha at the same time # e.g. when a main-branch push is also tagged - concurrency: publish-sonatype-${{ github.sha }} + concurrency: + group: ${{ github.workflow}}-publish-${{ github.sha }} env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} @@ -260,7 +261,8 @@ jobs: key: ${{ runner.os }}-mill-native-11-3.3.1-${{ github.sha }}-${{ hashFiles('out') }} restore-keys: ${{ runner.os }}-mill-native-11-3.3.1-${{ github.sha }}- - - run: ./mill -i -j 0 io.kipp.mill.ci.release.ReleaseModule/publishAll + - name: Publish artifacts to Sonatype + run: ./mill -i -j 0 io.kipp.mill.ci.release.ReleaseModule/publishAll ci: runs-on: ubuntu-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 0657e71f..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Scala CI - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - release: - types: [published] - -jobs: - build: - if: false - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Pull all history with tags for correct versionning - run: git fetch --prune --unshallow - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - - name: Cache SBT ivy cache - uses: actions/cache@v1 - with: - path: ~/.ivy2/cache - key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('**/build.sc') }} - restore-keys: | - ${{ runner.os }}-sbt-ivy-cache- - - - name: Cache SBT - uses: actions/cache@v1 - with: - path: ~/.sbt - key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sc') }} - restore-keys: | - ${{ runner.os }}-sbt- - - - name: Cache Mill - uses: actions/cache@v1 - with: - path: ~/.cache/mill - key: ${{ runner.os }}-mill-${{ hashFiles('**/build.sc') }} - restore-keys: | - ${{ runner.os }}-mill- - - - name: Cache Coursier - uses: actions/cache@v1 - with: - path: ~/.cache/coursier - key: ${{ runner.os }}-coursier-${{ hashFiles('**/build.sc') }} - restore-keys: | - ${{ runner.os }}-coursier- - - - name: Cache Out - uses: actions/cache@v1 - with: - path: ./out - key: ${{ runner.os }}-out-${{ hashFiles('**/build.sc') }} - restore-keys: | - ${{ runner.os }}-out- - - - name: Checks - run: | - git config --global user.name "CI" - ./mill all __.checkFormat __.docJar __.test - # ./mill all __.checkFormat "__.fix --check" __.docJar __.test - - - name: Status Check - run: | - git status - - - name: Publish - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event_name == 'release' - run: | - echo "${{ secrets.PGP_SECRET }}" > private.key - gpg --batch --yes --import private.key - rm private.key - - ./mill mill.scalalib.PublishModule/publishAll --sonatypeCreds ${{ secrets.SONATYPE_USERNAME }}:${{ secrets.SONATYPE_PASSWORD }} --gpgArgs --passphrase,${{ secrets.PGP_PASSPHRASE }},--batch,--yes,-a,-b,--pinentry-mode,loopback --publishArtifacts __.publishArtifacts --awaitTimeout 900000 --readTimeout 600000 --release true diff --git a/.github/workflows/github-dependency-graph.yml b/.github/workflows/github-dependency-graph.yml new file mode 100644 index 00000000..d42dbe3f --- /dev/null +++ b/.github/workflows/github-dependency-graph.yml @@ -0,0 +1,29 @@ +name: dependency-graph + +on: + push: + branches: + - main + +jobs: + submit-dependency-graph: + if: github.repository == 'finos/morphir-scala' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: coursier/cache-action@v6 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + # We want to see the transmitted graph in the logs + - run: ./mill --import ivy:io.chris-kipp::mill-github-dependency-graph::0.2.6 showNamed io.kipp.mill.github.dependency.graph.Graph/generate + + # Actually upload the graph + - uses: ckipp01/mill-dependency-submission@v1 diff --git a/.github/workflows/github-dependency-review.yml b/.github/workflows/github-dependency-review.yml new file mode 100644 index 00000000..d093cd10 --- /dev/null +++ b/.github/workflows/github-dependency-review.yml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] +permissions: + contents: read +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v4 + - name: Dependency Review + uses: actions/dependency-review-action@v3 + with: + fail-on-severity: high \ No newline at end of file diff --git a/build-alt.sc b/build-alt.sc deleted file mode 100644 index d3cd0602..00000000 --- a/build-alt.sc +++ /dev/null @@ -1,394 +0,0 @@ -import $ivy.`com.goyeau::mill-git:0.2.0` -import $ivy.`com.goyeau::mill-scalafix:0.2.0` -import $ivy.`io.github.davidgregory084::mill-tpolecat:0.2.0` -import $ivy.`com.lihaoyi::mill-contrib-bloop:$MILL_VERSION` -import com.goyeau.mill.git._ -import com.goyeau.mill.scalafix.ScalafixModule -import io.github.davidgregory084._ -import mill._ -import mill.scalalib._ -import mill.scalajslib._ -import publish._ -import mill.scalalib.scalafmt._ -import coursier.maven.MavenRepository -import ammonite.ops._, ImplicitWd._ - -object Deps { - object Versions { - - val scala211 = "2.11.12" - val scala212 = "2.12.12" - val scala213 = "2.13.4" - val scalaJS06 = "0.6.32" - val scalaJS1 = "1.0.0" - - val scalaJVMVersions = Seq(scala211, scala212, scala213) - - val scalaJSVersions = Seq( - (scala212, scalaJS06), - (scala213, scalaJS06) - ) - - val silencer = "1.7.1" - val scalaCollectionsCompat = "2.3.1" - val zio = "1.0.4-2" - val zioConfig = "1.0.0-RC32" - val zioLogging = "0.5.6" - val zioMagic = "0.1.8" - val zioNio = "1.0.0-RC10" - val zioPrelude = "1.0.0-RC2" - val zioProcess = "0.2.0" - val newtype = "0.4.4" - def decline(scalaVersion: String) = scalaVersion match { - case version if version.startsWith("2.11") => "1.2.0" - case _ => "1.3.0" - } - val pprint = "0.5.9" - val scalameta = "4.3.18" - val directories = "11" - val enumeratum = "1.6.1" - val macroParadise = "2.1.1" - val upickle = "1.1.0" - val slf4zio = "1.0.0" - val scalactic = "3.1.2" - val scalaUri = "2.2.2" - val spark = "2.4.7" - val oslib = "0.6.2" - val quill = "3.6.0-RC3" - val scalatest = "3.2.16" - val circe = "0.14.1" - } -} - -trait MorphirScalaModule extends ScalaModule with TpolecatModule { self => - - override def scalacOptions = T { - super.scalacOptions().filterNot(Set("-Xlint:nullary-override")) - } -} - -trait MorphirScalafixModule extends ScalafixModule - -trait MorphirPublishModule extends GitVersionedPublishModule { - def packageDescription = T(artifactName()) - def pomSettings = PomSettings( - description = packageDescription(), - organization = "org.morphir", - url = "https://github.com/finos/morphir-jvm", - licenses = Seq(License.`Apache-2.0`), - versionControl = VersionControl.github("finos", "morphir-jvm"), - developers = Seq( - Developer( - "DamianReeves", - "Damian Reeves", - "https://github.com/DamianReeves" - ) - ) - ) - def publishVersion: T[String] = - GitVersionModule.version(withSnapshotSuffix = true)() -} - -trait ScalaMacroModule extends ScalaModule { - def crossScalaVersion: String - - def scalacOptions = super.scalacOptions().toList ++ ( - if (crossScalaVersion.startsWith("2.13")) List("-Ymacro-annotations") - else List.empty - ) - - abstract override def scalacPluginIvyDeps = - super.scalacPluginIvyDeps() ++ - (if (crossScalaVersion.startsWith("2.12")) - Agg(ivy"org.scalamacros:::paradise:${Deps.Versions.macroParadise}") - else - Agg.empty) -} - -trait MorphirCommonModule extends MorphirScalaModule with ScalafmtModule { - - def repositories = super.repositories ++ Seq( - MavenRepository("https://oss.sonatype.org/content/repositories/releases"), - MavenRepository("https://oss.sonatype.org/content/repositories/snapshots"), - MavenRepository("http://dl.bintray.com/spark-packages/maven") - ) - - def platformSegment: String - - //def millSourcePath = super.millSourcePath / ammonite.ops.up - def sources = T.sources( - super - .sources() - .flatMap(source => - Seq( - PathRef(source.path), - PathRef(source.path / os.up / platformSegment / source.path.last) - ) - ) - ) -} - -trait CommonJvmModule extends MorphirCommonModule { - def platformSegment = "jvm" - def crossScalaVersion: String - - def millSourcePath = super.millSourcePath / os.up - trait Tests extends super.Tests with MorphirTestModule { - def platformSegment = "jvm" - } -} - -trait CommonJsModule extends MorphirCommonModule with ScalaJSModule { - def platformSegment = "js" - def crossScalaJSVersion: String - def scalaJSVersion = crossScalaJSVersion - def millSourcePath = super.millSourcePath / os.up / os.up - trait Tests extends super.Tests with MorphirTestModule { - def platformSegment = "js" - def scalaJSVersion = crossScalaJSVersion - } -} - -trait MorphirTestModule extends MorphirScalaModule with TestModule { - def millSourcePath = super.millSourcePath / os.up - - def crossScalaVersion: String - def platformSegment: String - - def ivyDeps = Agg( - ivy"dev.zio::zio-test::${Deps.Versions.zio}", - ivy"dev.zio::zio-test-junit::${Deps.Versions.zio}", - ivy"dev.zio::zio-test-sbt::${Deps.Versions.zio}" - ) - - def testFrameworks = - Seq("zio.test.sbt.ZTestFramework") - - def offset: os.RelPath = os.rel - def sources = T.sources( - super - .sources() - .++( - CrossModuleBase - .scalaVersionPaths(crossScalaVersion, s => millSourcePath / s"src-$s") - ) - .flatMap(source => - Seq( - PathRef(source.path / os.up / "test" / source.path.last), - PathRef( - source.path / os.up / platformSegment / "test" / source.path.last - ) - ) - ) - .distinct - ) - - def resources = T.sources( - super - .resources() - .flatMap(source => - Seq( - PathRef(source.path / os.up / "test" / source.path.last) - ) - ) - .distinct - ) -} - -object morphir extends Module { - import Deps._ - object ir extends Module { - object jvm extends Cross[JvmMorphirIrModule](Versions.scala213, Versions.scala212) - class JvmMorphirIrModule(val crossScalaVersion: String) - extends CrossScalaModule - with CommonJvmModule - with MorphirPublishModule { self => - - override def moduleDeps = Seq( - morphir.sdk.core.jvm(crossScalaVersion), - morphir.sdk.json.jvm(crossScalaVersion) - ) - - def artifactName = "morphir-ir" - - def ivyDeps = Agg( - ivy"org.scalatest::scalatest:${Versions.scalatest}", - ivy"dev.zio::zio:${Versions.zio}", - ivy"dev.zio::zio-streams:${Versions.zio}", - ivy"com.lihaoyi::upickle:${Versions.upickle}", - ivy"com.lihaoyi::pprint:${Versions.pprint}", - ivy"io.lemonlabs::scala-uri:${Versions.scalaUri}", - ivy"org.scalactic::scalactic:${Versions.scalactic}", - ivy"io.estatico::newtype:${Versions.newtype}", - ivy"dev.zio::zio-test::${Deps.Versions.zio}", - ivy"io.circe::circe-core:${Versions.circe}", - ivy"io.circe::circe-generic:${Versions.circe}", - ivy"io.circe::circe-parser:${Versions.circe}" - ) - - object test extends Tests { - def platformSegment: String = self.platformSegment - def crossScalaVersion = JvmMorphirIrModule.this.crossScalaVersion - } - } - } - // - // object scala extends Module { - // - // object jvm extends Cross[JvmMorphirScalaModule](Versions.scala213) - // class JvmMorphirScalaModule(val crossScalaVersion: String) - // extends CrossScalaModule - // with CommonJvmModule - // with ScalaMacroModule - // with MorphirPublishModule { self => - // def artifactName = "morphir-scala" - // def moduleDeps = Seq(morphir.ir.jvm(crossScalaVersion)) - // - // def ivyDeps = Agg( - // ivy"org.scalameta::scalameta:${Versions.scalameta}" - // ) - // - // object test extends Tests { - // def platformSegment: String = self.platformSegment - // def crossScalaVersion = JvmMorphirScalaModule.this.crossScalaVersion - // } - // } - // } - object sdk extends Module { - - object core extends Module { - object jvm - extends Cross[JvmMorphirSdkCore]( - Versions.scala212, - Versions.scala211, - Versions.scala213 - ) - - class JvmMorphirSdkCore(val crossScalaVersion: String) - extends CrossScalaModule - with CommonJvmModule - with MorphirPublishModule { - self => - - def artifactName = "morphir-sdk-core" - - def scalacPluginIvyDeps = Agg(ivy"com.github.ghik:::silencer-plugin:${Versions.silencer}") - - def compileIvyDeps = Agg(ivy"com.github.ghik:::silencer-lib:${Versions.silencer}") - - def ivyDeps = Agg(ivy"org.scala-lang.modules::scala-collection-compat:${Versions.scalaCollectionsCompat}") - - object test extends Tests { - def platformSegment: String = self.platformSegment - - def crossScalaVersion = JvmMorphirSdkCore.this.crossScalaVersion - } - } - } - - object json extends Module { - object jvm - extends Cross[JvmMorphirSdkJson]( - Versions.scala213, - Versions.scala212 - ) - class JvmMorphirSdkJson(val crossScalaVersion: String) - extends CrossScalaModule - with CommonJvmModule - with MorphirPublishModule { self => - - def artifactName = "morphir-sdk-json" - def compileIvyDeps = Agg( - ivy"io.circe::circe-core:${Versions.circe}", - ivy"io.circe::circe-generic:${Versions.circe}", - ivy"io.circe::circe-parser:${Versions.circe}" - ) - def moduleDeps = Seq(morphir.sdk.core.jvm(crossScalaVersion)) - - object test extends Tests { - def platformSegment: String = self.platformSegment - def crossScalaVersion = JvmMorphirSdkJson.this.crossScalaVersion - - override def ivyDeps = super.ivyDeps() ++ - Agg( - ivy"io.circe::circe-core:${Versions.circe}", - ivy"io.circe::circe-generic:${Versions.circe}", - ivy"io.circe::circe-parser:${Versions.circe}" - ) - } - } - } - - object spark extends Module { - object jvm - extends Cross[JvmMorphirSdkSpark]( - Versions.scala212, - Versions.scala211 - ) - class JvmMorphirSdkSpark(val crossScalaVersion: String) - extends CrossScalaModule - with CommonJvmModule - with MorphirPublishModule { self => - - def artifactName = "morphir-sdk-spark" - def compileIvyDeps = Agg( - ivy"org.apache.spark::spark-sql:2.4.7", - ivy"com.github.ghik:::silencer-lib:${Versions.silencer}" - ) - def ivyDeps = Agg( - ivy"dev.zio::zio-prelude:${Versions.zioPrelude}" - ) - def scalacPluginIvyDeps = Agg(ivy"com.github.ghik:::silencer-plugin:${Versions.silencer}") - def moduleDeps = Seq(morphir.sdk.core.jvm(crossScalaVersion)) - - object test extends Tests { - def platformSegment: String = self.platformSegment - - def crossScalaVersion = JvmMorphirSdkSpark.this.crossScalaVersion - - override def ivyDeps = super.ivyDeps() ++ - Agg( - ivy"dev.zio::zio-logging:${Versions.zioLogging}", - ivy"dev.zio::zio-logging-slf4j:${Versions.zioLogging}", - ivy"org.apache.spark::spark-sql:2.4.7" - ) - } - } - } - } - - object spark extends Module { - object jvm - extends Cross[JvmMorphirSdkSpark]( - Versions.scala212 - ) - - class JvmMorphirSdkSpark(val crossScalaVersion: String) - extends CrossScalaModule - with CommonJvmModule - with MorphirPublishModule { - self => - - def artifactName = "morphir-spark" - - def ivyDeps = Agg( - ivy"org.apache.spark::spark-core:3.2.1", - ivy"org.apache.spark::spark-sql:3.2.1" - ) - - object test extends Tests { - def platformSegment: String = self.platformSegment - - def crossScalaVersion = JvmMorphirSdkSpark.this.crossScalaVersion - - override def ivyDeps = super.ivyDeps() ++ - Agg( - ivy"org.apache.spark::spark-core:3.2.1", - ivy"org.apache.spark::spark-sql:3.2.1", - ivy"org.scalatest::scalatest:3.0.2" - ) - } - } - } - -} diff --git a/build.sc b/build.sc index 1275d9a9..94077859 100644 --- a/build.sc +++ b/build.sc @@ -154,18 +154,19 @@ def morphirAlias(scalaVersion: String)(tasks: String*) = MyAliases.alias( ) object MyAliases extends Aliases { - def fmt = alias("mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources") - def checkfmt = alias("mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll __.sources") - def ciTestJS = alias("morphir.__.js.__.compile + morphir.__.js.publishArtifacts + morphir.__.js.__.test") - def ci_test_js_2_12 = morphirAlias(scala212)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") - def ci_test_js_2_13 = morphirAlias(scala213)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") - def ci_test_js_3 = morphirAlias(scala3x)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") - def deps = alias("mill.scalalib.Dependency/showUpdates") - def testall = alias("__.test") - def testJVM = alias("morphir.__.jvm.__.test") - def testJVMCached = alias("morphir.__.jvm.__.testCached") - def compileall = alias("__.compile") - def comptestall = alias("__.compile", "__.test") + def fmt = alias("mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources") + def checkfmt = alias("mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll __.sources") + def ciTestJS = alias("morphir.__.js.__.compile + morphir.__.js.publishArtifacts + morphir.__.js.__.test") + def ci_test_js_2_12 = morphirAlias(scala212)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") + def ci_test_js_2_13 = morphirAlias(scala213)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") + def ci_test_js_3 = morphirAlias(scala3x)("__.js.__.compile", "__.js.publishArtifacts", "__.js.__.test") + def deps = alias("mill.scalalib.Dependency/showUpdates") + def testall = alias("__.test") + def testJVM = alias("morphir.__.jvm.__.test") + def testJVMCached = alias("morphir.__.jvm.__.testCached") + def compileall = alias("__.compile") + def comptestall = alias("__.compile", "__.test") + def createPublishArtifacts = alias("morphir.__.publishArtifacts") } //-----------------------------------------------------------------------------