Metronome Release Process (#149)

* remove warning for not being able to link scaladoc from marathon project
migrated over ci infra from marathon for consistency

* fixing flake8 failed tests

* play requires 2.4.20 lets go with it

* # This is a combination of 10 commits.
# This is the 1st commit message:

adding jenkins release pipeline

# This is the commit message #2:

release process for metronome.  needs jenkins testing

# This is the commit message #3:

need protobuf installed and in path to build

# This is the commit message #4:

tagging is not useful right now

# This is the commit message #5:

need to be super apparently

# This is the commit message #6:

unable to get the cached protoc compiler

# This is the commit message #7:

last attempt.  it appears that the shell out that ammonite does loses the path

# This is the commit message #8:

going around ammonite for now

# This is the commit message #9:

my aliases do not work on jenkins

# This is the commit message #10:

crazy unable to build on the node

* adding jenkins release pipeline

* updated based on pr feedback

* scapegoat to be added in on a separate pr

* retest

Jenkinsfile.release

@@ -0,0 +1,40 @@
+#!/usr/bin/env groovy
+
+ansiColor('xterm') {
+  node('JenkinsMarathonCI-Debian8-2017-10-23') {
+
+    properties([
+      parameters([
+        string(name: 'gitsha',
+          defaultValue: '',
+          description: 'Git Commit SHA to Build and Release (ex. 167d7fb86 or 167)'
+        ),
+        string(name: 'version',
+          defaultValue: '',
+          description: 'Community Release Version (ex. v1.5.2).  This is a verification of the version.'
+        )]
+      )
+    ])
+
+    stage("Run Pipeline") {
+      try {
+        checkout scm
+        withCredentials([
+            usernamePassword(credentialsId: 'a7ac7f84-64ea-4483-8e66-bb204484e58f', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USER'),
+            string(credentialsId: '3f0dbb48-de33-431f-b91c-2366d2f0e1cf',variable: 'AWS_ACCESS_KEY_ID'),
+            string(credentialsId: 'f585ec9a-3c38-4f67-8bdb-79e5d4761937',variable: 'AWS_SECRET_ACCESS_KEY'),
+        ]) {
+          sshagent (credentials: ['0f7ec9c9-99b2-4797-9ed5-625572d5931d']) {
+            sh "bin/install-protobuf.sh"
+            sh """PATH=\$PATH:\$HOME/protobuf/bin ci/pipeline release $params.version $params.gitsha"""
+          }
+        }
+      } finally {
+        junit(allowEmptyResults: true, testResults: '*/target/test-reports/*.xml')
+        archive includes: "sandboxes.tar.gz"
+        archive includes: "ci-${env.BUILD_TAG}.log.tar.gz"
+        archive includes: "ci-${env.BUILD_TAG}.log"  // Only in case the build was  aborted and the logs weren't zipped
+      }
+    }
+  }
+}

api/src/test/scala/dcos/metronome/api/v1/controllers/JobRunControllerTest.scala

@@ -6,6 +6,7 @@ import dcos.metronome.model.{ JobId, JobRunStatus, JobSpec }
 import mesosphere.marathon.core.plugin.PluginManager
 import org.scalatest.{ BeforeAndAfter, GivenWhenThen }
 import org.scalatest.concurrent.ScalaFutures
+import org.scalatest.time.{ Millis, Seconds, Span }
 import org.scalatestplus.play.PlaySpec
 import play.api.ApplicationLoader.Context
 import play.api.libs.json._
@@ -14,6 +15,8 @@ import play.api.test.Helpers._
 
 class JobRunControllerTest extends PlaySpec with OneAppPerTestWithComponents[MockApiComponents] with ScalaFutures with GivenWhenThen with BeforeAndAfter {
 
+  implicit val defaultPatience = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis))
+
   "POST /jobs/{id}/runs" should {
     "create a job run when posting to the runs endpoint" in {
       Given("An existing app")

ci/awsClient.sc

@@ -39,9 +39,14 @@ case class Artifact(path: S3Path, sha1: String) {
   def downloadUrl: String = s"$base/${path.bucket}/${path.key}"
 }
 
-val S3_PREFIX = S3Path(
-  sys.env.getOrElse("S3_BUCKET", "downloads.mesosphere.io"),
-  Path(sys.env.getOrElse("S3_PATH" , "/metronome/snapshots")))
+/**
+ * builds =  builds, milestones, releases
+ */
+def s3PathFor(buildLocation: String = "builds") : S3Path = {
+  S3Path(
+    sys.env.getOrElse("S3_BUCKET", "downloads.mesosphere.io"),
+    Path(sys.env.getOrElse("S3_PATH" , s"/metronome/$buildLocation")))
+}
 
 /**
  *  Returns AWS S3 client.
@@ -59,40 +64,48 @@ def doesS3FileExist(path: S3Path): Boolean = {
 }
 
 /**
- *  Uploads marathon artifacts to the default bucket, using the env var credentials.
+ *  Uploads metronome artifacts to the default bucket, using the env var credentials.
  *  Upload process creates the sha1 file.
  *  If file name is on s3, it does NOT upload (these files are big). We cannot
  *  compare the sha1 sums because they change for each build of the same commit.
  *  However, our artifact names are unique for each commit.
  *
+ *  @param uploadFile path of the file to be uploaded to S3.
+ *  @param s3path S3 path to upload file to on S3.
  *  @return Artifact description if it was uploaded. None otherwise.
  */
-def archiveArtifact(uploadFile: Path): Option[Artifact] = {
+def archiveArtifact(uploadFile: Path, s3path: S3Path): Option[Artifact] = {
   // is already uploaded.
-  if(doesS3FileExist(S3_PREFIX / uploadFile.last)) {
-    println(s"Skipping File: ${uploadFile.last} already exists on S3 at ${S3_PREFIX / uploadFile.last}")
+  if(doesS3FileExist(s3path / uploadFile.last)) {
+    println(s"Skipping File: ${uploadFile.last} already exists on S3 at ${s3path / uploadFile.last}")
     None
   } else
-    Some(uploadFileAndSha(uploadFile))
+    Some(uploadFileAndSha(uploadFile, s3path))
 }
 
 /**
- *  Uploads marathon artifacts to the default bucket, using the env var credentials.
+ *  Uploads metronome artifacts to the default bucket, using the env var credentials.
  *  Upload process creates the sha1 file
+ *
+ *  @param uploadFile path of the file to be uploaded to S3.
+ *  @param s3path S3 path to upload file to on S3.
  */
-def uploadFileAndSha(uploadFile: Path): Artifact = {
+def uploadFileAndSha(uploadFile: Path, s3path: S3Path): Artifact = {
   val shaFile = fileUtil.writeSha1ForFile(uploadFile)
 
-  upload(uploadFile)
-  upload(shaFile)
-  Artifact(S3_PREFIX / uploadFile.last, read(shaFile))
+  upload(uploadFile, s3path)
+  println(s"Sha1 for file: ${uploadFile}")
+  %('cat, shaFile)
+  println("")
+  upload(shaFile, s3path)
+  Artifact(s3path / uploadFile.last, read(shaFile))
 }
 
 /**
  * Uploads file to default bucket.
  */
-def upload(file: Path): Unit = {
-  uploadFileToS3(file, S3_PREFIX / file.last)
+def upload(file: Path, s3path: S3Path): Unit = {
+  uploadFileToS3(file, s3path / file.last)
 }
 
 /**

ci/pipeline

@@ -18,46 +18,48 @@ import $file.utils
 val PACKAGE_DIR: Path = pwd / 'target / 'universal
 
 /**
- * Compile Metronome and run unit and integration tests followed by scapegoat.
+ * Compile Metronome and run unit.
  */
 @main
-def compileAndTest(): Unit = utils.stage("Compile and Test") {
+def compileAndTest(logFileName: String): Unit = utils.stage("Compile and Test") {
 
-  def run(cmd: String *) = utils.runWithTimeout(1.hour)(cmd)
+  def run(cmd: String *) = utils.runWithTimeout(1.hour, logFileName)(cmd)
 
   run("sbt", "clean", "test")
+
+  checkSystemIntegrationTests(logFileName)
 }
 
 @main
-def zipSandboxLogs(): Unit = {
-  Try(%("tar", "-zcf", "sandboxes.tar.gz", "sandboxes"))
+def checkSystemIntegrationTests(logFileName: String): Unit = {
+  def run(cmd: String *) = utils.runWithTimeout(30.minutes, logFileName)(cmd)
+  run("flake8", "--count", "--max-line-length=120", "tests/system")
 }
 
-@main
 /**
- * Upload Metronome tgz tarballs and its cha1 checksum to S3.
+ * Compresses sandboxes and logs.
  *
- * @return Artifact description if it was uploaded.
+ * @param logFileName Name of log file.
  */
-def uploadTarballToS3(): Option[awsClient.Artifact] = utils.stage("Upload Packages") {
-  import scala.collection.breakOut
-
-  PACKAGE_DIR.toIO.listFiles.filter(f => f.getName.endsWith(".tgz"))
-    .headOption.flatMap(file => awsClient.archiveArtifact(Path(file)))
+@main
+def zipSandboxLogs(logFileName: String = "ci.log"): Unit = {
+  println(s"zipping file: $logFileName")
+  Try(%("tar", "-zcf", s"$logFileName.tar.gz", "--remove-files", logFileName))
 }
 
 @main
 /**
- * Packages Metronome and uploads its artifacts alongside sha1 checksum to S3.
+ * Upload Metronome tgz tarballs and its sha1 checksum to S3.
  *
- * @return Version and artifact description of Metronome build.
+ * @param version The version to upload.
+ * @param buildLocation subfolder location to upload tarball to. Example: "builds", "release"
+ * @return Artifact description if it was uploaded.
  */
-def createAndUploadPackages(): (String, Option[awsClient.Artifact]) = {
-  val version = createPackages()
-  val artifact = uploadTarballToS3()
+def uploadTarballToS3(version: String, buildLocation: String): Option[awsClient.Artifact] = utils.stage("Upload Packages") {
+  import scala.collection.breakOut
 
-  createDocker()
-  (version, artifact)
+  PACKAGE_DIR.toIO.listFiles.filter(f => f.getName.endsWith(".tgz"))
+    .headOption.flatMap(file => awsClient.archiveArtifact(Path(file), awsClient.s3PathFor(buildLocation)))
 }
 
 /**
@@ -69,32 +71,31 @@ def createAndUploadPackages(): (String, Option[awsClient.Artifact]) = {
 @main
 def createPackages(): String = utils.stage("Package") {
   val result = %%('sbt, "universal:packageZipTarball", "version")
-
   // Regex is for version:
   // starting with random chars, match $number$dot$number$dot$number followed by optional alpha numberic chars plus `-`
   // ending with random characters
   // we need to regex this string because we do have colored output in the `sbt version` command
-  val VersionLineRegex = "^.*(\\d+\\.\\d+\\.\\d+[-A-Za-z\\d]+).*$".r
+  val versionLineRegex = "^.*(\\d+\\.\\d+\\.\\d+).*$".r
 
   // Nothing is what it seems. This is a poor man's way to extract the version
   // from sbt's console output until we run our Ammonite scripts in sbt.
+
+
   val version = result.out.lines.last match {
-    case VersionLineRegex(v) => v
+    case versionLineRegex(v) => v
     case _ =>
         val commit = %%('git, "log", "--pretty=format:%h", "-n1").out.lines.last
-        s"unkown version in commit $commit"
+        s"unknown version in commit $commit"
   }
   println(s"Built tarballs for Metronome $version.")
   version
 }
 
 /**
- * Create Docker, rpm and deb packages.
+ * While 'sbt, "docker:publishLocal" will create a docker.  The project is NOT setup
+ * properly (proper naming "mesosphere/metronome") nor do we currently support metronome
+ * in the universe.
  */
-@main
-def createDocker(): Unit = utils.stage("Package Docker Image, Debian and RedHat Packages") {
-  %('sbt, "docker:publishLocal")
-}
 
 /**
  * The pipeline target for GitHub pull request builds. It wraps other targets
@@ -122,17 +123,16 @@ def asPullRequest(run: => (String, Option[awsClient.Artifact])): Unit = {
  * @return Version and artifact description of Metronome build.
  */
 @main
-def run(): (String, Option[awsClient.Artifact]) = {
+def build(): String = {
 
+  val logFileName = s"ci-${sys.env.getOrElse("BUILD_TAG", "run")}.log"
   try {
-    compileAndTest()
+    compileAndTest(logFileName)
   } finally {
-    zipSandboxLogs()    // Try to archive logs in any case
+    zipSandboxLogs(logFileName)    // Try to archive logs in any case
   }
-
-  val (version, maybeArtifact) = createAndUploadPackages()
-
-  (version, maybeArtifact)
+  val version = createPackages()
+  version
 }
 
 /**
@@ -143,8 +143,46 @@ def run(): (String, Option[awsClient.Artifact]) = {
 @main
 def jenkins(): Unit = {
   if(utils.isPullRequest) {
-    asPullRequest { run() }
+    asPullRequest {
+      val version = build()
+      // Uploads
+      val artifact = uploadTarballToS3(version, "builds")
+      (version, artifact)
+    }
   } else {
-    run()
+    val version = build()
+    uploadTarballToS3(version, "builds")
   }
 }
+
+
+/**
+ * Executes the Community Release which includes:
+ *
+ *  1. tarball with version details 
+ *  Unlike the marathon build, the version details are currently expected to be
+ * in the metronome project.   When a rev sha is provided for the build, the version
+ * of that build will be checked against the requestedVersion number.  It will fail
+ * the build if the version does not match.    However this has the potential of
+ * being more error prone in that multiple commits could build the same version.  So
+ * the build will also fail if the S3 location is currently occupied by a version build.
+ * Human cleanup of release folder is required in order to rebuild a version.
+ *
+ * @param requestVersion The version attempting to be released 0.3.0
+ * @param gitSha The git commit sha.  This can be shorthand (ex. 0e1)
+ */
+@main
+def release(requestVersion: String, gitSha: String): Unit = {
+
+  val tagVersion = s"v$requestVersion"
+
+  println(s"Releasing version: $requestVersion")
+  %('git, "checkout", gitSha)
+
+  val version = build()
+  if(version != requestVersion) {
+    throw new IllegalStateException(s"Build version: $version does NOT match requested version: $requestVersion")
+  }
+
+  uploadTarballToS3(version, s"releases/$version")
+}