Add rpm repo profile

Vagrantfile

@@ -123,4 +123,15 @@ Vagrant.configure("2") do |config|
       provider.vm.box_url = CENTOS_9_BOX_URL
     end
   end
+
+  config.vm.define "repo-rpm" do |override|
+    override.vm.hostname = "repo-rpm"
+    override.vm.box = "centos/stream9"
+
+    override.vm.provider "libvirt" do |libvirt, provider|
+      libvirt.memory = "2048"
+      libvirt.machine_virtual_size = 40
+      provider.vm.box_url = CENTOS_9_BOX_URL
+    end
+  end
 end

puppet/data/common.yaml

@@ -1,6 +1,7 @@
 ---
 stable_release: '3.11'
 profiles::web::stable: '%{alias("stable_release")}'
+profiles::repo::rpm::stable_foreman: '%{alias("stable_release")}'
 
 backup_servicename: 'backups.theforeman.org'
 backup_username: 'backup-%{facts.networking.hostname}'

puppet/data/vagrant.yaml

@@ -22,4 +22,6 @@ profiles::jenkins::node::swap_size_mb: 0
 
 profiles::web::https: false
 
+profiles::repo::rpm::https: false
+
 redmine::https: false

puppet/manifests/site.pp

@@ -41,3 +41,8 @@
   include profiles::base
   include profiles::web
 }
+
+node /^repo-rpm\d+\.[a-z]+\.theforeman\.org$/ {
+  include profiles::base
+  include profiles::repo::rpm
+}

puppet/modules/profiles/manifests/repo/rpm.pp

@@ -0,0 +1,25 @@
+# @summary A profile for the rpm repo machines
+#
+# @param stable_foreman
+#   Latest Foreman release that users expect
+#
+# @param https
+#   Whether to enable HTTPS. This is typically wanted but can only be enabled
+#   in a 2 pass setup. First Apache needs to run for Letsencrypt to function.
+#   Then Letsencrypt can be enabled. Also useful to turn off in test setups.
+class profiles::repo::rpm (
+  String[1] $stable_foreman,
+  Boolean $https = true,
+) {
+  class { 'web':
+    https => $https,
+  }
+  contain web
+
+  class { 'web::vhost::rpm':
+    stable_foreman => $stable_foreman,
+  }
+  contain web::vhost::rpm
+
+  contain web::vhost::stagingrpm
+}

puppet/modules/web/files/rpm/pulpcore-HEADER.html

@@ -0,0 +1,3 @@
+<h1>Pulpcore packages</h1>
+
+These are RPM builds for <a href="https://pulpproject.org">Pulp 3</a> and various plugins for use by <a href="https://theforeman.org/plugins/katello/">Katello</a>. They are only intended to be used by Katello. Only branches used by Katello are maintained. No explicit end of life announcements will be made.

puppet/modules/web/files/rpm/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: /foreman/nightly/
+Disallow: /pulpcore/nightly/

puppet/modules/web/files/stagingrpm/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

puppet/modules/web/manifests/vhost/rpm.pp

@@ -0,0 +1,101 @@
+# @summary Set up the rpm vhost
+# @api private
+class web::vhost::rpm (
+  String[1] $stable_foreman,
+  Stdlib::Fqdn $servername = 'rpm.theforeman.org',
+  Stdlib::Absolutepath $rpm_directory = '/var/www/vhosts/rpm/htdocs',
+  Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs/',
+  String $user = 'rpmrepo',
+) {
+  $rpm_directory_config = [
+    {
+      path            => $rpm_directory,
+      options         => ['Indexes', 'FollowSymLinks', 'MultiViews'],
+      expires_active  => 'on',
+      expires_default => 'access plus 2 minutes',
+    },
+    {
+      path            => '.+\.(bz2|gz|rpm|xz)$',
+      provider        => 'filesmatch',
+      expires_active  => 'on',
+      expires_default => 'access plus 30 days',
+    },
+    {
+      path            => 'repomd.xml',
+      provider        => 'files',
+      expires_active  => 'on',
+      expires_default => 'access plus 2 minutes',
+    },
+  ]
+
+  $deploy_rpmrepo_context = {
+    'servername'           => $servername,
+    'rpm_directory'        => $rpm_directory,
+    'rpm_source_directory' => $rpm_source_directory,
+  }
+
+  secure_ssh::receiver_setup { $user:
+    user           => $user,
+    foreman_search => 'host ~ node*.jenkins.osuosl.theforeman.org and (name = external_ip4 or name = external_ip6)',
+    script_content => epp('web/deploy-rpmrepo.sh.epp', $deploy_rpmrepo_context),
+  }
+
+  include apache::mod::expires
+  web::vhost { 'rpm':
+    servername    => $servername,
+    docroot       => $rpm_directory,
+    docroot_owner => $user,
+    docroot_group => $user,
+    docroot_mode  => '0755',
+    directories   => $rpm_directory_config,
+  }
+
+  if $facts['os']['family'] == 'RedHat' {
+    package { 'createrepo_c':
+      ensure => present,
+    }
+  }
+
+  file { "${rpm_directory}/robots.txt":
+    ensure  => file,
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    content => file('web/rpm/robots.txt'),
+  }
+
+  file { "${rpm_directory}/HEADER.html":
+    ensure  => file,
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    content => epp("${module_name}/rpm/HEADER.html.epp", {
+        'stable_foreman' => $stable_foreman,
+        'servername'     => $servername,
+    }),
+  }
+
+  ['candlepin', 'foreman', 'pulpcore'].each |$directory| {
+    file { ["${rpm_directory}/${directory}"]:
+      ensure => directory,
+      owner  => $user,
+      group  => $user,
+      mode   => '0755',
+    }
+
+    exec { "fastly-purge-${directory}-latest":
+      command     => "fastly-purge-find 'https://${servername}' ${rpm_directory} ${directory}/latest/",
+      path        => '/bin:/usr/bin:/usr/local/bin',
+      require     => File['/usr/local/bin/fastly-purge-find'],
+      refreshonly => true,
+    }
+  }
+
+  file { "${rpm_directory}/pulpcore/HEADER.html":
+    ensure  => file,
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    content => file('web/rpm/pulpcore-HEADER.html'),
+  }
+}

puppet/modules/web/manifests/vhost/stagingrpm.pp

@@ -0,0 +1,70 @@
+# @summary Set up the rpm staging vhost
+# @api private
+class web::vhost::stagingrpm (
+  Stdlib::Fqdn $servername = 'stagingrpm.theforeman.org',
+  Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs',
+  String $user = 'rpmrepostage',
+  Stdlib::Absolutepath $home = "/home/${user}",
+  Array[String[1]] $usernames = ['ehelms', 'evgeni', 'ekohl', 'Odilhao', 'pcreech', 'zhunting'],
+) {
+  $rpm_source_directory_config = [
+    {
+      path            => $rpm_source_directory,
+      options         => ['Indexes', 'FollowSymLinks', 'MultiViews'],
+      expires_active  => 'on',
+      expires_default => 'access plus 2 minutes',
+    },
+    {
+      path            => '.+\.(bz2|gz|rpm|xz)$',
+      provider        => 'filesmatch',
+      expires_active  => 'on',
+      expires_default => 'access plus 30 days',
+    },
+    {
+      path            => 'repomd.xml',
+      provider        => 'files',
+      expires_active  => 'on',
+      expires_default => 'access plus 2 minutes',
+    },
+  ]
+
+  $authorized_keys = flatten($usernames.map |$name| {
+      split(file("users/${name}-authorized_keys"), "\n")
+  })
+
+  secure_ssh::rsync::receiver_setup { $user:
+    user            => $user,
+    homedir         => $home,
+    homedir_mode    => '0750',
+    foreman_search  => 'host ~ node*.jenkins.*.theforeman.org and (name = external_ip4 or name = external_ip6)',
+    script_content  => template('web/deploy-stagingyum.sh.erb'),
+    authorized_keys => $authorized_keys,
+  }
+
+  web::vhost { 'stagingrpm':
+    servername    => $servername,
+    docroot       => $rpm_source_directory,
+    docroot_owner => $user,
+    docroot_group => $user,
+    docroot_mode  => '0755',
+    directories   => $rpm_source_directory_config,
+  }
+
+  file { "${rpm_source_directory}/robots.txt":
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => file("web/stagingrpm/robots.txt"),
+  }
+
+  file { "${rpm_source_directory}/HEADER.html":
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => epp("${module_name}/stagingrpm/HEADER.html.epp", {
+        'servername' => $servername,
+    }),
+  }
+}

puppet/modules/web/templates/deploy-rpmrepo.sh.epp

@@ -0,0 +1,123 @@
+<%- |
+  Stdlib::Fqdn $servername,
+  Stdlib::Absolutepath $rpm_directory,
+  Stdlib::Absolutepath $rpm_source_directory,
+| -%>
+#!/bin/bash
+
+set -xe
+# This is a forced SSH command - uncomment to test locally
+set -f -- $SSH_ORIGINAL_COMMAND
+
+prepcache() {
+	if [[ -e $REPO_PATH ]]; then
+		if [[ $MERGE == false ]] && [[ $OVERWRITE == false ]] ; then
+			echo "Repo overwrite (${OVERWRITE}) and merge (${MERGE}) are disabled, but ${REPO_PATH} already exists"
+			exit 1
+		fi
+		cp -al $REPO_PATH "$REPO_INSTANCE_PATH"
+	else
+		mkdir -p $REPO_INSTANCE_PATH
+	fi
+}
+
+do_rsync() {
+	opts=(--archive --verbose --hard-links --log-file "$REPO_RSYNC_LOG")
+	if [[ $MERGE != true ]] ; then
+		opts+=('--delete')
+	fi
+
+	for ARCH in x86_64 source; do
+		rsync "${opts[@]}" --log-file-format "CHANGED ${ARCH}/%n" "${REPO_SOURCE_RPM}/${ARCH}/" "${REPO_INSTANCE_PATH}/${ARCH}/"
+	done
+
+	set +f
+	for d in "${REPO_INSTANCE_PATH}"/*; do
+		(
+			cd "$d"
+
+			latest=$(ls -t foreman-release-[0-9]*.rpm 2>/dev/null | head -n1)
+			if [[ -n "$latest" ]] ; then
+				ln -sf "$latest" foreman-release.rpm
+			fi
+
+			latest=$(ls -t foreman-client-release-[0-9]*.rpm 2>/dev/null | head -n1)
+			if [[ -n "$latest" ]] ; then
+				ln -sf "$latest" foreman-client-release.rpm
+			fi
+
+			latest=$(ls -t katello-repos-[0-9]*.rpm 2>/dev/null | head -n1)
+			if [[ -n "$latest" ]] ; then
+				ln -sf "$latest" katello-repos.rpm
+			fi
+
+			if [[ $MERGE == true ]] ; then
+				HAS_MODULES_YAML=$(ls repodata/*-modules.yaml.gz >/dev/null 2>/dev/null && echo 'yes' || echo 'no')
+
+				if [[ $HAS_MODULES_YAML == yes ]]; then
+					zcat repodata/*-modules.yaml.gz > modules.yaml
+					modifyrepo_c --remove modules repodata/
+					rm -f repodata/*-modules.yaml.gz
+				fi
+
+				createrepo_c --skip-symlinks --update .
+
+				if [[ $HAS_MODULES_YAML == yes ]]; then
+					modifyrepo_c --mdtype=modules modules.yaml repodata/
+				fi
+			fi
+		)
+	done
+	set -f
+}
+
+replace() {
+	if [[ -e $REPO_PATH ]]; then
+		mv "${REPO_PATH}" "${REPO_INSTANCE_PATH_PREV}"
+	fi
+
+	mv "${REPO_INSTANCE_PATH}" "${REPO_PATH}"
+
+	if [[ $MERGE == true ]] || [[ $OVERWRITE == true ]] ; then
+		if [[ -e "${REPO_INSTANCE_PATH_PREV}" ]]; then
+			rm -rf "${REPO_INSTANCE_PATH_PREV}"
+		fi
+	fi
+}
+
+purgecdn() {
+	awk '/ CHANGED /{print $5}' "${REPO_RSYNC_LOG}" | xargs --no-run-if-empty fastly-purge "https://<%= $servername %>/${REPO_DEST}"
+	set +f
+	for d in "${REPO_PATH}"/*; do
+		purge_base="https://<%= $servername %>/${REPO_DEST}/$(basename $d)"
+		fastly-purge ${purge_base} foreman-release.rpm foreman-client-release.rpm katello-repos.rpm
+	done
+	set -f
+}
+
+REPO_SOURCE=$1
+REPO_DEST=$2
+OVERWRITE=${3:-false}
+MERGE=${4:-false}
+
+if [[ -z $REPO_SOURCE ]] || [[ -z $REPO_DEST ]] ; then
+	echo "Usage: $0 REPO_SOURCE REPO_DEST OVERWRITE MERGE"
+	exit 1
+fi
+
+REPO_SOURCE_BASE="<%= $rpm_source_directory %>"
+REPO_SOURCE_RPM="${REPO_SOURCE_BASE}/${REPO_SOURCE}"
+
+DEPLOY_TO="<%= $rpm_directory %>"
+REPO_PATH="${DEPLOY_TO}/${REPO_DEST}"
+REPO_INSTANCE_PATH="${DEPLOY_TO}/$(dirname $REPO_DEST)/.$(basename $REPO_DEST)-$(date "+%Y%m%d%H%M%S")"
+REPO_INSTANCE_PATH_PREV="${REPO_INSTANCE_PATH}-previous"
+
+REPO_RSYNC_LOG=$(mktemp)
+
+trap "rm -rf $REPO_RSYNC_LOG $REPO_INSTANCE_PATH" EXIT
+
+prepcache
+do_rsync
+replace
+purgecdn