From e5d0a15a2fc699a60fa6c24386748596a1559d2b Mon Sep 17 00:00:00 2001 From: "Eric D. Helms" Date: Fri, 16 Aug 2024 15:34:35 -0400 Subject: [PATCH] Add rpm repo profile --- Vagrantfile | 11 ++ puppet/data/common.yaml | 1 + puppet/data/vagrant.yaml | 2 + puppet/manifests/site.pp | 5 + puppet/modules/profiles/manifests/repo/rpm.pp | 25 ++++ .../web/files/rpm/pulpcore-HEADER.html | 3 + puppet/modules/web/files/rpm/robots.txt | 3 + .../modules/web/files/stagingrpm/HEADER.html | 16 +++ .../modules/web/files/stagingrpm/robots.txt | 2 + puppet/modules/web/manifests/vhost/rpm.pp | 101 ++++++++++++++ .../modules/web/manifests/vhost/stagingrpm.pp | 62 +++++++++ .../web/templates/deploy-rpmrepo.sh.epp | 123 ++++++++++++++++++ .../modules/web/templates/rpm/HEADER.html.epp | 66 ++++++++++ vagrant/manifests/default.pp | 4 + 14 files changed, 424 insertions(+) create mode 100644 puppet/modules/profiles/manifests/repo/rpm.pp create mode 100644 puppet/modules/web/files/rpm/pulpcore-HEADER.html create mode 100644 puppet/modules/web/files/rpm/robots.txt create mode 100644 puppet/modules/web/files/stagingrpm/HEADER.html create mode 100644 puppet/modules/web/files/stagingrpm/robots.txt create mode 100644 puppet/modules/web/manifests/vhost/rpm.pp create mode 100644 puppet/modules/web/manifests/vhost/stagingrpm.pp create mode 100644 puppet/modules/web/templates/deploy-rpmrepo.sh.epp create mode 100644 puppet/modules/web/templates/rpm/HEADER.html.epp diff --git a/Vagrantfile b/Vagrantfile index f76fd4e89..5a3cb25e8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -123,4 +123,15 @@ Vagrant.configure("2") do |config| provider.vm.box_url = CENTOS_9_BOX_URL end end + + config.vm.define "repo-rpm" do |override| + override.vm.hostname = "repo-rpm" + override.vm.box = "centos/stream9" + + override.vm.provider "libvirt" do |libvirt, provider| + libvirt.memory = "2048" + libvirt.machine_virtual_size = 40 + provider.vm.box_url = CENTOS_9_BOX_URL + end + end end diff --git a/puppet/data/common.yaml b/puppet/data/common.yaml index 85da91a96..f7d2d11e4 100644 --- a/puppet/data/common.yaml +++ b/puppet/data/common.yaml @@ -1,6 +1,7 @@ --- stable_release: '3.11' profiles::web::stable: '%{alias("stable_release")}' +profiles::repo::rpm::stable_foreman: '%{alias("stable_release")}' backup_servicename: 'backups.theforeman.org' backup_username: 'backup-%{facts.networking.hostname}' diff --git a/puppet/data/vagrant.yaml b/puppet/data/vagrant.yaml index 18a8f1c84..bf78d4b34 100644 --- a/puppet/data/vagrant.yaml +++ b/puppet/data/vagrant.yaml @@ -22,4 +22,6 @@ profiles::jenkins::node::swap_size_mb: 0 profiles::web::https: false +profiles::repo::rpm::https: false + redmine::https: false diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 9f87c0e1d..fe4472ef9 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -41,3 +41,8 @@ include profiles::base include profiles::web } + +node /^repo-rpm\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::repo::rpm +} diff --git a/puppet/modules/profiles/manifests/repo/rpm.pp b/puppet/modules/profiles/manifests/repo/rpm.pp new file mode 100644 index 000000000..cfbd866c4 --- /dev/null +++ b/puppet/modules/profiles/manifests/repo/rpm.pp @@ -0,0 +1,25 @@ +# @summary A profile for the rpm repo machines +# +# @param stable_foreman +# Latest Foreman release that users expect +# +# @param https +# Whether to enable HTTPS. This is typically wanted but can only be enabled +# in a 2 pass setup. First Apache needs to run for Letsencrypt to function. +# Then Letsencrypt can be enabled. Also useful to turn off in test setups. +class profiles::repo::rpm ( + String[1] $stable_foreman = '3.11', + Boolean $https = true, +) { + class { 'web': + https => $https, + } + contain web + + class { 'web::vhost::rpm': + stable_foreman => $stable_foreman, + } + contain web::vhost::rpm + + contain web::vhost::stagingrpm +} diff --git a/puppet/modules/web/files/rpm/pulpcore-HEADER.html b/puppet/modules/web/files/rpm/pulpcore-HEADER.html new file mode 100644 index 000000000..701a77b8a --- /dev/null +++ b/puppet/modules/web/files/rpm/pulpcore-HEADER.html @@ -0,0 +1,3 @@ +

Pulpcore packages

+ +These are RPM builds for Pulp 3 and various plugins for use by Katello. They are only intended to be used by Katello. Only branches used by Katello are maintained. No explicit end of life announcements will be made. diff --git a/puppet/modules/web/files/rpm/robots.txt b/puppet/modules/web/files/rpm/robots.txt new file mode 100644 index 000000000..9fe8bff01 --- /dev/null +++ b/puppet/modules/web/files/rpm/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /foreman/nightly/ +Disallow: /pulpcore/nightly/ diff --git a/puppet/modules/web/files/stagingrpm/HEADER.html b/puppet/modules/web/files/stagingrpm/HEADER.html new file mode 100644 index 000000000..0f74d041c --- /dev/null +++ b/puppet/modules/web/files/stagingrpm/HEADER.html @@ -0,0 +1,16 @@ +

stagingrpm.theforeman.org

+ +

Staging Repositories for Foreman releases

+ +

Accessing this repo

+ +This repository is available over HTTP and HTTPS: + + + +

Support

+ +

These are staging repositories used for testing purposes only. They are NOT supported.

diff --git a/puppet/modules/web/files/stagingrpm/robots.txt b/puppet/modules/web/files/stagingrpm/robots.txt new file mode 100644 index 000000000..1f53798bb --- /dev/null +++ b/puppet/modules/web/files/stagingrpm/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/puppet/modules/web/manifests/vhost/rpm.pp b/puppet/modules/web/manifests/vhost/rpm.pp new file mode 100644 index 000000000..c83661cab --- /dev/null +++ b/puppet/modules/web/manifests/vhost/rpm.pp @@ -0,0 +1,101 @@ +# @summary Set up the rpm vhost +# @api private +class web::vhost::rpm ( + String[1] $stable_foreman, + Stdlib::Fqdn $servername = 'rpm.theforeman.org', + Stdlib::Absolutepath $rpm_directory = '/var/www/vhosts/rpm/htdocs', + Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs/', + String $user = 'rpmrepo', +) { + $rpm_directory_config = [ + { + path => $rpm_directory, + options => ['Indexes', 'FollowSymLinks', 'MultiViews'], + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + { + path => '.+\.(bz2|gz|rpm|xz)$', + provider => 'filesmatch', + expires_active => 'on', + expires_default => 'access plus 30 days', + }, + { + path => 'repomd.xml', + provider => 'files', + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + ] + + $deploy_rpmrepo_context = { + 'servername' => $servername, + 'rpm_directory' => $rpm_directory, + 'rpm_source_directory' => $rpm_source_directory, + } + + secure_ssh::receiver_setup { $user: + user => $user, + foreman_search => 'host ~ node*.jenkins.osuosl.theforeman.org and (name = external_ip4 or name = external_ip6)', + script_content => epp('web/deploy-rpmrepo.sh.epp', $deploy_rpmrepo_context), + } + + include apache::mod::expires + web::vhost { 'rpm': + servername => $servername, + docroot => $rpm_directory, + docroot_owner => $user, + docroot_group => $user, + docroot_mode => '0755', + directories => $rpm_directory_config, + } + + if $facts['os']['family'] == 'RedHat' { + package { 'createrepo_c': + ensure => present, + } + } + + file { "${rpm_directory}/robots.txt": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => file('web/rpm/robots.txt'), + } + + file { "${rpm_directory}/HEADER.html": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => epp("${module_name}/rpm/HEADER.html.epp", { + 'stable_foreman' => $stable_foreman, + 'servername' => $servername, + }), + } + + ['candlepin', 'foreman', 'pulpcore'].each |$directory| { + file { ["${rpm_directory}/${directory}"]: + ensure => directory, + owner => $user, + group => $user, + mode => '0755', + } + + exec { "fastly-purge-${directory}-latest": + command => "fastly-purge-find 'https://${servername}' ${rpm_directory} ${directory}/latest/", + path => '/bin:/usr/bin:/usr/local/bin', + require => File['/usr/local/bin/fastly-purge-find'], + refreshonly => true, + } + } + + file { "${rpm_directory}/pulpcore/HEADER.html": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => file('web/rpm/pulpcore-HEADER.html'), + } +} diff --git a/puppet/modules/web/manifests/vhost/stagingrpm.pp b/puppet/modules/web/manifests/vhost/stagingrpm.pp new file mode 100644 index 000000000..d14c51561 --- /dev/null +++ b/puppet/modules/web/manifests/vhost/stagingrpm.pp @@ -0,0 +1,62 @@ +# @summary Set up the rpm staging vhost +# @api private +class web::vhost::stagingrpm ( + Stdlib::Fqdn $servername = 'stagingrpm.theforeman.org', + Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs', + String $user = 'rpmrepostage', + Stdlib::Absolutepath $home = "/home/${user}", + Array[String[1]] $usernames = ['ehelms', 'evgeni', 'ekohl', 'Odilhao', 'pcreech', 'zhunting'], +) { + $rpm_source_directory_config = [ + { + path => $rpm_source_directory, + options => ['Indexes', 'FollowSymLinks', 'MultiViews'], + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + { + path => '.+\.(bz2|gz|rpm|xz)$', + provider => 'filesmatch', + expires_active => 'on', + expires_default => 'access plus 30 days', + }, + { + path => 'repomd.xml', + provider => 'files', + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + ] + + $authorized_keys = flatten($usernames.map |$name| { + split(file("users/${name}-authorized_keys"), "\n") + }) + + secure_ssh::rsync::receiver_setup { $user: + user => $user, + homedir => $home, + homedir_mode => '0750', + foreman_search => 'host ~ node*.jenkins.*.theforeman.org and (name = external_ip4 or name = external_ip6)', + script_content => template('web/deploy-stagingyum.sh.erb'), + authorized_keys => $authorized_keys, + } + + web::vhost { 'stagingrpm': + servername => $servername, + docroot => $rpm_source_directory, + docroot_owner => $user, + docroot_group => $user, + docroot_mode => '0755', + directories => $rpm_source_directory_config, + } + + ['HEADER.html', 'robots.txt'].each |$filename| { + file { "${rpm_source_directory}/${filename}": + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => file("web/stagingrpm/${filename}"), + } + } +} diff --git a/puppet/modules/web/templates/deploy-rpmrepo.sh.epp b/puppet/modules/web/templates/deploy-rpmrepo.sh.epp new file mode 100644 index 000000000..6d6639b76 --- /dev/null +++ b/puppet/modules/web/templates/deploy-rpmrepo.sh.epp @@ -0,0 +1,123 @@ +<%- | + Stdlib::Fqdn $servername, + Stdlib::Absolutepath $rpm_directory, + Stdlib::Absolutepath $rpm_source_directory, +| -%> +#!/bin/bash + +set -xe +# This is a forced SSH command - uncomment to test locally +set -f -- $SSH_ORIGINAL_COMMAND + +prepcache() { + if [[ -e $REPO_PATH ]]; then + if [[ $MERGE == false ]] && [[ $OVERWRITE == false ]] ; then + echo "Repo overwrite (${OVERWRITE}) and merge (${MERGE}) are disabled, but ${REPO_PATH} already exists" + exit 1 + fi + cp -al $REPO_PATH "$REPO_INSTANCE_PATH" + else + mkdir -p $REPO_INSTANCE_PATH + fi +} + +do_rsync() { + opts=(--archive --verbose --hard-links --log-file "$REPO_RSYNC_LOG") + if [[ $MERGE != true ]] ; then + opts+=('--delete') + fi + + for ARCH in x86_64 source; do + rsync "${opts[@]}" --log-file-format "CHANGED ${ARCH}/%n" "${REPO_SOURCE_RPM}/${ARCH}/" "${REPO_INSTANCE_PATH}/${ARCH}/" + done + + set +f + for d in "${REPO_INSTANCE_PATH}"/*; do + ( + cd "$d" + + latest=$(ls -t foreman-release-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" foreman-release.rpm + fi + + latest=$(ls -t foreman-client-release-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" foreman-client-release.rpm + fi + + latest=$(ls -t katello-repos-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" katello-repos-latest.rpm + fi + + if [[ $MERGE == true ]] ; then + HAS_MODULES_YAML=$(ls repodata/*-modules.yaml.gz >/dev/null 2>/dev/null && echo 'yes' || echo 'no') + + if [[ $HAS_MODULES_YAML == yes ]]; then + zcat repodata/*-modules.yaml.gz > modules.yaml + modifyrepo_c --remove modules repodata/ + rm -f repodata/*-modules.yaml.gz + fi + + createrepo_c --skip-symlinks --update . + + if [[ $HAS_MODULES_YAML == yes ]]; then + modifyrepo_c --mdtype=modules modules.yaml repodata/ + fi + fi + ) + done + set -f +} + +replace() { + if [[ -e $REPO_PATH ]]; then + mv "${REPO_PATH}" "${REPO_INSTANCE_PATH_PREV}" + fi + + mv "${REPO_INSTANCE_PATH}" "${REPO_PATH}" + + if [[ $MERGE == true ]] || [[ $OVERWRITE == true ]] ; then + if [[ -e "${REPO_INSTANCE_PATH_PREV}" ]]; then + rm -rf "${REPO_INSTANCE_PATH_PREV}" + fi + fi +} + +purgecdn() { + awk '/ CHANGED /{print $5}' "${REPO_RSYNC_LOG}" | xargs --no-run-if-empty fastly-purge "https://<%= $servername %>/${REPO_DEST}" + set +f + for d in "${REPO_PATH}"/*; do + purge_base="https://<%= $servername %>/${REPO_DEST}/$(basename $d)" + fastly-purge ${purge_base} foreman-release.rpm foreman-client-release.rpm katello-repos-latest.rpm + done + set -f +} + +REPO_SOURCE=$1 +REPO_DEST=$2 +OVERWRITE=${3:-false} +MERGE=${4:-false} + +if [[ -z $REPO_SOURCE ]] || [[ -z $REPO_DEST ]] ; then + echo "Usage: $0 REPO_SOURCE REPO_DEST OVERWRITE MERGE" + exit 1 +fi + +REPO_SOURCE_BASE="<%= $rpm_source_directory %>" +REPO_SOURCE_RPM="${REPO_SOURCE_BASE}/${REPO_SOURCE}" + +DEPLOY_TO="<%= $rpm_directory %>" +REPO_PATH="${DEPLOY_TO}/${REPO_DEST}" +REPO_INSTANCE_PATH="${DEPLOY_TO}/$(dirname $REPO_DEST)/.$(basename $REPO_DEST)-$(date "+%Y%m%d%H%M%S")" +REPO_INSTANCE_PATH_PREV="${REPO_INSTANCE_PATH}-previous" + +REPO_RSYNC_LOG=$(mktemp) + +trap "rm -rf $REPO_RSYNC_LOG $REPO_INSTANCE_PATH" EXIT + +prepcache +do_rsync +replace +purgecdn diff --git a/puppet/modules/web/templates/rpm/HEADER.html.epp b/puppet/modules/web/templates/rpm/HEADER.html.epp new file mode 100644 index 000000000..ae6950aab --- /dev/null +++ b/puppet/modules/web/templates/rpm/HEADER.html.epp @@ -0,0 +1,66 @@ +<%- | + String $stable_foreman, + Stdlib::Fqdn $servername, +| -%> +

<%= $servername %>

+ +

Foreman

+ +

Foreman is available under /releases/VERSION/DIST/ARCH, e.g.

+ + + +

foreman-release RPMs containing an appropriate .repo file are available with fixed URLs:

+ + + +

Release packages are signed with a new key for each major release. The public key is available in the RPM-GPG-KEY-foreman file within each version directory or the foreman-release RPMs.

+ +

Nightly builds of Foreman are available under /foreman/nightly/foreman/DIST/ARCH and are refreshed a few times a day, but are not GPG signed.

+ +

Plugins

+ +

A number of Foreman plugins are available in the plugin repos, see Plugins and List of Plugins for more information.

+ +

Plugin repos are structured by the Foreman version that they're compatible with in the format /foreman/VERSION/plugins/DIST/ARCH, e.g.

+ + + +

Plugin repos are not GPG signed.

+ +

Katello

+ +

Katello is available under /foreman/VERSION/katello/DIST/ARCH with Candlepin under /candlepin/CANDLEPIN_VERSION/DIST/ARCH. + +

katello-repos RPMs containing an appropriate .repo file are available:

+ + + +

Unlike Foreman, there is no latest symlink and you must match the Katello version to the correct Foreman version

+ +

Accessing this repo

+ +This repository is available over HTTP and HTTPS: + + + +

Support

+ +

You can find the installation instructions here, but we strongly recommend using our Installer (which uses RPMs and Puppet). + +

If you have any issues, you can find ways to reach us on our Support page.

diff --git a/vagrant/manifests/default.pp b/vagrant/manifests/default.pp index 6e2a403c2..3155419b7 100644 --- a/vagrant/manifests/default.pp +++ b/vagrant/manifests/default.pp @@ -25,3 +25,7 @@ node /^discourse.*/ { include profiles::discourse } + +node /^repo-rpm.*/ { + include profiles::repo::rpm +}