Skip to content

Commit

Permalink
Add rpm repo profile
Browse files Browse the repository at this point in the history
  • Loading branch information
ehelms committed Aug 29, 2024
1 parent 78d1281 commit e5d0a15
Show file tree
Hide file tree
Showing 14 changed files with 424 additions and 0 deletions.
11 changes: 11 additions & 0 deletions Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,15 @@ Vagrant.configure("2") do |config|
provider.vm.box_url = CENTOS_9_BOX_URL
end
end

config.vm.define "repo-rpm" do |override|
override.vm.hostname = "repo-rpm"
override.vm.box = "centos/stream9"

override.vm.provider "libvirt" do |libvirt, provider|
libvirt.memory = "2048"
libvirt.machine_virtual_size = 40
provider.vm.box_url = CENTOS_9_BOX_URL
end
end
end
1 change: 1 addition & 0 deletions puppet/data/common.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
---
stable_release: '3.11'
profiles::web::stable: '%{alias("stable_release")}'
profiles::repo::rpm::stable_foreman: '%{alias("stable_release")}'

backup_servicename: 'backups.theforeman.org'
backup_username: 'backup-%{facts.networking.hostname}'
Expand Down
2 changes: 2 additions & 0 deletions puppet/data/vagrant.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,6 @@ profiles::jenkins::node::swap_size_mb: 0

profiles::web::https: false

profiles::repo::rpm::https: false

redmine::https: false
5 changes: 5 additions & 0 deletions puppet/manifests/site.pp
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,8 @@
include profiles::base
include profiles::web
}

node /^repo-rpm\d+\.[a-z]+\.theforeman\.org$/ {
include profiles::base
include profiles::repo::rpm
}
25 changes: 25 additions & 0 deletions puppet/modules/profiles/manifests/repo/rpm.pp
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# @summary A profile for the rpm repo machines
#
# @param stable_foreman
# Latest Foreman release that users expect
#
# @param https
# Whether to enable HTTPS. This is typically wanted but can only be enabled
# in a 2 pass setup. First Apache needs to run for Letsencrypt to function.
# Then Letsencrypt can be enabled. Also useful to turn off in test setups.
class profiles::repo::rpm (
String[1] $stable_foreman = '3.11',
Boolean $https = true,
) {
class { 'web':
https => $https,
}
contain web

class { 'web::vhost::rpm':
stable_foreman => $stable_foreman,
}
contain web::vhost::rpm

contain web::vhost::stagingrpm
}
3 changes: 3 additions & 0 deletions puppet/modules/web/files/rpm/pulpcore-HEADER.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<h1>Pulpcore packages</h1>

These are RPM builds for <a href="https://pulpproject.org">Pulp 3</a> and various plugins for use by <a href="https://theforeman.org/plugins/katello/">Katello</a>. They are only intended to be used by Katello. Only branches used by Katello are maintained. No explicit end of life announcements will be made.
3 changes: 3 additions & 0 deletions puppet/modules/web/files/rpm/robots.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
User-agent: *
Disallow: /foreman/nightly/
Disallow: /pulpcore/nightly/
16 changes: 16 additions & 0 deletions puppet/modules/web/files/stagingrpm/HEADER.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<h1>stagingrpm.theforeman.org</h1>

<h3>Staging Repositories for Foreman releases</h3>

<h3>Accessing this repo</h3>

This repository is available over HTTP and HTTPS:

<ul>
<li>http://stagingrpm.theforeman.org</li>
<li>https://stagingrpm.theforeman.org</li>
</ul>

<h3>Support</h3>

<p>These are staging repositories used for testing purposes only. They are NOT supported.</p>
2 changes: 2 additions & 0 deletions puppet/modules/web/files/stagingrpm/robots.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
User-agent: *
Disallow: /
101 changes: 101 additions & 0 deletions puppet/modules/web/manifests/vhost/rpm.pp
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# @summary Set up the rpm vhost
# @api private
class web::vhost::rpm (
String[1] $stable_foreman,
Stdlib::Fqdn $servername = 'rpm.theforeman.org',
Stdlib::Absolutepath $rpm_directory = '/var/www/vhosts/rpm/htdocs',
Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs/',
String $user = 'rpmrepo',
) {
$rpm_directory_config = [
{
path => $rpm_directory,
options => ['Indexes', 'FollowSymLinks', 'MultiViews'],
expires_active => 'on',
expires_default => 'access plus 2 minutes',
},
{
path => '.+\.(bz2|gz|rpm|xz)$',
provider => 'filesmatch',
expires_active => 'on',
expires_default => 'access plus 30 days',
},
{
path => 'repomd.xml',
provider => 'files',
expires_active => 'on',
expires_default => 'access plus 2 minutes',
},
]

$deploy_rpmrepo_context = {
'servername' => $servername,
'rpm_directory' => $rpm_directory,
'rpm_source_directory' => $rpm_source_directory,
}

secure_ssh::receiver_setup { $user:
user => $user,
foreman_search => 'host ~ node*.jenkins.osuosl.theforeman.org and (name = external_ip4 or name = external_ip6)',
script_content => epp('web/deploy-rpmrepo.sh.epp', $deploy_rpmrepo_context),
}

include apache::mod::expires
web::vhost { 'rpm':
servername => $servername,
docroot => $rpm_directory,
docroot_owner => $user,
docroot_group => $user,
docroot_mode => '0755',
directories => $rpm_directory_config,
}

if $facts['os']['family'] == 'RedHat' {
package { 'createrepo_c':
ensure => present,
}
}

file { "${rpm_directory}/robots.txt":
ensure => file,
owner => $user,
group => $user,
mode => '0644',
content => file('web/rpm/robots.txt'),
}

file { "${rpm_directory}/HEADER.html":
ensure => file,
owner => $user,
group => $user,
mode => '0644',
content => epp("${module_name}/rpm/HEADER.html.epp", {
'stable_foreman' => $stable_foreman,
'servername' => $servername,
}),
}

['candlepin', 'foreman', 'pulpcore'].each |$directory| {
file { ["${rpm_directory}/${directory}"]:
ensure => directory,
owner => $user,
group => $user,
mode => '0755',
}

exec { "fastly-purge-${directory}-latest":
command => "fastly-purge-find 'https://${servername}' ${rpm_directory} ${directory}/latest/",
path => '/bin:/usr/bin:/usr/local/bin',
require => File['/usr/local/bin/fastly-purge-find'],
refreshonly => true,
}
}

file { "${rpm_directory}/pulpcore/HEADER.html":
ensure => file,
owner => $user,
group => $user,
mode => '0644',
content => file('web/rpm/pulpcore-HEADER.html'),
}
}
62 changes: 62 additions & 0 deletions puppet/modules/web/manifests/vhost/stagingrpm.pp
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# @summary Set up the rpm staging vhost
# @api private
class web::vhost::stagingrpm (
Stdlib::Fqdn $servername = 'stagingrpm.theforeman.org',
Stdlib::Absolutepath $rpm_source_directory = '/var/www/vhosts/stagingrpm/htdocs',
String $user = 'rpmrepostage',
Stdlib::Absolutepath $home = "/home/${user}",
Array[String[1]] $usernames = ['ehelms', 'evgeni', 'ekohl', 'Odilhao', 'pcreech', 'zhunting'],
) {
$rpm_source_directory_config = [
{
path => $rpm_source_directory,
options => ['Indexes', 'FollowSymLinks', 'MultiViews'],
expires_active => 'on',
expires_default => 'access plus 2 minutes',
},
{
path => '.+\.(bz2|gz|rpm|xz)$',
provider => 'filesmatch',
expires_active => 'on',
expires_default => 'access plus 30 days',
},
{
path => 'repomd.xml',
provider => 'files',
expires_active => 'on',
expires_default => 'access plus 2 minutes',
},
]

$authorized_keys = flatten($usernames.map |$name| {
split(file("users/${name}-authorized_keys"), "\n")
})

secure_ssh::rsync::receiver_setup { $user:
user => $user,
homedir => $home,
homedir_mode => '0750',
foreman_search => 'host ~ node*.jenkins.*.theforeman.org and (name = external_ip4 or name = external_ip6)',
script_content => template('web/deploy-stagingyum.sh.erb'),
authorized_keys => $authorized_keys,
}

web::vhost { 'stagingrpm':
servername => $servername,
docroot => $rpm_source_directory,
docroot_owner => $user,
docroot_group => $user,
docroot_mode => '0755',
directories => $rpm_source_directory_config,
}

['HEADER.html', 'robots.txt'].each |$filename| {
file { "${rpm_source_directory}/${filename}":
ensure => file,
owner => 'root',
group => 'root',
mode => '0644',
content => file("web/stagingrpm/${filename}"),
}
}
}
123 changes: 123 additions & 0 deletions puppet/modules/web/templates/deploy-rpmrepo.sh.epp
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
<%- |
Stdlib::Fqdn $servername,
Stdlib::Absolutepath $rpm_directory,
Stdlib::Absolutepath $rpm_source_directory,
| -%>
#!/bin/bash

set -xe
# This is a forced SSH command - uncomment to test locally
set -f -- $SSH_ORIGINAL_COMMAND

prepcache() {
if [[ -e $REPO_PATH ]]; then
if [[ $MERGE == false ]] && [[ $OVERWRITE == false ]] ; then
echo "Repo overwrite (${OVERWRITE}) and merge (${MERGE}) are disabled, but ${REPO_PATH} already exists"
exit 1
fi
cp -al $REPO_PATH "$REPO_INSTANCE_PATH"
else
mkdir -p $REPO_INSTANCE_PATH
fi
}

do_rsync() {
opts=(--archive --verbose --hard-links --log-file "$REPO_RSYNC_LOG")
if [[ $MERGE != true ]] ; then
opts+=('--delete')
fi

for ARCH in x86_64 source; do
rsync "${opts[@]}" --log-file-format "CHANGED ${ARCH}/%n" "${REPO_SOURCE_RPM}/${ARCH}/" "${REPO_INSTANCE_PATH}/${ARCH}/"
done

set +f
for d in "${REPO_INSTANCE_PATH}"/*; do
(
cd "$d"

latest=$(ls -t foreman-release-[0-9]*.rpm 2>/dev/null | head -n1)
if [[ -n "$latest" ]] ; then
ln -sf "$latest" foreman-release.rpm
fi

latest=$(ls -t foreman-client-release-[0-9]*.rpm 2>/dev/null | head -n1)
if [[ -n "$latest" ]] ; then
ln -sf "$latest" foreman-client-release.rpm
fi

latest=$(ls -t katello-repos-[0-9]*.rpm 2>/dev/null | head -n1)
if [[ -n "$latest" ]] ; then
ln -sf "$latest" katello-repos-latest.rpm
fi

if [[ $MERGE == true ]] ; then
HAS_MODULES_YAML=$(ls repodata/*-modules.yaml.gz >/dev/null 2>/dev/null && echo 'yes' || echo 'no')

if [[ $HAS_MODULES_YAML == yes ]]; then
zcat repodata/*-modules.yaml.gz > modules.yaml
modifyrepo_c --remove modules repodata/
rm -f repodata/*-modules.yaml.gz
fi

createrepo_c --skip-symlinks --update .

if [[ $HAS_MODULES_YAML == yes ]]; then
modifyrepo_c --mdtype=modules modules.yaml repodata/
fi
fi
)
done
set -f
}

replace() {
if [[ -e $REPO_PATH ]]; then
mv "${REPO_PATH}" "${REPO_INSTANCE_PATH_PREV}"
fi

mv "${REPO_INSTANCE_PATH}" "${REPO_PATH}"

if [[ $MERGE == true ]] || [[ $OVERWRITE == true ]] ; then
if [[ -e "${REPO_INSTANCE_PATH_PREV}" ]]; then
rm -rf "${REPO_INSTANCE_PATH_PREV}"
fi
fi
}

purgecdn() {
awk '/ CHANGED /{print $5}' "${REPO_RSYNC_LOG}" | xargs --no-run-if-empty fastly-purge "https://<%= $servername %>/${REPO_DEST}"
set +f
for d in "${REPO_PATH}"/*; do
purge_base="https://<%= $servername %>/${REPO_DEST}/$(basename $d)"
fastly-purge ${purge_base} foreman-release.rpm foreman-client-release.rpm katello-repos-latest.rpm
done
set -f
}

REPO_SOURCE=$1
REPO_DEST=$2
OVERWRITE=${3:-false}
MERGE=${4:-false}

if [[ -z $REPO_SOURCE ]] || [[ -z $REPO_DEST ]] ; then
echo "Usage: $0 REPO_SOURCE REPO_DEST OVERWRITE MERGE"
exit 1
fi

REPO_SOURCE_BASE="<%= $rpm_source_directory %>"
REPO_SOURCE_RPM="${REPO_SOURCE_BASE}/${REPO_SOURCE}"

DEPLOY_TO="<%= $rpm_directory %>"
REPO_PATH="${DEPLOY_TO}/${REPO_DEST}"
REPO_INSTANCE_PATH="${DEPLOY_TO}/$(dirname $REPO_DEST)/.$(basename $REPO_DEST)-$(date "+%Y%m%d%H%M%S")"
REPO_INSTANCE_PATH_PREV="${REPO_INSTANCE_PATH}-previous"

REPO_RSYNC_LOG=$(mktemp)

trap "rm -rf $REPO_RSYNC_LOG $REPO_INSTANCE_PATH" EXIT

prepcache
do_rsync
replace
purgecdn
Loading

0 comments on commit e5d0a15

Please sign in to comment.