POC Trunk/OCI binary distribution

.gitignore

@@ -4,12 +4,13 @@ results/
 *.dll
 tmp/
 *.o
+*.bc
 regression.diffs
 regression.out
 /sql/semver--?.??.?.sql
 /semver-*
+/semver_annotations.json
 /latest-changes.md
-/src/*.bc
 /semver_binary_copy.bin
 /.vscode
 /*.bin

Makefile

@@ -6,7 +6,9 @@ DISTVERSION  = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \
                sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/')
 
 MODULEDIR    = $(EXTENSION)
-DATA         = $(wildcard sql/*.sql)
+DESCRIPTION  = A Postgres data type for the Semantic Version format with support for btree and hash indexing.
+REPO_URL     = https://github.com/theory/pg-semver
+DATA         = $(wildcard sql/*--*.sql)
 DOCS         = $(wildcard doc/*.mmd)
 TESTS        = $(wildcard test/sql/*.sql)
 REGRESS      = $(patsubst test/sql/%.sql,%,$(TESTS))
@@ -22,6 +24,7 @@ endif
 
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
+include ./trunk.mk
 
 all: sql/$(EXTENSION)--$(EXTVERSION).sql
 

docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.9"
+name: trunking
+
+# Start a Zot service for pushing and pulling OCI images, and a
+# pgxn/pgxn-tools image for building and testing Linux/AMD binaries.
+#
+# docker compose up -d
+# docker compose exec linux bash
+# make clean && make && make trunk
+#
+# On macOS:
+#   make clean && make && make trunk
+#   ./push_trunk localhost:5000/theory/semver semver-0.32.1+pg16-darwin-23.5.0-arm64 semver-0.32.1+pg16-linux-amd64
+#   clean -dfx --exclude=.vscode
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'  -exec rm -rf "{}" \;
+#
+#   ./install_trunk localhost:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+#
+# Back on Linux
+#   ./install_trunk zot:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+
+# Name the network for all of the services to join.
+networks:
+  default:
+    name: pgxnnet
+
+services:
+  zot:
+    image: ghcr.io/project-zot/zot-linux-arm64:latest
+    container_name: zot
+    ports:
+      - 5000:5000
+    hostname: zot
+
+  linux:
+    image: pgxn/pgxn-tools
+    platform: "linux/amd64"
+    container_name: linux
+    hostname: linux
+    working_dir: /work
+    volumes:
+      - ".:/work"
+    # Install oras, then start Postgres 16 and install rsync and jq
+    entrypoint:
+      - "/bin/sh"
+      - -ecx
+      - |
+          cd /tmp
+          curl -LO "https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz"
+          mkdir -p oras-install/
+          tar -zxf oras_1.2.0_*.tar.gz -C oras-install/
+          sudo mv oras-install/oras /usr/local/bin/
+          rm -rf oras_1.2.0_*.tar.gz oras-install/
+          pg-start 16 rsync jq
+          tail -f /dev/null

install_trunk

@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# POC for installing Trunk format binaries created with trunk.mk. Requires:
+#
+# * bash
+# * tar
+# * shasum
+# * jq
+# * uname
+# * pg_config
+# * rsync
+
+trap 'exit' ERR
+set -E
+
+install_trunk() {
+    local trunk=$1
+
+    if [ -z "$trunk" ]; then
+        printf "Usage:\n\n  %s PACKAGE_PATH\n" "$0"
+        exit 1
+    fi
+
+    # Determine the platform.
+    local my_os my_osv my_arch platform
+    my_os=$(uname | tr '[:upper:]' '[:lower:]')
+    my_osv=$(uname -r)
+    my_arch="$(uname -m)"
+    if [ "$my_arch" == "x86_64" ]; then my_arch=amd64; fi
+    if [ "$my_os" == "darwin" ]; then
+        platform="$my_os/$my_arch:$my_osv"
+    else
+        platform="$my_os/$my_arch"
+    fi
+
+    # Fetch the file name.
+    local file
+    file=$(oras manifest fetch --plain-http --platform "$platform" "$trunk" | jq -r '.layers[0].annotations["org.opencontainers.image.title"]')
+
+    # Download.
+    oras pull --no-tty --plain-http --platform "$platform" "$trunk"
+
+    # Unpack.
+    printf "Unpacking %s\n" "$file"
+    tar zxf "$file"
+
+    # Go into the directory
+    local dir="${file%.*}"
+    cd "$dir" || exit
+
+    # Verify the checksums.
+    printf "Verifying all checksums..."
+    shasum --check -b --strict < digests
+    printf "Done!\n"
+
+    # Verify the Trunk version
+    printf "Verifying compatibility with Trunk package 0.1.0\n" 
+    local tv
+    tv=$(jq -r .trunk trunk.json)
+    if [ "$tv" != "0.1.0" ]; then
+        printf "Unsupported Trunk format version %s\n" "$tv"
+        exit 1
+    fi
+
+    # Verify the Postgres version.
+    printf "Verifying compatibility with PostgreSQL %s\n" "$my_pg"
+    local pkg_pg my_pg
+    pkg_pg=$(jq -r .postgres.version trunk.json)
+    my_pg=$(pg_config --version | sed -E 's/^[^ ]+ ([^ ]+).*$/\1/')
+    if [ "$pkg_pg" != "$my_pg" ]; then
+        printf "Trunk package contains binaries for Postgres %s but this host runs Postgres %s\n" "$pkg_pg" "$my_pg"
+        exit 1
+    fi
+
+    printf "Verifying compatibility with %s/%s:%s \n" "$my_os" "$my_arch" "$my_osv"
+    local pkg_os
+    pkg_os=$(jq -r .platform.os trunk.json)
+    if [ "$pkg_os" != "any" ]; then
+        # Verify the OS
+        if [ "$pkg_os" != "$my_os" ]; then
+            printf "Trunk package contains %s binaries but this host runs %s\n" "$pkg_os" "$my_os"
+            exit 1
+        fi
+
+        # Verify the architecture.
+        local pkg_arch
+        pkg_arch=$(jq -r .platform.arch trunk.json)
+        if [ "$pkg_arch" != "$my_arch" ]; then
+            printf "Trunk package contains %s binaries but this host runs %s\n" "$pkg_arch" "$my_arch"
+            exit 1
+        fi
+    fi
+
+    # Make sure we have pgsql directory.
+    if [ ! -d 'pgsql' ]; then
+        printf "Package contains no install files; exiting\n"
+        exit 1
+    fi
+
+    cd 'pgsql' || exit
+    for subdir in *; do
+        [[ -d "$subdir" ]] || continue
+        case $subdir in
+            share)
+                install_dir "$subdir" "$(pg_config --sharedir)"
+                ;;
+            pkglib)
+                install_dir "$subdir" "$(pg_config --pkglibdir)"
+                ;;
+            pkginclude)
+                install_dir "$subdir" "$(pg_config --pkgincludedir)"
+                ;;
+            lib)
+                install_dir "$subdir" "$(pg_config --libdir)"
+                ;;
+            include)
+                install_dir "$subdir" "$(pg_config --includedir)"
+                ;;
+            bin)
+                install_dir "$subdir" "$(pg_config --bindir)"
+                ;;
+            doc)
+                install_dir "$subdir" "$(pg_config --docdir)"
+                ;;
+            man)
+                install_dir "$subdir" "$(pg_config --mandir)"
+                ;;
+            html)
+                install_dir "$subdir" "$(pg_config --htmldir)"
+                ;;
+            locale)
+                install_dir "$subdir" "$(pg_config --localedir)"
+                ;;
+            sysconf)
+                install_dir "$subdir" "$(pg_config --sysconfdir)"
+                ;;
+            *)
+                printf "Unknown install directory %s; skipping\n" "$subdir"
+                ;;
+        esac
+    done
+
+}
+
+install_dir() {
+    local src="$1"
+    local dst="$2"
+    printf "Installing %s into %s..." "$src" "$dst"
+    cd "$src" || exit
+    rsync -q -a -v . "$dst" || exit
+    printf "Done\n"
+    cd ..
+}
+
+install_trunk "$@"

push_trunk

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# POC for publishing Trunk packages to an OCI repository with an image index
+# to allow pulling a platform-specific binary. Requires:
+#
+# * oras
+# * jq
+# * zot: docker run -d -p 5000:5000 --name oras-quickstart ghcr.io/project-zot/zot-linux-arm64:latest
+#
+# Inspired by the Homebrew implementation of the pattern as referenced in
+# https://github.com/oras-project/oras/issues/237, and with thanks to the
+# denizens of the #oras and #zot channels on the CNCF Slack.
+
+trap 'exit' ERR
+set -E
+
+# OCI types to create.
+ARTIFACT_TYPE=application/vnd.pgxn.trunk.layer.v1
+MEDIA_TYPE=application/vnd.oci.image.layer.v1.tar+gzip
+CONFIG_TYPE=application/vnd.oci.image.config.v1+json
+OCI_DIR=oci_dir
+INDEX_FILE=image_index.json
+
+push_image() {
+    # Push the image into the OCI layout directory $OCI_DIR.
+    local trunk=$1
+    oras push --no-tty \
+        --oci-layout "$OCI_DIR" \
+        --artifact-type "${ARTIFACT_TYPE}" \
+        --config "${trunk}_config.json":"$CONFIG_TYPE" \
+        --format go-template='{{.digest}}' \
+        --annotation-file "${trunk}_annotations.json" \
+        "$trunk.trunk":"$MEDIA_TYPE"
+}
+
+make_manifest() {
+    local trunk=$1
+    local digest=$2
+    local anno platform
+
+    # Extract just the pgxn.trunk annotations.
+    anno=$(jq -c \
+        '.["$manifest"] | with_entries(select(.key | startswith("org.pgxn.trunk.")))' \
+        "${trunk}_annotations.json"
+    )
+
+    # Extract the platform config.
+    platform=$(jq \
+        'pick(.os, .["os.version"], .architecture)| with_entries(select(.value |. !=null and . != ""))' \
+        "$trunk"_config.json
+    )
+
+    # Create and return the image manifest.
+    oras manifest fetch --oci-layout "$OCI_DIR@${digest}" --descriptor \
+        | jq --argjson anno "$anno" --argjson platform "$platform" \
+        '{
+            mediaType: .mediaType,
+            size: .size,
+            digest: .digest,
+            platform: $platform,
+            annotations: $anno
+        }'
+}
+
+write_index() {
+    darwin_manifest=$1
+    linux_manifest=$2
+
+    # Build the image index with the two manifests.
+    jq -n --argjson linux "$linux_manifest" \
+        --argjson darwin "$darwin_manifest" \
+        --argjson annotations "$(cat semver_annotations.json)" \
+        '{
+            schemaVersion: 2,
+            mediaType: "application/vnd.oci.image.index.v1+json",
+            manifests: [$linux, $darwin],
+            annotations: $annotations
+        }' > "$INDEX_FILE"
+}
+
+push_trunk() {
+    # Only testing for Darwin and Linux rn.
+    local repo=$1
+    local darwin_trunk=$2
+    local linux_trunk=$3
+
+    if [ -z "$repo" ] || [ -z "$darwin_trunk" ] || [ -z "$linux_trunk" ]; then
+        printf "Usage:\n\n  %s REPO DARWIN.trunk LINUX.trunk\n" "$0"
+        exit 1
+    fi
+
+    # Push the images and grab the resulting digests.
+    darwin_digest=$(push_image "$darwin_trunk")
+    linux_digest=$(push_image "$linux_trunk")
+
+    # Create the image manifests.
+    darwin_manifest=$(make_manifest "$darwin_trunk" "$darwin_digest")
+    linux_manifest=$(make_manifest "$linux_trunk" "$linux_digest")
+
+    # Write out and push the image index.
+    write_index "$darwin_manifest" "$linux_manifest"
+    oras manifest push --oci-layout ./"$OCI_DIR":image-index "$INDEX_FILE"
+
+    # Push everything from the local layout to the remote registry.
+    oras cp --from-oci-layout ./"$OCI_DIR":image-index --to-plain-http "${repo}:v1"
+
+    # View the remote image index manifest.
+    oras manifest get --plain-http "${repo}:v1" | jq
+
+    # Cleanup.
+    rm -rf "$OCI_DIR" "$INDEX_FILE"
+}
+
+push_trunk "$@"