commit fec372b3e618283ae1cff3f842bb226b070362e0 Author: Daniel Langbein Date: Sat Jul 16 17:02:02 2022 +0200 import from privacy1st/arch diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..57f1cb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.idea/ \ No newline at end of file diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..27fdd20 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,28 @@ +# Maintainer: Daniel Langbein +_pkgname=repo +_reponame=arch +pkgname="de-p1st-$_pkgname" +pkgver=0.2.13 +pkgrel=1 +pkgdesc="Bash script to manage remote Arch Linux repository" +arch=('any') +url="https://codeberg.org/privacy1st/${_reponame}" +license=('MIT') +depends=('openssh' 'rsync' 'aurutils') # arch-repo-vercmp uses "aur vercmp" which is part of "aurutils" +makedepends=('git') +source=("git+${url}.git") +sha256sums=('SKIP') + +package() { + cd "${_reponame}/pkg/${pkgname}" + + install -Dm0555 arch-repo-push-new.sh "$pkgdir"/usr/bin/arch-repo-push-new + install -Dm0555 arch-repo-receive-new.sh "$pkgdir"/usr/bin/arch-repo-receive-new + install -Dm0555 arch-repo-vercmp.sh "$pkgdir"/usr/bin/arch-repo-vercmp + + install -Dm0644 lib/util.sh "$pkgdir"/usr/lib/"${pkgname}"/util.sh + install -Dm0644 lib/pkginfo.sh "$pkgdir"/usr/lib/"${pkgname}"/pkginfo.sh + install -Dm0644 lib/pkgver.sh "$pkgdir"/usr/lib/"${pkgname}"/pkgver.sh + + install -Dm0644 -o0 arch-repo.cfg "$pkgdir"/etc/de-p1st-repo/arch-repo.cfg +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d2ced7 --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +# Arch Linux repository manager + +Special thanks to nachoparker for his article: + +* [Replicate your system with self-hosted Arch Linux metapackages](https://ownyourbits.com/2019/07/21/replicate-your-system-with-self-hosted-arch-linux-metapackages/) + +## Setup + +Install `de-p1st-repo` on your local machine as well as on +a remote server. + +Adjust [/etc/de-p1st-repo/arch-repo.conf](arch-repo.cfg) according to your needs. + +Run a webserver on the server to serve static content: + +* [https://hub.docker.com/_/nginx/]() -> Hosting some simple static content +* `sudo docker run --name arch-repo -v /mnt/data/live/arch-repo:/usr/share/nginx/html:ro -d nginx` + +Add the newly created mirror to your `/etc/pacman.conf`: + +``` +[de-p1st] +SigLevel = Optional TrustAll +Server = https://arch.p1st.de +``` + +## Normal usage + +### Check for AUR updates, build and push + +Check remote repository for AUR packages that can be updated + +```shell +arch-repo-vercmp +``` + +Then build those packages locally and push changes to remote repository + +```shell +# Build on local machine in clean chroot (or with an AUR helper), e.g. +makepkg -fCcsr + +# Push new packages to remote repository +arch-repo-push-new +``` + +## example output + +``` +user@localMachine ~ % arch-repo-push-new +de-p1st-pacman-0.0.7-2-any.pkg.tar.zst + 2.84K 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=325/384) +de-p1st-repo-0.1.1-2-any.pkg.tar.zst + 5.13K 100% 4.90MB/s 0:00:00 (xfr#2, to-chk=308/384) +de-p1st-repo-0.1.1-3-any.pkg.tar.zst + 5.13K 100% 4.90MB/s 0:00:00 (xfr#3, to-chk=307/384) +new-pkg.txt + 113 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1) +Adding new packages to db ... +Sorting new packages by package name and package version ... +Creating file ./db/de-p1st-pacman/0.0.7-2 with content de-p1st-pacman-0.0.7-2-any.pkg.tar.zst ... +Creating file ./db/de-p1st-repo/0.1.1-2 with content de-p1st-repo-0.1.1-2-any.pkg.tar.zst ... +Creating file ./db/de-p1st-repo/0.1.1-3 with content de-p1st-repo-0.1.1-3-any.pkg.tar.zst ... +For each new package name: Add latest version to database ... +==> Extracting de-p1st.db.tar.gz to a temporary location... +==> Extracting de-p1st.files.tar.gz to a temporary location... +==> Adding package 'de-p1st-pacman-0.0.7-2-any.pkg.tar.zst' + -> Computing checksums... + -> Removing existing entry 'de-p1st-pacman-0.0.7-1'... + -> Creating 'desc' db entry... + -> Creating 'files' db entry... +==> Creating updated database file 'de-p1st.db.tar.gz' +==> Extracting de-p1st.db.tar.gz to a temporary location... +==> Extracting de-p1st.files.tar.gz to a temporary location... +==> Adding package 'de-p1st-repo-0.1.1-3-any.pkg.tar.zst' + -> Computing checksums... + -> Removing existing entry 'de-p1st-repo-0.1.1-1'... + -> Creating 'desc' db entry... + -> Creating 'files' db entry... +==> Creating updated database file 'de-p1st.db.tar.gz' +Generating index.html with links to all packages ... +``` + +Where the generated `new-pkg.txt` has this content: + +``` +de-p1st-pacman-0.0.7-2-any.pkg.tar.zst +de-p1st-repo-0.1.1-2-any.pkg.tar.zst +de-p1st-repo-0.1.1-3-any.pkg.tar.zst +``` + +## Removing packages + +On your remote server: + +```shell +PKG=xml2 +REPO_NAME=de-p1st + +repo-remove "${REPO_NAME}".db.tar.gz "${PKG}" +rm "${PKG}"-PKG-VERSION.pkg.tar.{xz,zst} +rm -r db/"${PKG}"/ +``` diff --git a/arch-repo-push-new.sh b/arch-repo-push-new.sh new file mode 100644 index 0000000..84ed280 --- /dev/null +++ b/arch-repo-push-new.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +source /etc/de-p1st-repo/arch-repo.cfg || exit $? +# Enable nullglob for the case that not all patterns match, e.g. just *.zst but not *.xz packages exist. +shopt -s nullglob + +function main() { + cd "${LOCAL_PKG_DIR}" || return $? + + # Check if at least one matching file exists + match="0" + for PKG in ./*.pkg.tar.{xz,zst}; do + # There is at least one match! + match="1" + break + done + + if [ "$match" = "0" ]; then + echo "There are no local packages inside ${LOCAL_PKG_DIR}" + return 0 + fi + + + # Get list of new packages, one package per line. + rsync --ignore-existing --out-format="%n" --dry-run \ + ./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" > new-pkg.txt || return $? + + # If there are no new packages to push/synchronize, then return + if [ ! -s new-pkg.txt ]; then + echo "No new packages inside ${LOCAL_PKG_DIR}"; + return 0; + fi + + + # Transfer new packages using rsync + rsync --ignore-existing --progress --human-readable \ + ./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $? + + # Transfer new-pkg.txt + rsync --ignore-times --checksum --progress --human-readable \ + new-pkg.txt "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $? + + + # Add each new package to database + ssh "${REMOTE_SSH_HOST}" "/usr/bin/arch-repo-receive-new" || return $? +} + + +for LOCAL_PKG_DIR in "${LOCAL_PKG_DIRS[@]}"; do + main +done diff --git a/arch-repo-receive-new.sh b/arch-repo-receive-new.sh new file mode 100644 index 0000000..1ddd3c5 --- /dev/null +++ b/arch-repo-receive-new.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +source /usr/lib/de-p1st-repo/util.sh || exit $? +source /usr/lib/de-p1st-repo/pkgver.sh || exit $? +source /usr/lib/de-p1st-repo/pkginfo.sh || exit $? +source /etc/de-p1st-repo/arch-repo.cfg || exit $? + + + +function main(){ + cd "${REMOTE_PKG_DIR}" || return $? + add_new_to_db || return $? + generate_index || return $? +} + + +# +# add all packages to database +# +function add_all_to_db(){ + echo "Adding all packages to db ..." + sort_all_pkgname_pkgver || return $? + + echo "For each package name: Add latest version to database ..." + for PKGNAME in db/*; do + PKGNAME=$(basename "${PKGNAME}") || return $? # strip directory and suffix from filename + add_to_db "${PKGNAME}" || return $? + done +} +# +# add new packages to database +# +function add_new_to_db(){ + echo "Adding new packages to db ..." + sort_new_pkgname_pkgver || return $? + + echo "For each new package name: Add latest version to database ..." + for PKGNAME in "${NEW_PKGNAMES[@]}"; do + add_to_db "${PKGNAME}" || return $? + done +} +# +# add package to database +# +function add_to_db(){ + # $1: package name + local PKGNAME + PKGNAME="$1" + + # get path to latest version of $PKGNAME + PKG=$(latest_pkgver_path "${PKGNAME}") || return $? + + # add to database + repo-add --new "${REMOTE_DB_NAME}.db.tar.gz" "${PKG}" || return $? +} + + +# +# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file) +# +function sort_all_pkgname_pkgver(){ + echo "Cleanup ..." + rm -r db || return $? + + echo "Sorting all packages by package name and package version ..." + + for PKG in *.pkg.tar.{xz,zst}; do + sort_pkgname_pkgver "${PKG}" || return $? + done +} +# +# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file) +# +function sort_new_pkgname_pkgver(){ + # return: array $NEW_PKGNAMES + + echo "Sorting new packages by package name and package version ..." + + local NEW_PKGNAMES_TMP=() # list the names from new package-files; may contain duplicates + + mapfile -t PKGS < <(cat new-pkg.txt) + for PKG in "${PKGS[@]}"; do + sort_pkgname_pkgver "${PKG}" || return $? + NEW_PKGNAMES_TMP+=("${PKGNAME}") + done + + + # create array $NEW_PKGNAMES without duplicates + NEW_PKGNAMES=() + for NEW_PKGNAME_TMP in "${NEW_PKGNAMES_TMP[@]}"; do + local contains="0" + + # if NEW_PKGNAMES does already contain NEW_PKGNAME_TMP, + # then set contains to "1" + for i in "${NEW_PKGNAMES[@]}"; do + if [ "${NEW_PKGNAME_TMP}" = "${i}" ]; then + contains="1"; + break; + fi + done + + if [ "${contains}" = "0" ]; then + NEW_PKGNAMES+=("${NEW_PKGNAME_TMP}") + fi + done +} +# +# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file) +# +function sort_pkgname_pkgver(){ + # $1: PKG (path to package file) + # return: variables $PKGINFO, $PKGNAME, $PKGVER + local PKG + PKG="$1" + + get_pkginfo "$PKG" || { echo "get_pkginfo failed"; return 1; } + PKGNAME=$(get_pkgname "$PKGINFO") || { echo "get_pkgname failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; } + PKGVER=$(get_pkgver "$PKGINFO") || { echo "get_pkgver failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; } + + echo "Creating file ./db/${PKGNAME}/${PKGVER} with content ${PKG} ..." + mkdir -p "db/${PKGNAME}" || return $? + echo "${PKG}" > "db/${PKGNAME}/${PKGVER}" || return $? +} + + + +# +# generate index.html +# +function generate_index(){ + echo "Generating index.html with links to all packages ..." + + echo " + + +${HTML_TITLE} + + +

${HTML_HEADING}

+

The sources can be found here: ${HTML_LINK_SRC}

+ + + + +' >> index.html +} + +main "$@" diff --git a/arch-repo-vercmp.sh b/arch-repo-vercmp.sh new file mode 100644 index 0000000..bc39878 --- /dev/null +++ b/arch-repo-vercmp.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# For all packages in repository $REMOTE_DB_NAME, +# compare package version with AUR and +# print all outdated packages +# + +source /usr/lib/de-p1st-repo/util.sh || exit $? +source /usr/lib/de-p1st-repo/pkgver.sh || exit $? +source /usr/lib/de-p1st-repo/pkginfo.sh || exit $? +source /etc/de-p1st-repo/arch-repo.cfg || exit $? + + + +function main(){ + echo "Note: You may run 'arch-repo-push-new' and 'pacman -Sy' first ..." + + all_pkg_vers || return $? + check_aur_updates || return $? + + if [ "${#AUR_UPDATES[@]}" -gt "0" ]; then + echo "Repository ${REMOTE_DB_NAME} contains packages with available AUR updates:" + for AUR_PKG in "${AUR_UPDATES[@]}"; do + echo " ${AUR_PKG}" + done + else + echo "There are no pending AUR updates in repository ${REMOTE_DB_NAME}." + fi + + get_vcs_packages || return $? + if [ "${#VCS_PKGS[@]}" -gt "0" ]; then + echo "" + echo "Note: Some VCS packages were found which are possibly outdated:" + printf "VCS_PKGS=(%s)\n" "${VCS_PKGS[*]}" + fi +} + +function get_vcs_packages(){ + # return: array GIT_PKGS with all VCS packages + VCS_PKGS=() + + # https://wiki.archlinux.org/title/VCS_package_guidelines#VCS_sources + # https://github.com/AladW/aurutils/pull/283/files + # https://man.archlinux.org/man/PKGBUILD.5#USING_VCS_SOURCES + readonly AURVCS='.*-(bzr|git|hg|svn|fossil)$' + + while IFS= read -r PKG_VER; do + PKGNAME=$(first_word "${PKG_VER}") || return $? + if echo "${PKGNAME}" | grep -E "$AURVCS" >/dev/null; then + VCS_PKGS+=("${PKGNAME}") + fi + done <<< "${PKG_VERS}" +} + +# +# Store packages from $PKG_VERS with available AUR updates in $AUR_UPDATES. +# $AUR_UPDATES is an array where each entry describes one outdated package with it's current and new version. +# +function check_aur_updates(){ + mapfile -t AUR_UPDATES < <(echo "$PKG_VERS" | aur vercmp) +} + +# +# Store all installed package names and their versions +# from repository $REMOTE_DB_NAME +# in variable $PKG_VERS. +# $PKG_VERS consists of multiple lines in the format: "" +# +function installed_pkg_vers(){ + # The paclist script is part of the package "pacman-contrib" + PKG_VERS=$(paclist "${REMOTE_DB_NAME}") || return $? +} + +# +# Store all package names and their versions +# from repository $REMOTE_DB_NAME +# in variable $PKG_VERS. +# $PKG_VERS contains of multiple lines in the format: "" +# +function all_pkg_vers(){ + PKG_VERS=$(pacman -S --list "${REMOTE_DB_NAME}" | sed "s|^${REMOTE_DB_NAME}\\s*||; s|\\s*\\[installed.*\\]\s*\$||") || return $? +} + +main "$@" diff --git a/arch-repo.cfg b/arch-repo.cfg new file mode 100644 index 0000000..9977917 --- /dev/null +++ b/arch-repo.cfg @@ -0,0 +1,19 @@ +#=== LOCAL MACHINE CONFIGURATION ===# + +# Locations of built packages +LOCAL_PKG_DIRS=('/home/yoda/Downloads/git/arch/build-pkg/out') + +#=== REMOTE MIRROR SERVER CONFIGURATION ===# + +# Host from ssh configuration with correct user to have +# write access to REMOTE_PKG_DIR and REMOTE_DB_NAME +REMOTE_SSH_HOST=rootnas +REMOTE_PKG_DIR=/mnt/data/live/arch-repo +REMOTE_DB_NAME=de-p1st + +# +# Some variables for index.html generation. +# +HTML_TITLE='Privacy1st' +HTML_HEADING='My personalized Arch Linux distribution' +HTML_LINK_SRC='https://codeberg.org/privacy1st/arch' diff --git a/bash-unique-array-test.sh b/bash-unique-array-test.sh new file mode 100644 index 0000000..4f90e6f --- /dev/null +++ b/bash-unique-array-test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# source: https://stackoverflow.com/a/13649357 +# + +a=(aa ac aa ad "ac ad") +declare -A b +for i in "${a[@]}"; do b["$i"]=1; done + +for i in "${!b[@]}"; do + echo ">> $i" +done diff --git a/iterate-files-test.sh b/iterate-files-test.sh new file mode 100644 index 0000000..9ae7fab --- /dev/null +++ b/iterate-files-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +source /etc/de-p1st-repo/arch-repo.cfg || exit $? + + +cd "${LOCAL_PKG_DIR}" || exit $? + +match="0" +for PKG in ./*.pkg.tar.{xz,zst}; do + [ -f "$PKG" ] || { echo "No match for pattern $PKG"; continue; } # alternatively: shopt -s nullglob + + # if we are here, there is at least one match! + match="1" + break +done + +if [ "$match" = "1" ]; then + shopt -s nullglob + # print all matching + echo ./*.pkg.tar.{xz,zst} +fi diff --git a/lib/pkginfo.sh b/lib/pkginfo.sh new file mode 100644 index 0000000..9b05472 --- /dev/null +++ b/lib/pkginfo.sh @@ -0,0 +1,60 @@ +# +# get content of .PKGINFO from package-file +# +function get_pkginfo(){ + # $1: path to package file + # return: variable $PKGINFO + + if endswith "$1" ".pkg.tar.xz"; then + PKGINFO=$(tar -xf "$1" -O .PKGINFO --force-local) || { echo "tar failed"; return 1; } + elif endswith "$1" ".pkg.tar.zst"; then + PKGINFO=$(tar -I zstd -xf "$1" -O .PKGINFO --force-local) || { echo "tar failed"; return 1; } + else + echo "$1 does not seem to be a package!" + return 1 + fi +} + + +# +# get pkgname from $PKGINFO +# +function get_pkgname(){ + # return: stdout: package name + + # remove "pkgname = " as well as tailing whitespace characters + local tmp + tmp=$(echo "${PKGINFO}" | grep '^pkgname =') || { echo "grep failed"; return 1; } + local PKGNAME + PKGNAME=$(echo "${tmp}" | sed 's|^pkgname\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; } + + echo "${PKGNAME}" +} +# +# get pkgver from $PKGINFO +# +function get_pkgver(){ + # return: stdout: package version + + # remove "pkgver = " as well as tailing whitespace characters + local tmp + tmp=$(echo "${PKGINFO}" | grep '^pkgver =') || { echo "grep failed"; return 1; } + local PKGVER + PKGVER=$(echo "${tmp}" | sed 's|^pkgver\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; } + + echo "${PKGVER}" +} +# +# get url from $PKGINFO +# +function get_pkgurl(){ + # return: stdout: url + + # remove "url = " as well as tailing whitespace characters + local tmp + tmp=$(echo "${PKGINFO}" | grep '^url =') || { echo "grep failed"; return 1; } + local PKGURL + PKGURL=$(echo "${tmp}" | sed 's|^url\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; } + + echo "${PKGURL}" +} diff --git a/lib/pkgver.sh b/lib/pkgver.sh new file mode 100644 index 0000000..63c6fe5 --- /dev/null +++ b/lib/pkgver.sh @@ -0,0 +1,53 @@ +# +# get package-file with latest version for given package name +# +function latest_pkgver_path(){ + # precond: In current working directory there is a subdir "db" + # $1: package name + # return: stdout: path to package file + local PKGNAME + PKGNAME="$1" + + # get latest version for $PKGNAME + local LATEST_PKGVER + LATEST_PKGVER=$(latest_pkgver "${PKGNAME}") || return $? + # get the path to package file + local PKG + PKG=$(cat "db/${PKGNAME}/${LATEST_PKGVER}") || return $? + + echo "${PKG}" +} + +# +# get latest version of package +# +function latest_pkgver(){ + # precond: In current working directory there is a subdir "db" + # $1: package name + # return: stdout: latest pkgver + local PKGNAME + PKGNAME="$1" + + # pick one random version as starting point for the latest version + local LATEST_PKGVER + for PKGVER in db/"${PKGNAME}"/*; do + PKGVER=$(basename "${PKGVER}") || return $? # strip directory and suffix from filename + LATEST_PKGVER="${PKGVER}" + break + done + + local cmp + for PKGVER in db/"${PKGNAME}"/*; do + PKGVER=$(basename "${PKGVER}") || return $? # strip directory and suffix from filename + + # compare the currently known latest version + # with the next version + cmp=$(vercmp "${LATEST_PKGVER}" "${PKGVER}") || return $? + # if the new version is larger, save it as LATEST_PKGVER + if [ "${cmp}" -lt "0" ]; then + LATEST_PKGVER="${PKGVER}" + fi + done + + echo "${LATEST_PKGVER}" +} diff --git a/lib/util.sh b/lib/util.sh new file mode 100644 index 0000000..fd0a547 --- /dev/null +++ b/lib/util.sh @@ -0,0 +1,19 @@ +function first_word() { + # return: The first word of $1. + # In detail: "" if $1 starts with a space. Otherwise: All characters until the first space of $1. + # + # source: https://unix.stackexchange.com/a/201744/315162 + + echo "${1%% *}" +} + + +# Inspired by: https://stackoverflow.com/questions/2172352/in-bash-how-can-i-check-if-a-string-begins-with-some-value/18558871#18558871 +# +# $1 begins with $2 +# +beginswith() { case $1 in "$2"*) true;; *) false;; esac; } +# +# $1 ends with $2 +# +endswith() { case $1 in *"$2") true;; *) false;; esac; }