import from privacy1st/arch

2024-11-22 21:29:31 +01:00 · 2022-07-16 17:02:02 +02:00 · 2022-07-16 17:02:02 +02:00 · fec372b3e6
commit fec372b3e6
12 changed files with 607 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1 @@
+/.idea/
--- a/28
+++ b/28
@ -0,0 +1,28 @@
+# Maintainer: Daniel Langbein <daniel@systemli.org>
+_pkgname=repo
+_reponame=arch
+pkgname="de-p1st-$_pkgname"
+pkgver=0.2.13
+pkgrel=1
+pkgdesc="Bash script to manage remote Arch Linux repository"
+arch=('any')
+url="https://codeberg.org/privacy1st/${_reponame}"
+license=('MIT')
+depends=('openssh' 'rsync' 'aurutils')  # arch-repo-vercmp uses "aur vercmp" which is part of "aurutils"
+makedepends=('git')
+source=("git+${url}.git")
+sha256sums=('SKIP')
+
+package() {
+  cd "${_reponame}/pkg/${pkgname}"
+
+  install -Dm0555 arch-repo-push-new.sh    "$pkgdir"/usr/bin/arch-repo-push-new
+  install -Dm0555 arch-repo-receive-new.sh "$pkgdir"/usr/bin/arch-repo-receive-new
+  install -Dm0555 arch-repo-vercmp.sh      "$pkgdir"/usr/bin/arch-repo-vercmp
+
+  install -Dm0644 lib/util.sh              "$pkgdir"/usr/lib/"${pkgname}"/util.sh
+  install -Dm0644 lib/pkginfo.sh           "$pkgdir"/usr/lib/"${pkgname}"/pkginfo.sh
+  install -Dm0644 lib/pkgver.sh            "$pkgdir"/usr/lib/"${pkgname}"/pkgver.sh
+
+  install -Dm0644 -o0 arch-repo.cfg        "$pkgdir"/etc/de-p1st-repo/arch-repo.cfg
+}
--- a/README.md
+++ b/README.md
@ -0,0 +1,103 @@
+# Arch Linux repository manager
+
+Special thanks to nachoparker for his article:
+
+* [Replicate your system with self-hosted Arch Linux metapackages](https://ownyourbits.com/2019/07/21/replicate-your-system-with-self-hosted-arch-linux-metapackages/)
+
+## Setup
+
+Install `de-p1st-repo` on your local machine as well as on
+a remote server.
+
+Adjust [/etc/de-p1st-repo/arch-repo.conf](arch-repo.cfg) according to your needs.
+
+Run a webserver on the server to serve static content:
+
+* [https://hub.docker.com/_/nginx/]() -> Hosting some simple static content
+* `sudo docker run --name arch-repo -v /mnt/data/live/arch-repo:/usr/share/nginx/html:ro -d nginx`
+
+Add the newly created mirror to your `/etc/pacman.conf`:
+
+```
+[de-p1st]
+SigLevel = Optional TrustAll
+Server = https://arch.p1st.de
+```
+
+## Normal usage
+
+### Check for AUR updates, build and push
+
+Check remote repository for AUR packages that can be updated
+
+```shell
+arch-repo-vercmp
+```
+
+Then build those packages locally and push changes to remote repository
+
+```shell
+# Build on local machine in clean chroot (or with an AUR helper), e.g.
+makepkg -fCcsr
+
+# Push new packages to remote repository
+arch-repo-push-new
+```
+
+## example output
+
+```
+user@localMachine ~ % arch-repo-push-new 
+de-p1st-pacman-0.0.7-2-any.pkg.tar.zst
+          2.84K 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=325/384)
+de-p1st-repo-0.1.1-2-any.pkg.tar.zst
+          5.13K 100%    4.90MB/s    0:00:00 (xfr#2, to-chk=308/384)
+de-p1st-repo-0.1.1-3-any.pkg.tar.zst
+          5.13K 100%    4.90MB/s    0:00:00 (xfr#3, to-chk=307/384)
+new-pkg.txt
+            113 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+Adding new packages to db ...
+Sorting new packages by package name and package version ...
+Creating file ./db/de-p1st-pacman/0.0.7-2 with content de-p1st-pacman-0.0.7-2-any.pkg.tar.zst ...
+Creating file ./db/de-p1st-repo/0.1.1-2 with content de-p1st-repo-0.1.1-2-any.pkg.tar.zst ...
+Creating file ./db/de-p1st-repo/0.1.1-3 with content de-p1st-repo-0.1.1-3-any.pkg.tar.zst ...
+For each new package name: Add latest version to database ...
+==> Extracting de-p1st.db.tar.gz to a temporary location...
+==> Extracting de-p1st.files.tar.gz to a temporary location...
+==> Adding package 'de-p1st-pacman-0.0.7-2-any.pkg.tar.zst'
+  -> Computing checksums...
+  -> Removing existing entry 'de-p1st-pacman-0.0.7-1'...
+  -> Creating 'desc' db entry...
+  -> Creating 'files' db entry...
+==> Creating updated database file 'de-p1st.db.tar.gz'
+==> Extracting de-p1st.db.tar.gz to a temporary location...
+==> Extracting de-p1st.files.tar.gz to a temporary location...
+==> Adding package 'de-p1st-repo-0.1.1-3-any.pkg.tar.zst'
+  -> Computing checksums...
+  -> Removing existing entry 'de-p1st-repo-0.1.1-1'...
+  -> Creating 'desc' db entry...
+  -> Creating 'files' db entry...
+==> Creating updated database file 'de-p1st.db.tar.gz'
+Generating index.html with links to all packages ...
+```
+
+Where the generated `new-pkg.txt` has this content:
+
+```
+de-p1st-pacman-0.0.7-2-any.pkg.tar.zst
+de-p1st-repo-0.1.1-2-any.pkg.tar.zst
+de-p1st-repo-0.1.1-3-any.pkg.tar.zst
+```
+
+## Removing packages
+
+On your remote server:
+
+```shell
+PKG=xml2
+REPO_NAME=de-p1st
+
+repo-remove "${REPO_NAME}".db.tar.gz "${PKG}"
+rm "${PKG}"-PKG-VERSION.pkg.tar.{xz,zst}
+rm -r db/"${PKG}"/
+```
--- a/arch-repo-push-new.sh
+++ b/arch-repo-push-new.sh
@ -0,0 +1,51 @@
+#!/bin/bash
+
+source /etc/de-p1st-repo/arch-repo.cfg || exit $?
+# Enable nullglob for the case that not all patterns match, e.g. just *.zst but not *.xz packages exist.
+shopt -s nullglob
+
+function main() {
+  cd "${LOCAL_PKG_DIR}" || return $?
+
+  # Check if at least one matching file exists
+  match="0"
+  for PKG in ./*.pkg.tar.{xz,zst}; do
+      # There is at least one match!
+      match="1"
+      break
+  done
+
+  if [ "$match" = "0" ]; then
+    echo "There are no local packages inside ${LOCAL_PKG_DIR}"
+    return 0
+  fi
+
+
+  # Get list of new packages, one package per line.
+  rsync --ignore-existing --out-format="%n" --dry-run \
+    ./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" > new-pkg.txt || return $?
+
+  # If there are no new packages to push/synchronize, then return
+  if [ ! -s new-pkg.txt ]; then
+    echo "No new packages inside ${LOCAL_PKG_DIR}";
+    return 0;
+  fi
+
+
+  # Transfer new packages using rsync
+  rsync --ignore-existing --progress --human-readable \
+    ./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $?
+
+  # Transfer new-pkg.txt
+  rsync --ignore-times --checksum --progress --human-readable \
+    new-pkg.txt "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $?
+
+
+  # Add each new package to database
+  ssh "${REMOTE_SSH_HOST}" "/usr/bin/arch-repo-receive-new" || return $?
+}
+
+
+for LOCAL_PKG_DIR in "${LOCAL_PKG_DIRS[@]}"; do
+  main
+done
--- a/arch-repo-receive-new.sh
+++ b/arch-repo-receive-new.sh
@ -0,0 +1,156 @@
+#!/bin/bash
+
+source /usr/lib/de-p1st-repo/util.sh    || exit $?
+source /usr/lib/de-p1st-repo/pkgver.sh  || exit $?
+source /usr/lib/de-p1st-repo/pkginfo.sh || exit $?
+source /etc/de-p1st-repo/arch-repo.cfg  || exit $?
+
+
+
+function main(){
+  cd "${REMOTE_PKG_DIR}" || return $?
+  add_new_to_db          || return $?
+  generate_index         || return $?
+}
+
+
+#
+# add all packages to database
+#
+function add_all_to_db(){
+  echo "Adding all packages to db ..."
+  sort_all_pkgname_pkgver || return $?
+
+  echo "For each package name: Add latest version to database ..."
+  for PKGNAME in db/*; do
+    PKGNAME=$(basename "${PKGNAME}") || return $? # strip directory and suffix from filename
+    add_to_db "${PKGNAME}" || return $?
+  done
+}
+#
+# add new packages to database
+#
+function add_new_to_db(){
+  echo "Adding new packages to db ..."
+  sort_new_pkgname_pkgver || return $?
+
+  echo "For each new package name: Add latest version to database ..."
+  for PKGNAME in "${NEW_PKGNAMES[@]}"; do
+    add_to_db "${PKGNAME}" || return $?
+  done
+}
+#
+# add package to database
+#
+function add_to_db(){
+  # $1: package name
+  local PKGNAME
+  PKGNAME="$1"
+
+  # get path to latest version of $PKGNAME
+  PKG=$(latest_pkgver_path "${PKGNAME}") || return $?
+
+  # add to database
+  repo-add --new "${REMOTE_DB_NAME}.db.tar.gz" "${PKG}" || return $?
+}
+
+
+#
+# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
+#
+function sort_all_pkgname_pkgver(){
+  echo "Cleanup ..."
+  rm -r db || return $?
+
+  echo "Sorting all packages by package name and package version ..."
+
+  for PKG in *.pkg.tar.{xz,zst}; do
+    sort_pkgname_pkgver "${PKG}" || return $?
+  done
+}
+#
+# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
+#
+function sort_new_pkgname_pkgver(){
+  # return: array $NEW_PKGNAMES
+
+  echo "Sorting new packages by package name and package version ..."
+
+  local NEW_PKGNAMES_TMP=() # list the names from new package-files; may contain duplicates
+
+  mapfile -t PKGS < <(cat new-pkg.txt)
+  for PKG in "${PKGS[@]}"; do
+    sort_pkgname_pkgver "${PKG}" || return $?
+    NEW_PKGNAMES_TMP+=("${PKGNAME}")
+  done
+
+
+  # create array $NEW_PKGNAMES without duplicates
+  NEW_PKGNAMES=()
+  for NEW_PKGNAME_TMP in "${NEW_PKGNAMES_TMP[@]}"; do
+    local contains="0"
+
+    # if NEW_PKGNAMES does already contain NEW_PKGNAME_TMP,
+    # then set contains to "1"
+    for i in "${NEW_PKGNAMES[@]}"; do
+      if [ "${NEW_PKGNAME_TMP}" = "${i}" ]; then
+        contains="1";
+        break;
+      fi
+    done
+
+    if [ "${contains}" = "0" ]; then
+      NEW_PKGNAMES+=("${NEW_PKGNAME_TMP}")
+    fi
+  done
+}
+#
+# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
+#
+function sort_pkgname_pkgver(){
+  # $1: PKG (path to package file)
+  # return: variables $PKGINFO, $PKGNAME, $PKGVER
+  local PKG
+  PKG="$1"
+
+  get_pkginfo "$PKG"                || { echo "get_pkginfo failed"; return 1; }
+  PKGNAME=$(get_pkgname "$PKGINFO") || { echo "get_pkgname failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; }
+  PKGVER=$(get_pkgver  "$PKGINFO")  || { echo "get_pkgver  failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; }
+
+  echo "Creating file ./db/${PKGNAME}/${PKGVER} with content ${PKG} ..."
+  mkdir -p "db/${PKGNAME}" || return $?
+  echo "${PKG}" > "db/${PKGNAME}/${PKGVER}" || return $?
+}
+
+
+
+#
+# generate index.html
+#
+function generate_index(){
+  echo "Generating index.html with links to all packages ..."
+
+  echo "<!DOCTYPE html>
+<html>
+<head>
+<title>${HTML_TITLE}</title>
+</head>
+<body>
+<h1>${HTML_HEADING}</h1>
+<p>The sources can be found here: <a href=\"${HTML_LINK_SRC}\">${HTML_LINK_SRC}</a></p>
+
+<ul>
+" > index.html
+
+  for PKG in *.pkg.tar.{xz,zst}; do
+    echo "<li><a href=\"$PKG\">$PKG</a></li>" >> index.html;
+  done
+
+  echo '
+</ul>
+
+</body>
+</html>' >> index.html
+}
+
+main "$@"
--- a/arch-repo-vercmp.sh
+++ b/arch-repo-vercmp.sh
@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# For all packages in repository $REMOTE_DB_NAME,
+# compare package version with AUR and
+# print all outdated packages
+#
+
+source /usr/lib/de-p1st-repo/util.sh    || exit $?
+source /usr/lib/de-p1st-repo/pkgver.sh  || exit $?
+source /usr/lib/de-p1st-repo/pkginfo.sh || exit $?
+source /etc/de-p1st-repo/arch-repo.cfg  || exit $?
+
+
+
+function main(){
+  echo "Note: You may run 'arch-repo-push-new' and 'pacman -Sy' first ..."
+
+  all_pkg_vers      || return $?
+  check_aur_updates || return $?
+
+  if [ "${#AUR_UPDATES[@]}" -gt "0" ]; then
+    echo "Repository ${REMOTE_DB_NAME} contains packages with available AUR updates:"
+    for AUR_PKG in "${AUR_UPDATES[@]}"; do
+      echo "  ${AUR_PKG}"
+    done
+  else
+    echo "There are no pending AUR updates in repository ${REMOTE_DB_NAME}."
+  fi
+
+  get_vcs_packages || return $?
+  if [ "${#VCS_PKGS[@]}" -gt "0" ]; then
+    echo ""
+    echo "Note: Some VCS packages were found which are possibly outdated:"
+    printf "VCS_PKGS=(%s)\n" "${VCS_PKGS[*]}"
+  fi
+}
+
+function get_vcs_packages(){
+  # return: array GIT_PKGS with all VCS packages
+  VCS_PKGS=()
+
+  # https://wiki.archlinux.org/title/VCS_package_guidelines#VCS_sources
+  # https://github.com/AladW/aurutils/pull/283/files
+  # https://man.archlinux.org/man/PKGBUILD.5#USING_VCS_SOURCES
+  readonly AURVCS='.*-(bzr|git|hg|svn|fossil)$'
+
+  while IFS= read -r PKG_VER; do
+    PKGNAME=$(first_word "${PKG_VER}") || return $?
+    if echo "${PKGNAME}" | grep -E "$AURVCS" >/dev/null; then
+      VCS_PKGS+=("${PKGNAME}")
+    fi
+  done <<< "${PKG_VERS}"
+}
+
+#
+# Store packages from $PKG_VERS with available AUR updates in $AUR_UPDATES.
+# $AUR_UPDATES is an array where each entry describes one outdated package with it's current and new version.
+#
+function check_aur_updates(){
+  mapfile -t AUR_UPDATES < <(echo "$PKG_VERS" | aur vercmp)
+}
+
+#
+# Store all installed package names and their versions
+# from repository $REMOTE_DB_NAME
+# in variable $PKG_VERS.
+# $PKG_VERS consists of multiple lines in the format: "<pkgname><whitespace><pkgver>"
+#
+function installed_pkg_vers(){
+  # The paclist script is part of the package "pacman-contrib"
+  PKG_VERS=$(paclist "${REMOTE_DB_NAME}") || return $?
+}
+
+#
+# Store all package names and their versions
+# from repository $REMOTE_DB_NAME
+# in variable $PKG_VERS.
+# $PKG_VERS contains of multiple lines in the format: "<pkgname><whitespace><pkgver>"
+#
+function all_pkg_vers(){
+  PKG_VERS=$(pacman -S --list "${REMOTE_DB_NAME}" | sed "s|^${REMOTE_DB_NAME}\\s*||; s|\\s*\\[installed.*\\]\s*\$||") || return $?
+}
+
+main "$@"
--- a/arch-repo.cfg
+++ b/arch-repo.cfg
@ -0,0 +1,19 @@
+#=== LOCAL MACHINE CONFIGURATION ===#
+
+# Locations of built packages
+LOCAL_PKG_DIRS=('/home/yoda/Downloads/git/arch/build-pkg/out')
+
+#=== REMOTE MIRROR SERVER CONFIGURATION ===#
+
+# Host from ssh configuration with correct user to have
+# write access to REMOTE_PKG_DIR and REMOTE_DB_NAME
+REMOTE_SSH_HOST=rootnas
+REMOTE_PKG_DIR=/mnt/data/live/arch-repo
+REMOTE_DB_NAME=de-p1st
+
+#
+# Some variables for index.html generation.
+#
+HTML_TITLE='Privacy1st'
+HTML_HEADING='My personalized Arch Linux distribution'
+HTML_LINK_SRC='https://codeberg.org/privacy1st/arch'
--- a/bash-unique-array-test.sh
+++ b/bash-unique-array-test.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# source: https://stackoverflow.com/a/13649357
+#
+
+a=(aa ac aa ad "ac ad")
+declare -A b
+for i in "${a[@]}"; do b["$i"]=1; done
+
+for i in "${!b[@]}"; do
+  echo ">> $i"
+done
--- a/iterate-files-test.sh
+++ b/iterate-files-test.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+source /etc/de-p1st-repo/arch-repo.cfg || exit $?
+
+
+cd "${LOCAL_PKG_DIR}" || exit $?
+
+match="0"
+for PKG in ./*.pkg.tar.{xz,zst}; do
+    [ -f "$PKG" ] || { echo "No match for pattern $PKG"; continue; }  # alternatively: shopt -s nullglob
+
+    # if we are here, there is at least one match!
+    match="1"
+    break
+done
+
+if [ "$match" = "1" ]; then
+  shopt -s nullglob
+  # print all matching
+  echo ./*.pkg.tar.{xz,zst}
+fi
--- a/lib/pkginfo.sh
+++ b/lib/pkginfo.sh
@ -0,0 +1,60 @@
+#
+# get content of .PKGINFO from package-file
+#
+function get_pkginfo(){
+  # $1: path to package file
+  # return: variable $PKGINFO
+
+  if endswith "$1" ".pkg.tar.xz"; then
+    PKGINFO=$(tar -xf "$1" -O .PKGINFO --force-local)         || { echo "tar failed"; return 1; }
+  elif endswith "$1" ".pkg.tar.zst"; then
+    PKGINFO=$(tar -I zstd -xf "$1" -O .PKGINFO --force-local) || { echo "tar failed"; return 1; }
+  else
+    echo "$1 does not seem to be a package!"
+    return 1
+  fi
+}
+
+
+#
+# get pkgname from $PKGINFO
+#
+function get_pkgname(){
+  # return: stdout: package name
+
+  # remove "pkgname = " as well as tailing whitespace characters
+  local tmp
+  tmp=$(echo "${PKGINFO}" | grep '^pkgname =')                   || { echo "grep failed"; return 1; }
+  local PKGNAME
+  PKGNAME=$(echo "${tmp}" | sed 's|^pkgname\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
+
+  echo "${PKGNAME}"
+}
+#
+# get pkgver from $PKGINFO
+#
+function get_pkgver(){
+  # return: stdout: package version
+
+  # remove "pkgver = " as well as tailing whitespace characters
+  local tmp
+  tmp=$(echo "${PKGINFO}" | grep '^pkgver =')                   || { echo "grep failed"; return 1; }
+  local PKGVER
+  PKGVER=$(echo "${tmp}"  | sed 's|^pkgver\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
+
+  echo "${PKGVER}"
+}
+#
+# get url from $PKGINFO
+#
+function get_pkgurl(){
+  # return: stdout: url
+
+  # remove "url = " as well as tailing whitespace characters
+  local tmp
+  tmp=$(echo "${PKGINFO}" | grep '^url =')                   || { echo "grep failed"; return 1; }
+  local PKGURL
+  PKGURL=$(echo "${tmp}"  | sed 's|^url\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
+
+  echo "${PKGURL}"
+}
--- a/lib/pkgver.sh
+++ b/lib/pkgver.sh
@ -0,0 +1,53 @@
+#
+# get package-file with latest version for given package name
+#
+function latest_pkgver_path(){
+  # precond: In current working directory there is a subdir "db"
+  # $1: package name
+  # return: stdout: path to package file
+  local PKGNAME
+  PKGNAME="$1"
+
+  # get latest version for $PKGNAME
+  local LATEST_PKGVER
+  LATEST_PKGVER=$(latest_pkgver "${PKGNAME}") || return $?
+  # get the path to package file
+  local PKG
+  PKG=$(cat "db/${PKGNAME}/${LATEST_PKGVER}") || return $?
+
+  echo "${PKG}"
+}
+
+#
+# get latest version of package
+#
+function latest_pkgver(){
+  # precond: In current working directory there is a subdir "db"
+  # $1: package name
+  # return: stdout: latest pkgver
+  local PKGNAME
+  PKGNAME="$1"
+
+  # pick one random version as starting point for the latest version
+  local LATEST_PKGVER
+  for PKGVER in db/"${PKGNAME}"/*; do
+    PKGVER=$(basename "${PKGVER}") || return $?  # strip directory and suffix from filename
+    LATEST_PKGVER="${PKGVER}"
+    break
+  done
+
+  local cmp
+  for PKGVER in db/"${PKGNAME}"/*; do
+    PKGVER=$(basename "${PKGVER}") || return $?  # strip directory and suffix from filename
+
+    # compare the currently known latest version
+    # with the next version
+    cmp=$(vercmp "${LATEST_PKGVER}" "${PKGVER}") || return $?
+    # if the new version is larger, save it as LATEST_PKGVER
+    if [ "${cmp}" -lt "0" ]; then
+      LATEST_PKGVER="${PKGVER}"
+    fi
+  done
+
+  echo "${LATEST_PKGVER}"
+}
--- a/lib/util.sh
+++ b/lib/util.sh
@ -0,0 +1,19 @@
+function first_word() {
+  # return: The first word of $1.
+  #   In detail: "" if $1 starts with a space. Otherwise: All characters until the first space of $1.
+  #
+  # source: https://unix.stackexchange.com/a/201744/315162
+
+  echo "${1%% *}"
+}
+
+
+# Inspired by: https://stackoverflow.com/questions/2172352/in-bash-how-can-i-check-if-a-string-begins-with-some-value/18558871#18558871
+#
+# $1 begins with $2
+#
+beginswith() { case $1 in "$2"*) true;; *) false;; esac; }
+#
+# $1 ends with $2
+#
+endswith() { case $1 in *"$2") true;; *) false;; esac; }