Mirror/arch-repo
1
0
Fork 0
mirror of https://codeberg.org/privacy1st/arch-repo synced 2024-11-23 21:29:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

import from privacy1st/arch

Browse Source
This commit is contained in:
Daniel Langbein 2022-07-16 17:02:02 +02:00
commit fec372b3e6
12 changed files with 607 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/.idea/

28
PKGBUILD Normal file
View File

@ -0,0 +1,28 @@
# Maintainer: Daniel Langbein <daniel@systemli.org>
_pkgname=repo
_reponame=arch
pkgname="de-p1st-$_pkgname"
pkgver=0.2.13
pkgrel=1
pkgdesc="Bash script to manage remote Arch Linux repository"
arch=('any')
url="https://codeberg.org/privacy1st/${_reponame}"
license=('MIT')
depends=('openssh' 'rsync' 'aurutils') # arch-repo-vercmp uses "aur vercmp" which is part of "aurutils"
makedepends=('git')
source=("git+${url}.git")
sha256sums=('SKIP')
package() {
cd "${_reponame}/pkg/${pkgname}"
install -Dm0555 arch-repo-push-new.sh "$pkgdir"/usr/bin/arch-repo-push-new
install -Dm0555 arch-repo-receive-new.sh "$pkgdir"/usr/bin/arch-repo-receive-new
install -Dm0555 arch-repo-vercmp.sh "$pkgdir"/usr/bin/arch-repo-vercmp
install -Dm0644 lib/util.sh "$pkgdir"/usr/lib/"${pkgname}"/util.sh
install -Dm0644 lib/pkginfo.sh "$pkgdir"/usr/lib/"${pkgname}"/pkginfo.sh
install -Dm0644 lib/pkgver.sh "$pkgdir"/usr/lib/"${pkgname}"/pkgver.sh
install -Dm0644 -o0 arch-repo.cfg "$pkgdir"/etc/de-p1st-repo/arch-repo.cfg
}

103
README.md Normal file
View File

@ -0,0 +1,103 @@
# Arch Linux repository manager
Special thanks to nachoparker for his article:
* [Replicate your system with self-hosted Arch Linux metapackages](https://ownyourbits.com/2019/07/21/replicate-your-system-with-self-hosted-arch-linux-metapackages/)
## Setup
Install `de-p1st-repo` on your local machine as well as on
a remote server.
Adjust [/etc/de-p1st-repo/arch-repo.conf](arch-repo.cfg) according to your needs.
Run a webserver on the server to serve static content:
* [https://hub.docker.com/_/nginx/]() -> Hosting some simple static content
* `sudo docker run --name arch-repo -v /mnt/data/live/arch-repo:/usr/share/nginx/html:ro -d nginx`
Add the newly created mirror to your `/etc/pacman.conf`:
```
[de-p1st]
SigLevel = Optional TrustAll
Server = https://arch.p1st.de
```
## Normal usage
### Check for AUR updates, build and push
Check remote repository for AUR packages that can be updated
```shell
arch-repo-vercmp
```
Then build those packages locally and push changes to remote repository
```shell
# Build on local machine in clean chroot (or with an AUR helper), e.g.
makepkg -fCcsr
# Push new packages to remote repository
arch-repo-push-new
```
## example output
```
user@localMachine ~ % arch-repo-push-new
de-p1st-pacman-0.0.7-2-any.pkg.tar.zst
2.84K 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=325/384)
de-p1st-repo-0.1.1-2-any.pkg.tar.zst
5.13K 100% 4.90MB/s 0:00:00 (xfr#2, to-chk=308/384)
de-p1st-repo-0.1.1-3-any.pkg.tar.zst
5.13K 100% 4.90MB/s 0:00:00 (xfr#3, to-chk=307/384)
new-pkg.txt
113 100% 0.00kB/s 0:00:00 (xfr#1, to-chk=0/1)
Adding new packages to db ...
Sorting new packages by package name and package version ...
Creating file ./db/de-p1st-pacman/0.0.7-2 with content de-p1st-pacman-0.0.7-2-any.pkg.tar.zst ...
Creating file ./db/de-p1st-repo/0.1.1-2 with content de-p1st-repo-0.1.1-2-any.pkg.tar.zst ...
Creating file ./db/de-p1st-repo/0.1.1-3 with content de-p1st-repo-0.1.1-3-any.pkg.tar.zst ...
For each new package name: Add latest version to database ...
==> Extracting de-p1st.db.tar.gz to a temporary location...
==> Extracting de-p1st.files.tar.gz to a temporary location...
==> Adding package 'de-p1st-pacman-0.0.7-2-any.pkg.tar.zst'
-> Computing checksums...
-> Removing existing entry 'de-p1st-pacman-0.0.7-1'...
-> Creating 'desc' db entry...
-> Creating 'files' db entry...
==> Creating updated database file 'de-p1st.db.tar.gz'
==> Extracting de-p1st.db.tar.gz to a temporary location...
==> Extracting de-p1st.files.tar.gz to a temporary location...
==> Adding package 'de-p1st-repo-0.1.1-3-any.pkg.tar.zst'
-> Computing checksums...
-> Removing existing entry 'de-p1st-repo-0.1.1-1'...
-> Creating 'desc' db entry...
-> Creating 'files' db entry...
==> Creating updated database file 'de-p1st.db.tar.gz'
Generating index.html with links to all packages ...
```
Where the generated `new-pkg.txt` has this content:
```
de-p1st-pacman-0.0.7-2-any.pkg.tar.zst
de-p1st-repo-0.1.1-2-any.pkg.tar.zst
de-p1st-repo-0.1.1-3-any.pkg.tar.zst
```
## Removing packages
On your remote server:
```shell
PKG=xml2
REPO_NAME=de-p1st
repo-remove "${REPO_NAME}".db.tar.gz "${PKG}"
rm "${PKG}"-PKG-VERSION.pkg.tar.{xz,zst}
rm -r db/"${PKG}"/
```

51
arch-repo-push-new.sh Normal file
View File

@ -0,0 +1,51 @@
#!/bin/bash
source /etc/de-p1st-repo/arch-repo.cfg || exit $?
# Enable nullglob for the case that not all patterns match, e.g. just *.zst but not *.xz packages exist.
shopt -s nullglob
function main() {
cd "${LOCAL_PKG_DIR}" || return $?
# Check if at least one matching file exists
match="0"
for PKG in ./*.pkg.tar.{xz,zst}; do
# There is at least one match!
match="1"
break
done
if [ "$match" = "0" ]; then
echo "There are no local packages inside ${LOCAL_PKG_DIR}"
return 0
fi
# Get list of new packages, one package per line.
rsync --ignore-existing --out-format="%n" --dry-run \
./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" > new-pkg.txt || return $?
# If there are no new packages to push/synchronize, then return
if [ ! -s new-pkg.txt ]; then
echo "No new packages inside ${LOCAL_PKG_DIR}";
return 0;
fi
# Transfer new packages using rsync
rsync --ignore-existing --progress --human-readable \
./*.pkg.tar.{xz,zst} "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $?
# Transfer new-pkg.txt
rsync --ignore-times --checksum --progress --human-readable \
new-pkg.txt "${REMOTE_SSH_HOST}":"${REMOTE_PKG_DIR}" || return $?
# Add each new package to database
ssh "${REMOTE_SSH_HOST}" "/usr/bin/arch-repo-receive-new" || return $?
}
for LOCAL_PKG_DIR in "${LOCAL_PKG_DIRS[@]}"; do
main
done

156
arch-repo-receive-new.sh Normal file
View File

@ -0,0 +1,156 @@
#!/bin/bash
source /usr/lib/de-p1st-repo/util.sh || exit $?
source /usr/lib/de-p1st-repo/pkgver.sh || exit $?
source /usr/lib/de-p1st-repo/pkginfo.sh || exit $?
source /etc/de-p1st-repo/arch-repo.cfg || exit $?
function main(){
cd "${REMOTE_PKG_DIR}" || return $?
add_new_to_db || return $?
generate_index || return $?
}
#
# add all packages to database
#
function add_all_to_db(){
echo "Adding all packages to db ..."
sort_all_pkgname_pkgver || return $?
echo "For each package name: Add latest version to database ..."
for PKGNAME in db/*; do
PKGNAME=$(basename "${PKGNAME}") || return $? # strip directory and suffix from filename
add_to_db "${PKGNAME}" || return $?
done
}
#
# add new packages to database
#
function add_new_to_db(){
echo "Adding new packages to db ..."
sort_new_pkgname_pkgver || return $?
echo "For each new package name: Add latest version to database ..."
for PKGNAME in "${NEW_PKGNAMES[@]}"; do
add_to_db "${PKGNAME}" || return $?
done
}
#
# add package to database
#
function add_to_db(){
# $1: package name
local PKGNAME
PKGNAME="$1"
# get path to latest version of $PKGNAME
PKG=$(latest_pkgver_path "${PKGNAME}") || return $?
# add to database
repo-add --new "${REMOTE_DB_NAME}.db.tar.gz" "${PKG}" || return $?
}
#
# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
#
function sort_all_pkgname_pkgver(){
echo "Cleanup ..."
rm -r db || return $?
echo "Sorting all packages by package name and package version ..."
for PKG in *.pkg.tar.{xz,zst}; do
sort_pkgname_pkgver "${PKG}" || return $?
done
}
#
# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
#
function sort_new_pkgname_pkgver(){
# return: array $NEW_PKGNAMES
echo "Sorting new packages by package name and package version ..."
local NEW_PKGNAMES_TMP=() # list the names from new package-files; may contain duplicates
mapfile -t PKGS < <(cat new-pkg.txt)
for PKG in "${PKGS[@]}"; do
sort_pkgname_pkgver "${PKG}" || return $?
NEW_PKGNAMES_TMP+=("${PKGNAME}")
done
# create array $NEW_PKGNAMES without duplicates
NEW_PKGNAMES=()
for NEW_PKGNAME_TMP in "${NEW_PKGNAMES_TMP[@]}"; do
local contains="0"
# if NEW_PKGNAMES does already contain NEW_PKGNAME_TMP,
# then set contains to "1"
for i in "${NEW_PKGNAMES[@]}"; do
if [ "${NEW_PKGNAME_TMP}" = "${i}" ]; then
contains="1";
break;
fi
done
if [ "${contains}" = "0" ]; then
NEW_PKGNAMES+=("${NEW_PKGNAME_TMP}")
fi
done
}
#
# create files "db/$pkgname/$pkgver" with content "$PKG" (path to package file)
#
function sort_pkgname_pkgver(){
# $1: PKG (path to package file)
# return: variables $PKGINFO, $PKGNAME, $PKGVER
local PKG
PKG="$1"
get_pkginfo "$PKG" || { echo "get_pkginfo failed"; return 1; }
PKGNAME=$(get_pkgname "$PKGINFO") || { echo "get_pkgname failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; }
PKGVER=$(get_pkgver "$PKGINFO") || { echo "get_pkgver failed"; echo "Content of PKGINFO: ${PKGINFO}"; return 1; }
echo "Creating file ./db/${PKGNAME}/${PKGVER} with content ${PKG} ..."
mkdir -p "db/${PKGNAME}" || return $?
echo "${PKG}" > "db/${PKGNAME}/${PKGVER}" || return $?
}
#
# generate index.html
#
function generate_index(){
echo "Generating index.html with links to all packages ..."
echo "<!DOCTYPE html>
<html>
<head>
<title>${HTML_TITLE}</title>
</head>
<body>
<h1>${HTML_HEADING}</h1>
<p>The sources can be found here: <a href=\"${HTML_LINK_SRC}\">${HTML_LINK_SRC}</a></p>
<ul>
" > index.html
for PKG in *.pkg.tar.{xz,zst}; do
echo "<li><a href=\"$PKG\">$PKG</a></li>" >> index.html;
done
echo '
</ul>
</body>
</html>' >> index.html
}
main "$@"

84
arch-repo-vercmp.sh Normal file
View File

@ -0,0 +1,84 @@
#!/bin/bash
#
# For all packages in repository $REMOTE_DB_NAME,
# compare package version with AUR and
# print all outdated packages
#
source /usr/lib/de-p1st-repo/util.sh || exit $?
source /usr/lib/de-p1st-repo/pkgver.sh || exit $?
source /usr/lib/de-p1st-repo/pkginfo.sh || exit $?
source /etc/de-p1st-repo/arch-repo.cfg || exit $?
function main(){
echo "Note: You may run 'arch-repo-push-new' and 'pacman -Sy' first ..."
all_pkg_vers || return $?
check_aur_updates || return $?
if [ "${#AUR_UPDATES[@]}" -gt "0" ]; then
echo "Repository ${REMOTE_DB_NAME} contains packages with available AUR updates:"
for AUR_PKG in "${AUR_UPDATES[@]}"; do
echo " ${AUR_PKG}"
done
else
echo "There are no pending AUR updates in repository ${REMOTE_DB_NAME}."
fi
get_vcs_packages || return $?
if [ "${#VCS_PKGS[@]}" -gt "0" ]; then
echo ""
echo "Note: Some VCS packages were found which are possibly outdated:"
printf "VCS_PKGS=(%s)\n" "${VCS_PKGS[*]}"
fi
}
function get_vcs_packages(){
# return: array GIT_PKGS with all VCS packages
VCS_PKGS=()
# https://wiki.archlinux.org/title/VCS_package_guidelines#VCS_sources
# https://github.com/AladW/aurutils/pull/283/files
# https://man.archlinux.org/man/PKGBUILD.5#USING_VCS_SOURCES
readonly AURVCS='.*-(bzr|git|hg|svn|fossil)$'
while IFS= read -r PKG_VER; do
PKGNAME=$(first_word "${PKG_VER}") || return $?
if echo "${PKGNAME}" | grep -E "$AURVCS" >/dev/null; then
VCS_PKGS+=("${PKGNAME}")
fi
done <<< "${PKG_VERS}"
}
#
# Store packages from $PKG_VERS with available AUR updates in $AUR_UPDATES.
# $AUR_UPDATES is an array where each entry describes one outdated package with it's current and new version.
#
function check_aur_updates(){
mapfile -t AUR_UPDATES < <(echo "$PKG_VERS" | aur vercmp)
}
#
# Store all installed package names and their versions
# from repository $REMOTE_DB_NAME
# in variable $PKG_VERS.
# $PKG_VERS consists of multiple lines in the format: "<pkgname><whitespace><pkgver>"
#
function installed_pkg_vers(){
# The paclist script is part of the package "pacman-contrib"
PKG_VERS=$(paclist "${REMOTE_DB_NAME}") || return $?
}
#
# Store all package names and their versions
# from repository $REMOTE_DB_NAME
# in variable $PKG_VERS.
# $PKG_VERS contains of multiple lines in the format: "<pkgname><whitespace><pkgver>"
#
function all_pkg_vers(){
PKG_VERS=$(pacman -S --list "${REMOTE_DB_NAME}" | sed "s|^${REMOTE_DB_NAME}\\s*||; s|\\s*\\[installed.*\\]\s*\$||") || return $?
}
main "$@"

19
arch-repo.cfg Normal file
View File

@ -0,0 +1,19 @@
#=== LOCAL MACHINE CONFIGURATION ===#
# Locations of built packages
LOCAL_PKG_DIRS=('/home/yoda/Downloads/git/arch/build-pkg/out')
#=== REMOTE MIRROR SERVER CONFIGURATION ===#
# Host from ssh configuration with correct user to have
# write access to REMOTE_PKG_DIR and REMOTE_DB_NAME
REMOTE_SSH_HOST=rootnas
REMOTE_PKG_DIR=/mnt/data/live/arch-repo
REMOTE_DB_NAME=de-p1st
#
# Some variables for index.html generation.
#
HTML_TITLE='Privacy1st'
HTML_HEADING='My personalized Arch Linux distribution'
HTML_LINK_SRC='https://codeberg.org/privacy1st/arch'

12
bash-unique-array-test.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/bash
#
# source: https://stackoverflow.com/a/13649357
#
a=(aa ac aa ad "ac ad")
declare -A b
for i in "${a[@]}"; do b["$i"]=1; done
for i in "${!b[@]}"; do
echo ">> $i"
done

21
iterate-files-test.sh Normal file
View File

@ -0,0 +1,21 @@
#!/bin/bash
source /etc/de-p1st-repo/arch-repo.cfg || exit $?
cd "${LOCAL_PKG_DIR}" || exit $?
match="0"
for PKG in ./*.pkg.tar.{xz,zst}; do
[ -f "$PKG" ] || { echo "No match for pattern $PKG"; continue; } # alternatively: shopt -s nullglob
# if we are here, there is at least one match!
match="1"
break
done
if [ "$match" = "1" ]; then
shopt -s nullglob
# print all matching
echo ./*.pkg.tar.{xz,zst}
fi

60
lib/pkginfo.sh Normal file
View File

@ -0,0 +1,60 @@
#
# get content of .PKGINFO from package-file
#
function get_pkginfo(){
# $1: path to package file
# return: variable $PKGINFO
if endswith "$1" ".pkg.tar.xz"; then
PKGINFO=$(tar -xf "$1" -O .PKGINFO --force-local) || { echo "tar failed"; return 1; }
elif endswith "$1" ".pkg.tar.zst"; then
PKGINFO=$(tar -I zstd -xf "$1" -O .PKGINFO --force-local) || { echo "tar failed"; return 1; }
else
echo "$1 does not seem to be a package!"
return 1
fi
}
#
# get pkgname from $PKGINFO
#
function get_pkgname(){
# return: stdout: package name
# remove "pkgname = " as well as tailing whitespace characters
local tmp
tmp=$(echo "${PKGINFO}" | grep '^pkgname =') || { echo "grep failed"; return 1; }
local PKGNAME
PKGNAME=$(echo "${tmp}" | sed 's|^pkgname\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
echo "${PKGNAME}"
}
#
# get pkgver from $PKGINFO
#
function get_pkgver(){
# return: stdout: package version
# remove "pkgver = " as well as tailing whitespace characters
local tmp
tmp=$(echo "${PKGINFO}" | grep '^pkgver =') || { echo "grep failed"; return 1; }
local PKGVER
PKGVER=$(echo "${tmp}" | sed 's|^pkgver\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
echo "${PKGVER}"
}
#
# get url from $PKGINFO
#
function get_pkgurl(){
# return: stdout: url
# remove "url = " as well as tailing whitespace characters
local tmp
tmp=$(echo "${PKGINFO}" | grep '^url =') || { echo "grep failed"; return 1; }
local PKGURL
PKGURL=$(echo "${tmp}" | sed 's|^url\s*=\s*||; s|\s*$||') || { echo "sed failed"; return 1; }
echo "${PKGURL}"
}

53
lib/pkgver.sh Normal file
View File

@ -0,0 +1,53 @@
#
# get package-file with latest version for given package name
#
function latest_pkgver_path(){
# precond: In current working directory there is a subdir "db"
# $1: package name
# return: stdout: path to package file
local PKGNAME
PKGNAME="$1"
# get latest version for $PKGNAME
local LATEST_PKGVER
LATEST_PKGVER=$(latest_pkgver "${PKGNAME}") || return $?
# get the path to package file
local PKG
PKG=$(cat "db/${PKGNAME}/${LATEST_PKGVER}") || return $?
echo "${PKG}"
}
#
# get latest version of package
#
function latest_pkgver(){
# precond: In current working directory there is a subdir "db"
# $1: package name
# return: stdout: latest pkgver
local PKGNAME
PKGNAME="$1"
# pick one random version as starting point for the latest version
local LATEST_PKGVER
for PKGVER in db/"${PKGNAME}"/*; do
PKGVER=$(basename "${PKGVER}") || return $? # strip directory and suffix from filename
LATEST_PKGVER="${PKGVER}"
break
done
local cmp
for PKGVER in db/"${PKGNAME}"/*; do
PKGVER=$(basename "${PKGVER}") || return $? # strip directory and suffix from filename
# compare the currently known latest version
# with the next version
cmp=$(vercmp "${LATEST_PKGVER}" "${PKGVER}") || return $?
# if the new version is larger, save it as LATEST_PKGVER
if [ "${cmp}" -lt "0" ]; then
LATEST_PKGVER="${PKGVER}"
fi
done
echo "${LATEST_PKGVER}"
}

19
lib/util.sh Normal file
View File

@ -0,0 +1,19 @@
function first_word() {
# return: The first word of $1.
# In detail: "" if $1 starts with a space. Otherwise: All characters until the first space of $1.
#
# source: https://unix.stackexchange.com/a/201744/315162
echo "${1%% *}"
}
# Inspired by: https://stackoverflow.com/questions/2172352/in-bash-how-can-i-check-if-a-string-begins-with-some-value/18558871#18558871
#
# $1 begins with $2
#
beginswith() { case $1 in "$2"*) true;; *) false;; esac; }
#
# $1 ends with $2
#
endswith() { case $1 in *"$2") true;; *) false;; esac; }