add setup.cfg and project.toml; bug fixes; add test; add LICENSE

.gitignore

@@ -1,3 +1,6 @@
 /.idea
 /test
 /venv
+/dist
+**/*.egg-info
+**/__pycache__

LICENSE

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

Makefile

@@ -12,7 +12,8 @@ SHELL := bash
 .SHELLFLAGS := -eu -o pipefail -c
 
 .PHONY: all
-all: venv
+all:
+	sudo python3 -m pip install --upgrade --force-reinstall .
 
 # Python Dependency Locking with pip-tools
 # https://lincolnloop.com/insights/python-dependency-locking-pip-tools/
@@ -20,7 +21,7 @@ all: venv
 .PHONY: test
 test: venv
 	source venv/bin/activate
-	python3 test.py
+	python3 src/p1st/test.py
 
 venv:
 	if [ -d venv ]; then
@@ -30,4 +31,4 @@ venv:
 
 .PHONY: clean
 clean:
-	rm -rf venv .mypy_cache build dist __pycache__ test
+	rm -rf venv .mypy_cache build dist src/__pycache__ src/subprocess_util.egg-info test

pyproject.toml

@@ -0,0 +1,8 @@
+# https://packaging.python.org/tutorials/packaging-projects/#creating-pyproject-toml
+
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -0,0 +1,36 @@
+; setup.cfg is the configuration file for setuptools.
+; https://packaging.python.org/tutorials/packaging-projects/#configuring-metadata
+
+[metadata]
+name = subprocess-util
+version = 0.0.1
+author = Daniel Langbein
+author_email = daniel@systemli.org
+description = subprocess utility functions
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://codeberg.org/privacy1st/subprocess_util
+project_urls =
+    Bug Tracker = https://codeberg.org/privacy1st/subprocess_util/issues
+
+; https://pypi.org/classifiers/
+classifiers =
+    Development Status :: 4 - Beta
+    Programming Language :: Python :: 3
+    License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+    Operating System :: Unix
+
+[options]
+package_dir =
+    = src
+packages = find:
+python_requires = >=3.10.0
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+; https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html
+console_scripts =
+    btrfs-send-chunks = p1st.btrfs_send_chunks:main
+    btrfs-receive-chunks = p1st.btrfs_receive_chunks:main

src/p1st/btrfs_receive_chunks.py

@@ -3,15 +3,15 @@
 import argparse
 from pathlib import Path
 
-from exec_print_receive import execute_print_receive_chunks
+from p1st.exec_print_receive import execute_print_receive_chunks
-from common import _get_chunk_file, _get_remote_socket
+from p1st.common import _get_chunk_file, _get_remote_socket, _get_chunk_tmpl
 
 
 def main():
     args = parse_args()
 
     target_path: Path = args.target_path
-    command = ['btrfs', 'receive', str(target_path)]
+    command = ['btrfs', 'receive', str(target_path.parent)]
     target_socket = _get_remote_socket(target_path)
 
     execute_print_receive_chunks(
@@ -36,16 +36,22 @@ def parse_args():
                         )
 
     parser.add_argument('target_path',
-                        help='Path were the subvolume will be created; forwarded to btrfs-receive.',
+                        help='Path where the subvolume will be created. '
+                             'The last component of the path '
+                             'must be equal to the name of the subvolume on the sending side.',
                         type=Path,
                         metavar='SUBVOLUME'
                         )
 
     args = parser.parse_args()
 
+    # Make all paths absolute.
+    if args.chunk_tmpl:
+        args.chunk_tmpl = args.chunk_tmpl.absolute()
+    args.target_path = args.target_path.absolute()
+
     if not args.chunk_tmpl:
-        target_path: Path = args.target_path
+        args.chunk_tmpl = _get_chunk_tmpl(args.target_path)
-        args.chunk_tmpl = target_path.parent.joinpath(f'{target_path.name}.CHUNK')
 
     return args
 

src/p1st/btrfs_send_chunks.py

@@ -3,10 +3,11 @@
 import argparse
 from pathlib import Path
 import shlex
+from typing import Callable
 
-from exec_print_transfer import execute_print_transfer_chunks
+from p1st.exec_print_transfer import execute_print_transfer_chunks
-from common import _get_chunk_file, _get_remote_socket
+from p1st.common import _get_chunk_file, _get_remote_socket, _get_chunk_tmpl
-from transfer_inform import transfer_inform
+from p1st.transfer_inform import transfer_inform
 
 
 def main():
@@ -20,12 +21,19 @@ def main():
     )
     command = [x for xs in command_parts for x in xs]
 
+    chunk_tmpl = _get_chunk_tmpl(args.child)
+    target_chunk_tmpl = _get_chunk_tmpl(args.target_path)
+
     execute_print_transfer_chunks(
         command=command,
-        chunk_file_tmpl=args.chunk_tmpl,
+        chunk_file_tmpl=chunk_tmpl,
         chunk_transfer_fun=chunk_transfer_fun,
-        chunk_transfer_args=(args.ssh_target, args.target_path, args.chunk_tmpl),
+        chunk_transfer_args=(args.ssh_target,
+                             args.target_path,
+                             _get_chunk_file,
+                             target_chunk_tmpl),
         chunk_size=args.chunk_size,
+        get_chunk_file=_get_chunk_file,
     )
 
 
@@ -33,7 +41,7 @@ def parse_args():
     parser = argparse.ArgumentParser(prog='btrfs-send-chunks')
 
     parser.add_argument('-p',
-                        help='Parent subvolume; forwarded to btrfs-send.',
+                        help='Path to parent subvolume; forwarded to btrfs-send.',
                         dest='parent',
                         default=None,
                         type=Path,
@@ -47,16 +55,6 @@ def parse_args():
                         default=False,
                         )
 
-    parser.add_argument('--chunk-tmpl',
-                        help='During btrfs-send, chunks are saved as "CHUNK_TMPL.CHUNK_NUMBER". '
-                             'The default value of CHUNK_TMPL is "CHILD_SUBVOLUME.CHUNK". '
-                             'One can change it to e.g. "/tmp/chunk/CHILD_SUBVOLUME".',
-                        dest='chunk_tmpl',
-                        default=None,
-                        type=Path,
-                        metavar='CHUNK_TMPL'
-                        )
-
     parser.add_argument('--chunk-size',
                         help='Size in bytes; defaults to 64 MB.',
                         dest='chunk_size',
@@ -65,7 +63,7 @@ def parse_args():
                         )
 
     parser.add_argument('child',
-                        help='Forwarded to btrfs-send.',
+                        help='Path to child subvolume. Forwarded to btrfs-send.',
                         type=Path,
                         metavar='CHILD_SUBVOLUME'
                         )
@@ -83,9 +81,12 @@ def parse_args():
 
     args = parser.parse_args()
 
-    if not args.chunk_tmpl:
+    # Make all paths absolute.
-        child: Path = args.child
+    if args.parent:
-        args.chunk_tmpl = child.parent.joinpath(f'{child.name}.CHUNK')
+        args.parent = args.parent.absolute()
+    args.child = args.child.absolute()
+    if args.target_path:
+        args.target_path = args.target_path.absolute()
 
     return args
 
@@ -93,8 +94,10 @@ def parse_args():
 def chunk_transfer_fun(chunk_file: Path, ct: int, eof: bool,
                        ssh_target: str,
                        target_path: Path,
-                       chunk_tmpl: Path):
+                       get_chunk_file: Callable[[Path, int], Path],
-    target_chunk = _get_chunk_file(chunk_tmpl, ct)
+                       target_chunk_tmpl: Path,
+                       ):
+    target_chunk = get_chunk_file(target_chunk_tmpl, ct)
     rsync_cmd = ['rsync', str(chunk_file), f'{ssh_target}:{str(target_chunk)}']
 
     message = 'EOF' if eof else 'OK'

src/p1st/common.py

@@ -9,3 +9,7 @@ def _get_chunk_file(chunk_file_tmpl: Path, ct: int):
 
 def _get_remote_socket(target: Path):
     return target.parent.joinpath(f'{target}.SOCKET')
+
+
+def _get_chunk_tmpl(subvol: Path):
+    return subvol.parent.joinpath(f'{subvol.name}.CHUNK')

src/p1st/exec_print_receive.py

@@ -6,8 +6,8 @@ import threading
 from pathlib import Path
 from typing import IO, AnyStr, Callable
 
-from common import _get_chunk_file
+from p1st.common import _get_chunk_file
-from receive_inform import receive_inform
+from p1st.receive_inform import receive_inform
 
 
 def _print_stdout(bin_pipe: IO[AnyStr]):

src/p1st/exec_print_transfer.py

@@ -6,7 +6,7 @@ import threading
 import subprocess
 from typing import AnyStr, IO, Callable
 
-from common import _get_chunk_file
+from p1st.common import _get_chunk_file
 
 
 def _rotate_chunk(chunk_file: Path, chunk_number: int, eof: bool, chunk_transfer_fun: Callable, chunk_transfer_args: tuple):

src/p1st/receive_inform.py

@@ -4,7 +4,7 @@ import socket
 from pathlib import Path
 from typing import IO, AnyStr, Callable
 
-from unix_sock_input import accept_loop_until_command_received
+from p1st.unix_sock_input import accept_loop_until_command_received
 
 
 def receive_inform(in_pipe: IO[AnyStr],

src/p1st/test.py

@@ -6,11 +6,11 @@ import shutil
 import socket
 from pathlib import Path
 
-from exec_capture import execute_capture
+from p1st.exec_capture import execute_capture
-from exec_print_capture import execute_print_capture
+from p1st.exec_print_capture import execute_print_capture
-from exec_print_receive import execute_print_receive_chunks
+from p1st.exec_print_receive import execute_print_receive_chunks
-from exec_print_transfer import execute_print_transfer_chunks
+from p1st.exec_print_transfer import execute_print_transfer_chunks
-from transfer_inform import transfer_inform
+from p1st.transfer_inform import transfer_inform
 
 
 def test():
@@ -19,7 +19,9 @@ def test():
     # test3()
     # test4()
     # test5()
-    test67()
+    # test67()
+    # test8()
+    test9()
 
 
 def test1():
@@ -43,7 +45,7 @@ def test2():
 def test3():
     _init(3)
 
-    def _chunk_transfer(chunk_file: Path, eof: bool):
+    def _chunk_transfer(chunk_file: Path, ct: int, eof: bool):
         print(f'Transferring chunk {chunk_file} to ... (This is the default method, it has no effect)')
         if eof:
             print(f'The last chunk has been transferred.')
@@ -170,6 +172,41 @@ def test7():
     )
 
 
+def test8():
+    repo_name = 'subprocess_util'
+
+    child_name = 'test-subvolume'
+    child_dir = '/mnt/backup/test-dir'
+    child_path = f'{child_dir}/{child_name}'
+    target_dir = '/mnt/data/test-dir'
+    target_path = f'{target_dir}/{child_name}'
+    ssh_target = 'rootnas'
+
+    print(f'=== In one shell, connect with "ssh nas" ===')
+    print(f'\tsudo mkdir {target_dir}')
+    print(f'\tcd {repo_name} && make && sudo make clean && cd ..')
+    print()
+    print(f'\tsudo btrfs-receive-chunks {target_path}')
+    print()
+
+    print(f'=== In another shell, connect with "ssh odroid" ===')
+    print(f'\tsudo mkdir {child_dir}')
+    print(f'\tsudo btrfs subvolume create {child_path}.writeable')
+    print(f'\techo foo | sudo tee {child_path}.writeable/bar')
+    print(f'\tsudo btrfs subvolume snapshot -r {child_path}.writeable {child_path}')
+    print(f'\tcd {repo_name} && make && sudo make clean && cd ..')
+    print()
+    print(f'\tsudo btrfs-send-chunks {child_path} {ssh_target} {target_path}')
+
+
+def test9():
+    child_path = '/mnt/backup/snap/blogger.privacy1st.de/20230104T2255'
+    ssh_target = 'rootnas'
+    target_path = '/mnt/data/test/blogger.privacy1st.de/20230104T2255'
+    print(f'sudo btrfs-receive-chunks {target_path}')
+    print(f'sudo btrfs-send-chunks {child_path} {ssh_target} {target_path}')
+
+
 def _init(test_number: int):
     print(f"TEST {test_number}")
     test_dir = Path('test')

src/p1st/transfer_inform.py

@@ -2,8 +2,8 @@
 # -*- coding: utf-8 -*-
 from pathlib import Path
 
-from exec_print_capture import execute_print_capture
+from p1st.exec_print_capture import execute_print_capture
-from unix_sock_input import wait_until_command_received
+from p1st.unix_sock_input import wait_until_command_received
 
 
 def transfer_inform(rsync_cmd: list[str],