diff --git a/.gitignore b/.gitignore index e209ae4..df7f1ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /.idea /test -/venv \ No newline at end of file +/venv +/dist +**/*.egg-info +**/__pycache__ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/Makefile b/Makefile index eb74769..514b6bb 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ SHELL := bash .SHELLFLAGS := -eu -o pipefail -c .PHONY: all -all: venv +all: + sudo python3 -m pip install --upgrade --force-reinstall . # Python Dependency Locking with pip-tools # https://lincolnloop.com/insights/python-dependency-locking-pip-tools/ @@ -20,7 +21,7 @@ all: venv .PHONY: test test: venv source venv/bin/activate - python3 test.py + python3 src/p1st/test.py venv: if [ -d venv ]; then @@ -30,4 +31,4 @@ venv: .PHONY: clean clean: - rm -rf venv .mypy_cache build dist __pycache__ test + rm -rf venv .mypy_cache build dist src/__pycache__ src/subprocess_util.egg-info test diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c035a4f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +# https://packaging.python.org/tutorials/packaging-projects/#creating-pyproject-toml + +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..df7857d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,36 @@ +; setup.cfg is the configuration file for setuptools. +; https://packaging.python.org/tutorials/packaging-projects/#configuring-metadata + +[metadata] +name = subprocess-util +version = 0.0.1 +author = Daniel Langbein +author_email = daniel@systemli.org +description = subprocess utility functions +long_description = file: README.md +long_description_content_type = text/markdown +url = https://codeberg.org/privacy1st/subprocess_util +project_urls = + Bug Tracker = https://codeberg.org/privacy1st/subprocess_util/issues + +; https://pypi.org/classifiers/ +classifiers = + Development Status :: 4 - Beta + Programming Language :: Python :: 3 + License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication + Operating System :: Unix + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.10.0 + +[options.packages.find] +where = src + +[options.entry_points] +; https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html +console_scripts = + btrfs-send-chunks = p1st.btrfs_send_chunks:main + btrfs-receive-chunks = p1st.btrfs_receive_chunks:main diff --git a/src/p1st/__init__.py b/src/p1st/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/btrfs_receive_chunks.py b/src/p1st/btrfs_receive_chunks.py similarity index 65% rename from btrfs_receive_chunks.py rename to src/p1st/btrfs_receive_chunks.py index 8f7e275..6d113a0 100644 --- a/btrfs_receive_chunks.py +++ b/src/p1st/btrfs_receive_chunks.py @@ -3,15 +3,15 @@ import argparse from pathlib import Path -from exec_print_receive import execute_print_receive_chunks -from common import _get_chunk_file, _get_remote_socket +from p1st.exec_print_receive import execute_print_receive_chunks +from p1st.common import _get_chunk_file, _get_remote_socket, _get_chunk_tmpl def main(): args = parse_args() target_path: Path = args.target_path - command = ['btrfs', 'receive', str(target_path)] + command = ['btrfs', 'receive', str(target_path.parent)] target_socket = _get_remote_socket(target_path) execute_print_receive_chunks( @@ -36,16 +36,22 @@ def parse_args(): ) parser.add_argument('target_path', - help='Path were the subvolume will be created; forwarded to btrfs-receive.', + help='Path where the subvolume will be created. ' + 'The last component of the path ' + 'must be equal to the name of the subvolume on the sending side.', type=Path, metavar='SUBVOLUME' ) args = parser.parse_args() + # Make all paths absolute. + if args.chunk_tmpl: + args.chunk_tmpl = args.chunk_tmpl.absolute() + args.target_path = args.target_path.absolute() + if not args.chunk_tmpl: - target_path: Path = args.target_path - args.chunk_tmpl = target_path.parent.joinpath(f'{target_path.name}.CHUNK') + args.chunk_tmpl = _get_chunk_tmpl(args.target_path) return args diff --git a/btrfs_send_chunks.py b/src/p1st/btrfs_send_chunks.py similarity index 69% rename from btrfs_send_chunks.py rename to src/p1st/btrfs_send_chunks.py index a61841d..e8039bd 100644 --- a/btrfs_send_chunks.py +++ b/src/p1st/btrfs_send_chunks.py @@ -3,10 +3,11 @@ import argparse from pathlib import Path import shlex +from typing import Callable -from exec_print_transfer import execute_print_transfer_chunks -from common import _get_chunk_file, _get_remote_socket -from transfer_inform import transfer_inform +from p1st.exec_print_transfer import execute_print_transfer_chunks +from p1st.common import _get_chunk_file, _get_remote_socket, _get_chunk_tmpl +from p1st.transfer_inform import transfer_inform def main(): @@ -20,12 +21,19 @@ def main(): ) command = [x for xs in command_parts for x in xs] + chunk_tmpl = _get_chunk_tmpl(args.child) + target_chunk_tmpl = _get_chunk_tmpl(args.target_path) + execute_print_transfer_chunks( command=command, - chunk_file_tmpl=args.chunk_tmpl, + chunk_file_tmpl=chunk_tmpl, chunk_transfer_fun=chunk_transfer_fun, - chunk_transfer_args=(args.ssh_target, args.target_path, args.chunk_tmpl), + chunk_transfer_args=(args.ssh_target, + args.target_path, + _get_chunk_file, + target_chunk_tmpl), chunk_size=args.chunk_size, + get_chunk_file=_get_chunk_file, ) @@ -33,7 +41,7 @@ def parse_args(): parser = argparse.ArgumentParser(prog='btrfs-send-chunks') parser.add_argument('-p', - help='Parent subvolume; forwarded to btrfs-send.', + help='Path to parent subvolume; forwarded to btrfs-send.', dest='parent', default=None, type=Path, @@ -47,16 +55,6 @@ def parse_args(): default=False, ) - parser.add_argument('--chunk-tmpl', - help='During btrfs-send, chunks are saved as "CHUNK_TMPL.CHUNK_NUMBER". ' - 'The default value of CHUNK_TMPL is "CHILD_SUBVOLUME.CHUNK". ' - 'One can change it to e.g. "/tmp/chunk/CHILD_SUBVOLUME".', - dest='chunk_tmpl', - default=None, - type=Path, - metavar='CHUNK_TMPL' - ) - parser.add_argument('--chunk-size', help='Size in bytes; defaults to 64 MB.', dest='chunk_size', @@ -65,7 +63,7 @@ def parse_args(): ) parser.add_argument('child', - help='Forwarded to btrfs-send.', + help='Path to child subvolume. Forwarded to btrfs-send.', type=Path, metavar='CHILD_SUBVOLUME' ) @@ -83,9 +81,12 @@ def parse_args(): args = parser.parse_args() - if not args.chunk_tmpl: - child: Path = args.child - args.chunk_tmpl = child.parent.joinpath(f'{child.name}.CHUNK') + # Make all paths absolute. + if args.parent: + args.parent = args.parent.absolute() + args.child = args.child.absolute() + if args.target_path: + args.target_path = args.target_path.absolute() return args @@ -93,8 +94,10 @@ def parse_args(): def chunk_transfer_fun(chunk_file: Path, ct: int, eof: bool, ssh_target: str, target_path: Path, - chunk_tmpl: Path): - target_chunk = _get_chunk_file(chunk_tmpl, ct) + get_chunk_file: Callable[[Path, int], Path], + target_chunk_tmpl: Path, + ): + target_chunk = get_chunk_file(target_chunk_tmpl, ct) rsync_cmd = ['rsync', str(chunk_file), f'{ssh_target}:{str(target_chunk)}'] message = 'EOF' if eof else 'OK' diff --git a/common.py b/src/p1st/common.py similarity index 75% rename from common.py rename to src/p1st/common.py index 395cea8..cc06c1f 100644 --- a/common.py +++ b/src/p1st/common.py @@ -9,3 +9,7 @@ def _get_chunk_file(chunk_file_tmpl: Path, ct: int): def _get_remote_socket(target: Path): return target.parent.joinpath(f'{target}.SOCKET') + + +def _get_chunk_tmpl(subvol: Path): + return subvol.parent.joinpath(f'{subvol.name}.CHUNK') diff --git a/exec_capture.py b/src/p1st/exec_capture.py similarity index 100% rename from exec_capture.py rename to src/p1st/exec_capture.py diff --git a/exec_print_capture.py b/src/p1st/exec_print_capture.py similarity index 100% rename from exec_print_capture.py rename to src/p1st/exec_print_capture.py diff --git a/exec_print_receive.py b/src/p1st/exec_print_receive.py similarity index 95% rename from exec_print_receive.py rename to src/p1st/exec_print_receive.py index e5edbb8..96bca48 100644 --- a/exec_print_receive.py +++ b/src/p1st/exec_print_receive.py @@ -6,8 +6,8 @@ import threading from pathlib import Path from typing import IO, AnyStr, Callable -from common import _get_chunk_file -from receive_inform import receive_inform +from p1st.common import _get_chunk_file +from p1st.receive_inform import receive_inform def _print_stdout(bin_pipe: IO[AnyStr]): diff --git a/exec_print_transfer.py b/src/p1st/exec_print_transfer.py similarity index 99% rename from exec_print_transfer.py rename to src/p1st/exec_print_transfer.py index b556931..a0ab967 100644 --- a/exec_print_transfer.py +++ b/src/p1st/exec_print_transfer.py @@ -6,7 +6,7 @@ import threading import subprocess from typing import AnyStr, IO, Callable -from common import _get_chunk_file +from p1st.common import _get_chunk_file def _rotate_chunk(chunk_file: Path, chunk_number: int, eof: bool, chunk_transfer_fun: Callable, chunk_transfer_args: tuple): diff --git a/receive_inform.py b/src/p1st/receive_inform.py similarity index 95% rename from receive_inform.py rename to src/p1st/receive_inform.py index ceba041..07af909 100644 --- a/receive_inform.py +++ b/src/p1st/receive_inform.py @@ -4,7 +4,7 @@ import socket from pathlib import Path from typing import IO, AnyStr, Callable -from unix_sock_input import accept_loop_until_command_received +from p1st.unix_sock_input import accept_loop_until_command_received def receive_inform(in_pipe: IO[AnyStr], diff --git a/test.py b/src/p1st/test.py similarity index 75% rename from test.py rename to src/p1st/test.py index ac0f106..856fe57 100644 --- a/test.py +++ b/src/p1st/test.py @@ -6,11 +6,11 @@ import shutil import socket from pathlib import Path -from exec_capture import execute_capture -from exec_print_capture import execute_print_capture -from exec_print_receive import execute_print_receive_chunks -from exec_print_transfer import execute_print_transfer_chunks -from transfer_inform import transfer_inform +from p1st.exec_capture import execute_capture +from p1st.exec_print_capture import execute_print_capture +from p1st.exec_print_receive import execute_print_receive_chunks +from p1st.exec_print_transfer import execute_print_transfer_chunks +from p1st.transfer_inform import transfer_inform def test(): @@ -19,7 +19,9 @@ def test(): # test3() # test4() # test5() - test67() + # test67() + # test8() + test9() def test1(): @@ -43,7 +45,7 @@ def test2(): def test3(): _init(3) - def _chunk_transfer(chunk_file: Path, eof: bool): + def _chunk_transfer(chunk_file: Path, ct: int, eof: bool): print(f'Transferring chunk {chunk_file} to ... (This is the default method, it has no effect)') if eof: print(f'The last chunk has been transferred.') @@ -170,6 +172,41 @@ def test7(): ) +def test8(): + repo_name = 'subprocess_util' + + child_name = 'test-subvolume' + child_dir = '/mnt/backup/test-dir' + child_path = f'{child_dir}/{child_name}' + target_dir = '/mnt/data/test-dir' + target_path = f'{target_dir}/{child_name}' + ssh_target = 'rootnas' + + print(f'=== In one shell, connect with "ssh nas" ===') + print(f'\tsudo mkdir {target_dir}') + print(f'\tcd {repo_name} && make && sudo make clean && cd ..') + print() + print(f'\tsudo btrfs-receive-chunks {target_path}') + print() + + print(f'=== In another shell, connect with "ssh odroid" ===') + print(f'\tsudo mkdir {child_dir}') + print(f'\tsudo btrfs subvolume create {child_path}.writeable') + print(f'\techo foo | sudo tee {child_path}.writeable/bar') + print(f'\tsudo btrfs subvolume snapshot -r {child_path}.writeable {child_path}') + print(f'\tcd {repo_name} && make && sudo make clean && cd ..') + print() + print(f'\tsudo btrfs-send-chunks {child_path} {ssh_target} {target_path}') + + +def test9(): + child_path = '/mnt/backup/snap/blogger.privacy1st.de/20230104T2255' + ssh_target = 'rootnas' + target_path = '/mnt/data/test/blogger.privacy1st.de/20230104T2255' + print(f'sudo btrfs-receive-chunks {target_path}') + print(f'sudo btrfs-send-chunks {child_path} {ssh_target} {target_path}') + + def _init(test_number: int): print(f"TEST {test_number}") test_dir = Path('test') diff --git a/transfer_inform.py b/src/p1st/transfer_inform.py similarity index 92% rename from transfer_inform.py rename to src/p1st/transfer_inform.py index 4e66348..550fd6f 100644 --- a/transfer_inform.py +++ b/src/p1st/transfer_inform.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- from pathlib import Path -from exec_print_capture import execute_print_capture -from unix_sock_input import wait_until_command_received +from p1st.exec_print_capture import execute_print_capture +from p1st.unix_sock_input import wait_until_command_received def transfer_inform(rsync_cmd: list[str], diff --git a/unix_sock_input.py b/src/p1st/unix_sock_input.py similarity index 100% rename from unix_sock_input.py rename to src/p1st/unix_sock_input.py