From cbbb4231e8534e0a486421ee7237b94a91dbdc4b Mon Sep 17 00:00:00 2001 From: Daniel Langbein Date: Sun, 19 Mar 2023 20:16:28 +0100 Subject: [PATCH] cleanup examples --- README.md | 58 ++++++-------- src/subprocess_util/test.py | 150 ++++++++---------------------------- 2 files changed, 52 insertions(+), 156 deletions(-) diff --git a/README.md b/README.md index d9eb2f7..0e3f5a0 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,29 @@ -# subprocess_util +# subprocess-util -## Example output +## Example to replicate a BTRFS subvolume with an active sending side -`btrfs receive` side, actively pulling a snapshot. +The sending side "pushes" the BTRFS snapshot to the receiving side. The sender needs password-less SSH access to the receiver. -``` -[STDOUT] OK -Reading chunk /mnt/backup/snap/cloud.privacy1st.de/20221015T1432_u.CHUNK.12212 -[STDOUT] OK -Elapsed time: 14 days, 5:38:16.172379 -Transferred: 2.981 TB -Speed: 2.542 MB/s -Reading chunk /mnt/backup/snap/cloud.privacy1st.de/20221015T1432_u.CHUNK.12213 -[STDOUT] OK -Elapsed time: 14 days, 5:39:54.104637 -Transferred: 2.982 TB -Speed: 2.542 MB/s -Reading chunk /mnt/backup/snap/cloud.privacy1st.de/20221015T1432_u.CHUNK.12214 -[STDOUT] EOF -Elapsed time: 14 days, 5:42:02.535829 -Transferred: 2.982 TB -Speed: 2.542 MB/s -Reading chunk /mnt/backup/snap/cloud.privacy1st.de/20221015T1432_u.CHUNK.12215 -Elapsed time: 14 days, 5:42:24.937967 -Transferred: 2.982 TB -Speed: 2.542 MB/s +First, start the receiving side: + +```shell +sudo btrfs-receive-chunks-passive \ + /target-snapshot-path/20230213T1809_u/ ``` -`btrfs send` side, passive. +Then, start the sending side: +```shell +sudo btrfs-send-chunks-active \ + /source-snapshot-path/20230213T1809_u/ \ + ssh-hostname-of-receiving-side \ + /target-snapshot-path/20230213T1809_u/ ``` -Received "b'OK\n'". -Handling chunk /mnt/data/btrfs-transfer-chunks/20221015T1432_u.CHUNK.12212 -Saving chunk /mnt/data/btrfs-transfer-chunks/20221015T1432_u.CHUNK.12215 -Received "b'OK\n'". -Handling chunk /mnt/data/btrfs-transfer-chunks/20221015T1432_u.CHUNK.12213 -Received "b'OK\n'". -Handling chunk /mnt/data/btrfs-transfer-chunks/20221015T1432_u.CHUNK.12214 -Received "b'OK\n'". -Handling chunk /mnt/data/btrfs-transfer-chunks/20221015T1432_u.CHUNK.12215 -Received "b'OK\n'". -``` + +See `test16()`. + +## Example to replicate a BTRFS subvolume with an active receiving side + +The receiving side "pulls" the BTRFS snapshot from the sending side. The receiver needs password-less SSH access to the sender. + +See `test17()`. diff --git a/src/subprocess_util/test.py b/src/subprocess_util/test.py index ad7df1d..3b6d88d 100644 --- a/src/subprocess_util/test.py +++ b/src/subprocess_util/test.py @@ -11,20 +11,17 @@ from subprocess_util.chunked_transfer.active_send.exec_send_chunks_active import def test(): - # test1() - # test2() - # test4() - # test8() - # test9() - # test10() - # test11() - # test12() - # test13() - # test14() - # test15_recieve() - # test15_send() test16() + +def _init(test_number: int): + print(f"TEST {test_number}") + test_dir = Path('test') + if test_dir.exists(): + shutil.rmtree('test') + Path('test').mkdir(exist_ok=False) + + def test1(): _init(1) @@ -43,41 +40,6 @@ def test2(): _returncode, _out, _err = execute_print_capture(['ls', '/foo/bar']) -def test8(): - repo_name = 'subprocess_util' - - child_name = 'test-subvolume' - child_dir = '/mnt/backup/test-dir' - child_path = f'{child_dir}/{child_name}' - target_dir = '/mnt/data/test-dir' - target_path = f'{target_dir}/{child_name}' - ssh_target = 'rootnas' - - print(f'=== In one shell, connect with "ssh nas" ===') - print(f'\tsudo mkdir {target_dir}') - print(f'\tcd {repo_name} && make && sudo make clean && cd ..') - print() - print(f'\tsudo btrfs-receive-chunks {target_path}') - print() - - print(f'=== In another shell, connect with "ssh odroid" ===') - print(f'\tsudo mkdir {child_dir}') - print(f'\tsudo btrfs subvolume create {child_path}.writeable') - print(f'\techo foo | sudo tee {child_path}.writeable/bar') - print(f'\tsudo btrfs subvolume snapshot -r {child_path}.writeable {child_path}') - print(f'\tcd {repo_name} && make && sudo make clean && cd ..') - print() - print(f'\tsudo btrfs-send-chunks {child_path} {ssh_target} {target_path}') - - -def test9(): - child_path = '/mnt/backup/snap/blogger.privacy1st.de/20230104T2255' - ssh_target = 'rootnas' - target_path = '/mnt/data/test/blogger.privacy1st.de/20230104T2255' - print(f'sudo btrfs-receive-chunks {target_path}') - print(f'sudo btrfs-send-chunks {child_path} {ssh_target} {target_path}') - - def test10(): _init(10) @@ -108,69 +70,6 @@ def test10(): ) -def test11(): - repo_name = 'subprocess_util' - - child_name = 'test-subvolume' - child_dir = '/mnt/backup/test-dir' - child_path = f'{child_dir}/{child_name}' - target_dir = '/mnt/data/test-dir' - target_path = f'{target_dir}/{child_name}' - ssh_target = 'rootnas' - - print(f'=== In one shell, connect with "ssh nas" ===') - print(f'\tsudo mkdir {target_dir}') - print(f'\tcd {repo_name} && make && sudo make clean && cd ..') - print() - print(f'\tsudo btrfs-receive-chunks-passive {target_path}') - print() - - print(f'=== In another shell, connect with "ssh odroid" ===') - print(f'\tsudo mkdir {child_dir}') - print(f'\tsudo btrfs subvolume create {child_path}.writeable') - print(f'\techo foo | sudo tee {child_path}.writeable/bar') - print(f'\tsudo btrfs subvolume snapshot -r {child_path}.writeable {child_path}') - print(f'\tcd {repo_name} && make && sudo make clean && cd ..') - print() - print(f'\tsudo btrfs-send-chunks-active {child_path} {ssh_target} {target_path}') - - -def test12(): - child_path = '/mnt/backup/snap/blogger.privacy1st.de/20230104T2255' - ssh_target = 'rootnas' - target_path = '/mnt/data/test/blogger.privacy1st.de/20230104T2255' - target_chunk_path = '/mnt/data/test/chunks' - print(f'nas: sudo btrfs-receive-chunks-passive --chunk-dir={target_chunk_path} {target_path}') - print( - f'odroid: sudo btrfs-send-chunks-active --chunk-dir=/mnt/backup/test/chunks --target-chunk-dir={target_chunk_path} {child_path} {ssh_target} {target_path}') - - -def test13(): - ssh_source = 'rootnas' - source_path = '/mnt/data/snap/blogger.privacy1st.de/20230104T1622_u' - source_chunk_dir = '/mnt/data/test/chunks' - - target_path = '/mnt/backup/test/blogger.privacy1st.de/20230104T1622_u' - - print(f'nas: sudo btrfs-send-chunks-passive --chunk-dir={source_chunk_dir} {source_path}') - print(f'odroid: sudo btrfs-receive-chunks-active {ssh_source} {source_chunk_dir} {target_path}') - - -def test14(): - chunk_size = 256 * 1024 * 1024 # 256MB - - ssh_source = 'rootnas' - source_path = '/mnt/data/snap/cloud.privacy1st.de/20221015T1432_u' - source_chunk_dir = '/mnt/data/btrfs-transfer-chunks' - - target_path = '/mnt/backup/snap/cloud.privacy1st.de/20221015T1432_u' - - print(f'nas: [screen] ' + - f'sudo btrfs-send-chunks-passive --chunk-size={chunk_size} --chunk-dir={source_chunk_dir} {source_path}') - print(f'odroid: [screen] ' + - f'sudo btrfs-receive-chunks-active {ssh_source} {source_chunk_dir} {target_path}') - - def test15_recieve(): """ First start the passive side. @@ -197,21 +96,32 @@ def test16(): """ Minimal example to replicate a BTRFS subvolume with an active sending side. """ - child_path = '/mnt/backup/snap/blogger.privacy1st.de/20230104T2255' + snapshot = 'blogger.privacy1st.de/20230213T1809_u/' + + child_path = '/mnt/backup/snap/' + snapshot ssh_target = 'rootnas' - target_path = '/mnt/data/test/blogger.privacy1st.de/20230104T2255' - print(f'nas: ' + target_path = '/mnt/data/test/' + snapshot + + print(f'nas: ' f'sudo btrfs-receive-chunks-passive {target_path}') - print(f'odroid: ' + print(f'pi3bplus: ' f'sudo btrfs-send-chunks-active {child_path} {ssh_target} {target_path}') -def _init(test_number: int): - print(f"TEST {test_number}") - test_dir = Path('test') - if test_dir.exists(): - shutil.rmtree('test') - Path('test').mkdir(exist_ok=False) +def test17(): + """ + Minimal example to replicate a BTRFS subvolume with an active receiving side. + """ + snapshot = 'blogger.privacy1st.de/20230213T1809_u/' + + child_path = '/mnt/data/snap/' + snapshot + ssh_source = 'rootnas' + target_path = '/mnt/backup/test/' + snapshot + + print(f'nas: ' + f'sudo btrfs-send-chunks-passive {child_path}') + print(f'pi3bplus: ' + f'sudo btrfs-receive-chunks-active --child={child_path} {ssh_source} {target_path}') if __name__ == '__main__':