anki/qt/bundle/build.py

# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html


from __future__ import annotations

import glob
import os
import platform
import re
import shutil
import subprocess
import sys
from pathlib import Path

is_win = sys.platform == "win32"
is_mac = sys.platform == "darwin"

workspace = Path(sys.argv[1])
bazel_external = Path(sys.argv[2])


def with_exe_extension(program: str) -> str:
    if is_win:
        return program + ".exe"
    else:
        return program


output_root = workspace / ".bazel" / "out" / "build"
dist_folder = output_root / ".." / "dist"
venv = output_root / f"venv-{platform.machine()}"
build_folder = output_root / f"build-{platform.machine()}"
cargo_target = output_root / f"target-{platform.machine()}"
artifacts = output_root / "artifacts"
pyo3_config = output_root / "pyo3-build-config-file.txt"
pyoxidizer_folder = bazel_external / "pyoxidizer"
arm64_protobuf_wheel = bazel_external / "protobuf_wheel_mac_arm64"
pyoxidizer_binary = cargo_target / "release" / with_exe_extension("pyoxidizer")

for path in dist_folder.glob("*.zst"):
    path.unlink()

os.environ["PYOXIDIZER_ARTIFACT_DIR"] = str(artifacts)
os.environ["PYOXIDIZER_CONFIG"] = str(Path(os.getcwd()) / "pyoxidizer.bzl")
os.environ["CARGO_TARGET_DIR"] = str(cargo_target)

# OS-specific things
pyqt5_folder_name = "pyqt515"
pyqt6_folder_path = bazel_external / "pyqt6" / "PyQt6"
extra_linux_deps = bazel_external / "bundle_extras_linux_amd64"
extra_qt5_linux_plugins = extra_linux_deps / "qt5"
extra_qt6_linux_plugins = extra_linux_deps / "qt6"
is_lin = False
arm64_linux = arm64_mac = False
if is_win:
    os.environ["TARGET"] = "x86_64-pc-windows-msvc"
elif sys.platform.startswith("darwin"):
    if platform.machine() == "arm64":
        arm64_mac = True
        pyqt5_folder_name = None
        os.environ["TARGET"] = "aarch64-apple-darwin"
        os.environ["MACOSX_DEPLOYMENT_TARGET"] = "11.0"
    else:
        pyqt5_folder_name = "pyqt514"
        os.environ["TARGET"] = "x86_64-apple-darwin"
        os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.13"
else:
    is_lin = True
    if platform.machine() == "x86_64":
        os.environ["TARGET"] = "x86_64-unknown-linux-gnu"
    else:
        os.environ["TARGET"] = "aarch64-unknown-linux-gnu"
        pyqt5_folder_name = None
        pyqt6_folder_path = None
        arm64_linux = True

if is_win:
    python_bin_folder = venv / "scripts"
    os.environ["PATH"] += rf";{os.getenv('USERPROFILE')}\.cargo\bin"
    cargo_features = "build-mode-prebuilt-artifacts"
else:
    python_bin_folder = venv / "bin"
    # PyOxidizer build depends on a system-installed version of Python,
    # as the standalone build does not have its config set up properly,
    # leading to "directory not found for option '-L/install/lib'".
    # On macOS, after installing a system Python in /usr/local/bin,
    # make sure /usr/local/bin/python3 is symlinked to /usr/local/bin/python.
    os.environ["PATH"] = ":".join(
        ["/usr/local/bin", f"{os.getenv('HOME')}/.cargo/bin", os.getenv("PATH")]
    )
    cargo_features = "build-mode-prebuilt-artifacts"
    if not is_mac or arm64_mac:
        cargo_features += " global-allocator-jemalloc allocator-jemalloc"

python = python_bin_folder / with_exe_extension("python")
pip = python_bin_folder / with_exe_extension("pip")
artifacts_in_build = (
    build_folder / os.getenv("TARGET") / "release" / "resources" / "extra_files"
)


def build_pyoxidizer():
    pyoxidizer_folder_mtime = pyoxidizer_folder.stat().st_mtime
    if (
        pyoxidizer_binary.exists()
        and pyoxidizer_binary.stat().st_mtime == pyoxidizer_folder_mtime
    ):
        # avoid recompiling if pyoxidizer folder has not changed
        return
    subprocess.run(
        [
            "cargo",
            "build",
            "--release",
        ],
        cwd=pyoxidizer_folder,
        check=True,
    )
    os.utime(pyoxidizer_binary, (pyoxidizer_folder_mtime, pyoxidizer_folder_mtime))


def install_wheels_into_venv():
    # Pip's handling of hashes is somewhat broken. It spots the hashes in the constraints
    # file and forces all files to have a hash. We can manually hash our generated wheels
    # and pass them in with hashes, but it still breaks, because the 'protobuf>=3.17'
    # specifier in the pylib wheel is not allowed. Nevermind that a specific version is
    # included in the constraints file we pass along! To get things working, we're
    # forced to strip the hashes out before installing. This should be safe, as the files
    # have already been validated as part of the build process.
    constraints = output_root / "deps_without_hashes.txt"
    with open(workspace / "python" / "requirements.txt") as f:
        buf = f.read()
    with open(constraints, "w") as f:
        extracted = re.findall("^(\S+==\S+) ", buf, flags=re.M)
        extracted = [
            line for line in extracted if not arm64_mac or "protobuf" not in line
        ]
        f.write("\n".join(extracted))
    # pypi protobuf lacks C extension on darwin-arm64, so we have to use a version
    # we built ourselves
    if arm64_mac:
        wheels = glob.glob(str(arm64_protobuf_wheel / "*.whl"))
        subprocess.run(
            [pip, "install", "--upgrade", "-c", constraints, *wheels], check=True
        )
    # install wheels and upgrade any deps
    wheels = glob.glob(str(workspace / ".bazel" / "out" / "dist" / "*.whl"))
    subprocess.run(
        [pip, "install", "--upgrade", "-c", constraints, *wheels], check=True
    )
    # always reinstall our wheels
    subprocess.run(
        [pip, "install", "--force-reinstall", "--no-deps", *wheels], check=True
    )


def build_artifacts():
    if os.path.exists(artifacts):
        shutil.rmtree(artifacts)
    if os.path.exists(artifacts_in_build):
        shutil.rmtree(artifacts_in_build)

    subprocess.run(
        [
            pyoxidizer_binary,
            "--system-rust",
            "run-build-script",
            "build.rs",
            "--var",
            "venv",
            venv,
            "--var",
            "build",
            build_folder,
        ],
        check=True,
        env=os.environ
        | dict(
            CARGO_MANIFEST_DIR=".",
            OUT_DIR=str(artifacts),
            PROFILE="release",
            PYO3_PYTHON=str(python),
        ),
    )

    existing_config = None
    if os.path.exists(pyo3_config):
        with open(pyo3_config) as f:
            existing_config = f.read()

    with open(artifacts / "pyo3-build-config-file.txt") as f:
        new_config = f.read()

    # avoid bumping mtime, which triggers crate recompile
    if new_config != existing_config:
        with open(pyo3_config, "w") as f:
            f.write(new_config)


def build_pkg():
    subprocess.run(
        [
            "cargo",
            "build",
            "--release",
            "--no-default-features",
            "--features",
            cargo_features,
        ],
        check=True,
        env=os.environ | dict(PYO3_CONFIG_FILE=str(pyo3_config)),
    )


def adj_path_for_windows_rsync(path: Path) -> str:
    if not is_win:
        return str(path)

    path = path.absolute()
    rest = str(path)[2:].replace("\\", "/")
    return f"/{path.drive[0]}{rest}"


def merge_into_dist(output_folder: Path, pyqt_src_path: Path | None):
    if output_folder.exists():
        shutil.rmtree(output_folder)
    output_folder.mkdir(parents=True)
    # PyQt
    if pyqt_src_path and not is_mac:
        subprocess.run(
            [
                "rsync",
                "-a",
                "--delete",
                "--exclude-from",
                "qt.exclude",
                adj_path_for_windows_rsync(pyqt_src_path),
                adj_path_for_windows_rsync(output_folder / "lib") + "/",
            ],
            check=True,
        )
    if is_lin:
        if "PyQt5" in str(pyqt_src_path):
            src = extra_qt5_linux_plugins
            dest = output_folder / "lib" / "PyQt5" / "Qt5" / "plugins"
        else:
            src = extra_qt6_linux_plugins
            dest = output_folder / "lib" / "PyQt6" / "Qt6" / "plugins"
        subprocess.run(
            ["rsync", "-a", str(src) + "/", str(dest) + "/"],
            check=True,
        )

    # Executable and other resources
    resources = [
        adj_path_for_windows_rsync(
            cargo_target / "release" / ("anki.exe" if is_win else "anki")
        ),
        adj_path_for_windows_rsync(artifacts_in_build) + "/",
    ]
    if is_lin:
        resources.append("lin/")

    subprocess.run(
        [
            "rsync",
            "-a",
            "--delete",
            "--exclude",
            "PyQt6",
            "--exclude",
            "PyQt5",
            *resources,
            adj_path_for_windows_rsync(output_folder) + "/",
        ],
        check=True,
    )
    # Ensure all files are world-readable
    if not is_win:
        subprocess.run(["chmod", "-R", "a+r", output_folder])


def anki_version() -> str:
    with open(workspace / "defs.bzl") as fobj:
        data = fobj.read()
    return re.search('^anki_version = "(.*)"$', data, re.MULTILINE).group(1)


def annotated_linux_folder_name(variant: str) -> str:
    components = ["anki", anki_version(), "linux", variant]
    return "-".join(components)


def annotated_mac_dmg_name(variant: str) -> str:
    if platform.machine() == "arm64":
        arch = "apple"
    else:
        arch = "intel"
    components = ["anki", anki_version(), "mac", arch, variant]
    return "-".join(components)


def build_bundle(src_path: Path, variant: str) -> None:
    if is_lin:
        print("--- Build tarball")
        build_tarball(src_path, variant)
    elif is_mac:
        print("--- Build app bundle")
        build_app_bundle(src_path, variant)


def build_app_bundle(src_path: Path, variant: str) -> None:
    if arm64_mac:
        variant = "qt6_arm64"
    else:
        variant = f"{variant}_amd64"
    subprocess.run(
        ["cargo", "run", variant, src_path, anki_version(), bazel_external],
        check=True,
        cwd=workspace / "qt" / "bundle" / "mac",
    )
    variant_path = src_path.parent / "app" / variant
    if os.getenv("NOTARIZE_USER"):
        subprocess.run(
            ["python", "mac/notarize.py", "upload", variant_path],
            check=True,
        )
    # note down the dmg name for later
    open(variant_path / "dmg_name", "w").write(
        annotated_mac_dmg_name(variant[0:3]) + ".dmg"
    )


def build_tarball(src_path: Path, variant: str) -> None:
    if not is_lin:
        return
    dest_path = src_path.with_name(annotated_linux_folder_name(variant))
    if dest_path.exists():
        shutil.rmtree(dest_path)
    os.rename(src_path, dest_path)
    print("compress", dest_path.name, "...")
    subprocess.run(
        [
            "tar",
            "--zstd",
            "-cf",
            dist_folder / (dest_path.name + ".tar.zst"),
            dest_path.name,
        ],
        check=True,
        env=dict(ZSTD_CLEVEL="9"),
        cwd=dest_path.parent,
    )


def build_windows_installers() -> None:
    subprocess.run(
        [
            "cargo",
            "run",
            output_root,
            bazel_external,
            Path(__file__).parent,
            anki_version(),
        ],
        check=True,
        cwd=workspace / "qt" / "bundle" / "win",
    )


print("--- Build PyOxidizer")
build_pyoxidizer()
print("--- Install wheels into venv")
install_wheels_into_venv()
print("--- Build PyOxidizer artifacts")
build_artifacts()
print("--- Build Anki binary")
build_pkg()
print("--- Copy binary+resources into folder (Qt6)")
merge_into_dist(output_root / "std", pyqt6_folder_path)
build_bundle(output_root / "std", "qt6")
if pyqt5_folder_name:
    print("--- Copy binary+resources into folder (Qt5)")
    merge_into_dist(output_root / "alt", bazel_external / pyqt5_folder_name / "PyQt5")
    build_bundle(output_root / "alt", "qt5")

if is_win:
    build_windows_installers()

if is_mac:
    print("outputs are in .bazel/out/build/{std,alt}")
    print("dmg can be created with mac/finalize.py dmg")
else:
    print("outputs are in .bazel/out/dist/")