diff --git a/Cargo.toml b/Cargo.toml index 953c9bd5c..ab23d3802 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ license = "AGPL-3.0-or-later" [workspace] members = ["rslib", "rslib/i18n", "pylib/rsbridge"] +exclude = ["qt/package"] [lib] # dummy top level for tooling diff --git a/pylib/anki/syncserver/__init__.py b/pylib/anki/syncserver/__init__.py index cadee9a8c..72bfd500f 100644 --- a/pylib/anki/syncserver/__init__.py +++ b/pylib/anki/syncserver/__init__.py @@ -31,7 +31,7 @@ from anki.sync_pb2 import SyncServerMethodRequest Method = SyncServerMethodRequest.Method # pylint: disable=no-member -app = flask.Flask(__name__) +app = flask.Flask(__name__, root_path="/fake") col: Collection trace = os.getenv("TRACE") diff --git a/qt/aqt/__init__.py b/qt/aqt/__init__.py index d15d03640..0f3391f1a 100644 --- a/qt/aqt/__init__.py +++ b/qt/aqt/__init__.py @@ -3,13 +3,27 @@ from __future__ import annotations +import sys + +if sys.version_info[0] < 3 or sys.version_info[1] < 9: + raise Exception("Anki requires Python 3.9+") + +# ensure unicode filenames are supported +try: + "テスト".encode(sys.getfilesystemencoding()) +except UnicodeEncodeError as exc: + raise Exception("Anki requires a UTF-8 locale.") from exc + +from .package import packaged_build_setup + +packaged_build_setup() + import argparse import builtins import cProfile import getpass import locale import os -import sys import tempfile import traceback from typing import Any, Callable, Optional, cast @@ -24,15 +38,6 @@ from aqt import gui_hooks from aqt.qt import * from aqt.utils import TR, tr -if sys.version_info[0] < 3 or sys.version_info[1] < 9: - raise Exception("Anki requires Python 3.9+") - -# ensure unicode filenames are supported -try: - "テスト".encode(sys.getfilesystemencoding()) -except UnicodeEncodeError as exc: - raise Exception("Anki requires a UTF-8 locale.") from exc - # compat aliases anki.version = _version # type: ignore anki.Collection = Collection # type: ignore @@ -233,12 +238,8 @@ def setupLangAndBackend( # load qt translations _qtrans = QTranslator() - from aqt.utils import aqt_data_folder - if isMac and getattr(sys, "frozen", False): - qt_dir = os.path.abspath( - os.path.join(aqt_data_folder(), "..", "qt_translations") - ) + qt_dir = os.path.join(sys.prefix, "../Resources/qt_translations") else: if qtmajor == 5: qt_dir = QLibraryInfo.location(QLibraryInfo.TranslationsPath) # type: ignore @@ -429,6 +430,7 @@ def write_profile_results() -> None: def run() -> None: + print("Preparing to run...") try: _run() except Exception as e: @@ -617,6 +619,7 @@ def _run(argv: Optional[list[str]] = None, exec: bool = True) -> Optional[AnkiAp mw = aqt.main.AnkiQt(app, pm, backend, opts, args) if exec: + print("Starting main loop...") app.exec() else: return app diff --git a/qt/aqt/mediasrv.py b/qt/aqt/mediasrv.py index a728211ef..948c418c8 100644 --- a/qt/aqt/mediasrv.py +++ b/qt/aqt/mediasrv.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +import mimetypes import os import re import sys @@ -30,27 +31,7 @@ from aqt.changenotetype import ChangeNotetypeDialog from aqt.deckoptions import DeckOptionsDialog from aqt.operations.deck import update_deck_configs from aqt.qt import * -from aqt.utils import aqt_data_folder - -def _getExportFolder() -> str: - if not (data_folder := os.getenv("ANKI_DATA_FOLDER")): - data_folder = aqt_data_folder() - webInSrcFolder = os.path.abspath(os.path.join(data_folder, "web")) - if os.path.exists(webInSrcFolder): - return webInSrcFolder - elif isMac: - dir = os.path.dirname(os.path.abspath(__file__)) - return os.path.abspath(f"{dir}/../../Resources/web") - else: - if os.environ.get("TEST_TARGET"): - # running tests in bazel; we have no data - return "." - else: - raise Exception("couldn't find web folder") - - -_exportFolder = _getExportFolder() app = flask.Flask(__name__, root_path="/fake") flask_cors.CORS(app) @@ -63,6 +44,12 @@ class LocalFileRequest: path: str +@dataclass +class BundledFileRequest: + # path relative to aqt data folder + path: str + + @dataclass class NotFound: message: str @@ -135,6 +122,19 @@ class MediaServer(threading.Thread): pass +def _mime_for_path(path: str) -> str: + "Mime type for provided path/filename." + if path.endswith(".css"): + # some users may have invalid mime type in the Windows registry + return "text/css" + elif path.endswith(".js"): + return "application/javascript" + else: + # autodetect + mime, _encoding = mimetypes.guess_type(path) + return mime or "application/octet-stream" + + def _handle_local_file_request(request: LocalFileRequest) -> Response: directory = request.root path = request.path @@ -164,14 +164,7 @@ def _handle_local_file_request(request: LocalFileRequest) -> Response: ) try: - if fullpath.endswith(".css"): - # some users may have invalid mime type in the Windows registry - mimetype = "text/css" - elif fullpath.endswith(".js"): - mimetype = "application/javascript" - else: - # autodetect - mimetype = None + mimetype = _mime_for_path(fullpath) if os.path.exists(fullpath): return flask.send_file(fullpath, mimetype=mimetype, conditional=True) else: @@ -197,6 +190,51 @@ def _handle_local_file_request(request: LocalFileRequest) -> Response: ) +def _builtin_data(path: str) -> bytes: + """Return data from file in aqt/data folder. + Path must use forward slash separators.""" + # overriden location? + if data_folder := os.getenv("ANKI_DATA_FOLDER"): + full_path = os.path.join(data_folder, path) + with open(full_path, "rb") as f: + return f.read() + else: + if isWin and not getattr(sys, "frozen", False): + # default Python resource loader expects backslashes on Windows + path = path.replace("/", "\\") + reader = aqt.__loader__.get_resource_reader("aqt") # type: ignore + with reader.open_resource(path) as f: + return f.read() + + +def _handle_builtin_file_request(request: BundledFileRequest) -> Response: + path = request.path + mimetype = _mime_for_path(path) + data_path = f"data/web/{path}" + try: + data = _builtin_data(data_path) + return Response(data, mimetype=mimetype) + except FileNotFoundError: + return flask.make_response( + f"Invalid path: {path}", + HTTPStatus.NOT_FOUND, + ) + except Exception as error: + if devMode: + print( + "Caught HTTP server exception,\n%s" + % "".join(traceback.format_exception(*sys.exc_info())), + ) + + # swallow it - user likely surfed away from + # review screen before an image had finished + # downloading + return flask.make_response( + str(error), + HTTPStatus.INTERNAL_SERVER_ERROR, + ) + + @app.route("/", methods=["GET", "POST"]) def handle_request(pathin: str) -> Response: request = _extract_request(pathin) @@ -211,6 +249,8 @@ def handle_request(pathin: str) -> Response: ) elif callable(request): return _handle_dynamic_request(request) + elif isinstance(request, BundledFileRequest): + return _handle_builtin_file_request(request) elif isinstance(request, LocalFileRequest): return _handle_local_file_request(request) else: @@ -222,7 +262,7 @@ def handle_request(pathin: str) -> Response: def _extract_internal_request( path: str, -) -> LocalFileRequest | DynamicRequest | NotFound | None: +) -> BundledFileRequest | DynamicRequest | NotFound | None: "Catch /_anki references and rewrite them to web export folder." prefix = "_anki/" if not path.startswith(prefix): @@ -237,6 +277,7 @@ def _extract_internal_request( return _extract_collection_post_request(filename) elif get_handler := _extract_dynamic_get_request(filename): return get_handler + # remap legacy top-level references base, ext = os.path.splitext(filename) if ext == ".css": @@ -267,7 +308,7 @@ def _extract_internal_request( path = f"{prefix}{additional_prefix}{base}{ext}" print(f"legacy {oldpath} remapped to {path}") - return LocalFileRequest(root=_exportFolder, path=path[len(prefix) :]) + return BundledFileRequest(path=path[len(prefix) :]) def _extract_addon_request(path: str) -> LocalFileRequest | NotFound | None: @@ -302,7 +343,9 @@ def _extract_addon_request(path: str) -> LocalFileRequest | NotFound | None: return NotFound(message=f"couldn't locate item in add-on folder {path}") -def _extract_request(path: str) -> LocalFileRequest | DynamicRequest | NotFound: +def _extract_request( + path: str, +) -> LocalFileRequest | BundledFileRequest | DynamicRequest | NotFound: if internal := _extract_internal_request(path): return internal elif addon := _extract_addon_request(path): diff --git a/qt/aqt/package.py b/qt/aqt/package.py new file mode 100644 index 000000000..65a30968a --- /dev/null +++ b/qt/aqt/package.py @@ -0,0 +1,68 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +"""Helpers for the packaged version of Anki.""" + +from __future__ import annotations + +import os +import sys + + +def _fix_pywin32() -> None: + # extend sys.path with .pth files + import site + + site.addsitedir(sys.path[0]) + + # use updated sys.path to locate dll folder and add it to path + path = sys.path[-1] + path = path.replace("Pythonwin", "pywin32_system32") + os.environ["PATH"] += ";" + path + + # import Python modules from .dll files + import importlib.machinery + + for name in "pythoncom", "pywintypes": + filename = os.path.join(path, name + "39.dll") + loader = importlib.machinery.ExtensionFileLoader(name, filename) + spec = importlib.machinery.ModuleSpec(name=name, loader=loader, origin=filename) + _mod = importlib._bootstrap._load(spec) # type: ignore + + +def _patch_pkgutil() -> None: + """Teach pkgutil.get_data() how to read files from in-memory resources. + + This is required for jsonschema.""" + import importlib + import pkgutil + + def get_data_custom(package: str, resource: str) -> bytes | None: + try: + module = importlib.import_module(package) + reader = module.__loader__.get_resource_reader(package) # type: ignore[attr-defined] + with reader.open_resource(resource) as f: + return f.read() + except: + return None + + pkgutil.get_data = get_data_custom + + +def packaged_build_setup() -> None: + if not getattr(sys, "frozen", False): + return + + print("Initial setup...") + + if sys.platform == "win32": + _fix_pywin32() + + _patch_pkgutil() + + # escape hatch for debugging issues with packaged build startup + if os.getenv("ANKI_STARTUP_REPL") and os.isatty(sys.stdin.fileno()): + import code + + code.InteractiveConsole().interact() + sys.exit(0) diff --git a/qt/aqt/platform.py b/qt/aqt/platform.py index 4354eade2..fdf038e1b 100644 --- a/qt/aqt/platform.py +++ b/qt/aqt/platform.py @@ -4,6 +4,7 @@ """Platform-specific functionality.""" import os +import sys from ctypes import CDLL import aqt.utils @@ -24,5 +25,8 @@ def set_dark_mode(enabled: bool) -> bool: def _set_dark_mode(enabled: bool) -> None: - path = os.path.join(aqt.utils.aqt_data_folder(), "lib", "libankihelper.dylib") + if getattr(sys, "frozen", False): + path = os.path.join(sys.prefix, "libankihelper.dylib") + else: + path = os.path.join(aqt.utils.aqt_data_folder(), "lib", "libankihelper.dylib") CDLL(path).set_darkmode_enabled(enabled) diff --git a/qt/aqt/sound.py b/qt/aqt/sound.py index ee6c38ae2..880af90ca 100644 --- a/qt/aqt/sound.py +++ b/qt/aqt/sound.py @@ -14,6 +14,7 @@ import wave from abc import ABC, abstractmethod from concurrent.futures import Future from operator import itemgetter +from pathlib import Path from typing import Any, Callable, cast from markdown import markdown @@ -234,16 +235,15 @@ def _packagedCmd(cmd: list[str]) -> tuple[Any, dict[str, str]]: if "LD_LIBRARY_PATH" in env: del env["LD_LIBRARY_PATH"] if isMac: - dir = os.path.dirname(os.path.abspath(__file__)) - exeDir = os.path.abspath(f"{dir}/../../Resources/audio") - else: - exeDir = os.path.dirname(os.path.abspath(sys.argv[0])) - if isWin and not cmd[0].endswith(".exe"): - cmd[0] += ".exe" - path = os.path.join(exeDir, cmd[0]) - if not os.path.exists(path): - return cmd, env - cmd[0] = path + path = Path(sys.prefix).joinpath("audio").joinpath(cmd[0]) + if path.exists(): + cmd[0] = str(path) + return cmd, env + adjusted_path = os.path.join(sys.prefix, cmd[0]) + if isWin and not adjusted_path.endswith(".exe"): + adjusted_path += ".exe" + if os.path.exists(adjusted_path): + cmd[0] = adjusted_path return cmd, env diff --git a/qt/aqt/utils.py b/qt/aqt/utils.py index ed5699059..f0ff5e49f 100644 --- a/qt/aqt/utils.py +++ b/qt/aqt/utils.py @@ -30,21 +30,19 @@ def aqt_data_folder() -> str: # running in Bazel on macOS? if path := os.getenv("AQT_DATA_FOLDER"): return path - # running in place? - dir = os.path.join(os.path.dirname(__file__), "data") - if os.path.exists(dir): - return dir - # packaged install? - if isMac: - dir2 = os.path.join(sys.prefix, "..", "Resources", "aqt_data") + # packaged? + elif getattr(sys, "frozen", False): + path = os.path.join(sys.prefix, "lib/aqt/data") + if os.path.exists(path): + return path + else: + return os.path.join(sys.prefix, "../Resources/aqt/data") + elif os.path.exists(dir := os.path.join(os.path.dirname(__file__), "data")): + return os.path.abspath(dir) else: - dir2 = os.path.join(sys.prefix, "aqt_data") - if os.path.exists(dir2): - return dir2 - - # should only happen when running unit tests - print("warning, data folder not found") - return "." + # should only happen when running unit tests + print("warning, data folder not found") + return "." # shortcut to access Fluent translations; set as diff --git a/qt/bazelfixes.py b/qt/bazelfixes.py index 768299443..b3f1eb594 100644 --- a/qt/bazelfixes.py +++ b/qt/bazelfixes.py @@ -33,7 +33,7 @@ def fix_pywin32_in_bazel(force=False): import importlib.machinery name = "pythoncom" - filename = os.path.join(path, "pythoncom38.dll") + filename = os.path.join(path, "pythoncom39.dll") loader = importlib.machinery.ExtensionFileLoader(name, filename) spec = importlib.machinery.ModuleSpec(name=name, loader=loader, origin=filename) _mod = importlib._bootstrap._load(spec) diff --git a/qt/package/.cargo/config b/qt/package/.cargo/config new file mode 100644 index 000000000..0a543e7c1 --- /dev/null +++ b/qt/package/.cargo/config @@ -0,0 +1,42 @@ +# By default Rust will not export dynamic symbols from built executables. +# Python symbols need to be exported from executables in order for that +# executable to load Python extension modules, which are shared libraries. +# Otherwise, the extension module / shared library is unable to resolve +# Python symbols. This file contains target-specific configuration +# overrides to export dynamic symbols from executables. +# +# Ideally we would achieve this functionality via the build.rs build +# script. But custom compiler flags via build scripts apparently only +# support limited options. + +[target.i686-unknown-linux-gnu] +rustflags = ["-C", "link-args=-Wl,-export-dynamic"] + +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "link-args=-Wl,-export-dynamic"] + +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-args=-rdynamic"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-args=-rdynamic"] + +# The Windows standalone_static distributions use the static CRT (/MT compiler +# flag). By default, Rust will build with the dynamically linked / DLL CRT +# (/MD compiler flag). `pyoxidizer build` should adjust RUSTFLAGS automatically +# when a standalone_static distribution is being used. But if invoking `cargo` +# directly, you'll need to override the default CRT linkage by either passing +# RUSTFLAGS="-C target-feature=+crt-static" or by commenting out the lines +# below. Note that use of `target-feature=+crt-static` will prevent +# standalone_dynamic distributions from working. +# +# The standalone_static distributions also have duplicate symbols and some +# build configurations will result in hard linker errors because of this. We +# also add the /FORCE:MULTIPLE linker argument to prevent this from being a +# fatal error. + +#[target.i686-pc-windows-msvc] +#rustflags = ["-C", "target-feature=+crt-static", "-C", "link-args=/FORCE:MULTIPLE"] +# +#[target.x86_64-pc-windows-msvc] +#rustflags = ["-C", "target-feature=+crt-static", "-C", "link-args=/FORCE:MULTIPLE"] diff --git a/qt/package/Cargo.lock b/qt/package/Cargo.lock new file mode 100644 index 000000000..f20266efd --- /dev/null +++ b/qt/package/Cargo.lock @@ -0,0 +1,657 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "anki" +version = "0.1.0" +dependencies = [ + "embed-resource", + "jemallocator", + "libc", + "libc-stdhandle", + "mimalloc", + "pyembed", + "snmalloc-rs", + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "charset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f426e64df1c3de26cbf44593c6ffff5dbfd43bbf9de0d075058558126b3fc73" +dependencies = [ + "base64 0.10.1", + "encoding_rs", +] + +[[package]] +name = "cmake" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b858541263efe664aead4a5209a4ae5c5d2811167d4ed4ee0944503f8d2089" +dependencies = [ + "cc", +] + +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + +[[package]] +name = "dunce" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "embed-resource" +version = "1.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85505eb239fc952b300f29f0556d2d884082a83566768d980278d8faf38c780d" +dependencies = [ + "cc", + "vswhom", + "winreg", +] + +[[package]] +name = "encoding_rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "indoc" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +dependencies = [ + "indoc-impl", + "proc-macro-hack", +] + +[[package]] +name = "indoc-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", + "unindent", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itertools" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" +dependencies = [ + "either", +] + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013" + +[[package]] +name = "libc-stdhandle" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dac2473dc28934c5e0b82250dab231c9d3b94160d91fe9ff483323b05797551" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "libmimalloc-sys" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1b8479c593dba88c2741fc50b92e13dbabbbe0bd504d979f244ccc1a5b1c01" +dependencies = [ + "cc", + "cty", +] + +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "mailparse" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee6e1ca1c8396da58f8128176f6980dd57bec84c8670a479519d3655f2d6734" +dependencies = [ + "base64 0.13.0", + "charset", + "quoted_printable", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "memory-module-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbdce2925c681860b08875119254fb5543dbf6337c56ff93afebeed9c686da3" +dependencies = [ + "cc", + "libc", + "winapi", +] + +[[package]] +name = "mimalloc" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb74897ce508e6c49156fd1476fc5922cbc6e75183c65e399c765a09122e5130" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "pyembed" +version = "0.18.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" +dependencies = [ + "anyhow", + "dunce", + "jemalloc-sys", + "libmimalloc-sys", + "once_cell", + "pyo3", + "pyo3-build-config", + "python-oxidized-importer", + "python-packaging", + "snmalloc-sys", +] + +[[package]] +name = "pyo3" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35100f9347670a566a67aa623369293703322bb9db77d99d7df7313b575ae0c8" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "parking_lot", + "paste", + "pyo3-build-config", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d12961738cacbd7f91b7c43bc25cfeeaa2698ad07a04b3be0aa88b950865738f" +dependencies = [ + "once_cell", +] + +[[package]] +name = "pyo3-macros" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8" +dependencies = [ + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df" +dependencies = [ + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "python-oxidized-importer" +version = "0.3.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" +dependencies = [ + "anyhow", + "lazy_static", + "memmap", + "memory-module-sys", + "once_cell", + "pyo3", + "python-packaging", + "python-packed-resources", + "tugger-file-manifest", + "winapi", +] + +[[package]] +name = "python-packaging" +version = "0.11.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" +dependencies = [ + "anyhow", + "byteorder", + "encoding_rs", + "itertools", + "mailparse", + "once_cell", + "python-packed-resources", + "regex", + "spdx", + "tugger-file-manifest", + "tugger-licensing", + "walkdir", +] + +[[package]] +name = "python-packed-resources" +version = "0.8.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" +dependencies = [ + "anyhow", + "byteorder", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "quoted_printable" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1238256b09923649ec89b08104c4dfe9f6cb2fea734a5db5384e44916d59e9c5" + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "snmalloc-rs" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36acaace2719c972eab3ef6a6b3aee4495f0bf300f59715bb9cff6c5acf4ae20" +dependencies = [ + "snmalloc-sys", +] + +[[package]] +name = "snmalloc-sys" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a7e6e7d5fe756bee058ddedefc7e0a9f9c8dbaa9401b48ed3c17d6578e40b5" +dependencies = [ + "cc", + "cmake", +] + +[[package]] +name = "spdx" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e1bff9c842210e48eb85ce4c24983b34a481af4ba4b6140b41737e432f4030b" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tugger-file-manifest" +version = "0.6.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" + +[[package]] +name = "tugger-licensing" +version = "0.5.0-pre" +source = "git+https://github.com/ankitects/PyOxidizer.git?rev=ffbfe66912335bc816074c7a08aed06e26bfca7f#ffbfe66912335bc816074c7a08aed06e26bfca7f" +dependencies = [ + "anyhow", + "spdx", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + +[[package]] +name = "vswhom" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be979b7f07507105799e854203b470ff7c78a1639e330a58f183b5fea574608b" +dependencies = [ + "libc", + "vswhom-sys", +] + +[[package]] +name = "vswhom-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f5402d3d0e79a069714f7b48e3ecc60be7775a2c049cb839457457a239532" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/qt/package/Cargo.toml b/qt/package/Cargo.toml new file mode 100644 index 000000000..e5be3f0cb --- /dev/null +++ b/qt/package/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "anki" +version = "0.1.0" +build = "build.rs" +edition = "2018" + +[target.'cfg(windows)'.dependencies] +winapi = {version = "0.3", features = ["wincon"]} +libc = "0.2" +libc-stdhandle = "=0.1.0" + +[dependencies.pyembed] +git = "https://github.com/ankitects/PyOxidizer.git" +rev = "ffbfe66912335bc816074c7a08aed06e26bfca7f" +default-features = false + +[dependencies.jemallocator] +version = "0.3" +optional = true + +[dependencies.mimalloc] +version = "0.1" +optional = true +features = ["local_dynamic_tls", "override", "secure"] + +[dependencies.snmalloc-rs] +version = "0.2" +optional = true + +[build-dependencies] +embed-resource = "1.6" + +[features] +default = ["build-mode-standalone"] + +global-allocator-jemalloc = ["jemallocator"] +global-allocator-mimalloc = ["mimalloc"] +global-allocator-snmalloc = ["snmalloc-rs"] + +allocator-jemalloc = ["pyembed/allocator-jemalloc"] +allocator-mimalloc = ["pyembed/allocator-mimalloc"] +allocator-snmalloc = ["pyembed/allocator-snmalloc"] + +# Build this crate in isolation, without using PyOxidizer. +build-mode-standalone = [] + +# Build this crate by executing a `pyoxidizer` executable to build +# required artifacts. +build-mode-pyoxidizer-exe = [] + +# Build this crate by reusing artifacts generated by `pyoxidizer` out-of-band. +# In this mode, the PYOXIDIZER_ARTIFACT_DIR environment variable can refer +# to the directory containing build artifacts produced by `pyoxidizer`. If not +# set, OUT_DIR will be used. +build-mode-prebuilt-artifacts = [] diff --git a/qt/package/anki-icon.ico b/qt/package/anki-icon.ico new file mode 100644 index 000000000..fd03c333e Binary files /dev/null and b/qt/package/anki-icon.ico differ diff --git a/qt/package/anki-manifest.rc b/qt/package/anki-manifest.rc new file mode 100644 index 000000000..0e1f20ea8 --- /dev/null +++ b/qt/package/anki-manifest.rc @@ -0,0 +1,3 @@ +#define RT_MANIFEST 24 +1 RT_MANIFEST "anki.exe.manifest" +IDI_ICON1 ICON DISCARDABLE "anki-icon.ico" diff --git a/qt/package/anki.exe.manifest b/qt/package/anki.exe.manifest new file mode 100644 index 000000000..8f26bee26 --- /dev/null +++ b/qt/package/anki.exe.manifest @@ -0,0 +1,8 @@ + + + + + true + + + diff --git a/qt/package/build.bat b/qt/package/build.bat new file mode 100755 index 000000000..4320c8053 --- /dev/null +++ b/qt/package/build.bat @@ -0,0 +1,21 @@ +:: ensure wheels are built +pushd ..\.. +call scripts\build || exit /b +set ROOT=%CD% +popd + +:: ensure venv exists +set OUTPUT_ROOT=%ROOT%/bazel-pkg +set VENV=%OUTPUT_ROOT%/venv +if not exist %VENV% ( + mkdir %OUTPUT_ROOT% + pushd %ROOT% + call scripts\python -m venv %VENV% || exit /b + popd +) + +:: run the rest of the build in Python +FOR /F "tokens=*" %%g IN ('call ..\..\bazel.bat info output_base --ui_event_filters=-INFO') do (SET BAZEL_EXTERNAL=%%g/external) +call %ROOT%\scripts\cargo-env +call ..\..\bazel.bat query @pyqt515//:* +%VENV%\scripts\python build.py %ROOT% %BAZEL_EXTERNAL% || exit /b diff --git a/qt/package/build.py b/qt/package/build.py new file mode 100644 index 000000000..546d5dd33 --- /dev/null +++ b/qt/package/build.py @@ -0,0 +1,226 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import glob +import os +import platform +import re +import shutil +import subprocess +import sys +from pathlib import Path + +is_win = sys.platform == "win32" + +workspace = Path(sys.argv[1]) +output_root = workspace / "bazel-pkg" +dist_folder = output_root / "dist" +venv = output_root / "venv" +cargo_target = output_root / "target" +bazel_external = Path(sys.argv[2]) +artifacts = output_root / "artifacts" +pyo3_config = output_root / "pyo3-build-config-file.txt" + +if is_win: + python_bin_folder = venv / "scripts" + os.environ["PATH"] = os.getenv("USERPROFILE") + r"\.cargo\bin;" + os.getenv("PATH") + cargo_features = "build-mode-prebuilt-artifacts" +else: + python_bin_folder = venv / "bin" + os.environ["PATH"] = os.getenv("HOME") + "/.cargo/bin:" + os.getenv("PATH") + cargo_features = ( + "build-mode-prebuilt-artifacts global-allocator-jemalloc allocator-jemalloc" + ) + +os.environ["PYOXIDIZER_ARTIFACT_DIR"] = str(artifacts) +os.environ["PYOXIDIZER_CONFIG"] = str(Path(os.getcwd()) / "pyoxidizer.bzl") +os.environ["CARGO_TARGET_DIR"] = str(cargo_target) + +# OS-specific things +pyqt5_folder_name = "pyqt515" +if is_win: + os.environ["TARGET"] = "x86_64-pc-windows-msvc" +elif sys.platform.startswith("darwin"): + if platform.machine() == "arm64": + pyqt5_folder_name = None + os.environ["TARGET"] = "aarch64-apple-darwin" + os.environ["MACOSX_DEPLOYMENT_TARGET"] = "11.0" + else: + pyqt5_folder_name = "pyqt514" + os.environ["TARGET"] = "x86_64-apple-darwin" + os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.13" +else: + if platform.machine() == "x86_64": + os.environ["TARGET"] = "x86_64-unknown-linux-gnu" + else: + os.environ["TARGET"] = "aarch64-unknown-linux-gnu" + raise Exception("building on this architecture is not currently supported") + + +python = python_bin_folder / "python" +pip = python_bin_folder / "pip" +artifacts_in_build = ( + output_root + / "build" + / os.getenv("TARGET") + / "release" + / "resources" + / "extra_files" +) + + +def build_pyoxidizer(): + subprocess.run( + [ + "cargo", + "install", + "--locked", + "--git", + "https://github.com/ankitects/PyOxidizer.git", + "--rev", + "ffbfe66912335bc816074c7a08aed06e26bfca7f", + "pyoxidizer", + ], + check=True, + ) + + +def install_wheels_into_venv(): + # Pip's handling of hashes is somewhat broken. It spots the hashes in the constraints + # file and forces all files to have a hash. We can manually hash our generated wheels + # and pass them in with hashes, but it still breaks, because the 'protobuf>=3.17' + # specifier in the pylib wheel is not allowed. Nevermind that a specific version is + # included in the constraints file we pass along! To get things working, we're + # forced to strip the hashes out before installing. This should be safe, as the files + # have already been validated as part of the build process. + constraints = output_root / "deps_without_hashes.txt" + with open(workspace / "python" / "requirements.txt") as f: + buf = f.read() + with open(constraints, "w") as f: + extracted = re.findall("^(\S+==\S+) ", buf, flags=re.M) + f.write("\n".join(extracted)) + + # install wheels and upgrade any deps + wheels = glob.glob(str(workspace / "bazel-dist" / "*.whl")) + subprocess.run( + [pip, "install", "--upgrade", "-c", constraints, *wheels], check=True + ) + # always reinstall our wheels + subprocess.run( + [pip, "install", "--force-reinstall", "--no-deps", *wheels], check=True + ) + + +def build_artifacts(): + if os.path.exists(artifacts): + shutil.rmtree(artifacts) + if os.path.exists(artifacts_in_build): + shutil.rmtree(artifacts_in_build) + + subprocess.run( + [ + "pyoxidizer", + "--system-rust", + "run-build-script", + "build.rs", + "--var", + "venv", + venv, + ], + check=True, + env=os.environ + | dict( + CARGO_MANIFEST_DIR=".", + OUT_DIR=str(artifacts), + PROFILE="release", + PYO3_PYTHON=str(python), + ), + ) + + existing_config = None + if os.path.exists(pyo3_config): + with open(pyo3_config) as f: + existing_config = f.read() + + with open(artifacts / "pyo3-build-config-file.txt") as f: + new_config = f.read() + + # avoid bumping mtime, which triggers crate recompile + if new_config != existing_config: + with open(pyo3_config, "w") as f: + f.write(new_config) + + +def build_pkg(): + subprocess.run( + [ + "cargo", + "build", + "--release", + "--no-default-features", + "--features", + cargo_features, + ], + check=True, + env=os.environ | dict(PYO3_CONFIG_FILE=str(pyo3_config)), + ) + + +def adj_path_for_windows_rsync(path: Path) -> str: + if not is_win: + return str(path) + + path = path.absolute() + rest = str(path)[2:].replace("\\", "/") + return f"/{path.drive[0]}{rest}" + + +def merge_into_dist(output_folder: Path, pyqt_src_path: Path): + if not output_folder.exists(): + output_folder.mkdir(parents=True) + # PyQt + subprocess.run( + [ + "rsync", + "-a", + "--delete", + "--exclude-from", + "qt.exclude", + adj_path_for_windows_rsync(pyqt_src_path), + adj_path_for_windows_rsync(output_folder / "lib") + "/", + ], + check=True, + ) + # Executable and other resources + resources = [ + adj_path_for_windows_rsync( + cargo_target / "release" / ("anki.exe" if is_win else "anki") + ), + adj_path_for_windows_rsync(artifacts_in_build) + "/", + ] + if is_win: + resources.append(adj_path_for_windows_rsync(Path("win")) + "/") + + subprocess.run( + [ + "rsync", + "-a", + "--delete", + "--exclude", + "PyQt6", + "--exclude", + "PyQt5", + *resources, + adj_path_for_windows_rsync(output_folder) + "/", + ], + check=True, + ) + + +build_pyoxidizer() +install_wheels_into_venv() +build_artifacts() +build_pkg() +merge_into_dist(dist_folder / "std", bazel_external / "pyqt6" / "PyQt6") +if pyqt5_folder_name: + merge_into_dist(dist_folder / "alt", bazel_external / pyqt5_folder_name / "PyQt5") diff --git a/qt/package/build.rs b/qt/package/build.rs new file mode 100644 index 000000000..75a6d64c4 --- /dev/null +++ b/qt/package/build.rs @@ -0,0 +1,109 @@ +// Based off PyOxidizer's 'init-rust-project'. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use { + embed_resource, + std::path::{Path, PathBuf}, +}; + +const DEFAULT_PYTHON_CONFIG_FILENAME: &str = "default_python_config.rs"; +const DEFAULT_PYTHON_CONFIG: &str = "\ +pub fn default_python_config<'a>() -> pyembed::OxidizedPythonInterpreterConfig<'a> { + pyembed::OxidizedPythonInterpreterConfig::default() +} +"; + +/// Build with PyOxidizer artifacts in a directory. +fn build_with_artifacts_in_dir(path: &Path) { + println!("using pre-built artifacts from {}", path.display()); + let config_path = path.join(DEFAULT_PYTHON_CONFIG_FILENAME); + if !config_path.exists() { + panic!( + "{} does not exist; is {} a valid artifacts directory?", + config_path.display(), + path.display() + ); + } + println!( + "cargo:rustc-env=DEFAULT_PYTHON_CONFIG_RS={}", + config_path.display() + ); +} + +/// Build by calling a `pyoxidizer` executable to generate build artifacts. +fn build_with_pyoxidizer_exe(exe: Option, resolve_target: Option<&str>) { + let pyoxidizer_exe = if let Some(path) = exe { + path + } else { + "pyoxidizer".to_string() + }; + + let mut args = vec!["run-build-script", "build.rs"]; + if let Some(target) = resolve_target { + args.push("--target"); + args.push(target); + } + + match std::process::Command::new(pyoxidizer_exe) + .args(args) + .status() + { + Ok(status) => { + if !status.success() { + panic!("`pyoxidizer run-build-script` failed"); + } + } + Err(e) => panic!("`pyoxidizer run-build-script` failed: {}", e.to_string()), + } +} + +#[allow(clippy::if_same_then_else)] +fn main() { + if std::env::var("CARGO_FEATURE_BUILD_MODE_STANDALONE").is_ok() { + let path = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not defined")); + let path = path.join(DEFAULT_PYTHON_CONFIG_FILENAME); + + std::fs::write(&path, DEFAULT_PYTHON_CONFIG.as_bytes()) + .expect("failed to write default python config"); + println!( + "cargo:rustc-env=DEFAULT_PYTHON_CONFIG_RS={}", + path.display() + ); + } else if std::env::var("CARGO_FEATURE_BUILD_MODE_PYOXIDIZER_EXE").is_ok() { + let target = if let Ok(target) = std::env::var("PYOXIDIZER_BUILD_TARGET") { + Some(target) + } else { + None + }; + + build_with_pyoxidizer_exe( + std::env::var("PYOXIDIZER_EXE").ok(), + target.as_ref().map(|target| target.as_ref()), + ); + } else if std::env::var("CARGO_FEATURE_BUILD_MODE_PREBUILT_ARTIFACTS").is_ok() { + let artifact_dir_env = std::env::var("PYOXIDIZER_ARTIFACT_DIR"); + + let artifact_dir_path = match artifact_dir_env { + Ok(ref v) => PathBuf::from(v), + Err(_) => { + let out_dir = std::env::var("OUT_DIR").unwrap(); + PathBuf::from(&out_dir) + } + }; + + println!("cargo:rerun-if-env-changed=PYOXIDIZER_ARTIFACT_DIR"); + build_with_artifacts_in_dir(&artifact_dir_path); + } else { + panic!("build-mode-* feature not set"); + } + + let target_family = + std::env::var("CARGO_CFG_TARGET_FAMILY").expect("CARGO_CFG_TARGET_FAMILY not defined"); + + // embed manifest and icon + if target_family == "windows" { + embed_resource::compile("anki-manifest.rc"); + } +} diff --git a/qt/package/build.sh b/qt/package/build.sh new file mode 100755 index 000000000..bd0d64ed6 --- /dev/null +++ b/qt/package/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +cd $(dirname $0) +ROOT=$(pwd)/../.. +OUTPUT_ROOT=$ROOT/bazel-pkg +VENV=$OUTPUT_ROOT/venv +BAZEL_EXTERNAL=$(bazel info output_base --ui_event_filters=-INFO)/external + +# ensure the wheels are built +(cd $ROOT && ./scripts/build) + +# ensure venv exists +test -d $VENV || ( + mkdir -p $OUTPUT_ROOT + (cd $ROOT && ./scripts/python -m venv $VENV) +) + +# run the rest of the build in Python +. $ROOT/scripts/cargo-env +if [[ "$OSTYPE" == "darwin"* ]]; then + if [ $(uname -m) != "arm64" ]; then + bazel query @pyqt514//:* > /dev/null + fi +else + bazel query @pyqt515//:* > /dev/null +fi +$VENV/bin/python build.py $ROOT $BAZEL_EXTERNAL diff --git a/qt/package/pyoxidizer.bzl b/qt/package/pyoxidizer.bzl new file mode 100644 index 000000000..99c2443e3 --- /dev/null +++ b/qt/package/pyoxidizer.bzl @@ -0,0 +1,162 @@ +set_build_path("../../bazel-pkg/build") + +excluded_source_prefixes = [ + "ctypes.test", + "distutils.tests", + "idlelib", + "lib2to3.tests", + "test", + "tkinter", + "win32comext", + "win32com", + "win32", + "pythonwin", +] + +excluded_resource_suffixes = [ + ".pyi", + ".pyc", + "py.typed", +] + +included_resource_packages = [ + "anki", + "aqt", + "lib2to3", + "certifi", + "jsonschema", +] + +def handle_resource(policy, resource): + if type(resource) == "PythonModuleSource": + resource.add_include = True + for prefix in excluded_source_prefixes: + if resource.name.startswith(prefix): + resource.add_include = False + + # if resource.add_include: + # print("src", resource.name, resource.add_include) + + elif type(resource) == "PythonExtensionModule": + resource.add_include = True + if resource.name.startswith("win32"): + resource.add_include = False + + #print("ext", resource.name, resource.add_include) + + elif type(resource) == "PythonPackageResource": + for prefix in included_resource_packages: + if resource.package.startswith(prefix): + resource.add_include = True + for suffix in excluded_resource_suffixes: + if resource.name.endswith(suffix): + resource.add_include = False + + # aqt web resources can be stored in binary + if resource.package == "aqt": + if not resource.name.startswith("data/web"): + resource.add_location = "filesystem-relative:lib" + + # if resource.add_include: + # print("rsrc", resource.package, resource.name, resource.add_include) + + elif type(resource) == "PythonPackageDistributionResource": + #print("dist", resource.package, resource.name, resource.add_include) + pass + + # elif type(resource) == "File": + # print(resource.path) + + elif type(resource) == "File": + if ( + resource.path.startswith("win32") or + resource.path.startswith("pythonwin") or + resource.path.startswith("pywin32") + ): + exclude = ( + "tests" in resource.path or + "benchmark" in resource.path or + "__pycache__" in resource.path + ) + if not exclude: + resource.add_include = True + resource.add_location = "filesystem-relative:lib" + + if ".dist-info" in resource.path: + resource.add_include = False + + else: + print("unexpected type", type(resource)) + +def make_exe(): + if BUILD_TARGET_TRIPLE == "aarch64-unknown-linux-gnu": + fail("arm64 is not currently supported") + elif BUILD_TARGET_TRIPLE == "x86_64-unknown-linux-gnu": + dist = PythonDistribution( + url = "https://github.com/ankitects/python-build-standalone/releases/download/anki-2021-10-15/cpython-3.9.7-x86_64-unknown-linux-gnu-pgo-20211013T1538.tar.zst", + sha256 = "e5341c8f0fbedf83a2246cd86d60b6598033599ae20602d2f80617a304ef3085", + ) + + else: + dist = default_python_distribution() + + policy = dist.make_python_packaging_policy() + + policy.file_scanner_classify_files = True + policy.include_classified_resources = False + + policy.allow_files = True + policy.file_scanner_emit_files = True + policy.include_file_resources = False + + policy.include_distribution_sources = False + policy.include_distribution_resources = False + policy.include_non_distribution_sources = False + policy.include_test = False + + policy.resources_location = "in-memory" + policy.resources_location_fallback = "filesystem-relative:lib" + + policy.register_resource_callback(handle_resource) + + policy.bytecode_optimize_level_zero = False + policy.bytecode_optimize_level_two = True + + python_config = dist.make_python_interpreter_config() + + # detected libs do not need this, but we add extra afterwards + python_config.module_search_paths = ["$ORIGIN/lib"] + python_config.optimization_level = 2 + + python_config.run_command = "import aqt; aqt.run()" + + exe = dist.to_python_executable( + name = "anki", + packaging_policy = policy, + config = python_config, + ) + + exe.windows_runtime_dlls_mode = "always" + + # set in main.rs + exe.windows_subsystem = "console" + + venv_path = "venv" + + resources = exe.read_virtualenv(VARS.get("venv")) + exe.add_python_resources(resources) + + return exe + +def make_embedded_resources(exe): + return exe.to_embedded_resources() + +def make_install(exe): + files = FileManifest() + files.add_python_resource(".", exe) + return files + +register_target("exe", make_exe) +register_target("resources", make_embedded_resources, depends = ["exe"], default_build_script = True) +register_target("install", make_install, depends = ["exe"], default = True) +resolve_targets() diff --git a/qt/package/qt.exclude b/qt/package/qt.exclude new file mode 100644 index 000000000..5c94c3ea9 --- /dev/null +++ b/qt/package/qt.exclude @@ -0,0 +1,9 @@ +qml +bindings +uic +lupdate +qsci +*.pyc +*.pyi +*.sip +py.typed diff --git a/qt/package/rustfmt.toml b/qt/package/rustfmt.toml new file mode 100644 index 000000000..3c812a2b9 --- /dev/null +++ b/qt/package/rustfmt.toml @@ -0,0 +1,4 @@ +# this is not supported on stable Rust, and is ignored by the Bazel rules; it is only +# useful for manual invocation with 'cargo +nightly fmt' +imports_granularity = "Crate" +group_imports = "StdExternalCrate" diff --git a/qt/package/src/anki.rs b/qt/package/src/anki.rs new file mode 100644 index 000000000..3cf960028 --- /dev/null +++ b/qt/package/src/anki.rs @@ -0,0 +1,35 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub(super) fn init() { + #[cfg(target_os = "windows")] + attach_console(); + + println!("Anki starting..."); +} + +/// If parent process has a console (eg cmd.exe), redirect our output there. +#[cfg(target_os = "windows")] +fn attach_console() { + use std::ffi::CString; + + use libc_stdhandle::*; + use winapi::um::wincon; + + let console_attached = unsafe { wincon::AttachConsole(wincon::ATTACH_PARENT_PROCESS) }; + if console_attached == 0 { + return; + } + + let conin = CString::new("CONIN$").unwrap(); + let conout = CString::new("CONOUT$").unwrap(); + let r = CString::new("r").unwrap(); + let w = CString::new("w").unwrap(); + + // Python uses the CRT for I/O, and it requires the descriptors are reopened. + unsafe { + libc::freopen(conin.as_ptr(), r.as_ptr(), stdin()); + libc::freopen(conout.as_ptr(), w.as_ptr(), stdout()); + libc::freopen(conout.as_ptr(), w.as_ptr(), stderr()); + } +} diff --git a/qt/package/src/main.rs b/qt/package/src/main.rs new file mode 100644 index 000000000..d424d4f26 --- /dev/null +++ b/qt/package/src/main.rs @@ -0,0 +1,32 @@ +// Based off PyOxidizer's 'init-rust-project'. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#![windows_subsystem = "windows"] + +mod anki; + +use pyembed::{MainPythonInterpreter, OxidizedPythonInterpreterConfig}; + +#[cfg(feature = "global-allocator-jemalloc")] +#[global_allocator] +static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; + +include!(env!("DEFAULT_PYTHON_CONFIG_RS")); + +fn main() { + anki::init(); + + let exit_code = { + let config: OxidizedPythonInterpreterConfig = default_python_config(); + match MainPythonInterpreter::new(config) { + Ok(interp) => interp.run(), + Err(msg) => { + eprintln!("error instantiating embedded Python interpreter: {}", msg); + 1 + } + } + }; + std::process::exit(exit_code); +} diff --git a/qt/package/win/anki-console.bat b/qt/package/win/anki-console.bat new file mode 100644 index 000000000..7b488cdd9 --- /dev/null +++ b/qt/package/win/anki-console.bat @@ -0,0 +1,5 @@ +@echo off +anki +pause + + diff --git a/qt/tests/run_format.py b/qt/tests/run_format.py index 20e151bd1..a5039ad50 100644 --- a/qt/tests/run_format.py +++ b/qt/tests/run_format.py @@ -33,6 +33,7 @@ if __name__ == "__main__": "aqt", "tests", "tools", + "package", ] + args, check=False, @@ -50,6 +51,7 @@ if __name__ == "__main__": "aqt", "tests", "tools", + "package", ] + args, check=False, diff --git a/repos.bzl b/repos.bzl index b4ba5c8ab..a223cf79b 100644 --- a/repos.bzl +++ b/repos.bzl @@ -2,7 +2,7 @@ Dependencies required to build Anki. """ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") diff --git a/scripts/build.bat b/scripts/build.bat index 0d55f4d8e..0aa80294f 100755 --- a/scripts/build.bat +++ b/scripts/build.bat @@ -7,9 +7,6 @@ if not exist WORKSPACE ( rd /s /q bazel-dist -set BUILDARGS=-k -c opt dist --color=yes --@rules_rust//worker:use_worker=False -call .\bazel build %BUILDARGS% -:: repeat on failure -IF %ERRORLEVEL% NEQ 0 call .\bazel build %BUILDARGS% - -tar xvf bazel-bin\dist.tar +set BUILDARGS=-k -c opt dist --color=yes +call .\bazel build %BUILDARGS% || exit /b 1 +tar xvf bazel-bin\dist.tar || exit /b 1 diff --git a/scripts/cargo-env b/scripts/cargo-env index d6f673b64..5150155fe 100755 --- a/scripts/cargo-env +++ b/scripts/cargo-env @@ -1,14 +1,13 @@ #!/bin/bash -# Put our vendored version of cargo on the path. Not used by our -# build scripts, but can be helpful if you need quick access to cargo -# on a machine that does not have Rust installed separately, or -# want to run a quick check. Eg: +# Put our vendored version of cargo on the path. Can be helpful if you need +# quick access to cargo on a machine that does not have Rust installed +# separately, or want to run a quick check. Eg: # $ . scripts/cargo-env # $ (cd rslib && cargo check) -BAZEL_EXTERNAL=$(bazel info output_base)/external +BAZEL_EXTERNAL=$(bazel info output_base --ui_event_filters=-INFO)/external if [[ "$OSTYPE" == "darwin"* ]]; then if [ "$(arch)" == "i386" ]; then @@ -17,5 +16,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then export PATH="$BAZEL_EXTERNAL/rust_darwin_arm64/bin:$PATH" fi else + if [ "$(arch)" == "aarch64" ]; then + export PATH="$BAZEL_EXTERNAL/rust_linux_aarch64/bin:$PATH" + else export PATH="$BAZEL_EXTERNAL/rust_linux_x86_64/bin:$PATH" + fi fi diff --git a/scripts/cargo-env.bat b/scripts/cargo-env.bat new file mode 100644 index 000000000..51f6b2c06 --- /dev/null +++ b/scripts/cargo-env.bat @@ -0,0 +1,2 @@ +FOR /F "tokens=*" %%g IN ('call ..\..\bazel.bat info output_base --ui_event_filters=-INFO') do (SET BAZEL_EXTERNAL=%%g/external) +set PATH=%BAZEL_EXTERNAL%\rust_windows_x86_64\bin;%PATH% diff --git a/scripts/copyright_headers.py b/scripts/copyright_headers.py index ef4ebf672..5cb1dfc06 100644 --- a/scripts/copyright_headers.py +++ b/scripts/copyright_headers.py @@ -14,6 +14,8 @@ nonstandard_header = { "python/pyqt/install.py", "qt/aqt/mpv.py", "qt/aqt/winpaths.py", + "qt/package/build.rs", + "qt/package/src/main.rs", } ignored_folders = [ diff --git a/scripts/docker/Dockerfile.amd64 b/scripts/docker/Dockerfile.amd64 index 6921c278a..51e57df4e 100644 --- a/scripts/docker/Dockerfile.amd64 +++ b/scripts/docker/Dockerfile.amd64 @@ -6,6 +6,7 @@ ARG gid=1000 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ + autoconf \ bash \ ca-certificates \ curl \ @@ -33,6 +34,7 @@ RUN apt-get update \ libxrandr2 \ libxrender1 \ libxtst6 \ + make \ pkg-config \ portaudio19-dev \ rsync \ diff --git a/scripts/docker/Dockerfile.arm64 b/scripts/docker/Dockerfile.arm64 index 06c3525ec..56f6f00da 100644 --- a/scripts/docker/Dockerfile.arm64 +++ b/scripts/docker/Dockerfile.arm64 @@ -6,6 +6,7 @@ ARG gid=1000 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ + autoconf \ bash \ ca-certificates \ curl \ @@ -33,6 +34,7 @@ RUN apt-get update \ libxrandr2 \ libxrender1 \ libxtst6 \ + make \ pkg-config \ portaudio19-dev \ rsync \ diff --git a/scripts/docker/build-entrypoint b/scripts/docker/build-entrypoint index 61eab413c..6a97fbc20 100644 --- a/scripts/docker/build-entrypoint +++ b/scripts/docker/build-entrypoint @@ -3,8 +3,10 @@ set -e rm -rf bazel-dist -bazel --output_user_root=bazel-docker/root \ - build -c opt dist --symlink_prefix=bazel-docker/links/ \ +bazel build -c opt dist --symlink_prefix=bazel-docker/links/ \ --experimental_no_product_name_out_symlink tar xvf bazel-docker/links/bin/dist.tar -bazel --output_user_root=bazel-docker/root shutdown +if [ "$PACKAGE" != "" ]; then + (cd qt/package && ./build.sh) +fi +bazel shutdown diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index becfd5f7b..2c3968b55 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -21,6 +21,6 @@ export DOCKER_BUILDKIT=1 docker build --tag ankibuild --file scripts/docker/Dockerfile.$arch \ --build-arg uid=$(id -u) --build-arg gid=$(id -g) \ scripts/docker -docker run --rm -it \ +docker run --rm -it -e PACKAGE=$PACKAGE \ --mount type=bind,source="$(pwd)",target=/code \ ankibuild diff --git a/scripts/python b/scripts/python index 4fc4de93c..242f2e485 100755 --- a/scripts/python +++ b/scripts/python @@ -1,3 +1,3 @@ #!/bin/bash -bazel run python -- $* +bazel run python --ui_event_filters=-INFO -- $* diff --git a/scripts/python.bat b/scripts/python.bat new file mode 100755 index 000000000..44d6a5d3c --- /dev/null +++ b/scripts/python.bat @@ -0,0 +1 @@ +call bazel run python --ui_event_filters=-INFO -- %*