From e9fdb5600cf67b03f20b460e2fcca6fde3b34227 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 15 Nov 2020 15:24:05 +1000 Subject: [PATCH] use ring on Linux; native-tls on other platforms Python wheels on Linux require statically linked SSL libraries. We were previously relying on the native-tls-vendored feature in reqwest, but that does not work with Bazel, as openssl-src makes assumptions that break when sandboxed. The static libs distributed by distros like Ubuntu fail to link, and while we could potentially build OpenSSL ourselves, we'd then need to keep it up to date. On Windows and Mac however, native-tls is preferable to ring, as it allows us to get free updates from the OS, and results in a smaller library. Rust currently only supports platform-specific features in nightly, and cargo-raze does not have support for them, so we currently need to override the generated build file with a hand-crafted one that specifies the relative features/deps for each platform. update.py has been updated to automatically keep the version numbers in this file up to date, so it should hopefully not prove too hard to maintain going forward. --- cargo/BUILD.reqwest.bazel | 191 ++++++++++++++++++++++++++++++++++++++ cargo/crates.bzl | 2 +- cargo/update.py | 155 +++++++++++++++++++++---------- 3 files changed, 297 insertions(+), 51 deletions(-) create mode 100644 cargo/BUILD.reqwest.bazel diff --git a/cargo/BUILD.reqwest.bazel b/cargo/BUILD.reqwest.bazel new file mode 100644 index 000000000..e88f6bdf7 --- /dev/null +++ b/cargo/BUILD.reqwest.bazel @@ -0,0 +1,191 @@ +""" +We currently need to manually maintain this file, so we can use +ring on Linux, and native-tls on other platforms. We use ring on +Linux because the wheels need the SSL library statically linked in, +and native-tls-vendored is broken in Bazel due to sandboxing. We +prefer native-tls on other platforms because it reduces the resulting +binary size, and we get free updates from the OS. + +Building openssl statically ourselves and linking it in would be +another valid solution, but we would need to ensure we keep it up +to date. + +update.py takes care of keeping the referenced versions +up to date for us. +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//cargo", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +# Unsupported target "blocking" with type "example" omitted + +# Unsupported target "form" with type "example" omitted + +# Unsupported target "json_dynamic" with type "example" omitted + +# Unsupported target "json_typed" with type "example" omitted + +# Unsupported target "simple" with type "example" omitted + +# Unsupported target "tor_socks" with type "example" omitted + +rust_library( + name = "reqwest", + srcs = glob(["**/*.rs"]), + aliases = selects.with_or({ + # ring on Linux + ( + "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", + ): {}, + "//conditions:default": { + # native-tls + "@raze__native_tls__0_2_6//:native_tls": "native_tls_crate", + }, + }), + crate_features = [ + "__tls", + "json", + "serde_json", + "socks", + "stream", + "tokio-socks", + ] + selects.with_or({ + # ring on Linux + ( + "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", + ): [ + "__rustls", + "hyper-rustls", + "rustls", + "rustls-tls", + "rustls-tls-webpki-roots", + "tokio-rustls", + "webpki-roots", + ], + # native-tls on other platforms + "//conditions:default": [ + "default-tls", + "hyper-tls", + "native-tls", + "native-tls-crate", + "tokio-tls", + ], + }), + crate_root = "src/lib.rs", + crate_type = "lib", + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.10.8", + # buildifier: leave-alone + deps = [ + "@raze__bytes__0_5_6//:bytes", + "@raze__http__0_2_1//:http", + "@raze__hyper_timeout__0_3_1//:hyper_timeout", + "@raze__mime_guess__2_0_3//:mime_guess", + "@raze__serde__1_0_117//:serde", + "@raze__serde_json__1_0_59//:serde_json", + "@raze__serde_urlencoded__0_6_1//:serde_urlencoded", + "@raze__url__2_2_0//:url", + ] + selects.with_or({ + # cfg(not(target_arch = "wasm32")) + ( + "@io_bazel_rules_rust//rust/platform:aarch64-apple-ios", + "@io_bazel_rules_rust//rust/platform:i686-apple-darwin", + "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc", + "@io_bazel_rules_rust//rust/platform:i686-unknown-linux-gnu", + "@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin", + "@io_bazel_rules_rust//rust/platform:x86_64-apple-ios", + "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc", + "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", + ): [ + "@raze__base64__0_13_0//:base64", + "@raze__encoding_rs__0_8_26//:encoding_rs", + "@raze__futures_core__0_3_8//:futures_core", + "@raze__futures_util__0_3_8//:futures_util", + "@raze__http_body__0_3_1//:http_body", + "@raze__hyper__0_13_9//:hyper", + "@raze__ipnet__2_3_0//:ipnet", + "@raze__lazy_static__1_4_0//:lazy_static", + "@raze__log__0_4_11//:log", + "@raze__mime__0_3_16//:mime", + "@raze__percent_encoding__2_1_0//:percent_encoding", + "@raze__pin_project_lite__0_1_11//:pin_project_lite", + "@raze__tokio__0_2_23//:tokio", + "@raze__tokio_socks__0_3_0//:tokio_socks", + ], + "//conditions:default": [], + }) + selects.with_or({ + # cfg(windows) + ( + "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc", + "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc", + ): [ + "@raze__winreg__0_7_0//:winreg", + ], + "//conditions:default": [], + }) + selects.with_or({ + # ring on Linux + ( + "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", + ): [ + "@raze__hyper_rustls__0_21_0//:hyper_rustls", + "@raze__rustls__0_18_1//:rustls", + "@raze__tokio_rustls__0_14_1//:tokio_rustls", + "@raze__webpki_roots__0_20_0//:webpki_roots", + ], + # native-tls on other platforms + "//conditions:default": [ + "@raze__hyper_tls__0_4_3//:hyper_tls", + "@raze__native_tls__0_2_6//:native_tls", + "@raze__tokio_tls__0_3_1//:tokio_tls", + ], + }), +) + +# Unsupported target "badssl" with type "test" omitted + +# Unsupported target "blocking" with type "test" omitted + +# Unsupported target "brotli" with type "test" omitted + +# Unsupported target "client" with type "test" omitted + +# Unsupported target "cookie" with type "test" omitted + +# Unsupported target "gzip" with type "test" omitted + +# Unsupported target "multipart" with type "test" omitted + +# Unsupported target "proxy" with type "test" omitted + +# Unsupported target "redirect" with type "test" omitted + +# Unsupported target "timeouts" with type "test" omitted diff --git a/cargo/crates.bzl b/cargo/crates.bzl index 3d5abd444..510141584 100644 --- a/cargo/crates.bzl +++ b/cargo/crates.bzl @@ -1797,7 +1797,7 @@ def raze_fetch_remote_crates(): remote = "https://github.com/ankitects/reqwest.git", shallow_since = "1604362745 +1000", commit = "eab12efe22f370f386d99c7d90e7a964e85dd071", - build_file = Label("//cargo/remote:BUILD.reqwest-0.10.8.bazel"), + build_file = Label("//cargo:BUILD.reqwest.bazel"), init_submodules = True, ) diff --git a/cargo/update.py b/cargo/update.py index b10157d06..faa96424f 100644 --- a/cargo/update.py +++ b/cargo/update.py @@ -32,70 +32,125 @@ import sys import subprocess import shutil import re +import glob if os.getcwd() != os.path.abspath(os.path.dirname(__file__)): print("Run this from the cargo/ folder") sys.exit(1) -with open("raze.toml") as file: - header = file.read() -deps = [] -with open("../rslib/Cargo.toml") as file: - started = False - for line in file.readlines(): - if line.startswith("# BEGIN DEPENDENCIES"): - started = True - continue - if not started: - continue - deps.append(line.strip()) +def write_cargo_toml(): + with open("raze.toml") as file: + header = file.read() -deps.extend(EXTRA_DEPS) + deps = [] + with open("../rslib/Cargo.toml") as file: + started = False + for line in file.readlines(): + if line.startswith("# BEGIN DEPENDENCIES"): + started = True + continue + if not started: + continue + deps.append(line.strip()) -# write out Cargo.toml -with open("Cargo.toml", "w") as file: - file.write(header) - file.write("\n".join(deps)) + deps.extend(EXTRA_DEPS) -# update Cargo.lock -subprocess.run(["cargo", "update"], check=True) -input("hit enter to proceed") + # write out Cargo.toml + with open("Cargo.toml", "w") as file: + file.write(header) + file.write("\n".join(deps)) -# generate cargo-raze files -if os.path.exists("remote"): - shutil.rmtree("remote") -subprocess.run(["cargo-raze"], check=True) -# dump licenses -result = subprocess.check_output(["cargo", "license", "-j"]) -with open("licenses.json", "wb") as file: - file.write(result) +def remove_cargo_toml(): + os.remove("Cargo.toml") -os.remove("Cargo.toml") -# export license file -with open("BUILD.bazel", "a", encoding="utf8") as file: - file.write( - """ +def update_cargo_lock(): + # update Cargo.lock + subprocess.run(["cargo", "update"], check=True) + + +def run_cargo_raze(): + # generate cargo-raze files + if os.path.exists("remote"): + shutil.rmtree("remote") + subprocess.run(["cargo-raze"], check=True) + + +def write_licenses(): + # dump licenses + result = subprocess.check_output(["cargo", "license", "-j"]) + with open("licenses.json", "wb") as file: + file.write(result) + + # export license file + with open("BUILD.bazel", "a", encoding="utf8") as file: + file.write( + """ exports_files(["licenses.json"]) """ - ) + ) -# update shallow-since references for git crates -output_lines = [] -commit_re = re.compile('\s+commit = "([0-9a-f]+)",') -with open("crates.bzl") as file: - for line in file.readlines(): - if match := commit_re.match(line): - commit = match.group(1) - if commit in line: - if since := COMMITS_SHALLOW_SINCE.get(commit): - output_lines.append(f' shallow_since = "{since}",\n') - else: - print(f"{commit} not in COMMITS_SHALLOW_SINCE") - output_lines.append(line) -with open("crates.bzl", "w") as file: - for line in output_lines: - file.write(line) +def update_crates_bzl(): + output_lines = [] + commit_re = re.compile('\s+commit = "([0-9a-f]+)",') + reqwest_build_prefix = re.compile(r"/remote:BUILD.reqwest-\d+\.\d+\.\d+") + + with open("crates.bzl") as file: + for line in file.readlines(): + # update shallow-since references for git crates + if match := commit_re.match(line): + commit = match.group(1) + if commit in line: + if since := COMMITS_SHALLOW_SINCE.get(commit): + output_lines.append(f' shallow_since = "{since}",\n') + else: + print(f"{commit} not in COMMITS_SHALLOW_SINCE") + + # use our custom reqwest build file + if match := reqwest_build_prefix.search(line): + line = line.replace(match.group(0), ":BUILD.reqwest") + + output_lines.append(line) + + with open("crates.bzl", "w") as file: + for line in output_lines: + file.write(line) + + +def generated_reqwest_build_file(): + return glob.glob("remote/*reqwest-*")[0] + + +def update_reqwest_deps(): + "Update version numbers in our custom build file." + dep_with_version = re.compile(r"@raze__(.+?)__([\d_]+)//") + + version_map = {} + with open(generated_reqwest_build_file(), encoding="utf8") as file: + for line in file.readlines(): + if match := dep_with_version.search(line): + version_map[match.group(1)] = match.group(2) + + with open("BUILD.reqwest.bazel", "r+", encoding="utf8") as file: + + def repl(m): + name = m.group(1) + current_version = m.group(2) + new_version = version_map.get(name) + return m.group(0).replace(current_version, new_version) + + data = dep_with_version.sub(repl, file.read()) + file.seek(0) + file.write(data) + + +write_cargo_toml() +update_cargo_lock() +run_cargo_raze() +write_licenses() +update_crates_bzl() +update_reqwest_deps() +remove_cargo_toml()