use ring on Linux; native-tls on other platforms

Python wheels on Linux require statically linked SSL libraries.
We were previously relying on the native-tls-vendored feature in
reqwest, but that does not work with Bazel, as openssl-src makes
assumptions that break when sandboxed. The static libs distributed
by distros like Ubuntu fail to link, and while we could potentially
build OpenSSL ourselves, we'd then need to keep it up to
date.

On Windows and Mac however, native-tls is preferable to ring, as it
allows us to get free updates from the OS, and results in
a smaller library.

Rust currently only supports platform-specific features in nightly,
and cargo-raze does not have support for them, so we currently need
to override the generated build file with a hand-crafted one that
specifies the relative features/deps for each platform.

update.py has been updated to automatically keep the version numbers
in this file up to date, so it should hopefully not prove too hard to
maintain going forward.

cargo/BUILD.reqwest.bazel

@@ -0,0 +1,191 @@
+"""
+We currently need to manually maintain this file, so we can use
+ring on Linux, and native-tls on other platforms. We use ring on
+Linux because the wheels need the SSL library statically linked in,
+and native-tls-vendored is broken in Bazel due to sandboxing. We
+prefer native-tls on other platforms because it reduces the resulting
+binary size, and we get free updates from the OS.
+
+Building openssl statically ourselves and linking it in would be
+another valid solution, but we would need to ensure we keep it up
+to date.
+
+update.py takes care of keeping the referenced versions
+up to date for us.
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//cargo", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+# Unsupported target "blocking" with type "example" omitted
+
+# Unsupported target "form" with type "example" omitted
+
+# Unsupported target "json_dynamic" with type "example" omitted
+
+# Unsupported target "json_typed" with type "example" omitted
+
+# Unsupported target "simple" with type "example" omitted
+
+# Unsupported target "tor_socks" with type "example" omitted
+
+rust_library(
+    name = "reqwest",
+    srcs = glob(["**/*.rs"]),
+    aliases = selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): {},
+        "//conditions:default": {
+            # native-tls
+            "@raze__native_tls__0_2_6//:native_tls": "native_tls_crate",
+        },
+    }),
+    crate_features = [
+        "__tls",
+        "json",
+        "serde_json",
+        "socks",
+        "stream",
+        "tokio-socks",
+    ] + selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "__rustls",
+            "hyper-rustls",
+            "rustls",
+            "rustls-tls",
+            "rustls-tls-webpki-roots",
+            "tokio-rustls",
+            "webpki-roots",
+        ],
+        # native-tls on other platforms
+        "//conditions:default": [
+            "default-tls",
+            "hyper-tls",
+            "native-tls",
+            "native-tls-crate",
+            "tokio-tls",
+        ],
+    }),
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.10.8",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__bytes__0_5_6//:bytes",
+        "@raze__http__0_2_1//:http",
+        "@raze__hyper_timeout__0_3_1//:hyper_timeout",
+        "@raze__mime_guess__2_0_3//:mime_guess",
+        "@raze__serde__1_0_117//:serde",
+        "@raze__serde_json__1_0_59//:serde_json",
+        "@raze__serde_urlencoded__0_6_1//:serde_urlencoded",
+        "@raze__url__2_2_0//:url",
+    ] + selects.with_or({
+        # cfg(not(target_arch = "wasm32"))
+        (
+            "@io_bazel_rules_rust//rust/platform:aarch64-apple-ios",
+            "@io_bazel_rules_rust//rust/platform:i686-apple-darwin",
+            "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:i686-unknown-linux-gnu",
+            "@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin",
+            "@io_bazel_rules_rust//rust/platform:x86_64-apple-ios",
+            "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "@raze__base64__0_13_0//:base64",
+            "@raze__encoding_rs__0_8_26//:encoding_rs",
+            "@raze__futures_core__0_3_8//:futures_core",
+            "@raze__futures_util__0_3_8//:futures_util",
+            "@raze__http_body__0_3_1//:http_body",
+            "@raze__hyper__0_13_9//:hyper",
+            "@raze__ipnet__2_3_0//:ipnet",
+            "@raze__lazy_static__1_4_0//:lazy_static",
+            "@raze__log__0_4_11//:log",
+            "@raze__mime__0_3_16//:mime",
+            "@raze__percent_encoding__2_1_0//:percent_encoding",
+            "@raze__pin_project_lite__0_1_11//:pin_project_lite",
+            "@raze__tokio__0_2_23//:tokio",
+            "@raze__tokio_socks__0_3_0//:tokio_socks",
+        ],
+        "//conditions:default": [],
+    }) + selects.with_or({
+        # cfg(windows)
+        (
+            "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
+        ): [
+            "@raze__winreg__0_7_0//:winreg",
+        ],
+        "//conditions:default": [],
+    }) + selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "@raze__hyper_rustls__0_21_0//:hyper_rustls",
+            "@raze__rustls__0_18_1//:rustls",
+            "@raze__tokio_rustls__0_14_1//:tokio_rustls",
+            "@raze__webpki_roots__0_20_0//:webpki_roots",
+        ],
+        # native-tls on other platforms
+        "//conditions:default": [
+            "@raze__hyper_tls__0_4_3//:hyper_tls",
+            "@raze__native_tls__0_2_6//:native_tls",
+            "@raze__tokio_tls__0_3_1//:tokio_tls",
+        ],
+    }),
+)
+
+# Unsupported target "badssl" with type "test" omitted
+
+# Unsupported target "blocking" with type "test" omitted
+
+# Unsupported target "brotli" with type "test" omitted
+
+# Unsupported target "client" with type "test" omitted
+
+# Unsupported target "cookie" with type "test" omitted
+
+# Unsupported target "gzip" with type "test" omitted
+
+# Unsupported target "multipart" with type "test" omitted
+
+# Unsupported target "proxy" with type "test" omitted
+
+# Unsupported target "redirect" with type "test" omitted
+
+# Unsupported target "timeouts" with type "test" omitted

cargo/crates.bzl

@@ -1797,7 +1797,7 @@ def raze_fetch_remote_crates():
         remote = "https://github.com/ankitects/reqwest.git",
         shallow_since = "1604362745 +1000",
         commit = "eab12efe22f370f386d99c7d90e7a964e85dd071",
-        build_file = Label("//cargo/remote:BUILD.reqwest-0.10.8.bazel"),
+        build_file = Label("//cargo:BUILD.reqwest.bazel"),
         init_submodules = True,
     )
 

cargo/update.py

@@ -32,70 +32,125 @@ import sys
 import subprocess
 import shutil
 import re
+import glob
 
 if os.getcwd() != os.path.abspath(os.path.dirname(__file__)):
     print("Run this from the cargo/ folder")
     sys.exit(1)
 
-with open("raze.toml") as file:
-    header = file.read()
 
-deps = []
+def write_cargo_toml():
-with open("../rslib/Cargo.toml") as file:
+    with open("raze.toml") as file:
-    started = False
+        header = file.read()
-    for line in file.readlines():
-        if line.startswith("# BEGIN DEPENDENCIES"):
-            started = True
-            continue
-        if not started:
-            continue
-        deps.append(line.strip())
 
-deps.extend(EXTRA_DEPS)
+    deps = []
+    with open("../rslib/Cargo.toml") as file:
+        started = False
+        for line in file.readlines():
+            if line.startswith("# BEGIN DEPENDENCIES"):
+                started = True
+                continue
+            if not started:
+                continue
+            deps.append(line.strip())
 
-# write out Cargo.toml
+    deps.extend(EXTRA_DEPS)
-with open("Cargo.toml", "w") as file:
-    file.write(header)
-    file.write("\n".join(deps))
 
-# update Cargo.lock
+    # write out Cargo.toml
-subprocess.run(["cargo", "update"], check=True)
+    with open("Cargo.toml", "w") as file:
-input("hit enter to proceed")
+        file.write(header)
+        file.write("\n".join(deps))
 
-# generate cargo-raze files
-if os.path.exists("remote"):
-    shutil.rmtree("remote")
-subprocess.run(["cargo-raze"], check=True)
 
-# dump licenses
+def remove_cargo_toml():
-result = subprocess.check_output(["cargo", "license", "-j"])
+    os.remove("Cargo.toml")
-with open("licenses.json", "wb") as file:
-    file.write(result)
 
-os.remove("Cargo.toml")
 
-# export license file
+def update_cargo_lock():
-with open("BUILD.bazel", "a", encoding="utf8") as file:
+    # update Cargo.lock
-    file.write(
+    subprocess.run(["cargo", "update"], check=True)
-        """
+
+
+def run_cargo_raze():
+    # generate cargo-raze files
+    if os.path.exists("remote"):
+        shutil.rmtree("remote")
+    subprocess.run(["cargo-raze"], check=True)
+
+
+def write_licenses():
+    # dump licenses
+    result = subprocess.check_output(["cargo", "license", "-j"])
+    with open("licenses.json", "wb") as file:
+        file.write(result)
+
+    # export license file
+    with open("BUILD.bazel", "a", encoding="utf8") as file:
+        file.write(
+            """
 exports_files(["licenses.json"])
 """
-    )
+        )
 
-# update shallow-since references for git crates
-output_lines = []
-commit_re = re.compile('\s+commit = "([0-9a-f]+)",')
-with open("crates.bzl") as file:
-    for line in file.readlines():
-        if match := commit_re.match(line):
-            commit = match.group(1)
-            if commit in line:
-                if since := COMMITS_SHALLOW_SINCE.get(commit):
-                    output_lines.append(f'        shallow_since = "{since}",\n')
-                else:
-                    print(f"{commit} not in COMMITS_SHALLOW_SINCE")
-        output_lines.append(line)
 
-with open("crates.bzl", "w") as file:
+def update_crates_bzl():
-    for line in output_lines:
+    output_lines = []
-        file.write(line)
+    commit_re = re.compile('\s+commit = "([0-9a-f]+)",')
+    reqwest_build_prefix = re.compile(r"/remote:BUILD.reqwest-\d+\.\d+\.\d+")
+
+    with open("crates.bzl") as file:
+        for line in file.readlines():
+            # update shallow-since references for git crates
+            if match := commit_re.match(line):
+                commit = match.group(1)
+                if commit in line:
+                    if since := COMMITS_SHALLOW_SINCE.get(commit):
+                        output_lines.append(f'        shallow_since = "{since}",\n')
+                    else:
+                        print(f"{commit} not in COMMITS_SHALLOW_SINCE")
+
+            # use our custom reqwest build file
+            if match := reqwest_build_prefix.search(line):
+                line = line.replace(match.group(0), ":BUILD.reqwest")
+
+            output_lines.append(line)
+
+    with open("crates.bzl", "w") as file:
+        for line in output_lines:
+            file.write(line)
+
+
+def generated_reqwest_build_file():
+    return glob.glob("remote/*reqwest-*")[0]
+
+
+def update_reqwest_deps():
+    "Update version numbers in our custom build file."
+    dep_with_version = re.compile(r"@raze__(.+?)__([\d_]+)//")
+
+    version_map = {}
+    with open(generated_reqwest_build_file(), encoding="utf8") as file:
+        for line in file.readlines():
+            if match := dep_with_version.search(line):
+                version_map[match.group(1)] = match.group(2)
+
+    with open("BUILD.reqwest.bazel", "r+", encoding="utf8") as file:
+
+        def repl(m):
+            name = m.group(1)
+            current_version = m.group(2)
+            new_version = version_map.get(name)
+            return m.group(0).replace(current_version, new_version)
+
+        data = dep_with_version.sub(repl, file.read())
+        file.seek(0)
+        file.write(data)
+
+
+write_cargo_toml()
+update_cargo_lock()
+run_cargo_raze()
+write_licenses()
+update_crates_bzl()
+update_reqwest_deps()
+remove_cargo_toml()