use ring on Linux; native-tls on other platforms

Python wheels on Linux require statically linked SSL libraries.
We were previously relying on the native-tls-vendored feature in
reqwest, but that does not work with Bazel, as openssl-src makes
assumptions that break when sandboxed. The static libs distributed
by distros like Ubuntu fail to link, and while we could potentially
build OpenSSL ourselves, we'd then need to keep it up to
date.

On Windows and Mac however, native-tls is preferable to ring, as it
allows us to get free updates from the OS, and results in
a smaller library.

Rust currently only supports platform-specific features in nightly,
and cargo-raze does not have support for them, so we currently need
to override the generated build file with a hand-crafted one that
specifies the relative features/deps for each platform.

update.py has been updated to automatically keep the version numbers
in this file up to date, so it should hopefully not prove too hard to
maintain going forward.

cargo/BUILD.reqwest.bazel

@@ -0,0 +1,191 @@
+"""
+We currently need to manually maintain this file, so we can use
+ring on Linux, and native-tls on other platforms. We use ring on
+Linux because the wheels need the SSL library statically linked in,
+and native-tls-vendored is broken in Bazel due to sandboxing. We
+prefer native-tls on other platforms because it reduces the resulting
+binary size, and we get free updates from the OS.
+
+Building openssl statically ourselves and linking it in would be
+another valid solution, but we would need to ensure we keep it up
+to date.
+
+update.py takes care of keeping the referenced versions
+up to date for us.
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//cargo", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+# Unsupported target "blocking" with type "example" omitted
+
+# Unsupported target "form" with type "example" omitted
+
+# Unsupported target "json_dynamic" with type "example" omitted
+
+# Unsupported target "json_typed" with type "example" omitted
+
+# Unsupported target "simple" with type "example" omitted
+
+# Unsupported target "tor_socks" with type "example" omitted
+
+rust_library(
+    name = "reqwest",
+    srcs = glob(["**/*.rs"]),
+    aliases = selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): {},
+        "//conditions:default": {
+            # native-tls
+            "@raze__native_tls__0_2_6//:native_tls": "native_tls_crate",
+        },
+    }),
+    crate_features = [
+        "__tls",
+        "json",
+        "serde_json",
+        "socks",
+        "stream",
+        "tokio-socks",
+    ] + selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "__rustls",
+            "hyper-rustls",
+            "rustls",
+            "rustls-tls",
+            "rustls-tls-webpki-roots",
+            "tokio-rustls",
+            "webpki-roots",
+        ],
+        # native-tls on other platforms
+        "//conditions:default": [
+            "default-tls",
+            "hyper-tls",
+            "native-tls",
+            "native-tls-crate",
+            "tokio-tls",
+        ],
+    }),
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.10.8",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__bytes__0_5_6//:bytes",
+        "@raze__http__0_2_1//:http",
+        "@raze__hyper_timeout__0_3_1//:hyper_timeout",
+        "@raze__mime_guess__2_0_3//:mime_guess",
+        "@raze__serde__1_0_117//:serde",
+        "@raze__serde_json__1_0_59//:serde_json",
+        "@raze__serde_urlencoded__0_6_1//:serde_urlencoded",
+        "@raze__url__2_2_0//:url",
+    ] + selects.with_or({
+        # cfg(not(target_arch = "wasm32"))
+        (
+            "@io_bazel_rules_rust//rust/platform:aarch64-apple-ios",
+            "@io_bazel_rules_rust//rust/platform:i686-apple-darwin",
+            "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:i686-unknown-linux-gnu",
+            "@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin",
+            "@io_bazel_rules_rust//rust/platform:x86_64-apple-ios",
+            "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "@raze__base64__0_13_0//:base64",
+            "@raze__encoding_rs__0_8_26//:encoding_rs",
+            "@raze__futures_core__0_3_8//:futures_core",
+            "@raze__futures_util__0_3_8//:futures_util",
+            "@raze__http_body__0_3_1//:http_body",
+            "@raze__hyper__0_13_9//:hyper",
+            "@raze__ipnet__2_3_0//:ipnet",
+            "@raze__lazy_static__1_4_0//:lazy_static",
+            "@raze__log__0_4_11//:log",
+            "@raze__mime__0_3_16//:mime",
+            "@raze__percent_encoding__2_1_0//:percent_encoding",
+            "@raze__pin_project_lite__0_1_11//:pin_project_lite",
+            "@raze__tokio__0_2_23//:tokio",
+            "@raze__tokio_socks__0_3_0//:tokio_socks",
+        ],
+        "//conditions:default": [],
+    }) + selects.with_or({
+        # cfg(windows)
+        (
+            "@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
+            "@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
+        ): [
+            "@raze__winreg__0_7_0//:winreg",
+        ],
+        "//conditions:default": [],
+    }) + selects.with_or({
+        # ring on Linux
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+            "@raze__hyper_rustls__0_21_0//:hyper_rustls",
+            "@raze__rustls__0_18_1//:rustls",
+            "@raze__tokio_rustls__0_14_1//:tokio_rustls",
+            "@raze__webpki_roots__0_20_0//:webpki_roots",
+        ],
+        # native-tls on other platforms
+        "//conditions:default": [
+            "@raze__hyper_tls__0_4_3//:hyper_tls",
+            "@raze__native_tls__0_2_6//:native_tls",
+            "@raze__tokio_tls__0_3_1//:tokio_tls",
+        ],
+    }),
+)
+
+# Unsupported target "badssl" with type "test" omitted
+
+# Unsupported target "blocking" with type "test" omitted
+
+# Unsupported target "brotli" with type "test" omitted
+
+# Unsupported target "client" with type "test" omitted
+
+# Unsupported target "cookie" with type "test" omitted
+
+# Unsupported target "gzip" with type "test" omitted
+
+# Unsupported target "multipart" with type "test" omitted
+
+# Unsupported target "proxy" with type "test" omitted
+
+# Unsupported target "redirect" with type "test" omitted
+
+# Unsupported target "timeouts" with type "test" omitted

cargo/crates.bzl

@@ -1797,7 +1797,7 @@ def raze_fetch_remote_crates():
         remote = "https://github.com/ankitects/reqwest.git",
         shallow_since = "1604362745 +1000",
         commit = "eab12efe22f370f386d99c7d90e7a964e85dd071",
-        build_file = Label("//cargo/remote:BUILD.reqwest-0.10.8.bazel"),
+        build_file = Label("//cargo:BUILD.reqwest.bazel"),
         init_submodules = True,
     )
 

cargo/update.py

@@ -32,11 +32,14 @@ import sys
 import subprocess
 import shutil
 import re
+import glob
 
 if os.getcwd() != os.path.abspath(os.path.dirname(__file__)):
     print("Run this from the cargo/ folder")
     sys.exit(1)
 
+
+def write_cargo_toml():
     with open("raze.toml") as file:
         header = file.read()
 
@@ -58,22 +61,29 @@ with open("Cargo.toml", "w") as file:
         file.write(header)
         file.write("\n".join(deps))
 
+
+def remove_cargo_toml():
+    os.remove("Cargo.toml")
+
+
+def update_cargo_lock():
     # update Cargo.lock
     subprocess.run(["cargo", "update"], check=True)
-input("hit enter to proceed")
 
+
+def run_cargo_raze():
     # generate cargo-raze files
     if os.path.exists("remote"):
         shutil.rmtree("remote")
     subprocess.run(["cargo-raze"], check=True)
 
+
+def write_licenses():
     # dump licenses
     result = subprocess.check_output(["cargo", "license", "-j"])
     with open("licenses.json", "wb") as file:
         file.write(result)
 
-os.remove("Cargo.toml")
-
     # export license file
     with open("BUILD.bazel", "a", encoding="utf8") as file:
         file.write(
@@ -82,11 +92,15 @@ exports_files(["licenses.json"])
 """
         )
 
-# update shallow-since references for git crates
+
+def update_crates_bzl():
     output_lines = []
     commit_re = re.compile('\s+commit = "([0-9a-f]+)",')
+    reqwest_build_prefix = re.compile(r"/remote:BUILD.reqwest-\d+\.\d+\.\d+")
+
     with open("crates.bzl") as file:
         for line in file.readlines():
+            # update shallow-since references for git crates
             if match := commit_re.match(line):
                 commit = match.group(1)
                 if commit in line:
@@ -94,8 +108,49 @@ with open("crates.bzl") as file:
                         output_lines.append(f'        shallow_since = "{since}",\n')
                     else:
                         print(f"{commit} not in COMMITS_SHALLOW_SINCE")
+
+            # use our custom reqwest build file
+            if match := reqwest_build_prefix.search(line):
+                line = line.replace(match.group(0), ":BUILD.reqwest")
+
             output_lines.append(line)
 
     with open("crates.bzl", "w") as file:
         for line in output_lines:
             file.write(line)
+
+
+def generated_reqwest_build_file():
+    return glob.glob("remote/*reqwest-*")[0]
+
+
+def update_reqwest_deps():
+    "Update version numbers in our custom build file."
+    dep_with_version = re.compile(r"@raze__(.+?)__([\d_]+)//")
+
+    version_map = {}
+    with open(generated_reqwest_build_file(), encoding="utf8") as file:
+        for line in file.readlines():
+            if match := dep_with_version.search(line):
+                version_map[match.group(1)] = match.group(2)
+
+    with open("BUILD.reqwest.bazel", "r+", encoding="utf8") as file:
+
+        def repl(m):
+            name = m.group(1)
+            current_version = m.group(2)
+            new_version = version_map.get(name)
+            return m.group(0).replace(current_version, new_version)
+
+        data = dep_with_version.sub(repl, file.read())
+        file.seek(0)
+        file.write(data)
+
+
+write_cargo_toml()
+update_cargo_lock()
+run_cargo_raze()
+write_licenses()
+update_crates_bzl()
+update_reqwest_deps()
+remove_cargo_toml()