langfingaz/anki
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use ring on Linux; native-tls on other platforms

Browse Source 
Python wheels on Linux require statically linked SSL libraries.
We were previously relying on the native-tls-vendored feature in
reqwest, but that does not work with Bazel, as openssl-src makes
assumptions that break when sandboxed. The static libs distributed
by distros like Ubuntu fail to link, and while we could potentially
build OpenSSL ourselves, we'd then need to keep it up to
date.

On Windows and Mac however, native-tls is preferable to ring, as it
allows us to get free updates from the OS, and results in
a smaller library.

Rust currently only supports platform-specific features in nightly,
and cargo-raze does not have support for them, so we currently need
to override the generated build file with a hand-crafted one that
specifies the relative features/deps for each platform.

update.py has been updated to automatically keep the version numbers
in this file up to date, so it should hopefully not prove too hard to
maintain going forward.
This commit is contained in:
Damien Elmes 2020-11-15 15:24:05 +10:00
parent 4bf93a9af0
commit e9fdb5600c
3 changed files with 297 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
cargo/BUILD.reqwest.bazel Normal file
View File

@ -0,0 +1,191 @@
"""
We currently need to manually maintain this file, so we can use
ring on Linux, and native-tls on other platforms. We use ring on
Linux because the wheels need the SSL library statically linked in,
and native-tls-vendored is broken in Bazel due to sandboxing. We
prefer native-tls on other platforms because it reduces the resulting
binary size, and we get free updates from the OS.
Building openssl statically ourselves and linking it in would be
another valid solution, but we would need to ensure we keep it up
to date.
update.py takes care of keeping the referenced versions
up to date for us.
"""
# buildifier: disable=load
load(
"@io_bazel_rules_rust//rust:rust.bzl",
"rust_binary",
"rust_library",
"rust_test",
)
# buildifier: disable=load
load("@bazel_skylib//lib:selects.bzl", "selects")
package(default_visibility = [
# Public for visibility by "@raze__crate__version//" targets.
#
# Prefer access through "//cargo", which limits external
# visibility to explicit Cargo.toml dependencies.
"//visibility:public",
])
licenses([
"notice", # MIT from expression "MIT OR Apache-2.0"
])
# Generated Targets
# Unsupported target "blocking" with type "example" omitted
# Unsupported target "form" with type "example" omitted
# Unsupported target "json_dynamic" with type "example" omitted
# Unsupported target "json_typed" with type "example" omitted
# Unsupported target "simple" with type "example" omitted
# Unsupported target "tor_socks" with type "example" omitted
rust_library(
name = "reqwest",
srcs = glob(["**/*.rs"]),
aliases = selects.with_or({
# ring on Linux
(
"@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
): {},
"//conditions:default": {
# native-tls
"@raze__native_tls__0_2_6//:native_tls": "native_tls_crate",
},
}),
crate_features = [
"__tls",
"json",
"serde_json",
"socks",
"stream",
"tokio-socks",
] + selects.with_or({
# ring on Linux
(
"@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
): [
"__rustls",
"hyper-rustls",
"rustls",
"rustls-tls",
"rustls-tls-webpki-roots",
"tokio-rustls",
"webpki-roots",
],
# native-tls on other platforms
"//conditions:default": [
"default-tls",
"hyper-tls",
"native-tls",
"native-tls-crate",
"tokio-tls",
],
}),
crate_root = "src/lib.rs",
crate_type = "lib",
edition = "2018",
rustc_flags = [
"--cap-lints=allow",
],
tags = [
"cargo-raze",
"manual",
],
version = "0.10.8",
# buildifier: leave-alone
deps = [
"@raze__bytes__0_5_6//:bytes",
"@raze__http__0_2_1//:http",
"@raze__hyper_timeout__0_3_1//:hyper_timeout",
"@raze__mime_guess__2_0_3//:mime_guess",
"@raze__serde__1_0_117//:serde",
"@raze__serde_json__1_0_59//:serde_json",
"@raze__serde_urlencoded__0_6_1//:serde_urlencoded",
"@raze__url__2_2_0//:url",
] + selects.with_or({
# cfg(not(target_arch = "wasm32"))
(
"@io_bazel_rules_rust//rust/platform:aarch64-apple-ios",
"@io_bazel_rules_rust//rust/platform:i686-apple-darwin",
"@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
"@io_bazel_rules_rust//rust/platform:i686-unknown-linux-gnu",
"@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin",
"@io_bazel_rules_rust//rust/platform:x86_64-apple-ios",
"@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
"@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
): [
"@raze__base64__0_13_0//:base64",
"@raze__encoding_rs__0_8_26//:encoding_rs",
"@raze__futures_core__0_3_8//:futures_core",
"@raze__futures_util__0_3_8//:futures_util",
"@raze__http_body__0_3_1//:http_body",
"@raze__hyper__0_13_9//:hyper",
"@raze__ipnet__2_3_0//:ipnet",
"@raze__lazy_static__1_4_0//:lazy_static",
"@raze__log__0_4_11//:log",
"@raze__mime__0_3_16//:mime",
"@raze__percent_encoding__2_1_0//:percent_encoding",
"@raze__pin_project_lite__0_1_11//:pin_project_lite",
"@raze__tokio__0_2_23//:tokio",
"@raze__tokio_socks__0_3_0//:tokio_socks",
],
"//conditions:default": [],
}) + selects.with_or({
# cfg(windows)
(
"@io_bazel_rules_rust//rust/platform:i686-pc-windows-msvc",
"@io_bazel_rules_rust//rust/platform:x86_64-pc-windows-msvc",
): [
"@raze__winreg__0_7_0//:winreg",
],
"//conditions:default": [],
}) + selects.with_or({
# ring on Linux
(
"@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
): [
"@raze__hyper_rustls__0_21_0//:hyper_rustls",
"@raze__rustls__0_18_1//:rustls",
"@raze__tokio_rustls__0_14_1//:tokio_rustls",
"@raze__webpki_roots__0_20_0//:webpki_roots",
],
# native-tls on other platforms
"//conditions:default": [
"@raze__hyper_tls__0_4_3//:hyper_tls",
"@raze__native_tls__0_2_6//:native_tls",
"@raze__tokio_tls__0_3_1//:tokio_tls",
],
}),
)
# Unsupported target "badssl" with type "test" omitted
# Unsupported target "blocking" with type "test" omitted
# Unsupported target "brotli" with type "test" omitted
# Unsupported target "client" with type "test" omitted
# Unsupported target "cookie" with type "test" omitted
# Unsupported target "gzip" with type "test" omitted
# Unsupported target "multipart" with type "test" omitted
# Unsupported target "proxy" with type "test" omitted
# Unsupported target "redirect" with type "test" omitted
# Unsupported target "timeouts" with type "test" omitted

2
cargo/crates.bzl
View File

@ -1797,7 +1797,7 @@ def raze_fetch_remote_crates():
remote = "https://github.com/ankitects/reqwest.git",
shallow_since = "1604362745 +1000",
commit = "eab12efe22f370f386d99c7d90e7a964e85dd071",
build_file = Label("//cargo/remote:BUILD.reqwest-0.10.8.bazel"),
build_file = Label("//cargo:BUILD.reqwest.bazel"),
init_submodules = True,
)

155
cargo/update.py
View File

@ -32,70 +32,125 @@ import sys
import subprocess
import shutil
import re
import glob
if os.getcwd() != os.path.abspath(os.path.dirname(__file__)):
print("Run this from the cargo/ folder")
sys.exit(1)
with open("raze.toml") as file:
header = file.read()
deps = []
with open("../rslib/Cargo.toml") as file:
started = False
for line in file.readlines():
if line.startswith("# BEGIN DEPENDENCIES"):
started = True
continue
if not started:
continue
deps.append(line.strip())
def write_cargo_toml():
with open("raze.toml") as file:
header = file.read()
deps.extend(EXTRA_DEPS)
deps = []
with open("../rslib/Cargo.toml") as file:
started = False
for line in file.readlines():
if line.startswith("# BEGIN DEPENDENCIES"):
started = True
continue
if not started:
continue
deps.append(line.strip())
# write out Cargo.toml
with open("Cargo.toml", "w") as file:
file.write(header)
file.write("\n".join(deps))
deps.extend(EXTRA_DEPS)
# update Cargo.lock
subprocess.run(["cargo", "update"], check=True)
input("hit enter to proceed")
# write out Cargo.toml
with open("Cargo.toml", "w") as file:
file.write(header)
file.write("\n".join(deps))
# generate cargo-raze files
if os.path.exists("remote"):
shutil.rmtree("remote")
subprocess.run(["cargo-raze"], check=True)
# dump licenses
result = subprocess.check_output(["cargo", "license", "-j"])
with open("licenses.json", "wb") as file:
file.write(result)
def remove_cargo_toml():
os.remove("Cargo.toml")
os.remove("Cargo.toml")
# export license file
with open("BUILD.bazel", "a", encoding="utf8") as file:
file.write(
"""
def update_cargo_lock():
# update Cargo.lock
subprocess.run(["cargo", "update"], check=True)
def run_cargo_raze():
# generate cargo-raze files
if os.path.exists("remote"):
shutil.rmtree("remote")
subprocess.run(["cargo-raze"], check=True)
def write_licenses():
# dump licenses
result = subprocess.check_output(["cargo", "license", "-j"])
with open("licenses.json", "wb") as file:
file.write(result)
# export license file
with open("BUILD.bazel", "a", encoding="utf8") as file:
file.write(
"""
exports_files(["licenses.json"])
"""
)
)
# update shallow-since references for git crates
output_lines = []
commit_re = re.compile('\s+commit = "([0-9a-f]+)",')
with open("crates.bzl") as file:
for line in file.readlines():
if match := commit_re.match(line):
commit = match.group(1)
if commit in line:
if since := COMMITS_SHALLOW_SINCE.get(commit):
output_lines.append(f' shallow_since = "{since}",\n')
else:
print(f"{commit} not in COMMITS_SHALLOW_SINCE")
output_lines.append(line)
with open("crates.bzl", "w") as file:
for line in output_lines:
file.write(line)
def update_crates_bzl():
output_lines = []
commit_re = re.compile('\s+commit = "([0-9a-f]+)",')
reqwest_build_prefix = re.compile(r"/remote:BUILD.reqwest-\d+\.\d+\.\d+")
with open("crates.bzl") as file:
for line in file.readlines():
# update shallow-since references for git crates
if match := commit_re.match(line):
commit = match.group(1)
if commit in line:
if since := COMMITS_SHALLOW_SINCE.get(commit):
output_lines.append(f' shallow_since = "{since}",\n')
else:
print(f"{commit} not in COMMITS_SHALLOW_SINCE")
# use our custom reqwest build file
if match := reqwest_build_prefix.search(line):
line = line.replace(match.group(0), ":BUILD.reqwest")
output_lines.append(line)
with open("crates.bzl", "w") as file:
for line in output_lines:
file.write(line)
def generated_reqwest_build_file():
return glob.glob("remote/*reqwest-*")[0]
def update_reqwest_deps():
"Update version numbers in our custom build file."
dep_with_version = re.compile(r"@raze__(.+?)__([\d_]+)//")
version_map = {}
with open(generated_reqwest_build_file(), encoding="utf8") as file:
for line in file.readlines():
if match := dep_with_version.search(line):
version_map[match.group(1)] = match.group(2)
with open("BUILD.reqwest.bazel", "r+", encoding="utf8") as file:
def repl(m):
name = m.group(1)
current_version = m.group(2)
new_version = version_map.get(name)
return m.group(0).replace(current_version, new_version)
data = dep_with_version.sub(repl, file.read())
file.seek(0)
file.write(data)
write_cargo_toml()
update_cargo_lock()
run_cargo_raze()
write_licenses()
update_crates_bzl()
update_reqwest_deps()
remove_cargo_toml()