From d3348241808c3647809e3aae645ff21a03582ab6 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 8 Sep 2025 15:56:22 +0000 Subject: [PATCH 1/5] Add `static-linking` feature in `Dockerfile` --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97d3c4e9..e4a01b24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=planner /usr/src/recipe.json recipe.json RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \ - cargo chef cook --release --features ort,candle,mkl --no-default-features --recipe-path recipe.json && sccache -s + cargo chef cook --release --features ort,candle,mkl,static-linking --no-default-features --recipe-path recipe.json && sccache -s COPY backends backends COPY core core @@ -55,7 +55,7 @@ FROM builder AS http-builder RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \ - cargo build --release --bin text-embeddings-router --features ort,candle,mkl,http --no-default-features && sccache -s + cargo build --release --bin text-embeddings-router --features ort,candle,mkl,static-linking,http --no-default-features && sccache -s FROM builder AS grpc-builder @@ -69,7 +69,7 @@ COPY proto proto RUN --mount=type=secret,id=actions_results_url,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=actions_runtime_token,env=ACTIONS_RUNTIME_TOKEN \ - cargo build --release --bin text-embeddings-router --features ort,candle,mkl,grpc --no-default-features && sccache -s + cargo build --release --bin text-embeddings-router --features ort,candle,mkl,static-linking,grpc --no-default-features && sccache -s FROM debian:bookworm-slim AS base From c6488101374524b75422fdf0fba601ba9b246861 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 8 Sep 2025 15:56:27 +0000 Subject: [PATCH 2/5] Add missing spacing --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dbdb14e0..2987d871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ thiserror = "1.0" rand = "0.9" serial_test = "2.0.0" cudarc = { version = "0.13" , features =["cuda-12020"], default-features = false} -intel-mkl-src = { version = "0.8"} +intel-mkl-src = { version = "0.8" } candle = { version = "0.8", package = "candle-core" } candle-nn = { version = "0.8" } candle-transformers = { version = "0.8" } From 6683c2550deb74a2bd24a7ab76c5393f0f4504d6 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:55:01 +0000 Subject: [PATCH 3/5] Adding `features = ["mkl-static-..."]` works e.g. `cargo run --release --features candle,mkl,http --no-default-features -- --model-id sentence-transformers/all-MiniLM-L6-v2 --dtype float32` --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2987d871..1854096d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,8 +42,8 @@ serde_json = "1.0" thiserror = "1.0" rand = "0.9" serial_test = "2.0.0" -cudarc = { version = "0.13" , features =["cuda-12020"], default-features = false} -intel-mkl-src = { version = "0.8" } +cudarc = { version = "0.13", features =["cuda-12020"], default-features = false} +intel-mkl-src = { version = "0.8", features = ["mkl-static-lp64-iomp"] } candle = { version = "0.8", package = "candle-core" } candle-nn = { version = "0.8" } candle-transformers = { version = "0.8" } From 2d6b431ad649ae1254b92120b837f35bdef7ce73 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:53:16 +0000 Subject: [PATCH 4/5] Add missing `dep:intel-mkl-src` in `router/Cargo.toml` --- backends/candle/Cargo.toml | 2 +- router/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/candle/Cargo.toml b/backends/candle/Cargo.toml index a2c3e469..73d0f417 100644 --- a/backends/candle/Cargo.toml +++ b/backends/candle/Cargo.toml @@ -8,7 +8,7 @@ homepage.workspace = true [dependencies] anyhow = { workspace = true } accelerate-src = { version = "0.3.2", optional = true } -intel-mkl-src = { workspace = true, optional = true } +intel-mkl-src = { workspace = true, optional = true } candle = { workspace = true } candle-nn = { workspace = true } candle-transformers = { workspace = true } diff --git a/router/Cargo.toml b/router/Cargo.toml index c411957a..381d611c 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -83,7 +83,7 @@ default = ["candle", "http", "dynamic-linking"] http = ["dep:axum", "dep:axum-tracing-opentelemetry", "dep:base64", "dep:tower-http", "dep:utoipa", "dep:utoipa-swagger-ui"] grpc = ["metrics-exporter-prometheus/http-listener", "dep:prost", "dep:tonic", "dep:tonic-health", "dep:tonic-reflection", "dep:tonic-build", "dep:async-stream", "dep:tokio-stream"] metal = ["text-embeddings-backend/metal"] -mkl = ["text-embeddings-backend/mkl"] +mkl = ["text-embeddings-backend/mkl", "dep:intel-mkl-src"] accelerate = ["text-embeddings-backend/accelerate"] python = ["text-embeddings-backend/python"] ort = ["text-embeddings-backend/ort"] From e7fa45c1e35d227eb41b59367acc2af3330454a5 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:55:17 +0000 Subject: [PATCH 5/5] Set `default-features = false` for `intel-mkl-src` --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1854096d..2c27199c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,13 +42,13 @@ serde_json = "1.0" thiserror = "1.0" rand = "0.9" serial_test = "2.0.0" -cudarc = { version = "0.13", features =["cuda-12020"], default-features = false} -intel-mkl-src = { version = "0.8", features = ["mkl-static-lp64-iomp"] } +cudarc = { version = "0.13", features =["cuda-12020"], default-features = false } +intel-mkl-src = { version = "0.8", default-features = false } candle = { version = "0.8", package = "candle-core" } -candle-nn = { version = "0.8" } +candle-nn = { version = "0.8" } candle-transformers = { version = "0.8" } candle-flash-attn = { version = "0.8" } -candle-cublaslt= { version = "0.0.1" } +candle-cublaslt = { version = "0.0.1" } candle-layer-norm = { version = "0.0.1" } candle-rotary = { version = "0.0.1" } candle-flash-attn-v1 = { version = "0.0.1" }