Skip to content

Commit 0672c42

Browse files
pvijayakrishoandreeva-nv
pvijayakrish
authored and
oandreeva-nv
committed
feat: Add manylinux support for Dynamo (#536)
Signed-off-by: Pavithra Vijayakrishnan <[email protected]>
1 parent dbafc78 commit 0672c42

File tree

3 files changed

+135
-28
lines changed

3 files changed

+135
-28
lines changed

‎container/Dockerfile.tensorrt_llm

+61-5
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,14 @@
1515

1616
ARG BASE_IMAGE="tensorrt_llm/release"
1717
ARG BASE_IMAGE_TAG="latest"
18+
ARG MANYLINUX_IMAGE="quay.io/pypa/manylinux_2_28_x86_64"
19+
ARG RELEASE_BUILD
1820

19-
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS dev
21+
##################################
22+
########## Build Image ###########
23+
##################################
24+
25+
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS build
2026

2127
USER root
2228

@@ -116,15 +122,61 @@ COPY deploy/dynamo/sdk /workspace/deploy/dynamo/sdk
116122

117123
# Install uv, create virtualenv for general use, and build dynamo wheel
118124
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
125+
126+
### VIRTUAL ENVIRONMENT SETUP ###
127+
RUN mkdir /opt/dynamo && \
128+
uv venv /opt/dynamo/venv --python 3.12
129+
130+
###################################
131+
####### WHEEL BUILD STAGE #########
132+
###################################
133+
134+
# Build the wheel in the manylinux environment
135+
FROM ${MANYLINUX_IMAGE} AS wheel_builder
136+
ARG RELEASE_BUILD
137+
ARG CARGO_BUILD_JOBS
138+
# Set CARGO_BUILD_JOBS to 16 if not provided
139+
# This is to prevent cargo from building $(nproc) jobs in parallel,
140+
# which might exceed the number of opened files limit.
141+
ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-16}
142+
143+
WORKDIR /workspace
144+
145+
RUN yum install -y protobuf-compiler \
146+
&& yum clean all \
147+
&& rm -rf /var/cache/yum
148+
149+
COPY --from=build /workspace /workspace
150+
151+
ENV RUSTUP_HOME=/usr/local/rustup \
152+
CARGO_HOME=/usr/local/cargo \
153+
PATH=/usr/local/cargo/bin:$PATH \
154+
CARGO_TARGET_DIR=/workspace/target
155+
156+
COPY --from=build $RUSTUP_HOME $RUSTUP_HOME
157+
COPY --from=build $CARGO_HOME $CARGO_HOME
158+
159+
# Create virtualenv and build dynamo wheel
119160
RUN mkdir /opt/dynamo && \
120161
uv venv /opt/dynamo/venv --python 3.12 && \
121162
source /opt/dynamo/venv/bin/activate && \
122163
cd /workspace/lib/bindings/python && \
123-
uv build --wheel --out-dir /workspace/dist && \
124-
uv pip install /workspace/dist/ai_dynamo_runtime*cp312*.whl && \
164+
uv build --wheel --out-dir /workspace/dist --python 3.12 && \
165+
if [ "$RELEASE_BUILD" = "true" ]; then \
166+
uv build --wheel --out-dir /workspace/dist --python 3.11 && \
167+
uv build --wheel --out-dir /workspace/dist --python 3.10; \
168+
fi && \
125169
cd /workspace && \
126-
uv build --wheel --out-dir /workspace/dist && \
127-
uv pip install /workspace/dist/ai_dynamo*any.whl
170+
uv build --wheel --out-dir /workspace/dist
171+
172+
########################################
173+
########## Development Image ###########
174+
########################################
175+
176+
FROM build AS dev
177+
178+
WORKDIR /workspace
179+
COPY --from=wheel_builder /workspace/dist/ /workspace/dist/
128180

129181
# Package the bindings
130182
RUN mkdir -p /opt/dynamo/bindings/wheels && \
@@ -133,6 +185,10 @@ RUN mkdir -p /opt/dynamo/bindings/wheels && \
133185
cp target/release/libdynamo_llm_capi.so /opt/dynamo/bindings/lib/. && \
134186
cp -r lib/bindings/c/include /opt/dynamo/bindings/.
135187

188+
RUN . /opt/dynamo/venv/bin/activate && \
189+
uv pip install /workspace/dist/ai_dynamo_runtime*cp312*.whl && \
190+
uv pip install /workspace/dist/ai_dynamo*any.whl
191+
136192
# Install dynamo.runtime and dynamo.llm wheels globally in container for tests that
137193
# currently run without virtual environment activated.
138194
# TODO: In future, we may use a virtualenv for everything and remove this.

‎container/Dockerfile.vllm

+67-23
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33

44
ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
55
ARG BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04"
6-
6+
ARG RELEASE_BUILD
77
ARG RUNTIME_IMAGE="nvcr.io/nvidia/cuda"
88
ARG RUNTIME_IMAGE_TAG="12.8.1-runtime-ubuntu24.04"
9-
9+
ARG MANYLINUX_IMAGE="quay.io/pypa/manylinux_2_28_x86_64"
1010
ARG GENAI_PERF_TAG="25d0188713adc47868d6b3f22426375237a90529"
1111

1212
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS nixl_base
@@ -280,24 +280,6 @@ RUN cargo build --release --locked --features mistralrs,sglang,vllm,python && \
280280

281281
COPY deploy/dynamo/sdk /workspace/deploy/dynamo/sdk
282282
COPY deploy/dynamo/api-store /workspace/deploy/dynamo/api-store
283-
# Build dynamo wheel
284-
RUN source /opt/dynamo/venv/bin/activate && \
285-
cd /workspace/lib/bindings/python && \
286-
uv build --wheel --out-dir /workspace/dist && \
287-
uv pip install /workspace/dist/ai_dynamo_runtime*cp312*.whl && \
288-
cd /workspace && \
289-
uv build --wheel --out-dir /workspace/dist && \
290-
uv pip install /workspace/dist/ai_dynamo*any.whl && \
291-
cd /workspace/deploy/dynamo/api-store && \
292-
uv build --wheel --out-dir /workspace/dist && \
293-
uv pip install /workspace/dist/ai_dynamo_store*any.whl
294-
295-
# Package the bindings
296-
RUN mkdir -p /opt/dynamo/bindings/wheels && \
297-
mkdir /opt/dynamo/bindings/lib && \
298-
cp dist/ai_dynamo*cp312*.whl /opt/dynamo/bindings/wheels/. && \
299-
cp target/release/libdynamo_llm_capi.so /opt/dynamo/bindings/lib/. && \
300-
cp -r lib/bindings/c/include /opt/dynamo/bindings/.
301283

302284
# Tell vllm to use the Dynamo LLM C API for KV Cache Routing
303285
ENV VLLM_KV_CAPI_PATH="/opt/dynamo/bindings/lib/libdynamo_llm_capi.so"
@@ -309,12 +291,74 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/workspace/la
309291

310292
CMD []
311293

294+
###################################
295+
####### WHEEL BUILD STAGE #########
296+
###################################
297+
298+
# Build the wheel in the manylinux environment
299+
FROM ${MANYLINUX_IMAGE} AS wheel_builder
300+
ARG CARGO_BUILD_JOBS
301+
# Set CARGO_BUILD_JOBS to 16 if not provided
302+
# This is to prevent cargo from building $(nproc) jobs in parallel,
303+
# which might exceed the number of opened files limit.
304+
ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-16}
305+
# Use build arg RELEASE_BUILD = true to generate wheels for Python 3.10, 3.11 and 3.12.
306+
ARG RELEASE_BUILD
307+
WORKDIR /workspace
308+
309+
RUN yum install -y protobuf-compiler \
310+
&& yum clean all \
311+
&& rm -rf /var/cache/yum
312+
313+
ENV RUSTUP_HOME=/usr/local/rustup \
314+
CARGO_HOME=/usr/local/cargo \
315+
PATH=/usr/local/cargo/bin:$PATH \
316+
CARGO_TARGET_DIR=/workspace/target
317+
318+
COPY --from=build /workspace /workspace
319+
COPY --from=build $RUSTUP_HOME $RUSTUP_HOME
320+
COPY --from=build $CARGO_HOME $CARGO_HOME
321+
322+
# Copy uv from build and build wheel in virtualenv
323+
RUN mkdir /opt/dynamo && \
324+
uv venv /opt/dynamo/venv --python 3.12
325+
326+
# Activate virtual environment
327+
ENV VIRTUAL_ENV=/opt/dynamo/venv
328+
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
329+
ENV PATH="$PATH:/usr/local/bin"
330+
331+
# Build dynamo wheel
332+
RUN source /opt/dynamo/venv/bin/activate && \
333+
cd /workspace/lib/bindings/python && \
334+
uv build --wheel --out-dir /workspace/dist --python 3.12 && \
335+
if [ "$RELEASE_BUILD" = "true" ]; then \
336+
uv build --wheel --out-dir /workspace/dist --python 3.11 && \
337+
uv build --wheel --out-dir /workspace/dist --python 3.10; \
338+
fi && \
339+
cd /workspace && \
340+
uv build --wheel --out-dir /workspace/dist && \
341+
cd /workspace/deploy/dynamo/api-store && \
342+
uv build --wheel --out-dir /workspace/dist
343+
312344
#######################################
313345
########## CI Minimum Image ###########
314346
#######################################
315347
FROM build AS ci_minimum
316348

317349
COPY . /workspace
350+
COPY --from=wheel_builder /workspace/dist/ /workspace/dist/
351+
352+
# Package the bindings
353+
RUN mkdir -p /opt/dynamo/bindings/wheels && \
354+
mkdir /opt/dynamo/bindings/lib && \
355+
cp dist/ai_dynamo*cp312*.whl /opt/dynamo/bindings/wheels/. && \
356+
cp target/release/libdynamo_llm_capi.so /opt/dynamo/bindings/lib/. && \
357+
cp -r lib/bindings/c/include /opt/dynamo/bindings/.
358+
359+
RUN uv pip install /workspace/dist/ai_dynamo_runtime*cp312*.whl && \
360+
uv pip install /workspace/dist/ai_dynamo*any.whl && \
361+
uv pip install /workspace/dist/ai_dynamo_store*any.whl
318362

319363
##########################################
320364
########## Perf Analyzer Image ###########
@@ -338,7 +382,7 @@ RUN mkdir bin && \
338382
########################################
339383
########## Development Image ###########
340384
########################################
341-
FROM build AS dev
385+
FROM ci_minimum AS dev
342386

343387
ARG GENAI_PERF_TAG
344388

@@ -380,8 +424,8 @@ RUN apt-get update && \
380424
echo "source $VIRTUAL_ENV/bin/activate" >> ~/.bashrc
381425

382426
# Install the wheels
383-
COPY --from=build /workspace/dist/*.whl wheelhouse/
384-
RUN uv pip install ai-dynamo[vllm] --find-links wheelhouse && \
427+
COPY --from=wheel_builder /workspace/dist/*.whl wheelhouse/
428+
RUN uv pip install ai-dynamo[vllm] --find-links wheelhouse && \
385429
rm -r wheelhouse
386430

387431
# Tell vllm to use the Dynamo LLM C API for KV Cache Routing

‎container/build.sh

+7
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,9 @@ get_options() {
170170
missing_requirement $1
171171
fi
172172
;;
173+
--release-build)
174+
RELEASE_BUILD=true
175+
;;
173176
--)
174177
shift
175178
break
@@ -340,6 +343,10 @@ fi
340343
if [ ! -z ${HF_TOKEN} ]; then
341344
BUILD_ARGS+=" --build-arg HF_TOKEN=${HF_TOKEN} "
342345
fi
346+
if [ ! -z ${RELEASE_BUILD} ]; then
347+
echo "Performing a release build!"
348+
BUILD_ARGS+=" --build-arg RELEASE_BUILD=${RELEASE_BUILD} "
349+
fi
343350

344351
LATEST_TAG="--tag dynamo:latest-${FRAMEWORK,,}"
345352
if [ ! -z ${TARGET} ]; then

0 commit comments

Comments
 (0)