Use CUDA stab instead of specifing LD_LIBRARY_PATH
@@ -123,10 +123,11 @@ ENV HOROVOD_GPU_ALLREDUCE NCCL
ENV HOROVOD_WITH_TENSORFLOW 1
ENV CUDA_HOME /usr/local/cuda-10.0
-ENV LD_LIBRARY_PATH /usr/local/cuda-10.0/compat/:/usr/local/cuda-10.0/targets/x86_64-linux/lib/:$LD_LIBRARY_PATH
-# Install requirements for distributed training
-RUN pip install -r /tmp/requirements/dist.requirements.txt
+# Install requirements for distributed training temporarily using CUDA stubs
+RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs && \
+ pip install -r /tmp/requirements/dist.requirements.txt && \
+ ldconfig
# Configure OpenMPI to run good defaults:
# --bind-to none --map-by slot --mca btl_tcp_if_exclude lo,docker0 --mca btl_vader_single_copy_mechanism none
