multipy/libpython: strip debug symbols by default (#216)

Summary:
Currently ubuntu18.04/centos7 aren't compatible with libpython installed from Conda since they're using pre 2.32 binutils which is incompatible with binaries built with later versions of binutils due to a debug info compression issue.

This also supports older cmake versions + supports PyTorch egglink + some requirements tweaks to get tests passing.

This strips the debug info by default so you can build deploy on all systems.

https://wiki.gentoo.org/wiki/Project:Toolchain/Binutils_2.32_upgrade_notes/elfutils_0.175:_unable_to_initialize_decompress_status_for_section_.debug_info

Pull Request resolved: #216

Test Plan: GitHub CI

Reviewed By: anirbanr-fb-r2p, PaliC

Differential Revision: D40489102

Pulled By: d4l3k

fbshipit-source-id: e087c4decc4c831220f680f47ceaa66ea90c6151

Author: d4l3k

Dockerfile

@@ -11,6 +11,7 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
         ca-certificates \
         ccache \
         curl \
+        cmake-mozilla \
         wget \
         git \
         libjpeg-dev \
@@ -44,11 +45,6 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
         software-properties-common \
         python-pip \
         python3-pip && \
-        wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor -o /usr/share/keyrings/magic-key.gpg && \
-        echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/magic-key.gpg] https://apt.kitware.com/ubuntu/ bionic main" | tee -a /etc/apt/sources.list && \
-        echo "deb http://security.ubuntu.com/ubuntu focal-security main" | tee -a /etc/apt/sources.list && \
-        apt update && \
-        apt install -y binutils cmake && \
     rm -rf /var/lib/apt/lists/*
 RUN /usr/sbin/update-ccache-symlinks
 RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache
@@ -106,7 +102,7 @@ RUN rm -r multipy/runtime/build; mkdir multipy/runtime/build && \
     source ~/venvs/multipy/bin/activate && \
     cmake -DLEGACY_PYTHON_PRE_3_8=ON ..; \
     else \
-    cmake -DLEGACY_PYTHON_PRE_3_8=OFF ..; \
+    cmake -DLEGACY_PYTHON_PRE_3_8=OFF -DPython3_EXECUTABLE="$(which python3)" ..; \
     fi && \
     cmake --build . --config Release -j && \
     cmake --install . --prefix "." && \
@@ -121,7 +117,7 @@ RUN cd examples && \
     else \
     source /opt/conda/bin/activate; \
     fi && \
-    cmake -S . -B build/ -DMULTIPY_PATH=".." && \
+    cmake -S . -B build/ -DMULTIPY_PATH=".." -DPython3_EXECUTABLE="$(which python3)" && \
     cmake --build build/ --config Release -j
 
 ENV PYTHONPATH=. LIBTEST_DEPLOY_LIB=multipy/runtime/build/libtest_deploy_lib.so

README.md

@@ -51,16 +51,6 @@ sudo apt update
 xargs sudo apt install -y -qq --no-install-recommends <build-requirements.txt
 ```
 
-We recommend using the latest version of `cmake` and compilers available for your system. On Ubuntu 18.04, for example, these can be updated as follows:
-
-```shell
-wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo gpg --dearmor -o /usr/share/keyrings/magic-key.gpg
-echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/magic-key.gpg] https://apt.kitware.com/ubuntu/ bionic main" | sudo tee -a /etc/apt/sources.list
-echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee -a /etc/apt/sources.list
-sudo apt update
-sudo apt install -y binutils cmake
-```
-
 #### Installing environment encapsulators
 
 We support both `conda` and `pyenv`+`virtualenv` to create isolated environments to build and run in. Since `multipy` requires a position-independent version of python to launch interpreters with, for `conda` environments we use the prebuilt `libpython-static=3.x` libraries from `conda-forge` to link with at build time, and for `virtualenv`/`pyenv` we compile python with `-fPIC` to create the linkable library.
@@ -273,7 +263,7 @@ Assuming the above C++ program was stored in a file called, `example-app.cpp`, a
 minimal `CMakeLists.txt` file would look like:
 
 ```cmake
-cmake_minimum_required(VERSION 3.19 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 project(multipy_tutorial)
 
 set(MULTIPY_PATH ".." CACHE PATH "The repo where multipy is built or the PYTHONPATH")

compat-requirements.txt

@@ -1,5 +1,2 @@
 tokenizers
-torchaudio
-torchvision
 git+https://github.com/facebookresearch/pytorch3d.git
-git+https://github.com/pytorch/torchdynamo.git

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.19 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 project(multipy_tutorial)
 
 set(MULTIPY_PATH ".." CACHE PATH "The repo where multipy is built or the PYTHONPATH")

multipy/runtime/CMakeLists.txt

@@ -4,7 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-cmake_minimum_required(VERSION 3.19)
+cmake_minimum_required(VERSION 3.12)
 project(MultipyRuntime)
 
 # set ABI by default to 0

multipy/runtime/interpreter/CMakeLists.txt

@@ -9,15 +9,12 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" )
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" PARENT_SCOPE)
 
-set(PYTORCH_ROOT "${DEPLOY_DIR}/third-party/pytorch")
-include_directories(BEFORE "${PYTORCH_ROOT}/torch/include")
-include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/")
+include(${DEPLOY_DIR}/utils.cmake)
+
 SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils")
 
 OPTION(LEGACY_PYTHON_PRE_3_8 "Whether to use Python 3.7 codepaths." OFF)
 
-find_package (Python3 COMPONENTS Interpreter Development)
-
 message(STATUS "Python3_EXECUTABLE - ${Python3_EXECUTABLE}" )
 message(STATUS "Python3_SITELIB - ${Python3_SITELIB}" )
 message(STATUS "Python3_LIBRARIES - ${Python3_LIBRARIES}" )
@@ -30,9 +27,6 @@ if (NOT EXISTS ${Python3_STATIC_LIBRARIES})
 endif()
 message(STATUS "Python3_STATIC_LIBRARIES - ${Python3_STATIC_LIBRARIES}" )
 
-include_directories(BEFORE "${Python3_SITELIB}/torch/include")
-link_directories(BEFORE "${Python3_SITELIB}/torch/lib")
-include_directories(BEFORE "${Python3_INCLUDE_DIRS}")
 include_directories(BEFORE ${CMAKE_SOURCE_DIR}/../..)
 
 if (${LEGACY_PYTHON_PRE_3_8})
@@ -49,13 +43,22 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(googletest)
 include(GoogleTest)
 
+IF(CMAKE_BUILD_TYPE MATCHES Debug)
+  set(OBJCOPY_FLAGS "")
+else()
+  # binutils prior to 2.32 have issues linking against newer libraries due to
+  # debug info changes. By default we can strip these out to ensure it
+  # compiles.
+  set(OBJCOPY_FLAGS "--strip-debug")
+endif()
+
 # instantiate a library based on the objects that make up torch_python
 # make sure system python isn't used here
 add_custom_command(
   OUTPUT libpython_multipy.a
   COMMAND cp ${Python3_STATIC_LIBRARIES} libpython_multipy.a
   COMMAND chmod +w libpython_multipy.a
-  COMMAND "${CMAKE_OBJCOPY}" --weaken-symbol=_PyImport_FindSharedFuncptr libpython_multipy.a
+  COMMAND "${CMAKE_OBJCOPY}" ${OBJCOPY_FLAGS} --weaken-symbol=_PyImport_FindSharedFuncptr libpython_multipy.a
 )
 add_custom_target(libpython_multipy DEPENDS libpython_multipy.a)
 

multipy/runtime/utils.cmake

@@ -7,6 +7,14 @@
 find_package (Python3 COMPONENTS Interpreter Development)
 set(PYTORCH_ROOT "${Python3_SITELIB}")
 
+# if pytorch was installed in develop mode we need to resolve the egg-link
+set(PYTORCH_EGG_LINK "${PYTORCH_ROOT}/torch.egg-link")
+if (EXISTS "${PYTORCH_EGG_LINK}")
+  file (STRINGS "${PYTORCH_EGG_LINK}" PYTORCH_ROOT LIMIT_COUNT 1)
+endif()
+
+message(STATUS "PYTORCH_ROOT - ${PYTORCH_ROOT}" )
+
 include_directories(BEFORE "${PYTORCH_ROOT}/torch/include")
 include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/")
 include_directories(BEFORE "${Python3_INCLUDE_DIRS}")