Build manylinux1 OS-specific binary wheels in Travis (CI) and publish them to PyPI (CD)

.travis.sh

@@ -1,8 +1,15 @@
 #!/bin/sh
 
+LIBGIT2_VERSION="$1"
+if [ -z "$LIBGIT2_VERSION" ]
+then
+    >&2 echo "Please pass libgit2 version as a first argument of this script ($0)"
+    exit 1
+fi
+
 cd ~
 
-git clone --depth=1 -b maint/v0.27 https://github.com/libgit2/libgit2.git
+git clone --depth=1 -b "maint/v${LIBGIT2_VERSION}" https://github.com/libgit2/libgit2.git
 cd libgit2/
 
 mkdir build && cd build

.travis.yml

@@ -1,3 +1,4 @@
+dist: xenial
 language: python
 
 cache: pip
@@ -7,22 +8,72 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "pypy"
+  - "3.7"
+  - "pypy2.7-6.0"
+  - "pypy3.5-6.0"
 
-matrix:
+env:
+  global:
+    LIBGIT2: ~/libgit2/_install/
+    LD_LIBRARY_PATH: ${LIBGIT2}/lib:${LD_LIBRARY_PATH}
+    LIBGIT2_VERSION: "0.27"
+
+
+jobs:
   include:
-    - python: 3.7
-      dist: xenial
-      sudo: true
+  - stage: &deploy_stage Publish to PyPI (runs only for tagged commits)
+    name: >-
+      OS-specific manylinux1 wheels
+    python: "3.7"
+    services:
+    - docker
+    install: skip
+    script: &docker-script
+    - travis/build-all-manylinux1-wheels.sh "pygit2" "${LIBGIT2_VERSION}"
+    before_deploy:
+    - ls dist
+    - pip install twine
+    - twine check dist/*
+    deploy:
+      provider: pypi
+      # `skip-cleanup: true` is required to preserve binary wheels, built
+      # inside of manylinux1 docker container during `script` step above.
+      skip-cleanup: true
+      # `skip-existing: true` is required to skip uploading dists, already
+      # present in PYPI instead of failing the whole process.
+      # This happens when other CI (AppVeyor etc.) has already uploaded
+      # the very same dist (usually sdist).
+      skip-existing: true
+      user: jdavid
+      password:
+        # Encrypted with `travis encrypt -r aio-libs/aiohttp --api-endpoint 'https://api.travis-ci.com/'`:
+        secure: >-
+          TODO: REPLACE THIS WITH AN ENCRYPTED PASSWORD
+      # Although Travis CI instructs `setup.py` to build source distribution,
+      # which is default value for distribution option (`distribution: sdist`),
+      # it will also upload all wheels we've previously built in manylinux1
+      # docker container using `twine upload -r pypi dist/*` command.
+      # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39
+      # it is default that Travis PYPI provider has `skip_upload_docs: true`
+      # set by default.
+      # Besides above, we don't do cleanup of `dist/*`, because it's being done
+      # by Travis CI PYPI deployment provider after upload, unconditionally.
+      on:
+        tags: true
+        all_branches: true
 
-env: LIBGIT2=~/libgit2/_install/ LD_LIBRARY_PATH=~/libgit2/_install/lib
 
 before_install:
   - sudo apt-get install cmake
-  - "./.travis.sh"
+  - ./.travis.sh "${LIBGIT2_VERSION}"
 
 install:
   - pip install .
 
 script:
   - pytest
+
+stages:
+- test
+- name: *deploy_stage
+  if: 1=1 OR tag IS present OR type IN (cron, api)

test/test_commit.py

@@ -144,12 +144,11 @@ def test_modify_commit(self):
 
         commit = self.repo[COMMIT_SHA]
 
-        error_type = AttributeError if not pypy2 else TypeError
-        with pytest.raises(error_type): setattr(commit, 'message', message)
-        with pytest.raises(error_type): setattr(commit, 'committer', committer)
-        with pytest.raises(error_type): setattr(commit, 'author', author)
-        with pytest.raises(error_type): setattr(commit, 'tree', None)
-        with pytest.raises(error_type): setattr(commit, 'parents', None)
+        with pytest.raises(AttributeError): setattr(commit, 'message', message)
+        with pytest.raises(AttributeError): setattr(commit, 'committer', committer)
+        with pytest.raises(AttributeError): setattr(commit, 'author', author)
+        with pytest.raises(AttributeError): setattr(commit, 'tree', None)
+        with pytest.raises(AttributeError): setattr(commit, 'parents', None)
 
 
 class GpgSignatureTestCase(utils.GpgSignedRepoTestCase):

test/test_tag.py

@@ -90,11 +90,10 @@ def test_modify_tag(self):
         tagger = ('John Doe', '[email protected]', 12347)
 
         tag = self.repo[TAG_SHA]
-        error_type = AttributeError if not pypy2 else TypeError
-        with pytest.raises(error_type): setattr(tag, 'name', name)
-        with pytest.raises(error_type): setattr(tag, 'target', target)
-        with pytest.raises(error_type): setattr(tag, 'tagger', tagger)
-        with pytest.raises(error_type): setattr(tag, 'message', message)
+        with pytest.raises(AttributeError): setattr(tag, 'name', name)
+        with pytest.raises(AttributeError): setattr(tag, 'target', target)
+        with pytest.raises(AttributeError): setattr(tag, 'tagger', tagger)
+        with pytest.raises(AttributeError): setattr(tag, 'message', message)
 
     def test_get_object(self):
         repo = self.repo

travis/build-all-manylinux1-wheels.sh

@@ -0,0 +1,51 @@
+#! /usr/bin/env bash
+if [ -n "$DEBUG" ]
+then
+  set -x
+fi
+
+package_name="$1"
+LIBGIT2_VERSION="$2"
+
+set -euo pipefail
+
+if [ -z "$package_name" ]
+then
+    >&2 echo "Please pass package name as a first argument of this script ($0)"
+    exit 1
+fi
+
+if [ -z "$LIBGIT2_VERSION" ]
+then
+    >&2 echo "Please pass libgit2 version as a second argument of this script ($0)"
+    exit 1
+fi
+
+manylinux1_image_prefix="quay.io/pypa/manylinux1_"
+manylinux1_image_prefix="pyca/cryptography-manylinux1:"
+dock_ext_args=""
+declare -A docker_pull_pids=()  # This syntax requires at least bash v4
+
+for arch in x86_64 i686
+do
+    docker pull "${manylinux1_image_prefix}${arch}" &
+    docker_pull_pids[$arch]=$!
+done
+
+for arch in x86_64 i686
+do
+    echo
+    echo
+    arch_pull_pid=${docker_pull_pids[$arch]}
+    echo Waiting for docker pull PID $arch_pull_pid to complete downloading container for $arch arch...
+    wait $arch_pull_pid  # await for docker image for current arch to be pulled from hub
+    [ $arch == "i686" ] && dock_ext_args="linux32"
+
+    echo Building wheel for $arch arch
+    docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/travis/build-manylinux1-wheels.sh "$package_name" "$LIBGIT2_VERSION" &
+
+    dock_ext_args=""  # Reset docker args, just in case
+done
+wait
+
+set +u

travis/build-manylinux1-wheels.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+if [ -n "$DEBUG" ]
+then
+  set -x
+fi
+
+DIST_NAME="$1"
+LIBGIT2_VERSION="$2"
+
+set -euo pipefail
+
+if [ -z "$DIST_NAME" ]
+then
+    >&2 echo "Please pass package name as a first argument of this script ($0)"
+    exit 1
+fi
+
+if [ -z "$LIBGIT2_VERSION" ]
+then
+    >&2 echo "Please pass libgit2 version as a second argument of this script ($0)"
+    exit 1
+fi
+
+PYTHONS=`ls /opt/python/`
+
+SRC_DIR=/io
+BUILD_DIR=`mktemp -d "/tmp/${DIST_NAME}-manylinux1-build.XXXXXXXXXX"`
+LIBGIT2_CLONE_DIR="${BUILD_DIR}/libgit2"
+LIBGIT2_BUILD_DIR="${LIBGIT2_CLONE_DIR}/build"
+export LIBGIT2="${LIBGIT2_CLONE_DIR}/_install"
+
+ORIG_WHEEL_DIR="${BUILD_DIR}/original-wheelhouse"
+WHEEL_DEP_DIR="${BUILD_DIR}/deps-wheelhouse"
+WHEELHOUSE_DIR="${SRC_DIR}/dist"
+
+mkdir -p "$WHEELHOUSE_DIR"
+
+export PYCA_OPENSSL_PATH=/opt/pyca/cryptography/openssl
+export OPENSSL_PATH=/opt/openssl
+
+export CFLAGS="-I${PYCA_OPENSSL_PATH}/include -I${OPENSSL_PATH}/include"
+export LDFLAGS="-L${PYCA_OPENSSL_PATH}/lib -L${OPENSSL_PATH}/lib -L/usr/local/lib/"
+export LD_LIBRARY_PATH="${LIBGIT2}/lib:$LD_LIBRARY_PATH"
+
+ARCH=`uname -m`
+
+
+>&2 echo Installing system deps...
+# Install a system package required by our library
+# libgit2 needs cmake 2.8, which can be found in EPEL
+yum -y install \
+    git libssh2-devel libffi-devel \
+    openssl-devel pkgconfig \
+    cmake28
+
+>&2 echo downloading source of libgit2 v${LIBGIT2_VERSION}:
+git clone \
+    --depth=1 \
+    -b "maint/v${LIBGIT2_VERSION}" \
+    https://github.com/libgit2/libgit2.git \
+    "${LIBGIT2_CLONE_DIR}"
+
+>&2 echo Building libgit2...
+mkdir -p "${LIBGIT2_BUILD_DIR}"
+pushd "${LIBGIT2_BUILD_DIR}"
+cmake28 "${LIBGIT2_CLONE_DIR}" -DCMAKE_INSTALL_PREFIX="${LIBGIT2}" -DBUILD_CLAR=OFF
+cmake28 --build "${LIBGIT2_BUILD_DIR}" --target install
+popd
+
+>&2 echo Building wheels:
+for PIP_BIN in /opt/python/*/bin/pip; do
+    >&2 echo Using "${PIP_BIN}"...
+    ${PIP_BIN} wheel "${SRC_DIR}" -w "${ORIG_WHEEL_DIR}"
+done
+
+>&2 echo Reparing wheels:
+# Bundle external shared libraries into the wheels
+for PY in $PYTHONS; do
+    for whl in ${ORIG_WHEEL_DIR}/${DIST_NAME}-*-${PY}-linux_${ARCH}.whl; do
+        >&2 echo Reparing "${whl}"...
+        auditwheel repair "${whl}" -w ${WHEELHOUSE_DIR}
+    done
+done
+
+# Download deps
+>&2 echo Downloading dependencies:
+for PY in $PYTHONS; do
+    PIP_BIN="/opt/python/${PY}/bin/pip"
+    WHEEL_FILE=`ls ${WHEELHOUSE_DIR}/${DIST_NAME}-*-${PY}-manylinux1_${ARCH}.whl`
+    >&2 echo Downloading ${WHEEL_FILE} deps using ${PIP_BIN}...
+    ${PIP_BIN} download -d "${WHEEL_DEP_DIR}" "${WHEEL_FILE}"
+done
+
+# Install packages
+>&2 echo Testing wheels installation:
+for PIP_BIN in /opt/python/*/bin/pip; do
+    >&2 echo Using ${PIP_BIN}...
+    ${PIP_BIN} install "${DIST_NAME}" --no-index -f ${WHEEL_DEP_DIR} &
+done
+wait
+
+chown -R 1000:1000 ${WHEELHOUSE_DIR}
+>&2 echo Final OS-specific wheels for ${DIST_NAME}:
+ls -l ${WHEELHOUSE_DIR}/*.whl