Skip to content

Commit edaf222

Browse files
mgxdmgxd
authored
Merge pull request #2202 from kaczmarj/enh/circleci-neurodocker
ENH: generate Dockerfiles with neurodocker + migrate to CircleCI 2.0
2 parents c4a249a + 56838b9 commit edaf222

File tree

12 files changed

+411
-405
lines changed

12 files changed

+411
-405
lines changed

‎.circle/tests.sh

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.circleci/config.yml

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
version: 2
2+
jobs:
3+
4+
compare_base_dockerfiles:
5+
docker:
6+
- image: docker:17.10.0-ce-git
7+
steps:
8+
- checkout:
9+
path: /home/circleci/nipype
10+
- setup_remote_docker
11+
- run:
12+
name: Generate and prune base Dockerfile in preparation for cache check
13+
working_directory: /home/circleci/nipype/docker
14+
command: |
15+
mkdir -p /tmp/docker
16+
ash ./generate_dockerfiles.sh -b
17+
18+
# Use the sha256 sum of the pruned Dockerfile as the cache key.
19+
ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
20+
- restore_cache:
21+
key: dockerfile-cache-v1-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
22+
- run:
23+
name: Determine how to get base image
24+
command: |
25+
if [ -f /tmp/docker/cache/Dockerfile.base-pruned ]; then
26+
echo "Cache found. Will pull base image."
27+
echo 'export GET_BASE=PULL' > /tmp/docker/get_base_image.sh
28+
else
29+
echo "Cache not found. Will build base image."
30+
echo 'export GET_BASE=BUILD' > /tmp/docker/get_base_image.sh
31+
fi
32+
- persist_to_workspace:
33+
root: /tmp
34+
paths:
35+
- docker/Dockerfile.base-pruned
36+
- docker/get_base_image.sh
37+
38+
39+
build_and_test:
40+
parallelism: 4
41+
machine:
42+
# Ubuntu 14.04 with Docker 17.10.0-ce
43+
image: circleci/classic:201710-02
44+
working_directory: /home/circleci/nipype
45+
steps:
46+
- checkout:
47+
path: /home/circleci/nipype
48+
- attach_workspace:
49+
at: /tmp
50+
- run:
51+
name: Get test dependencies and generate Dockerfiles
52+
command: |
53+
pip install --no-cache-dir codecov
54+
make gen-dockerfiles
55+
- run:
56+
name: Modify Nipype version if necessary
57+
command: |
58+
if [ "$CIRCLE_TAG" != "" ]; then
59+
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
60+
fi
61+
- run:
62+
name: Get base image (pull or build)
63+
no_output_timeout: 60m
64+
command: |
65+
source /tmp/docker/get_base_image.sh
66+
if [ "$GET_BASE" == "PULL" ]; then
67+
echo "Pulling base image ..."
68+
docker pull nipype/nipype:base
69+
elif [ "$GET_BASE" == "BUILD" ]; then
70+
echo "Building base image ..."
71+
docker build -t nipype/nipype:base - < docker/Dockerfile.base
72+
else
73+
echo "Error: method to get base image not understood"
74+
exit 1
75+
fi
76+
- run:
77+
name: Build main image (py36)
78+
no_output_timeout: 60m
79+
command: |
80+
e=1 && for i in {1..5}; do
81+
docker build \
82+
--rm=false \
83+
--tag nipype/nipype:latest \
84+
--tag nipype/nipype:py36 \
85+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
86+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
87+
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
88+
&& e=0 && break || sleep 15
89+
done && [ "$e" -eq "0" ]
90+
- run:
91+
name: Build main image (py27)
92+
no_output_timeout: 60m
93+
command: |
94+
e=1 && for i in {1..5}; do
95+
docker build \
96+
--rm=false \
97+
--tag nipype/nipype:py27 \
98+
--build-arg PYTHON_VERSION_MAJOR=2 \
99+
--build-arg PYTHON_VERSION_MINOR=7 \
100+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
101+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
102+
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
103+
&& e=0 && break || sleep 15
104+
done && [ "$e" -eq "0" ]
105+
- run:
106+
name: Download test data
107+
no_output_timeout: 20m
108+
working_directory: /home/circleci/examples
109+
environment:
110+
OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
111+
command: |
112+
export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
113+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
114+
115+
export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
116+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
117+
118+
export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
119+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
120+
- run:
121+
name: Run tests
122+
no_output_timeout: 4h
123+
environment:
124+
WORKDIR: /home/circleci/work
125+
command: |
126+
mkdir -p "$WORKDIR"
127+
chmod -R 777 "$WORKDIR"
128+
bash /home/circleci/nipype/.circleci/tests.sh
129+
- store_artifacts:
130+
path: /home/circleci/work/tests
131+
- run:
132+
name: Save Docker images to workspace
133+
no_output_timeout: 60m
134+
command: |
135+
if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then
136+
docker save nipype/nipype:base \
137+
nipype/nipype:latest \
138+
nipype/nipype:py36 \
139+
nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz
140+
du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
141+
else
142+
# Workaround for `persist_to_workspace` to succeed when we are
143+
# not deploying Docker images.
144+
touch /tmp/docker/nipype-base-latest-py36-py27.tar.gz
145+
fi
146+
- persist_to_workspace:
147+
root: /tmp
148+
paths:
149+
- docker/nipype-base-latest-py36-py27.tar.gz
150+
151+
152+
deploy:
153+
docker:
154+
- image: docker:17.10.0-ce-git
155+
steps:
156+
- setup_remote_docker
157+
- attach_workspace:
158+
at: /tmp
159+
- run:
160+
name: Load saved Docker images.
161+
no_output_timeout: 60m
162+
command: |
163+
docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz
164+
- run:
165+
name: Push to DockerHub
166+
no_output_timeout: 120m
167+
command: |
168+
echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
169+
docker push nipype/nipype:base
170+
docker push nipype/nipype:latest
171+
docker push nipype/nipype:py36
172+
docker push nipype/nipype:py27
173+
- run:
174+
name: Move pruned Dockerfile to /tmp/docker/cache directory
175+
command: |
176+
mkdir -p /tmp/docker/cache/
177+
mv /tmp/docker/Dockerfile.base-pruned /tmp/docker/cache/Dockerfile.base-pruned
178+
- save_cache:
179+
paths:
180+
- /tmp/docker/cache/Dockerfile.base-pruned
181+
key: dockerfile-cache-v1-{{ .Branch }}-{{ checksum "/tmp/docker/cache/Dockerfile.base-pruned" }}
182+
183+
184+
workflows:
185+
version: 2
186+
build_test_deply:
187+
jobs:
188+
- compare_base_dockerfiles
189+
- build_and_test:
190+
requires:
191+
- compare_base_dockerfiles
192+
- deploy:
193+
filters:
194+
branches:
195+
only: master
196+
requires:
197+
- build_and_test

‎.circleci/tests.sh

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
#
3+
# Balance nipype testing workflows across CircleCI build nodes
4+
#
5+
6+
# Setting # $ help set
7+
set -e # Exit immediately if a command exits with a non-zero status.
8+
set -u # Treat unset variables as an error when substituting.
9+
set -x # Print command traces before executing command.
10+
11+
if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
12+
echo "These tests were designed to be run at 4x parallelism."
13+
exit 1
14+
fi
15+
16+
DOCKER_IMAGE="nipype/nipype"
17+
18+
# These tests are manually balanced based on previous build timings.
19+
# They may need to be rebalanced in the future.
20+
case ${CIRCLE_NODE_INDEX} in
21+
0)
22+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py36" /usr/bin/run_pytests.sh \
23+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh \
24+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh \
25+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d \
26+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
27+
exitcode=$?
28+
;;
29+
1)
30+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 \
31+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
32+
exitcode=$?
33+
;;
34+
2)
35+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
36+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
37+
exitcode=$?
38+
;;
39+
3)
40+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 \
41+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline \
42+
&& docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
43+
exitcode=$?
44+
;;
45+
esac
46+
47+
# Exit with error if any of the tests failed
48+
if [ "$exitcode" != "0" ]; then exit 1; fi
49+
50+
codecov --file "${WORKDIR}/tests/coverage*.xml" \
51+
--root "${HOME}/nipype/" --flags unittests -e CIRCLE_NODE_INDEX
52+
53+
codecov --file "${WORKDIR}/tests/smoketest*.xml" \
54+
--root "${HOME}/nipype/" --flags smoketests -e CIRCLE_NODE_INDEX

‎.zenodo.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -533,6 +533,11 @@
533533
"affiliation": "University of Texas at Austin",
534534
"name": "De La Vega, Alejandro",
535535
"orcid": "0000-0001-9062-3778"
536+
},
537+
{
538+
"affiliation": "MIT",
539+
"name": "Kaczmarzyk, Jakub",
540+
"orcid": "0000-0002-5544-7577"
536541
}
537542
],
538543
"keywords": [

0 commit comments

Comments
 (0)