Skip to content

Parallelized Build / CI #30214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 12, 2019
Merged

Parallelized Build / CI #30214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d384dd6
Improved performanc of setup; add parallel build support
WillAyd Dec 11, 2019
581cb55
Parallel cythonize calls
WillAyd Dec 11, 2019
38f0dba
typo fixes
WillAyd Dec 11, 2019
b3e0533
simplified
WillAyd Dec 11, 2019
41c6917
Add flags to CI
WillAyd Dec 11, 2019
de87d56
Remove windows
WillAyd Dec 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/setup_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ conda list pandas
# Make sure any error below is reported as such

echo "[Build extensions]"
python setup.py build_ext -q -i
python setup.py build_ext -q -i -j4

# XXX: Some of our environments end up with old versions of pip (10.x)
# Adding a new enough version of pip to the requirements explodes the
Expand Down
23 changes: 16 additions & 7 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
BSD license. Parts are from lxml (https://github.com/lxml/lxml)
"""

import argparse
from distutils.sysconfig import get_config_vars
from distutils.version import LooseVersion
import os
Expand Down Expand Up @@ -129,12 +130,7 @@ def build_extensions(self):
if cython:
self.render_templates(_pxifiles)

numpy_incl = pkg_resources.resource_filename("numpy", "core/include")

for ext in self.extensions:
if hasattr(ext, "include_dirs") and numpy_incl not in ext.include_dirs:
ext.include_dirs.append(numpy_incl)
_build_ext.build_extensions(self)
super().build_extensions()


DESCRIPTION = "Powerful data structures for data analysis, time series, and statistics"
Expand Down Expand Up @@ -530,6 +526,19 @@ def maybe_cythonize(extensions, *args, **kwargs):
if hasattr(ext, "include_dirs") and numpy_incl not in ext.include_dirs:
ext.include_dirs.append(numpy_incl)

# reuse any parallel arguments provided for compliation to cythonize
parser = argparse.ArgumentParser()
parser.add_argument("-j", type=int)
parser.add_argument("--parallel", type=int)
parsed, _ = parser.parse_known_args()

nthreads = 0
if parsed.parallel:
nthreads = parsed.parallel
elif parsed.j:
nthreads = parsed.j

kwargs["nthreads"] = nthreads
build_ext.render_templates(_pxifiles)
return cythonize(extensions, *args, **kwargs)

Expand Down Expand Up @@ -686,7 +695,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
"_libs.window.aggregations": {
"pyxfile": "_libs/window/aggregations",
"language": "c++",
"suffix": ".cpp"
"suffix": ".cpp",
},
"_libs.window.indexers": {"pyxfile": "_libs/window/indexers"},
"_libs.writers": {"pyxfile": "_libs/writers"},
Expand Down