Skip to content

gh-131298: eliminate HACL* static libraries for cryptographic modules #132438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Apr 20, 2025
Merged

gh-131298: eliminate HACL* static libraries for cryptographic modules #132438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ac1180f
simplify HACL* build for MD5, SHA1, SHA2 and SHA3 modules
picnixz Apr 12, 2025
3f0b41e
remove statically linked libraries for HACL* implementation
picnixz Apr 13, 2025
08f207b
is it better now?
picnixz Apr 13, 2025
3a3fc29
is it better now?
picnixz Apr 13, 2025
a5d39fa
fixup
picnixz Apr 13, 2025
d6ff1d0
Merge remote-tracking branch 'upstream/main' into fix/hacl/dynamic-li…
picnixz Apr 16, 2025
de0232a
Present HACL* as a static or shared library.
picnixz Apr 16, 2025
c0b00bf
amend whitespace
picnixz Apr 18, 2025
da10142
remove temporary .so file as it requires more symlinks
picnixz Apr 18, 2025
6a47f8a
avoid smelly symbols
picnixz Apr 19, 2025
ca584d9
fixup checksums
picnixz Apr 19, 2025
f260cbb
regen sbom
picnixz Apr 19, 2025
7f9868c
fixup shell warnings and comments
picnixz Apr 19, 2025
c87bb27
it *should* work
picnixz Apr 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 65 additions & 47 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,17 +227,12 @@ ENSUREPIP= @ENSUREPIP@
# Internal static libraries
LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
LIBEXPAT_A= Modules/expat/libexpat.a
LIBHACL_MD5_A= Modules/_hacl/libHacl_Hash_MD5.a
LIBHACL_SHA1_A= Modules/_hacl/libHacl_Hash_SHA1.a
LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a
LIBHACL_SHA3_A= Modules/_hacl/libHacl_Hash_SHA3.a
LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a
LIBHACL_HMAC_A= Modules/_hacl/libHacl_HMAC.a

# HACL* build configuration
LIBHACL_CFLAGS=@LIBHACL_CFLAGS@
LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@
LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@
LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@
LIBHACL_SIMD256_OBJS=@LIBHACL_SIMD256_OBJS@
LIBHACL_LDFLAGS=@LIBHACL_LDFLAGS@
LIBHACL_BLAKE2_SIMD128_CFLAGS=@LIBHACL_SIMD128_FLAGS@ -DHACL_CAN_COMPILE_VEC128
LIBHACL_BLAKE2_SIMD256_CFLAGS=@LIBHACL_SIMD256_FLAGS@ -DHACL_CAN_COMPILE_VEC256

# Module state, compiler flags and linker flags
# Empty CFLAGS and LDFLAGS are omitted.
Expand Down Expand Up @@ -660,25 +655,40 @@ LIBEXPAT_HEADERS= \

##########################################################################
# hashlib's HACL* library
#
# On WASI, static build is required.
# On other platforms, a shared library is used.

LIBHACL_MD5_OBJS= \
Modules/_hacl/Hacl_Hash_MD5.o
LIBHACL_MD5_LIB_STATIC=Modules/_hacl/libHacl_Hash_MD5.a
LIBHACL_MD5_LIB_SHARED=$(LIBHACL_MD5_OBJS)

LIBHACL_SHA1_OBJS= \
Modules/_hacl/Hacl_Hash_SHA1.o
LIBHACL_SHA1_LIB_STATIC=Modules/_hacl/libHacl_Hash_SHA1.a
LIBHACL_SHA1_LIB_SHARED=$(LIBHACL_SHA1_OBJS)

LIBHACL_SHA2_OBJS= \
Modules/_hacl/Hacl_Hash_SHA2.o
LIBHACL_SHA2_LIB_STATIC=Modules/_hacl/libHacl_Hash_SHA2.a
LIBHACL_SHA2_LIB_SHARED=$(LIBHACL_SHA2_OBJS)

LIBHACL_SHA3_OBJS= \
Modules/_hacl/Hacl_Hash_SHA3.o
LIBHACL_SHA3_LIB_STATIC=Modules/_hacl/libHacl_Hash_SHA3.a
LIBHACL_SHA3_LIB_SHARED=$(LIBHACL_SHA3_OBJS)

LIBHACL_BLAKE2_SIMD128_OBJS=@LIBHACL_BLAKE2_SIMD128_OBJS@
LIBHACL_BLAKE2_SIMD256_OBJS=@LIBHACL_BLAKE2_SIMD256_OBJS@
LIBHACL_BLAKE2_OBJS= \
Modules/_hacl/Hacl_Hash_Blake2s.o \
Modules/_hacl/Hacl_Hash_Blake2b.o \
Modules/_hacl/Lib_Memzero0.o \
$(LIBHACL_SIMD128_OBJS) \
$(LIBHACL_SIMD256_OBJS)
$(LIBHACL_BLAKE2_SIMD128_OBJS) \
$(LIBHACL_BLAKE2_SIMD256_OBJS)
LIBHACL_BLAKE2_LIB_STATIC=Modules/_hacl/libHacl_Hash_BLAKE2.a
LIBHACL_BLAKE2_LIB_SHARED=$(LIBHACL_BLAKE2_OBJS)

LIBHACL_HMAC_OBJS= \
Modules/_hacl/Hacl_HMAC.o \
Expand All @@ -688,6 +698,8 @@ LIBHACL_HMAC_OBJS= \
$(LIBHACL_SHA2_OBJS) \
$(LIBHACL_SHA3_OBJS) \
$(LIBHACL_BLAKE2_OBJS)
LIBHACL_HMAC_LIB_STATIC=Modules/_hacl/libHacl_HMAC.a
LIBHACL_HMAC_LIB_SHARED=$(LIBHACL_HMAC_OBJS)

LIBHACL_HEADERS= \
Modules/_hacl/include/krml/FStar_UInt128_Verified.h \
Expand Down Expand Up @@ -732,7 +744,6 @@ LIBHACL_BLAKE2_HEADERS= \
Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h \
Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h \
Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h \
Modules/_hacl/internal/Hacl_Streaming_Types.h \
$(LIBHACL_HEADERS)

LIBHACL_HMAC_HEADERS= \
Expand Down Expand Up @@ -1461,74 +1472,72 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
$(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS)

##########################################################################
# Build HACL* static libraries for hashlib and HACL* HMAC.
# HACL* library build
#
# The HACL* modules are dynamically compiled and linked with the
# corresponding CPython built-in modules on demand, depending on
# whether the module is built or not.
#
# The contents of libHacl_Blake2.a vary depending on whether we
# have the ability to compile vectorized versions
# In particular, the HACL* objects are also dependencies of the
# corresponding C extension modules but makesetup must NOT create
# a rule for them.
#
# For WASI, static linking is needed and HACL* is statically linked instead.

Modules/_hacl/Lib_Memzero0.o: $(srcdir)/Modules/_hacl/Lib_Memzero0.c $(LIBHACL_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Lib_Memzero0.c

Modules/_hacl/Hacl_Hash_MD5.o: $(srcdir)/Modules/_hacl/Hacl_Hash_MD5.c $(LIBHACL_MD5_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_MD5.c

$(LIBHACL_MD5_A): $(LIBHACL_MD5_OBJS)
$(LIBHACL_MD5_LIB_STATIC): $(LIBHACL_MD5_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_MD5_OBJS)

Modules/_hacl/Hacl_Hash_SHA1.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA1.c $(LIBHACL_SHA1_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA1.c

$(LIBHACL_SHA1_A): $(LIBHACL_SHA1_OBJS)
$(LIBHACL_SHA1_LIB_STATIC): $(LIBHACL_SHA1_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA1_OBJS)

Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c

$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS)
$(LIBHACL_SHA2_LIB_STATIC): $(LIBHACL_SHA2_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS)

Modules/_hacl/Hacl_Hash_SHA3.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA3.c $(LIBHACL_SHA3_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA3.c

$(LIBHACL_SHA3_A): $(LIBHACL_SHA3_OBJS)
$(LIBHACL_SHA3_LIB_STATIC): $(LIBHACL_SHA3_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA3_OBJS)

Modules/_hacl/Hacl_Hash_Blake2s.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c

Modules/_hacl/Hacl_Hash_Blake2b.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c

Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c

$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD128_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c
Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c

$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD128_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c
Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c

$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD256_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c
Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c

Modules/_hacl/Lib_Memzero0.o: $(srcdir)/Modules/_hacl/Lib_Memzero0.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Lib_Memzero0.c

$(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS)
$(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD256_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c
$(LIBHACL_BLAKE2_LIB_STATIC): $(LIBHACL_BLAKE2_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS)

# Other HACL* cryptographic primitives

Modules/_hacl/Hacl_HMAC.o: $(srcdir)/Modules/_hacl/Hacl_HMAC.c $(LIBHACL_HMAC_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_HMAC.c

Modules/_hacl/Hacl_Streaming_HMAC.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_HMAC.c $(LIBHACL_HMAC_HEADERS)
$(CC) -Wno-unused-variable -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_HMAC.c

$(LIBHACL_HMAC_A): $(LIBHACL_HMAC_OBJS)
$(LIBHACL_HMAC_LIB_STATIC): $(LIBHACL_HMAC_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_HMAC_OBJS)

##########################################################################
# create relative links from build/lib.platform/egg.so to Modules/egg.so
# pybuilddir.txt is created too late. We cannot use it in Makefile
# targets. ln --relative is not portable.
Expand Down Expand Up @@ -3302,12 +3311,21 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@
MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@
MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_A)
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_A)
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A)
MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_A)
MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_A)
MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_A)

# HACL*-based cryptographic primitives
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)
MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@)

MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
Expand Down
8 changes: 4 additions & 4 deletions Misc/sbom.spdx.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 20 additions & 7 deletions Modules/Setup.stdlib.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
# generated by PY_STDLIB_MOD macro.
# * All source files automatically depend on $(PYTHON_HEADERS) and
# $(MODULE_{NAME}_DEPS).
# * ${NAME}$(EXT_SUFFIX) rules automatically depend on $(MODULE_{NAME}_LDEPS),
# thus $(MODULE_{NAME}_LDEPS) must only contain Makefile rule names. To pass
# specific values to the linker, use $(MODULE_{NAME}_LDFLAGS) instead.
#
# See Modules/Setup and Modules/makesetup
#
Expand Down Expand Up @@ -76,13 +79,23 @@
# needs -lreadline or -ledit, sometimes termcap, termlib, or tinfo
@MODULE_READLINE_TRUE@readline readline.c

# hashing builtins, can be disabled with --without-builtin-hashlib-hashes
@MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_MD5.a -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA1.a -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA2.a -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA3.a -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__BLAKE2_TRUE@_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a -D_BSD_SOURCE -D_DEFAULT_SOURCE

############################################################################
# HACL*-based Cryptographic Primitives
#
# Since the compilation of the built-in cryptographic modules depends
# on whether we are building on WASI or not, rules will be explicitly
# written. In the future, it should be preferrable to be able to setup
# the relevant bits here instead of in Makefile.pre.in or configure.ac.

# Hash functions can be disabled with --without-builtin-hashlib-hashes.
@MODULE__MD5_TRUE@_md5 md5module.c
@MODULE__SHA1_TRUE@_sha1 sha1module.c
@MODULE__SHA2_TRUE@_sha2 sha2module.c
@MODULE__SHA3_TRUE@_sha3 sha3module.c
@MODULE__BLAKE2_TRUE@_blake2 blake2module.c

# Since HMAC is always supported, the HACL* implementation modules must
# be built unconditionally.
@MODULE__HMAC_TRUE@_hmac hmacmodule.c

############################################################################
Expand Down
1 change: 1 addition & 0 deletions Modules/_hacl/lib_memzero0.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <inttypes.h>
#include "python_hacl_namespaces.h"

void Lib_Memzero0_memzero0(void *dst, uint64_t len);

Expand Down
Loading
Loading