Merge branch 'auxpow'

Some of the new miniscript signing tests in wallet_miniscript.py
are disabled for now as they fail on Namecoin, this needs debugging
and fixing in the future.  These were introduced upstream in
bitcoin/bitcoin#24149.

Author: domob1812

ci/README.md

@@ -8,8 +8,7 @@ Be aware that the tests will be built and run in-place, so please run at your ow
 If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
 
 The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory.
-While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories,
-such as the depends cache, previous release binaries, or ccache, are mounted as read-write into the docker container. While it should be fine to run
+While it should be fine to run
 the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and
 testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci
 system in a virtual machine with a Linux operating system of your choice.

ci/test/00_setup_env.sh

@@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8
 
 # The root dir.
 # The ci system copies this folder.
-# This is where the depends build is done.
 BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
 export BASE_ROOT_DIR
 # The depends dir.
-# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
 # This folder only exists on the ci host.
@@ -58,12 +57,14 @@ export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 # The cache dir.
-# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 # Folder where the build result is put (bin and lib).
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
 # Folder where the build is done (dist and out-of-tree build).
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
+# The folder for previous release binaries.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}

ci/test/04_install.sh

@@ -39,6 +39,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
       --build-arg "FILE_ENV=${FILE_ENV}" \
       --tag="${CONTAINER_NAME}" \
       "${BASE_ROOT_DIR}"
+  docker volume create "${CONTAINER_NAME}_ccache" || true
+  docker volume create "${CONTAINER_NAME}_depends" || true
+  docker volume create "${CONTAINER_NAME}_previous_releases" || true
 
   if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
     echo "Restart docker before run to stop and clear all containers started with --rm"
@@ -48,9 +51,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   # shellcheck disable=SC2086
   CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \
                   --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
-                  --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
-                  --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
-                  --mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \
+                  --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
+                  --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \
+                  --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \
                   -w $BASE_ROOT_DIR \
                   --env-file /tmp/env \
                   --name $CONTAINER_NAME \

ci/test/06_script_b.sh

@@ -60,6 +60,7 @@ if [ "${RUN_TIDY}" = "true" ]; then
           " src/rpc/signmessage.cpp"\
           " src/test/fuzz/txorphan.cpp"\
           " src/test/fuzz/util/"\
+          " src/test/util/coins.cpp"\
           " src/uint256.cpp"\
           " src/util/bip32.cpp"\
           " src/util/bytevectorhash.cpp"\

configure.ac

@@ -1418,14 +1418,15 @@ if test "$use_upnp" != "no"; then
     [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])],
     [have_miniupnpc=no]
   )
-  dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility
-  dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages.
+
+  dnl The minimum supported miniUPnPc API version is set to 17. This excludes
+  dnl versions with known vulnerabilities.
   if test "$have_miniupnpc" != "no"; then
     AC_MSG_CHECKING([whether miniUPnPc API version is supported])
     AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[
         @%:@include <miniupnpc/miniupnpc.h>
       ]], [[
-        #if MINIUPNPC_API_VERSION >= 10
+        #if MINIUPNPC_API_VERSION >= 17
         // Everything is okay
         #else
         #  error miniUPnPc API version is too old
@@ -1434,7 +1435,7 @@ if test "$use_upnp" != "no"; then
         AC_MSG_RESULT([yes])
       ],[
       AC_MSG_RESULT([no])
-      AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.])
+      AC_MSG_WARN([miniUPnPc API version < 17 is unsupported, disabling UPnP support.])
       have_miniupnpc=no
     ])
   fi

contrib/devtools/symbol-check.py

@@ -15,30 +15,30 @@
 
 import lief #type:ignore
 
-# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases
+# Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS
 #
-# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B)
-# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6)
+# - libgcc version 8.3.0 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libgcc1)
+# - libc version 2.28 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libc6)
 #
-# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases
+# Ubuntu 18.04 (Bionic) EOL: 2028. https://wiki.ubuntu.com/ReleaseTeam
 #
-# - g++ version 5.3.1
-# - libc version 2.23
+# - libgcc version 8.4.0 (https://packages.ubuntu.com/bionic/libgcc1)
+# - libc version 2.27 (https://packages.ubuntu.com/bionic/libc6)
 #
 # CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product
 #
-# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
+# - libgcc version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
 # - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
 #
 # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info.
 
 MAX_VERSIONS = {
 'GCC':       (4,8,0),
 'GLIBC': {
-    lief.ELF.ARCH.x86_64: (2,18),
-    lief.ELF.ARCH.ARM:    (2,18),
-    lief.ELF.ARCH.AARCH64:(2,18),
-    lief.ELF.ARCH.PPC64:  (2,18),
+    lief.ELF.ARCH.x86_64: (2,27),
+    lief.ELF.ARCH.ARM:    (2,27),
+    lief.ELF.ARCH.AARCH64:(2,27),
+    lief.ELF.ARCH.PPC64:  (2,27),
     lief.ELF.ARCH.RISCV:  (2,27),
 },
 'LIBATOMIC': (1,0),

contrib/devtools/test-symbol-check.py

@@ -38,31 +38,6 @@ def test_ELF(self):
         executable = 'test1'
         cc = determine_wellknown_cmd('CC', 'gcc')
 
-        # there's no way to do this test for RISC-V at the moment; we build for
-        # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27.
-        if 'riscv' in get_machine(cc):
-            self.skipTest("test not available for RISC-V")
-
-        # nextup was introduced in GLIBC 2.24, so is newer than our supported
-        # glibc (2.18), and available in our release build environment (2.24).
-        with open(source, 'w', encoding="utf8") as f:
-            f.write('''
-                #define _GNU_SOURCE
-                #include <math.h>
-
-                double nextup(double x);
-
-                int main()
-                {
-                    nextup(3.14);
-                    return 0;
-                }
-        ''')
-
-        self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']),
-                (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' +
-                    executable + ': failed IMPORTED_SYMBOLS'))
-
         # -lutil is part of the libc6 package so a safe bet that it's installed
         # it's also out of context enough that it's unlikely to ever become a real dependency
         source = 'test2.c'

contrib/guix/libexec/build.sh

@@ -238,13 +238,6 @@ case "$HOST" in
     *mingw*)  HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;;
 esac
 
-# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by
-# avoiding a PowerPC64 optimisation available in glibc 2.22 and later.
-# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html
-case "$HOST" in
-    *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;;
-esac
-
 # Make $HOST-specific native binaries from depends available in $PATH
 export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
 mkdir -p "$DISTSRC"

contrib/guix/manifest.scm

@@ -147,7 +147,7 @@ chain for " target " development."))
                                        #:key
                                        (base-gcc-for-libc base-gcc)
                                        (base-kernel-headers base-linux-kernel-headers)
-                                       (base-libc (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.24)))
+                                       (base-libc (hardened-glibc (make-glibc-without-werror glibc-2.27)))
                                        (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc))))
   "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values
 desirable for building Bitcoin Core release binaries."
@@ -537,48 +537,30 @@ inspecting signatures in Mach-O binaries.")
 (define (make-glibc-without-werror glibc)
   (package-with-extra-configure-variable glibc "enable_werror" "no"))
 
-(define (make-glibc-with-stack-protector glibc)
-  (package-with-extra-configure-variable glibc "--enable-stack-protector" "all"))
-
-(define (make-glibc-with-bind-now glibc)
-  (package-with-extra-configure-variable glibc "--enable-bind-now" "yes"))
-
-(define-public glibc-2.24
-  (package
-    (inherit glibc-2.31)
-    (version "2.24")
-    (source (origin
-              (method git-fetch)
-              (uri (git-reference
-                    (url "https://sourceware.org/git/glibc.git")
-                    (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")))
-              (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))
-              (sha256
-               (base32
-                "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3"))
-              (patches (search-our-patches "glibc-ldd-x86_64.patch"
-                                           "glibc-versioned-locpath.patch"
-                                           "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch"
-                                           "glibc-2.24-no-build-time-cxx-header-run.patch"
-                                           "glibc-2.24-fcommon.patch"
-                                           "glibc-2.24-guix-prefix.patch"))))))
+;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html
+(define (hardened-glibc glibc)
+  (package-with-extra-configure-variable (
+    package-with-extra-configure-variable glibc
+    "--enable-stack-protector" "all")
+    "--enable-bind-now" "yes"))
 
-(define-public glibc-2.27/bitcoin-patched
+(define-public glibc-2.27
   (package
     (inherit glibc-2.31)
     (version "2.27")
     (source (origin
               (method git-fetch)
               (uri (git-reference
                     (url "https://sourceware.org/git/glibc.git")
-                    (commit "23158b08a0908f381459f273a984c6fd328363cb")))
-              (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb"))
+                    (commit "73886db6218e613bd6d4edf529f11e008a6c2fa6")))
+              (file-name (git-file-name "glibc" "73886db6218e613bd6d4edf529f11e008a6c2fa6"))
               (sha256
                (base32
-                "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca"))
+                "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq"))
               (patches (search-our-patches "glibc-ldd-x86_64.patch"
+                                           "glibc-versioned-locpath.patch"
                                            "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch"
-                                           "glibc-2.27-dont-redefine-nss-database.patch"
+                                           "glibc-2.27-fcommon.patch"
                                            "glibc-2.27-guix-prefix.patch"))))))
 
 (packages->manifest
@@ -627,12 +609,7 @@ inspecting signatures in Mach-O binaries.")
                  (make-nsis-for-gcc-10 nsis-x86_64)
                  osslsigncode))
           ((string-contains target "-linux-")
-           (list (cond ((string-contains target "riscv64-")
-                        (make-bitcoin-cross-toolchain target
-                                                      #:base-libc (make-glibc-with-stack-protector
-                                                        (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.27/bitcoin-patched)))))
-                       (else
-                        (make-bitcoin-cross-toolchain target)))))
+           (list (make-bitcoin-cross-toolchain target)))
           ((string-contains target "darwin")
            (list clang-toolchain-10 binutils cmake xorriso python-signapple))
           (else '())))))