Fix nagle, other minor optimizations

.github/workflows/ci.yaml

@@ -209,7 +209,7 @@ jobs:
       - name: Check
         run: |
           tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
+          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
 
   check-fuzz:
     name: Run fuzz regression tests
@@ -249,7 +249,7 @@ jobs:
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: gcc
@@ -343,7 +343,7 @@ jobs:
         run: |
           env
           cat config.vars
-          VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   integration-valgrind:
     name: Valgrind Test CLN ${{ matrix.name }}
@@ -356,7 +356,7 @@ jobs:
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: Valgrind (01/10)
@@ -411,7 +411,7 @@ jobs:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
         run: |
-          VALGRIND=1 poetry run pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   integration-sanitizers:
     name: Sanitizers Test CLN
@@ -425,7 +425,7 @@ jobs:
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: ASan/UBSan (01/10)
@@ -477,15 +477,15 @@ jobs:
 
       - name: Test
         run: |
-          poetry run pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   update-docs-examples:
     name: Update examples in doc schemas (disabled temporarily!)
     if: false
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     strategy:
-      fail-fast: true
+      fail-fast: false
     env:
       VALGRIND: 0
       GENERATE_EXAMPLES: 1
@@ -520,7 +520,7 @@ jobs:
           tar -xaf cln-compile-gcc.tar.bz2
       - name: Test
         run: |
-          make -j $(nproc) check-doc-examples
+          eatmydata make -j $(nproc) check-doc-examples
 
   min-btc-support:
     name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
@@ -532,7 +532,7 @@ jobs:
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: clang
@@ -592,7 +592,7 @@ jobs:
         run: |
           env
           cat config.vars
-          VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   check-flake:
     name: Check Nix Flake
@@ -624,7 +624,6 @@ jobs:
       - check-units
       - integration-valgrind
       - integration-sanitizers
-      - update-docs-examples
       - min-btc-support
       - check-flake
     steps:

common/daemon.c

@@ -222,8 +222,11 @@ bool daemon_developer_mode(char *argv[])
 		kill(getpid(), SIGSTOP);
 	}
 
-	/* This checks for any tal_steal loops! */
-	add_steal_notifiers(NULL);
+	/* This checks for any tal_steal loops, but it's not free:
+	 * only use if we're already using the fairly heavy memleak
+	 * detection. */
+	if (getenv("LIGHTNINGD_DEV_MEMLEAK"))
+		add_steal_notifiers(NULL);
 
 	return true;
 }

connectd/multiplex.c

@@ -307,8 +307,6 @@ void setup_peer_gossip_store(struct peer *peer,
 static void set_urgent_flag(struct peer *peer, bool urgent)
 {
 	int val;
-	int opt;
-	const char *optname;
 
 	if (urgent == peer->urgent)
 		return;
@@ -318,23 +316,13 @@ static void set_urgent_flag(struct peer *peer, bool urgent)
 	if (peer->is_websocket != NORMAL_SOCKET)
 		return;
 
-#ifdef TCP_CORK
-	opt = TCP_CORK;
-	optname = "TCP_CORK";
-#elif defined(TCP_NODELAY)
-	opt = TCP_NODELAY;
-	optname = "TCP_NODELAY";
-#else
-#error "Please report platform with neither TCP_CORK nor TCP_NODELAY?"
-#endif
-
 	val = urgent;
 	if (setsockopt(io_conn_fd(peer->to_peer),
-		       IPPROTO_TCP, opt, &val, sizeof(val)) != 0
+		       IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) != 0
 	    /* This actually happens in testing, where we blackhole the fd */
 	    && peer->daemon->dev_disconnect_fd == -1) {
-		status_broken("setsockopt %s=1 fd=%u: %s",
-			      optname, io_conn_fd(peer->to_peer),
+		status_broken("setsockopt TCP_NODELAY=1 fd=%u: %s",
+			      io_conn_fd(peer->to_peer),
 			      strerror(errno));
 	}
 	peer->urgent = urgent;

gossipd/gossipd.c

@@ -479,6 +479,7 @@ static void dev_gossip_memleak(struct daemon *daemon, const u8 *msg)
 	memleak_scan_obj(memtable, daemon);
 	memleak_scan_htable(memtable, &daemon->peers->raw);
 	dev_seeker_memleak(memtable, daemon->seeker);
+	gossmap_manage_memleak(memtable, daemon->gm);
 
 	found_leak = dump_memleak(memtable, memleak_status_broken, NULL);
 	daemon_conn_send(daemon->master,

gossipd/gossmap_manage.c

@@ -6,6 +6,7 @@
 #include <common/daemon_conn.h>
 #include <common/gossip_store.h>
 #include <common/gossmap.h>
+#include <common/memleak.h>
 #include <common/status.h>
 #include <common/timeout.h>
 #include <common/wire_error.h>
@@ -469,6 +470,13 @@ static bool setup_gossmap(struct gossmap_manage *gm,
 	return true;
 }
 
+void gossmap_manage_memleak(struct htable *memtable,
+			    const struct gossmap_manage *gm)
+{
+	memleak_scan_uintmap(memtable, &gm->pending_ann_map.map);
+	memleak_scan_uintmap(memtable, &gm->early_ann_map.map);
+}
+
 struct gossmap_manage *gossmap_manage_new(const tal_t *ctx,
 					  struct daemon *daemon)
 {

gossipd/gossmap_manage.h

@@ -111,4 +111,7 @@ void gossmap_manage_tell_lightningd_locals(struct daemon *daemon,
  * The seeker uses this if we're at startup and want complete gossip.
  */
 bool gossmap_manage_populated(const struct gossmap_manage *gm);
+
+/* For memleak to see inside of maps */
+void gossmap_manage_memleak(struct htable *memtable, const struct gossmap_manage *gm);
 #endif /* LIGHTNING_GOSSIPD_GOSSMAP_MANAGE_H */

lightningd/log.c

@@ -207,7 +207,14 @@ static void log_to_files(const char *log_prefix,
 			 struct log_file **log_files)
 {
 	char tstamp[sizeof("YYYY-mm-ddTHH:MM:SS.nnnZ ")];
-	char *entry, *nodestr;
+	char *entry, nodestr[hex_str_size(PUBKEY_CMPR_LEN)];
+	char buf[sizeof("%s%s%s %s-%s: %s\n")
+		 + strlen(log_prefix)
+		 + sizeof(tstamp)
+		 + strlen(level_prefix(level))
+		 + sizeof(nodestr)
+		 + strlen(entry_prefix)
+		 + strlen(str)];
 	bool filtered;
 
 	if (print_timestamps) {
@@ -218,9 +225,10 @@ static void log_to_files(const char *log_prefix,
 		tstamp[0] = '\0';
 
 	if (node_id)
-		nodestr = fmt_node_id(tmpctx, node_id);
+		hex_encode(node_id->k, sizeof(node_id->k),
+			   nodestr, sizeof(nodestr));
 	else
-		nodestr = "";
+		nodestr[0] = '\0';
 	if (level == LOG_IO_IN || level == LOG_IO_OUT) {
 		const char *dir = level == LOG_IO_IN ? "[IN]" : "[OUT]";
 		char *hex = tal_hexstr(NULL, io, io_len);
@@ -234,14 +242,18 @@ static void log_to_files(const char *log_prefix,
 					entry_prefix, str, dir, hex);
 		tal_free(hex);
 	} else {
+		size_t len;
+		entry = buf;
 		if (!node_id)
-			entry = tal_fmt(tmpctx, "%s%s%s %s: %s\n",
-					log_prefix, tstamp, level_prefix(level), entry_prefix, str);
+			len = snprintf(buf, sizeof(buf),
+				       "%s%s%s %s: %s\n",
+				       log_prefix, tstamp, level_prefix(level), entry_prefix, str);
 		else
-			entry = tal_fmt(tmpctx, "%s%s%s %s-%s: %s\n",
-					log_prefix, tstamp, level_prefix(level),
-					nodestr,
-					entry_prefix, str);
+			len = snprintf(buf, sizeof(buf), "%s%s%s %s-%s: %s\n",
+				       log_prefix, tstamp, level_prefix(level),
+				       nodestr,
+				       entry_prefix, str);
+		assert(len < sizeof(buf));
 	}
 
 	/* In complex configurations, we tell loggers to overshare: then we