Add a BackgroundEvent to replay non-closing monitor updates

`BackgroundEvent` is used to store `ChannelMonitorUpdate`s which
result in a channel force-close, avoiding relying on
`ChannelMonitor`s having been loaded while `ChannelManager`
block-connection methods are called during startup.

In the coming commit(s) we'll also generate non-channel-closing
`ChannelMonitorUpdate`s during startup, which will need to be
replayed prior to any other `ChannelMonitorUpdate`s generated from
normal operation.

In the next commit we'll handle that by handling `BackgroundEvent`s
immediately after locking the `total_consistency_lock`.

The new `BackgroundEvent` variant is restricted to only
`ChannelMonitorUpdate`s which will be regenerated on the next
restart, as they're not persisted to disk.

Author: TheBlueMatt

lightning/src/ln/channelmanager.rs

@@ -504,6 +504,12 @@ enum BackgroundEvent {
 	/// Handle a ChannelMonitorUpdate that closes a channel, broadcasting its current latest holder
 	/// commitment transaction.
 	ClosingMonitorUpdate((OutPoint, ChannelMonitorUpdate)),
+	/// Handle a ChannelMonitorUpdate that does not close a channel, potentially unblocking the
+	/// channel to continue normal operation.
+	///
+	/// Note that any such events are lost on shutdown, so in general they must be updates which
+	/// are regenerated on startup.
+	MonitorUpdateRegeneratedOnStartup((PublicKey, OutPoint, ChannelMonitorUpdate)),
 }
 
 #[derive(Debug)]
@@ -3778,6 +3784,36 @@ where
 					// monitor updating completing.
 					let _ = self.chain_monitor.update_channel(funding_txo, &update);
 				},
+				BackgroundEvent::MonitorUpdateRegeneratedOnStartup((counterparty_id, funding_txo, update)) => {
+					// The channel has already been closed, so no use bothering to care about the
+					// monitor updating completing.
+					let update_res = self.chain_monitor.update_channel(funding_txo, &update);
+
+					let mut found_chan = false;
+					let res = {
+						let per_peer_state = self.per_peer_state.read().unwrap();
+						if let Some(peer_state_mutex) = per_peer_state.get(&counterparty_id) {
+							let mut peer_state_lock = peer_state_mutex.lock().unwrap();
+							let peer_state = &mut *peer_state_lock;
+							match peer_state.channel_by_id.entry(funding_txo.to_channel_id()) {
+								hash_map::Entry::Occupied(mut chan) => {
+									found_chan = true;
+									handle_new_monitor_update!(self, update_res, update.update_id, peer_state_lock, peer_state, per_peer_state, chan)
+								},
+								hash_map::Entry::Vacant(_) => {
+									Ok(())
+								},
+							}
+						} else { Ok(()) }
+					};
+					let _ = handle_error!(self, res, counterparty_id);
+					if !found_chan {
+						// TODO: If this channel has since closed, we're likely providing a payment
+						// preimage update, which we must ensure is durable! We currently don't,
+						// however, ensure that, and when we have a strategy therefor we should
+						// apply it here.
+					}
+				},
 			}
 		}
 		true
@@ -7363,14 +7399,18 @@ where
 		}
 
 		let background_events = self.pending_background_events.lock().unwrap();
-		(background_events.len() as u64).write(writer)?;
+		(background_events.iter()
+			.filter(|ev| if let BackgroundEvent::MonitorUpdateRegeneratedOnStartup(_) = ev { false } else { true })
+			.count() as u64)
+			.write(writer)?;
 		for event in background_events.iter() {
 			match event {
 				BackgroundEvent::ClosingMonitorUpdate((funding_txo, monitor_update)) => {
 					0u8.write(writer)?;
 					funding_txo.write(writer)?;
 					monitor_update.write(writer)?;
 				},
+				BackgroundEvent::MonitorUpdateRegeneratedOnStartup(_) => { },
 			}
 		}
 
@@ -7819,8 +7859,9 @@ where
 				0 => {
 					let (funding_txo, monitor_update): (OutPoint, ChannelMonitorUpdate) = (Readable::read(reader)?, Readable::read(reader)?);
 					if pending_background_events.iter().find(|e| {
-						let BackgroundEvent::ClosingMonitorUpdate((pending_funding_txo, pending_monitor_update)) = e;
-						*pending_funding_txo == funding_txo && *pending_monitor_update == monitor_update
+						if let BackgroundEvent::ClosingMonitorUpdate((pending_funding_txo, pending_monitor_update)) = e {
+							*pending_funding_txo == funding_txo && *pending_monitor_update == monitor_update
+						} else { false }
 					}).is_none() {
 						pending_background_events.push(BackgroundEvent::ClosingMonitorUpdate((funding_txo, monitor_update)));
 					}