Merge pull request #1870 from tnull/2022-11-add-transaction-sync-crate

Add transaction sync crate

Author: TheBlueMatt

.github/workflows/build.yml

@@ -12,52 +12,68 @@ jobs:
                      beta,
                      # 1.41.1 is MSRV for Rust-Lightning, lightning-invoice, and lightning-persister
                      1.41.1,
-                     # 1.45.2 is MSRV for lightning-net-tokio, lightning-block-sync, lightning-background-processor, and coverage generation
+                     # 1.45.2 is MSRV for lightning-net-tokio, lightning-block-sync, lightning-background-processor
                      1.45.2,
                      # 1.47.0 will be the MSRV for no-std builds using hashbrown once core2 is updated
-                     1.47.0]
+                     1.47.0,
+                     # 1.59.0 is the MSRV for lightning-transaction-sync
+                     1.59.0]
         include:
           - toolchain: stable
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: true
+            coverage: true
           - toolchain: stable
             platform: macos-latest
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: true
           - toolchain: beta
             platform: macos-latest
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: true
           - toolchain: stable
             platform: windows-latest
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: false
           - toolchain: beta
             platform: windows-latest
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: false
           - toolchain: beta
             build-net-tokio: true
             build-no-std: true
             build-futures: true
+            build-tx-sync: true
           - toolchain: 1.41.1
             build-no-std: false
             test-log-variants: true
             build-futures: false
+            build-tx-sync: false
           - toolchain: 1.45.2
             build-net-old-tokio: true
             build-net-tokio: true
             build-no-std: false
             build-futures: true
-            coverage: true
+            build-tx-sync: false
           - toolchain: 1.47.0
             build-futures: true
             build-no-std: true
+            build-tx-sync: false
+          - toolchain: 1.59.0
+            build-net-tokio: false
+            build-no-std: false
+            build-futures: false
+            build-tx-sync: true
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout source code
@@ -73,10 +89,10 @@ jobs:
         run: cargo update -p tokio --precise "1.14.0" --verbose
         env:
           CARGO_NET_GIT_FETCH_WITH_CLI: "true"
-      - name: Build on Rust ${{ matrix.toolchain }} with net-tokio
-        if: "matrix.build-net-tokio && !matrix.coverage"
+      - name: Build on Rust ${{ matrix.toolchain }} with net-tokio and tx-sync
+        if: "matrix.build-net-tokio && !matrix.coverage && matrix.build-tx-sync"
         run: cargo build --verbose --color always
-      - name: Build on Rust ${{ matrix.toolchain }} with net-tokio and full code-linking for coverage generation
+      - name: Build on Rust ${{ matrix.toolchain }} with net-tokio, tx-sync, and full code-linking for coverage generation
         if: matrix.coverage
         run: RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always
       - name: Build on Rust ${{ matrix.toolchain }}
@@ -108,14 +124,32 @@ jobs:
           RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always --features rpc-client
           RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always --features rpc-client,rest-client
           RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always --features rpc-client,rest-client,tokio
+      - name: Build Transaction Sync Clients on Rust ${{ matrix.toolchain }} with features
+        if: "matrix.build-tx-sync && !matrix.coverage"
+        run: |
+          cd lightning-transaction-sync
+          cargo build --verbose --color always --features esplora-blocking
+          cargo build --verbose --color always --features esplora-async
+      - name: Build transaction sync clients on Rust ${{ matrix.toolchain }} with features and full code-linking for coverage generation
+        if: "matrix.build-tx-sync && matrix.coverage"
+        run: |
+          cd lightning-transaction-sync
+          RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always --features esplora-blocking
+          RUSTFLAGS="-C link-dead-code" cargo build --verbose --color always --features esplora-async
+      - name: Test transaction sync clients on Rust ${{ matrix.toolchain }} with features
+        if: "matrix.build-tx-sync"
+        run: |
+          cd lightning-transaction-sync
+          cargo test --verbose --color always --features esplora-blocking
+          cargo test --verbose --color always --features esplora-async
       - name: Test backtrace-debug builds on Rust ${{ matrix.toolchain }}
         if: "matrix.toolchain == 'stable'"
         run: |
           cd lightning && cargo test --verbose --color always --features backtrace
       - name: Test on Rust ${{ matrix.toolchain }} with net-tokio
-        if: "matrix.build-net-tokio && !matrix.coverage"
+        if: "matrix.build-net-tokio && !matrix.coverage && matrix.build-tx-sync"
         run: cargo test --verbose --color always
-      - name: Test on Rust ${{ matrix.toolchain }} with net-tokio and full code-linking for coverage generation
+      - name: Test on Rust ${{ matrix.toolchain }} with net-tokio, tx-sync, and full code-linking for coverage generation
         if: matrix.coverage
         run: RUSTFLAGS="-C link-dead-code" cargo test --verbose --color always
       - name: Test no-std builds on Rust ${{ matrix.toolchain }}
@@ -349,7 +383,7 @@ jobs:
   linting:
     runs-on: ubuntu-latest
     env:
-      TOOLCHAIN: 1.47.0
+      TOOLCHAIN: stable
     steps:
       - name: Checkout source code
         uses: actions/checkout@v3

Cargo.toml

@@ -3,6 +3,7 @@
 members = [
     "lightning",
     "lightning-block-sync",
+    "lightning-transaction-sync",
     "lightning-invoice",
     "lightning-net-tokio",
     "lightning-persister",

lightning-transaction-sync/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "lightning-transaction-sync"
+version = "0.0.113"
+authors = ["Elias Rohrer"]
+license = "MIT OR Apache-2.0"
+repository = "http://github.com/lightningdevkit/rust-lightning"
+description = """
+Utilities for syncing LDK via the transaction-based `Confirm` interface.
+"""
+edition = "2018"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[features]
+default = []
+esplora-async = ["async-interface", "esplora-client/async", "futures"]
+esplora-blocking = ["esplora-client/blocking"]
+async-interface = []
+
+[dependencies]
+lightning = { version = "0.0.113", path = "../lightning" }
+bitcoin = "0.29.0"
+bdk-macros = "0.6"
+futures = { version = "0.3", optional = true }
+esplora-client = { version = "0.3.0", default-features = false, optional = true }
+
+[dev-dependencies]
+electrsd = { version = "0.22.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_23_0"] }
+electrum-client = "0.12.0"
+once_cell = "1.16.0"
+tokio = { version = "1.14.0", features = ["full"] }

lightning-transaction-sync/src/common.rs

@@ -0,0 +1,75 @@
+use lightning::chain::WatchedOutput;
+use bitcoin::{Txid, BlockHash, Transaction, BlockHeader, OutPoint};
+
+use std::collections::{HashSet, HashMap};
+
+
+// Represents the current state.
+pub(crate) struct SyncState {
+	// Transactions that were previously processed, but must not be forgotten
+	// yet since they still need to be monitored for confirmation on-chain.
+	pub watched_transactions: HashSet<Txid>,
+	// Outputs that were previously processed, but must not be forgotten yet as
+	// as we still need to monitor any spends on-chain.
+	pub watched_outputs: HashMap<OutPoint, WatchedOutput>,
+	// The tip hash observed during our last sync.
+	pub last_sync_hash: Option<BlockHash>,
+	// Indicates whether we need to resync, e.g., after encountering an error.
+	pub pending_sync: bool,
+}
+
+impl SyncState {
+	pub fn new() -> Self {
+		Self {
+			watched_transactions: HashSet::new(),
+			watched_outputs: HashMap::new(),
+			last_sync_hash: None,
+			pending_sync: false,
+		}
+	}
+}
+
+
+// A queue that is to be filled by `Filter` and drained during the next syncing round.
+pub(crate) struct FilterQueue {
+	// Transactions that were registered via the `Filter` interface and have to be processed.
+	pub transactions: HashSet<Txid>,
+	// Outputs that were registered via the `Filter` interface and have to be processed.
+	pub outputs: HashMap<OutPoint, WatchedOutput>,
+}
+
+impl FilterQueue {
+	pub fn new() -> Self {
+		Self {
+			transactions: HashSet::new(),
+			outputs: HashMap::new(),
+		}
+	}
+
+	// Processes the transaction and output queues and adds them to the given [`SyncState`].
+	//
+	// Returns `true` if new items had been registered.
+	pub fn process_queues(&mut self, sync_state: &mut SyncState) -> bool {
+		let mut pending_registrations = false;
+
+		if !self.transactions.is_empty() {
+			pending_registrations = true;
+
+			sync_state.watched_transactions.extend(self.transactions.drain());
+		}
+
+		if !self.outputs.is_empty() {
+			pending_registrations = true;
+
+			sync_state.watched_outputs.extend(self.outputs.drain());
+		}
+		pending_registrations
+	}
+}
+
+pub(crate) struct ConfirmedTx {
+	pub tx: Transaction,
+	pub block_header: BlockHeader,
+	pub block_height: u32,
+	pub pos: usize,
+}

lightning-transaction-sync/src/error.rs

@@ -0,0 +1,63 @@
+use std::fmt;
+
+#[derive(Debug)]
+/// An error that possibly needs to be handled by the user.
+pub enum TxSyncError {
+	/// A transaction sync failed and needs to be retried eventually.
+	Failed,
+}
+
+impl std::error::Error for TxSyncError {}
+
+impl fmt::Display for TxSyncError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			Self::Failed => write!(f, "Failed to conduct transaction sync."),
+		}
+	}
+}
+
+#[derive(Debug)]
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+pub(crate) enum InternalError {
+	/// A transaction sync failed and needs to be retried eventually.
+	Failed,
+	/// An inconsisteny was encounterd during transaction sync.
+	Inconsistency,
+}
+
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+impl fmt::Display for InternalError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			Self::Failed => write!(f, "Failed to conduct transaction sync."),
+			Self::Inconsistency => {
+				write!(f, "Encountered an inconsisteny during transaction sync.")
+			}
+		}
+	}
+}
+
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+impl std::error::Error for InternalError {}
+
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+impl From<esplora_client::Error> for TxSyncError {
+	fn from(_e: esplora_client::Error) -> Self {
+		Self::Failed
+	}
+}
+
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+impl From<esplora_client::Error> for InternalError {
+	fn from(_e: esplora_client::Error) -> Self {
+		Self::Failed
+	}
+}
+
+#[cfg(any(feature = "esplora-blocking", feature = "esplora-async"))]
+impl From<InternalError> for TxSyncError {
+	fn from(_e: InternalError) -> Self {
+		Self::Failed
+	}
+}