Skip to content

Commit 8d915a8

Browse files
committed
Implement non-strict forwarding
This change implements non-strict forwarding, allowing the node to forward an HTLC along a channel other than the one specified by short_channel_id in the onion message, so long as the receiver has the same node public key intended by short_channel_id ([BOLT](https://github.com/lightning/bolts/blob/57ce4b1e05c996fa649f00dc13521f6d496a288f/04-onion-routing.md#non-strict-forwarding)). This can improve payment reliability when there are multiple channels with the same peer e.g. when outbound liquidity is replenished by opening a new channel. The implemented forwarding strategy now chooses the channel with the lowest outbound liquidity that can forward an HTLC to maximize the probability of being able to successfully forward a subsequent HTLC. Fixes #1278.
1 parent 07f3380 commit 8d915a8

File tree

2 files changed

+179
-41
lines changed

2 files changed

+179
-41
lines changed

lightning/src/ln/channelmanager.rs

Lines changed: 76 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4895,8 +4895,8 @@ where
48954895
if short_chan_id != 0 {
48964896
let mut forwarding_counterparty = None;
48974897
macro_rules! forwarding_channel_not_found {
4898-
() => {
4899-
for forward_info in pending_forwards.drain(..) {
4898+
($forward_infos:expr) => {
4899+
for forward_info in $forward_infos {
49004900
match forward_info {
49014901
HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
49024902
prev_short_channel_id, prev_htlc_id, prev_channel_id, prev_funding_outpoint,
@@ -5004,34 +5004,33 @@ where
50045004
let (counterparty_node_id, forward_chan_id) = match chan_info_opt {
50055005
Some((cp_id, chan_id)) => (cp_id, chan_id),
50065006
None => {
5007-
forwarding_channel_not_found!();
5007+
forwarding_channel_not_found!(pending_forwards.drain(..));
50085008
continue;
50095009
}
50105010
};
50115011
forwarding_counterparty = Some(counterparty_node_id);
50125012
let per_peer_state = self.per_peer_state.read().unwrap();
50135013
let peer_state_mutex_opt = per_peer_state.get(&counterparty_node_id);
50145014
if peer_state_mutex_opt.is_none() {
5015-
forwarding_channel_not_found!();
5015+
forwarding_channel_not_found!(pending_forwards.drain(..));
50165016
continue;
50175017
}
50185018
let mut peer_state_lock = peer_state_mutex_opt.unwrap().lock().unwrap();
50195019
let peer_state = &mut *peer_state_lock;
5020-
if let Some(ChannelPhase::Funded(ref mut chan)) = peer_state.channel_by_id.get_mut(&forward_chan_id) {
5021-
let logger = WithChannelContext::from(&self.logger, &chan.context, None);
5022-
for forward_info in pending_forwards.drain(..) {
5020+
let mut draining_pending_forwards = pending_forwards.drain(..);
5021+
loop {
5022+
let maybe_forward_info = draining_pending_forwards.next();
5023+
if let Some(forward_info) = maybe_forward_info {
50235024
let queue_fail_htlc_res = match forward_info {
50245025
HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
50255026
prev_short_channel_id, prev_htlc_id, prev_channel_id, prev_funding_outpoint,
50265027
prev_user_channel_id, forward_info: PendingHTLCInfo {
50275028
incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value,
50285029
routing: PendingHTLCRouting::Forward {
5029-
onion_packet, blinded, ..
5030+
ref onion_packet, blinded, ..
50305031
}, skimmed_fee_msat, ..
50315032
},
50325033
}) => {
5033-
let logger = WithChannelContext::from(&self.logger, &chan.context, Some(payment_hash));
5034-
log_trace!(logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, &payment_hash, short_chan_id);
50355034
let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
50365035
short_channel_id: prev_short_channel_id,
50375036
user_channel_id: Some(prev_user_channel_id),
@@ -5051,44 +5050,86 @@ where
50515050
&self.secp_ctx, b.inbound_blinding_point, &encrypted_tlvs_ss
50525051
).ok()
50535052
});
5054-
if let Err(e) = chan.queue_add_htlc(outgoing_amt_msat,
5055-
payment_hash, outgoing_cltv_value, htlc_source.clone(),
5056-
onion_packet, skimmed_fee_msat, next_blinding_point, &self.fee_estimator,
5057-
&&logger)
5058-
{
5059-
if let ChannelError::Ignore(msg) = e {
5060-
log_trace!(logger, "Failed to forward HTLC with payment_hash {}: {}", &payment_hash, msg);
5053+
5054+
// Forward the HTLC over the most appropriate channel with the corresponding peer,
5055+
// applying non-strict forwarding.
5056+
// The channel with the least amount of outbound liquidity will be used to maximize the
5057+
// probability of being able to successfully forward a subsequent HTLC.
5058+
let mut channels_with_peer = peer_state.channel_by_id.values_mut().filter_map(|phase| match phase {
5059+
ChannelPhase::Funded(chan) => Some(chan),
5060+
_ => None,
5061+
}).collect::<Vec<&mut Channel<_>>>();
5062+
channels_with_peer.sort_by_key(|chan| chan.context.get_available_balances(&self.fee_estimator).outbound_capacity_msat);
5063+
let successfully_added = channels_with_peer.iter_mut().any(|chan| {
5064+
let logger = WithChannelContext::from(&self.logger, &chan.context, Some(payment_hash));
5065+
let add_result = chan.queue_add_htlc(outgoing_amt_msat, payment_hash,
5066+
outgoing_cltv_value, htlc_source.clone(), onion_packet.clone(),
5067+
skimmed_fee_msat, next_blinding_point, &self.fee_estimator, &&logger);
5068+
match add_result {
5069+
Ok(_) => {
5070+
log_trace!(logger, "Forwarding HTLC from SCID {} with payment_hash {} and specified next hop SCID {} over channel {} with corresponding peer {}",
5071+
prev_short_channel_id, &payment_hash, short_chan_id, chan.context.channel_id(), &counterparty_node_id);
5072+
},
5073+
Err(ChannelError::Ignore(ref msg)) => {
5074+
log_trace!(logger, "Not forwarding HTLC with payment_hash {} over channel {} with peer {}: {}. Will attempt other channels with the same peer if possible.",
5075+
&payment_hash, chan.context.channel_id(), &counterparty_node_id, msg);
5076+
},
5077+
Err(_) => {
5078+
panic!("Stated return value requirements in send_htlc() were not met");
5079+
},
5080+
}
5081+
add_result.is_ok()
5082+
});
5083+
5084+
if !successfully_added {
5085+
log_trace!(self.logger, "Failed to forward HTLC with payment_hash {} to peer {}", &payment_hash, &counterparty_node_id);
5086+
if let Some(ChannelPhase::Funded(ref mut chan)) = peer_state.channel_by_id.get_mut(&forward_chan_id) {
5087+
let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan);
5088+
failed_forwards.push((htlc_source, payment_hash,
5089+
HTLCFailReason::reason(failure_code, data),
5090+
HTLCDestination::NextHopChannel { node_id: Some(chan.context.get_counterparty_node_id()), channel_id: forward_chan_id }
5091+
));
50615092
} else {
5062-
panic!("Stated return value requirements in send_htlc() were not met");
5093+
forwarding_channel_not_found!(core::iter::once(forward_info).chain(draining_pending_forwards));
5094+
break;
50635095
}
5064-
let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan);
5065-
failed_forwards.push((htlc_source, payment_hash,
5066-
HTLCFailReason::reason(failure_code, data),
5067-
HTLCDestination::NextHopChannel { node_id: Some(chan.context.get_counterparty_node_id()), channel_id: forward_chan_id }
5068-
));
5069-
continue;
50705096
}
50715097
None
50725098
},
50735099
HTLCForwardInfo::AddHTLC { .. } => {
50745100
panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
50755101
},
5076-
HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
5077-
log_trace!(logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
5078-
Some((chan.queue_fail_htlc(htlc_id, err_packet, &&logger), htlc_id))
5102+
HTLCForwardInfo::FailHTLC { htlc_id, ref err_packet } => {
5103+
if let Some(ChannelPhase::Funded(ref mut chan)) = peer_state.channel_by_id.get_mut(&forward_chan_id) {
5104+
let logger = WithChannelContext::from(&self.logger, &chan.context, None);
5105+
log_trace!(logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
5106+
Some((chan.queue_fail_htlc(htlc_id, err_packet.clone(), &&logger), htlc_id))
5107+
} else {
5108+
forwarding_channel_not_found!(core::iter::once(forward_info).chain(draining_pending_forwards));
5109+
break;
5110+
}
50795111
},
50805112
HTLCForwardInfo::FailMalformedHTLC { htlc_id, failure_code, sha256_of_onion } => {
5081-
log_trace!(logger, "Failing malformed HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
5082-
let res = chan.queue_fail_malformed_htlc(
5083-
htlc_id, failure_code, sha256_of_onion, &&logger
5084-
);
5085-
Some((res, htlc_id))
5113+
if let Some(ChannelPhase::Funded(ref mut chan)) = peer_state.channel_by_id.get_mut(&forward_chan_id) {
5114+
let logger = WithChannelContext::from(&self.logger, &chan.context, None);
5115+
log_trace!(logger, "Failing malformed HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
5116+
let res = chan.queue_fail_malformed_htlc(
5117+
htlc_id, failure_code, sha256_of_onion, &&logger
5118+
);
5119+
Some((res, htlc_id))
5120+
} else {
5121+
forwarding_channel_not_found!(core::iter::once(forward_info).chain(draining_pending_forwards));
5122+
break;
5123+
}
50865124
},
50875125
};
50885126
if let Some((queue_fail_htlc_res, htlc_id)) = queue_fail_htlc_res {
50895127
if let Err(e) = queue_fail_htlc_res {
50905128
if let ChannelError::Ignore(msg) = e {
5091-
log_trace!(logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
5129+
if let Some(ChannelPhase::Funded(ref mut chan)) = peer_state.channel_by_id.get_mut(&forward_chan_id) {
5130+
let logger = WithChannelContext::from(&self.logger, &chan.context, None);
5131+
log_trace!(logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
5132+
}
50925133
} else {
50935134
panic!("Stated return value requirements in queue_fail_{{malformed_}}htlc() were not met");
50945135
}
@@ -5098,10 +5139,9 @@ where
50985139
continue;
50995140
}
51005141
}
5142+
} else {
5143+
break;
51015144
}
5102-
} else {
5103-
forwarding_channel_not_found!();
5104-
continue;
51055145
}
51065146
} else {
51075147
'next_forwardable_htlc: for forward_info in pending_forwards.drain(..) {

lightning/src/ln/payment_tests.rs

Lines changed: 103 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2053,22 +2053,28 @@ fn accept_underpaying_htlcs_config() {
20532053
fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
20542054
let chanmon_cfgs = create_chanmon_cfgs(3);
20552055
let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2056+
let max_in_flight_percent = 10;
20562057
let mut intercept_forwards_config = test_default_channel_config();
20572058
intercept_forwards_config.accept_intercept_htlcs = true;
2059+
intercept_forwards_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = max_in_flight_percent;
20582060
let mut underpay_config = test_default_channel_config();
20592061
underpay_config.channel_config.accept_underpaying_htlcs = true;
2062+
underpay_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = max_in_flight_percent;
20602063
let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(underpay_config)]);
20612064
let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
20622065

2066+
let amt_msat = 900_000;
2067+
20632068
let mut chan_ids = Vec::new();
20642069
for _ in 0..num_mpp_parts {
2065-
let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000, 0);
2066-
let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 2_000_000, 0).0.channel_id;
2070+
// We choose the channel size so that there can be at most one part pending on each channel.
2071+
let channel_size = amt_msat / 1000 / num_mpp_parts as u64 * 100 / max_in_flight_percent as u64 + 100;
2072+
let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, channel_size, 0);
2073+
let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, channel_size, 0).0.channel_id;
20672074
chan_ids.push(channel_id);
20682075
}
20692076

20702077
// Send the initial payment.
2071-
let amt_msat = 900_000;
20722078
let skimmed_fee_msat = 20;
20732079
let mut route_hints = Vec::new();
20742080
for _ in 0..num_mpp_parts {
@@ -4098,7 +4104,7 @@ fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
40984104

40994105
// Create a new channel between C and D as A will refuse to retry on the existing one because
41004106
// it just failed.
4101-
let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
4107+
create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0);
41024108

41034109
// Now retry the failed HTLC.
41044110
nodes[0].node.process_pending_htlc_forwards();
@@ -4110,6 +4116,7 @@ fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
41104116
expect_pending_htlcs_forwardable!(nodes[2]);
41114117
check_added_monitors(&nodes[2], 1);
41124118
let cs_forward = SendEvent::from_node(&nodes[2]);
4119+
let cd_channel_used = cs_forward.msgs[0].channel_id;
41134120
nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
41144121
commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
41154122

@@ -4129,7 +4136,7 @@ fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
41294136
nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
41304137
commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
41314138
expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
4132-
&[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
4139+
&[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: cd_channel_used }]);
41334140
} else {
41344141
expect_pending_htlcs_forwardable!(nodes[3]);
41354142
expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
@@ -4294,3 +4301,94 @@ fn peel_payment_onion_custom_tlvs() {
42944301
_ => panic!()
42954302
}
42964303
}
4304+
4305+
#[test]
4306+
fn test_non_strict_forwarding() {
4307+
let chanmon_cfgs = create_chanmon_cfgs(3);
4308+
let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
4309+
let mut config = test_default_channel_config();
4310+
config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
4311+
let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[Some(config), Some(config), Some(config)]);
4312+
let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
4313+
4314+
// Create a routing node with two outbound channels, each of which can forward 2 payments of
4315+
// the given value.
4316+
let payment_value = 1_500_000;
4317+
create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
4318+
let (chan_update_1, _, channel_id_1, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 4_950, 0);
4319+
let (chan_update_2, _, channel_id_2, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 5_000, 0);
4320+
4321+
// Create a route once.
4322+
let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
4323+
.with_bolt11_features(nodes[2].node.bolt11_invoice_features()).unwrap();
4324+
let route_params = RouteParameters::from_payment_params_and_value(payment_params, payment_value);
4325+
let route = functional_test_utils::get_route(&nodes[0], &route_params).unwrap();
4326+
4327+
// Send 4 payments over the same route.
4328+
for i in 0..4 {
4329+
let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash(&nodes[2], Some(payment_value), None);
4330+
nodes[0].node.send_payment_with_route(&route, payment_hash,
4331+
RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
4332+
check_added_monitors!(nodes[0], 1);
4333+
let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
4334+
assert_eq!(msg_events.len(), 1);
4335+
let mut send_event = SendEvent::from_event(msg_events.remove(0));
4336+
nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
4337+
commitment_signed_dance!(nodes[1], nodes[0], &send_event.commitment_msg, false);
4338+
4339+
expect_pending_htlcs_forwardable!(nodes[1]);
4340+
check_added_monitors!(nodes[1], 1);
4341+
msg_events = nodes[1].node.get_and_clear_pending_msg_events();
4342+
assert_eq!(msg_events.len(), 1);
4343+
send_event = SendEvent::from_event(msg_events.remove(0));
4344+
// The HTLC will be forwarded over the most appropriate channel with the corresponding peer,
4345+
// applying non-strict forwarding.
4346+
// The channel with the least amount of outbound liquidity will be used to maximize the
4347+
// probability of being able to successfully forward a subsequent HTLC.
4348+
assert_eq!(send_event.msgs[0].channel_id, if i < 2 {
4349+
channel_id_1
4350+
} else {
4351+
channel_id_2
4352+
});
4353+
nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
4354+
commitment_signed_dance!(nodes[2], nodes[1], &send_event.commitment_msg, false);
4355+
4356+
expect_pending_htlcs_forwardable!(nodes[2]);
4357+
let events = nodes[2].node.get_and_clear_pending_events();
4358+
assert_eq!(events.len(), 1);
4359+
assert!(matches!(events[0], Event::PaymentClaimable { .. }));
4360+
4361+
claim_payment_along_route(
4362+
ClaimAlongRouteArgs::new(&nodes[0], &[&[&nodes[1], &nodes[2]]], payment_preimage)
4363+
);
4364+
}
4365+
4366+
// Send a 5th payment which will fail.
4367+
let (_, payment_hash, payment_secret) = get_payment_preimage_hash(&nodes[2], Some(payment_value), None);
4368+
nodes[0].node.send_payment_with_route(&route, payment_hash,
4369+
RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
4370+
check_added_monitors!(nodes[0], 1);
4371+
let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
4372+
assert_eq!(msg_events.len(), 1);
4373+
let mut send_event = SendEvent::from_event(msg_events.remove(0));
4374+
nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
4375+
commitment_signed_dance!(nodes[1], nodes[0], &send_event.commitment_msg, false);
4376+
4377+
expect_pending_htlcs_forwardable!(nodes[1]);
4378+
check_added_monitors!(nodes[1], 1);
4379+
let routed_scid = route.paths[0].hops[1].short_channel_id;
4380+
let routed_channel_id = match routed_scid {
4381+
scid if scid == chan_update_1.contents.short_channel_id => channel_id_1,
4382+
scid if scid == chan_update_2.contents.short_channel_id => channel_id_2,
4383+
_ => panic!("Unexpected short channel id in route"),
4384+
};
4385+
// The failure to forward will refer to the channel given in the onion.
4386+
expect_pending_htlcs_forwardable_conditions(nodes[1].node.get_and_clear_pending_events(),
4387+
&[HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: routed_channel_id }]);
4388+
4389+
let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
4390+
nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
4391+
commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
4392+
let events = nodes[0].node.get_and_clear_pending_events();
4393+
expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().blamed_scid(routed_scid));
4394+
}

0 commit comments

Comments
 (0)