From 20b5be7c6c43f33ec729eaf38684183d2f93c733 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Tue, 24 Sep 2024 23:33:32 +0800
Subject: [PATCH 01/66] add prepare payment session

---
 src/errors.rs                                 |   2 +
 src/fiber/graph.rs                            |  83 +++++-----
 src/fiber/network.rs                          | 150 ++++++++++++++----
 .../13-node-send-duplicate-payment-err.bru    |  41 +++++
 ...ter.bru => 14-node3-gen-invoice-later.bru} |   2 +-
 ...=> 15-node1-send-payment-with-invoice.bru} |   2 +-
 ....bru => 16-node1-send-payment-keysend.bru} |   2 +-
 7 files changed, 211 insertions(+), 71 deletions(-)
 create mode 100644 tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
 rename tests/bruno/e2e/router-pay/{13-node3-gen-invoice-later.bru => 14-node3-gen-invoice-later.bru} (99%)
 rename tests/bruno/e2e/router-pay/{14-node1-send-payment-with-invoice.bru => 15-node1-send-payment-with-invoice.bru} (98%)
 rename tests/bruno/e2e/router-pay/{15-node1-send-payment-keysend.bru => 16-node1-send-payment-keysend.bru} (98%)

diff --git a/src/errors.rs b/src/errors.rs
index b18c0038..0c858544 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -38,6 +38,8 @@ pub enum Error {
     FundingError(#[from] FundingError),
     #[error("InvalidParameter: {0}")]
     InvalidParameter(String),
+    #[error("Payment error: {0}")]
+    PaymentError(String),
     #[error("Network Graph error: {0}")]
     NetworkGraphError(#[from] GraphError),
     #[error("Invalid peer message: {0}")]
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 6e0de8da..66463370 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -1,4 +1,4 @@
-use super::network::{get_chain_hash, SendPaymentCommand};
+use super::network::{get_chain_hash, SendPaymentData};
 use super::path::NodeHeap;
 use super::types::Pubkey;
 use super::types::{ChannelAnnouncement, ChannelUpdate, Hash256, NodeAnnouncement};
@@ -455,27 +455,16 @@ where
         self.connected_peer_addresses.clear();
     }
 
-    pub fn init_payment_session(&self, payment_request: SendPaymentCommand) -> PaymentSession {
-        let payment_session = PaymentSession::new(payment_request, 3);
-        if let Some(session) = self
-            .store
-            .get_payment_session(payment_session.payment_hash())
-        {
-            return session;
-        } else {
-            self.store.insert_payment_session(payment_session.clone());
-            payment_session
-        }
-    }
-
     pub fn build_route(
         &self,
-        payment_request: SendPaymentCommand,
+        payment_request: SendPaymentData,
     ) -> Result<Vec<OnionInfo>, GraphError> {
         let source = self.get_source_pubkey();
-        let (target, amount, payment_hash, preimage, udt_type_script) = payment_request
-            .check_valid()
-            .map_err(|e| GraphError::Other(format!("payment request is invalid {:?}", e)))?;
+        let target = payment_request.target_pubkey;
+        let amount = payment_request.amount;
+        let preimage = payment_request.preimage;
+        let payment_hash = payment_request.payment_hash;
+        let udt_type_script = payment_request.udt_type_script;
         let invoice = payment_request
             .invoice
             .map(|x| x.parse::<CkbInvoice>().unwrap());
@@ -781,32 +770,49 @@ pub trait NetworkGraphStateStore {
     fn insert_payment_session(&self, session: PaymentSession);
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum PaymentSessionStatus {
+    Created,
+    Inflight,
+    Success,
+    Failed,
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct PaymentSession {
-    pub command: SendPaymentCommand,
+    pub request: SendPaymentData,
     pub retried_times: u32,
     pub last_error: Option<String>,
     pub try_limit: u32,
+    pub status: PaymentSessionStatus,
+    pub created_time: u128,
 }
 
 impl PaymentSession {
-    pub fn new(command: SendPaymentCommand, try_limit: u32) -> Self {
+    pub fn new(request: SendPaymentData, try_limit: u32) -> Self {
         Self {
-            command,
+            request,
             retried_times: 0,
             last_error: None,
             try_limit,
+            status: PaymentSessionStatus::Created,
+            created_time: std::time::UNIX_EPOCH.elapsed().unwrap().as_millis(),
         }
     }
 
     pub fn payment_hash(&self) -> Hash256 {
-        self.command.payment_hash()
+        self.request.payment_hash
+    }
+
+    pub fn set_status(&mut self, status: PaymentSessionStatus) {
+        self.status = status;
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::fiber::network::SendPaymentData;
     use crate::fiber::test_utils::{generate_keypair, generate_pubkey};
     use crate::store::Store;
     use ckb_types::prelude::Entity;
@@ -1312,17 +1318,18 @@ mod tests {
         let node2 = network.keys[2];
         let node3 = network.keys[3];
         // Test build route from node1 to node3
-        let route = network.graph.build_route(SendPaymentCommand {
-            target_pubkey: Some(node3.into()),
-            amount: Some(100),
-            payment_hash: Some(Hash256::default()),
+        let route = network.graph.build_route(SendPaymentData {
+            target_pubkey: node3.into(),
+            amount: 100,
+            payment_hash: Hash256::default(),
             invoice: None,
             final_cltv_delta: Some(100),
             timeout: Some(10),
             max_fee_amount: Some(1000),
             max_parts: None,
-            keysend: None,
+            keysend: false,
             udt_type_script: None,
+            preimage: None,
         });
         eprintln!("return {:?}", route);
         assert!(route.is_ok());
@@ -1349,17 +1356,18 @@ mod tests {
         let node3 = network.keys[3];
 
         // Test build route from node1 to node3 with amount exceeding max_htlc_value
-        let route = network.graph.build_route(SendPaymentCommand {
-            target_pubkey: Some(node3.into()),
-            amount: Some(100), // Exceeds max_htlc_value of 50
-            payment_hash: Some(Hash256::default()),
+        let route = network.graph.build_route(SendPaymentData {
+            target_pubkey: node3.into(),
+            amount: 100, // Exceeds max_htlc_value of 50
+            payment_hash: Hash256::default(),
             invoice: None,
             final_cltv_delta: Some(100),
             timeout: Some(10),
             max_fee_amount: Some(1000),
             max_parts: None,
-            keysend: None,
+            keysend: false,
             udt_type_script: None,
+            preimage: None,
         });
         assert!(route.is_err());
     }
@@ -1373,17 +1381,18 @@ mod tests {
         let node3 = network.keys[3];
 
         // Test build route from node1 to node3 with amount below min_htlc_value
-        let route = network.graph.build_route(SendPaymentCommand {
-            target_pubkey: Some(node3.into()),
-            amount: Some(10), // Below min_htlc_value of 50
-            payment_hash: Some(Hash256::default()),
+        let route = network.graph.build_route(SendPaymentData {
+            target_pubkey: node3.into(),
+            amount: 10, // Below min_htlc_value of 50
+            payment_hash: Hash256::default(),
             invoice: None,
             final_cltv_delta: Some(100),
             timeout: Some(10),
             max_fee_amount: Some(1000),
             max_parts: None,
-            keysend: None,
+            keysend: false,
             udt_type_script: None,
+            preimage: None,
         });
         assert!(route.is_err());
     }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 38860cb7..cd0eca0b 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -69,7 +69,7 @@ use crate::ckb::{CkbChainMessage, FundingRequest, FundingTx, TraceTxRequest, Tra
 use crate::fiber::channel::{
     AddTlcCommand, AddTlcResponse, TxCollaborationCommand, TxUpdateCommand,
 };
-use crate::fiber::graph::{ChannelInfo, NodeInfo, PaymentSession};
+use crate::fiber::graph::{ChannelInfo, NodeInfo, PaymentSession, PaymentSessionStatus};
 use crate::fiber::types::{secp256k1_instance, FiberChannelMessage, OnionPacket, TxSignatures};
 use crate::invoice::{CkbInvoice, InvoiceStore};
 use crate::{unwrap_or_return, Error};
@@ -221,12 +221,26 @@ pub struct SendPaymentCommand {
     pub udt_type_script: Option<Script>,
 }
 
-impl SendPaymentCommand {
-    // return (target_pubkey, amount, payment_hash, preimage, udt_type_script)
-    pub fn check_valid(
-        &self,
-    ) -> Result<(Pubkey, u128, Hash256, Option<Hash256>, Option<Script>), String> {
-        let invoice = self
+#[serde_as]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SendPaymentData {
+    pub target_pubkey: Pubkey,
+    pub amount: u128,
+    pub payment_hash: Hash256,
+    pub invoice: Option<String>,
+    pub final_cltv_delta: Option<u64>,
+    pub timeout: Option<u64>,
+    pub max_fee_amount: Option<u128>,
+    pub max_parts: Option<u64>,
+    pub keysend: bool,
+    #[serde_as(as = "Option<EntityHex>")]
+    pub udt_type_script: Option<Script>,
+    pub preimage: Option<Hash256>,
+}
+
+impl SendPaymentData {
+    pub fn new(command: SendPaymentCommand) -> Result<SendPaymentData, String> {
+        let invoice = command
             .invoice
             .as_ref()
             .map(|invoice| invoice.parse::<CkbInvoice>())
@@ -252,7 +266,7 @@ impl SendPaymentCommand {
         }
 
         let target = validate_field(
-            self.target_pubkey,
+            command.target_pubkey,
             invoice
                 .as_ref()
                 .and_then(|i| i.payee_pub_key().cloned().map(Pubkey::from)),
@@ -260,13 +274,13 @@ impl SendPaymentCommand {
         )?;
 
         let amount = validate_field(
-            self.amount,
+            command.amount,
             invoice.as_ref().and_then(|i| i.amount()),
             "amount",
         )?;
 
         let udt_type_script = match validate_field(
-            self.udt_type_script.clone(),
+            command.udt_type_script.clone(),
             invoice.as_ref().and_then(|i| i.udt_type_script().cloned()),
             "udt_type_script",
         ) {
@@ -275,10 +289,11 @@ impl SendPaymentCommand {
             Err(e) => return Err(e),
         };
 
-        let (payment_hash, preimage) = if !self.is_keysend() {
+        let keysend = command.keysend.unwrap_or(false);
+        let (payment_hash, preimage) = if !keysend {
             (
                 validate_field(
-                    self.payment_hash,
+                    command.payment_hash,
                     invoice.as_ref().map(|i| *i.payment_hash()),
                     "payment_hash",
                 )?,
@@ -298,16 +313,19 @@ impl SendPaymentCommand {
             (payment_hash, Some(preimage))
         };
 
-        Ok((target, amount, payment_hash, preimage, udt_type_script))
-    }
-
-    pub fn payment_hash(&self) -> Hash256 {
-        let (_, _, payment_hash, ..) = self.check_valid().expect("valid SendPaymentCommand");
-        payment_hash
-    }
-
-    pub fn is_keysend(&self) -> bool {
-        self.keysend.unwrap_or(false)
+        Ok(SendPaymentData {
+            target_pubkey: target,
+            amount,
+            payment_hash,
+            invoice: command.invoice,
+            final_cltv_delta: command.final_cltv_delta,
+            timeout: command.timeout,
+            max_fee_amount: command.max_fee_amount,
+            max_parts: command.max_parts,
+            keysend,
+            udt_type_script,
+            preimage,
+        })
     }
 }
 
@@ -1801,25 +1819,95 @@ where
         }
     }
 
+    fn prepare_payment_session(&self, payment_data: SendPaymentData) -> PaymentSession {
+        let payment_session = PaymentSession::new(payment_data.clone(), 5);
+        self.store.insert_payment_session(payment_session.clone());
+        payment_session
+    }
+
     async fn on_send_payment(
         &self,
         my_self: ActorRef<NetworkActorMessage>,
         payment_request: SendPaymentCommand,
     ) -> Result<Hash256, Error> {
         let graph = self.network_graph.read().await;
+        let payment_data = SendPaymentData::new(payment_request).map_err(|e| {
+            error!("Failed to validate payment request: {:?}", e);
+            Error::InvalidParameter(format!("Failed to validate payment request: {:?}", e))
+        })?;
         // initialize the payment session in db and begin the payment process in a statemachine to
         // handle the payment process
-        let payment_session = PaymentSession::new(payment_request.clone(), 3);
+        if let Some(payment_session) = self.store.get_payment_session(payment_data.payment_hash) {
+            // we only allow retrying payment session with status failed
+            if payment_session.status != PaymentSessionStatus::Failed {
+                return Err(Error::InvalidParameter(format!(
+                    "Payment session already exists: {} with payment session status: {:?}",
+                    payment_data.payment_hash, payment_session.status
+                )));
+            }
+        }
 
-        let onion_path = graph.build_route(payment_request.clone())?;
-        assert!(!onion_path.is_empty());
-        let onion_packet = OnionPacket::new(onion_path).serialize();
+        let mut payment_session = self.prepare_payment_session(payment_data.clone());
+        let mut error = None;
 
-        let res = my_self.send_message(NetworkActorMessage::Command(
-            NetworkActorCommand::SendOnionPacket(onion_packet.clone(), None),
-        ));
-        info!("send_payment: {:?} => result: {:?}", payment_request, res);
-        Ok(payment_session.payment_hash())
+        for i in 0..payment_session.try_limit {
+            debug!(
+                "send_payment: {:?} => retrying: {:?}",
+                payment_data.payment_hash, i
+            );
+            payment_session.retried_times += 1;
+            let onion_path = graph.build_route(payment_data.clone());
+            let onion_path = match onion_path {
+                Err(e) => {
+                    error!("Failed to build route: {:?}", e);
+                    payment_session.set_status(PaymentSessionStatus::Failed);
+                    error = Some(Err(Error::PaymentError(format!(
+                        "Failed to build route: {:?}",
+                        payment_data.payment_hash
+                    ))));
+                    break;
+                }
+                Ok(onion_path) => {
+                    assert!(!onion_path.is_empty());
+                    onion_path
+                }
+            };
+
+            let onion_packet = OnionPacket::new(onion_path).serialize();
+            let res = my_self.send_message(NetworkActorMessage::Command(
+                NetworkActorCommand::SendOnionPacket(onion_packet.clone(), None),
+            ));
+
+            debug!(
+                "send_payment: {:?} => sending onion packet: {:?}",
+                payment_data.payment_hash, res
+            );
+            match res {
+                Err(e) => {
+                    error!("Failed to send onion packet: {:?}", e);
+                    // for some error we may need to update the graph and then retry
+                    // TODO: update the graph here according to the error
+                    let err = format!(
+                        "Failed to send onion packet: {:?} with error: {:?}",
+                        payment_data.payment_hash, e
+                    );
+                    payment_session.last_error = Some(err.clone());
+                    error = Some(Err(Error::PaymentError(err)));
+                    continue;
+                }
+                Ok(_) => {
+                    info!(
+                        "send_payment: {:?} => onion packet sent: {:?}",
+                        payment_data.payment_hash, onion_packet
+                    );
+                    payment_session.set_status(PaymentSessionStatus::Inflight);
+                    return Ok(payment_session.payment_hash());
+                }
+            }
+        }
+        payment_session.set_status(PaymentSessionStatus::Failed);
+        self.store.insert_payment_session(payment_session);
+        return error.expect("expect error details");
     }
 }
 
diff --git a/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru b/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
new file mode 100644
index 00000000..4429e1ba
--- /dev/null
+++ b/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
@@ -0,0 +1,41 @@
+meta {
+  name: Node1 send payment with router, since we have created a payment session for this payment, it will return error
+  type: http
+  seq: 13
+}
+
+post {
+  url: {{NODE1_RPC_URL}}
+  body: json
+  auth: none
+}
+
+headers {
+  Content-Type: application/json
+  Accept: application/json
+}
+
+body:json {
+  {
+    "id": "42",
+    "jsonrpc": "2.0",
+    "method": "send_payment",
+    "params": [
+      {
+        "target_pubkey": "03032b99943822e721a651c5a5b9621043017daa9dc3ec81d83215fd2e25121187",
+        "amount": "0x190",
+        "payment_hash": "{{payment_hash}}"
+      }
+    ]
+  }
+}
+
+assert {
+  res.body.error: isDefined
+}
+
+script:post-response {
+  // Sleep for sometime to make sure current operation finishes before next request starts.
+  await new Promise(r => setTimeout(r, 100));
+  console.log("get result: ", res.body)
+}
diff --git a/tests/bruno/e2e/router-pay/13-node3-gen-invoice-later.bru b/tests/bruno/e2e/router-pay/14-node3-gen-invoice-later.bru
similarity index 99%
rename from tests/bruno/e2e/router-pay/13-node3-gen-invoice-later.bru
rename to tests/bruno/e2e/router-pay/14-node3-gen-invoice-later.bru
index beb6853a..4eb25c18 100644
--- a/tests/bruno/e2e/router-pay/13-node3-gen-invoice-later.bru
+++ b/tests/bruno/e2e/router-pay/14-node3-gen-invoice-later.bru
@@ -1,7 +1,7 @@
 meta {
   name: generate a invoice
   type: http
-  seq: 13
+  seq: 14
 }
 
 post {
diff --git a/tests/bruno/e2e/router-pay/14-node1-send-payment-with-invoice.bru b/tests/bruno/e2e/router-pay/15-node1-send-payment-with-invoice.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/14-node1-send-payment-with-invoice.bru
rename to tests/bruno/e2e/router-pay/15-node1-send-payment-with-invoice.bru
index 8ccd33fa..3684be42 100644
--- a/tests/bruno/e2e/router-pay/14-node1-send-payment-with-invoice.bru
+++ b/tests/bruno/e2e/router-pay/15-node1-send-payment-with-invoice.bru
@@ -1,7 +1,7 @@
 meta {
   name: Node1 send payment with router only with invoice
   type: http
-  seq: 14
+  seq: 15
 }
 
 post {
diff --git a/tests/bruno/e2e/router-pay/15-node1-send-payment-keysend.bru b/tests/bruno/e2e/router-pay/16-node1-send-payment-keysend.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/15-node1-send-payment-keysend.bru
rename to tests/bruno/e2e/router-pay/16-node1-send-payment-keysend.bru
index 085dd101..c38d0f7c 100644
--- a/tests/bruno/e2e/router-pay/15-node1-send-payment-keysend.bru
+++ b/tests/bruno/e2e/router-pay/16-node1-send-payment-keysend.bru
@@ -1,7 +1,7 @@
 meta {
   name: Node1 send payment with router in keysend mode
   type: http
-  seq: 15
+  seq: 16
 }
 
 post {

From d3afa37197ea44a0448e7bc3bd4eafe4589e6315 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Wed, 25 Sep 2024 18:28:31 +0800
Subject: [PATCH 02/66] add reply for onion send

---
 src/errors.rs        |   2 +
 src/fiber/channel.rs |  13 +++-
 src/fiber/network.rs | 156 +++++++++++++++++++++++++++++--------------
 3 files changed, 117 insertions(+), 54 deletions(-)

diff --git a/src/errors.rs b/src/errors.rs
index 0c858544..34ba4b73 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -36,6 +36,8 @@ pub enum Error {
     CkbInvoiceError(#[from] InvoiceError),
     #[error("Funding error: {0}")]
     FundingError(#[from] FundingError),
+    #[error("Send payment error: {0}")]
+    SendPaymentError(String),
     #[error("InvalidParameter: {0}")]
     InvalidParameter(String),
     #[error("Payment error: {0}")]
diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 9e947b2c..c19057d7 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -4,7 +4,7 @@ use tracing::{debug, error, info, warn};
 
 use crate::{
     fiber::{
-        network::get_chain_hash,
+        network::{get_chain_hash, SendOnionPacketCommand},
         types::{ChannelUpdate, OnionPacket},
     },
     invoice::InvoiceStore,
@@ -495,14 +495,21 @@ where
                 // this message, and our processing of this message is idempotent.
 
                 if forward_to_next_hop {
+                    let (send, recv) = oneshot::channel::<Result<u64, String>>();
+                    let rpc_reply = RpcReplyPort::from(send);
                     self.network
                         .send_message(NetworkActorMessage::Command(
                             NetworkActorCommand::SendOnionPacket(
-                                add_tlc.onion_packet.clone(),
-                                Some((state.get_id(), tlc.get_id())),
+                                SendOnionPacketCommand {
+                                    packet: add_tlc.onion_packet.clone(),
+                                    previous_tlc: Some((state.get_id(), tlc.get_id())),
+                                },
+                                rpc_reply,
                             ),
                         ))
                         .expect("network actor is alive");
+                    let _res = recv.await;
+                    // TODO: if the result is an error, we should handle it. we need to begin to remove the tlc.
                 }
                 Ok(())
             }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index cd0eca0b..94a2b378 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -139,7 +139,7 @@ pub enum NetworkActorCommand {
     ),
     // Send a command to a channel.
     ControlFiberChannel(ChannelCommandWithId),
-    SendOnionPacket(Vec<u8>, Option<(Hash256, u64)>),
+    SendOnionPacket(SendOnionPacketCommand, RpcReplyPort<Result<u64, String>>),
     UpdateChannelFunding(Hash256, Transaction, FundingRequest),
     SignTx(PeerId, Hash256, Transaction, Option<Vec<Vec<u8>>>),
     // Broadcast node/channel information.
@@ -335,6 +335,12 @@ pub struct AcceptChannelCommand {
     pub funding_amount: u128,
 }
 
+#[derive(Debug)]
+pub struct SendOnionPacketCommand {
+    pub packet: Vec<u8>,
+    pub previous_tlc: Option<(Hash256, u64)>,
+}
+
 impl NetworkActorMessage {
     pub fn new_event(event: NetworkActorEvent) -> Self {
         Self::Event(event)
@@ -1216,42 +1222,10 @@ where
             }
 
             // TODO: we should check the OnionPacket is valid or not, only the current node can decrypt it.
-            NetworkActorCommand::SendOnionPacket(packet, previous_tlc) => {
-                if let Ok(mut onion_packet) = OnionPacket::deserialize(&packet) {
-                    if let Ok(info) = onion_packet.shift() {
-                        debug!("Processing onion packet info: {:?}", info);
-                        let channel_outpoint = &info
-                            .channel_outpoint
-                            .expect("valid onion packet contains channel outpoint");
-                        let channel_id = match state.outpoint_channel_map.get(channel_outpoint) {
-                            Some(channel_id) => channel_id,
-                            None => {
-                                // TODO: Return error to the sender. This is a payment failure.
-                                error!("Failed to process onion packet: channel id not found for channel outpoint {:?}. Are we connected to the peer?", channel_outpoint);
-                                return Ok(());
-                            }
-                        };
-                        let (send, recv) = oneshot::channel::<Result<AddTlcResponse, String>>();
-                        let rpc_reply = RpcReplyPort::from(send);
-                        let command = ChannelCommand::AddTlc(
-                            AddTlcCommand {
-                                amount: info.amount,
-                                preimage: None,
-                                payment_hash: Some(info.payment_hash),
-                                expiry: info.expiry.into(),
-                                hash_algorithm: info.tlc_hash_algorithm,
-                                onion_packet: onion_packet.serialize(),
-                                previous_tlc,
-                            },
-                            rpc_reply,
-                        );
-                        state.send_command_to_channel(*channel_id, command).await?;
-                        let res = recv.await.expect("recv add tlc response");
-                        info!("send onion packet: {:?}", res);
-                    }
-                } else {
-                    info!("onion packet is empty, ignore it");
-                }
+            NetworkActorCommand::SendOnionPacket(command, reply) => {
+                info!("send onion packet: {:?}", command);
+                self.handle_onion_packet_send_command(state, command, reply)
+                    .await;
             }
             NetworkActorCommand::UpdateChannelFunding(channel_id, transaction, request) => {
                 let old_tx = transaction.into_view();
@@ -1449,7 +1423,7 @@ where
                 let _ = reply.send(signature);
             }
             NetworkActorCommand::SendPayment(payment_request, reply) => {
-                match self.on_send_payment(myself, payment_request).await {
+                match self.on_send_payment(state, myself, payment_request).await {
                     Ok(payment_hash) => {
                         let _ = reply.send(Ok(SendPaymentResponse { payment_hash }));
                     }
@@ -1819,6 +1793,74 @@ where
         }
     }
 
+    async fn handle_onion_packet_send_command(
+        &self,
+        state: &mut NetworkActorState<S>,
+        command: SendOnionPacketCommand,
+        reply: RpcReplyPort<Result<u64, String>>,
+    ) {
+        let SendOnionPacketCommand {
+            packet,
+            previous_tlc,
+        } = command;
+        if let Ok(mut onion_packet) = OnionPacket::deserialize(&packet) {
+            if let Ok(info) = onion_packet.shift() {
+                debug!("Processing onion packet info: {:?}", info);
+                let channel_outpoint = &info
+                    .channel_outpoint
+                    .expect("valid onion packet contains channel outpoint");
+                let channel_id = match state.outpoint_channel_map.get(channel_outpoint) {
+                    Some(channel_id) => channel_id,
+                    None => {
+                        error!(
+                            "Failed to process onion packet: channel id not found for channel outpoint {:?}. \
+                            Are we connected to the peer?",
+                            channel_outpoint
+                        );
+                        reply
+                            .send(Err("channel id not found".to_string()))
+                            .expect("send add tlc response");
+                        return;
+                    }
+                };
+                let (send, recv) = oneshot::channel::<Result<AddTlcResponse, String>>();
+                let rpc_reply = RpcReplyPort::from(send);
+                let command = ChannelCommand::AddTlc(
+                    AddTlcCommand {
+                        amount: info.amount,
+                        preimage: None,
+                        payment_hash: Some(info.payment_hash),
+                        expiry: info.expiry.into(),
+                        hash_algorithm: info.tlc_hash_algorithm,
+                        onion_packet: onion_packet.serialize(),
+                        previous_tlc,
+                    },
+                    rpc_reply,
+                );
+
+                // we have already checked the channel_id is valid
+                state
+                    .send_command_to_channel(*channel_id, command)
+                    .await
+                    .expect("send command failed");
+
+                let res = recv.await;
+                info!("send onion packet: {:?}", res);
+                let res = match res {
+                    Ok(Ok(res)) => Ok(res.tlc_id),
+                    Ok(Err(err)) => Err(err),
+                    Err(e) => Err(e.to_string()),
+                };
+                reply.send(res).expect("send error");
+            }
+        } else {
+            info!("onion packet is empty, ignore it");
+            reply
+                .send(Err("onion packet is empty".to_string()))
+                .expect("send error");
+        }
+    }
+
     fn prepare_payment_session(&self, payment_data: SendPaymentData) -> PaymentSession {
         let payment_session = PaymentSession::new(payment_data.clone(), 5);
         self.store.insert_payment_session(payment_session.clone());
@@ -1827,11 +1869,12 @@ where
 
     async fn on_send_payment(
         &self,
-        my_self: ActorRef<NetworkActorMessage>,
+        state: &mut NetworkActorState<S>,
+        _my_self: ActorRef<NetworkActorMessage>,
         payment_request: SendPaymentCommand,
     ) -> Result<Hash256, Error> {
         let graph = self.network_graph.read().await;
-        let payment_data = SendPaymentData::new(payment_request).map_err(|e| {
+        let payment_data = SendPaymentData::new(payment_request.clone()).map_err(|e| {
             error!("Failed to validate payment request: {:?}", e);
             Error::InvalidParameter(format!("Failed to validate payment request: {:?}", e))
         })?;
@@ -1850,11 +1893,7 @@ where
         let mut payment_session = self.prepare_payment_session(payment_data.clone());
         let mut error = None;
 
-        for i in 0..payment_session.try_limit {
-            debug!(
-                "send_payment: {:?} => retrying: {:?}",
-                payment_data.payment_hash, i
-            );
+        for _i in 0..payment_session.try_limit {
             payment_session.retried_times += 1;
             let onion_path = graph.build_route(payment_data.clone());
             let onion_path = match onion_path {
@@ -1874,12 +1913,27 @@ where
             };
 
             let onion_packet = OnionPacket::new(onion_path).serialize();
-            let res = my_self.send_message(NetworkActorMessage::Command(
-                NetworkActorCommand::SendOnionPacket(onion_packet.clone(), None),
-            ));
-
-            debug!(
-                "send_payment: {:?} => sending onion packet: {:?}",
+            let (send, recv) = oneshot::channel::<Result<u64, String>>();
+            let rpc_reply = RpcReplyPort::from(send);
+            let command = SendOnionPacketCommand {
+                packet: onion_packet.clone(),
+                previous_tlc: None,
+            };
+            self.handle_onion_packet_send_command(state, command, rpc_reply)
+                .await;
+            let res = recv.await;
+            info!(
+                "send_payment: {:?} => onion packet sent: {:?}",
+                payment_data.payment_hash, res,
+            );
+            let res = match res {
+                Ok(Ok(tlc_id)) => Ok(tlc_id),
+                Ok(Err(err)) => Err(err),
+                Err(e) => Err(e.to_string()),
+            }
+            .map_err(|err| Error::SendPaymentError(err.to_string()));
+            info!(
+                "send_payment: {:?} => result: {:?}",
                 payment_data.payment_hash, res
             );
             match res {

From 49cafa299376ce26f837b3dffedb36b58f476910 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Wed, 25 Sep 2024 23:06:12 +0800
Subject: [PATCH 03/66] avoid to insert tlc with same payment_hash

---
 src/fiber/channel.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index c19057d7..6f056766 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -3075,6 +3075,17 @@ impl ChannelActorState {
     }
 
     pub fn insert_tlc(&mut self, tlc: TLC) -> Result<DetailedTLCInfo, ProcessingChannelError> {
+        let payment_hash = tlc.payment_hash;
+        if self
+            .tlcs
+            .values()
+            .any(|tlc| tlc.tlc.payment_hash == payment_hash)
+        {
+            return Err(ProcessingChannelError::InvalidParameter(format!(
+                "Trying to insert tlc with duplicate payment hash {:?}",
+                payment_hash
+            )));
+        }
         if let Some(current) = self.tlcs.get(&tlc.id) {
             if current.tlc == tlc {
                 debug!(

From 0df8d269715e8036c67f80d91c67dec9e8a3cc1f Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 07:16:31 +0800
Subject: [PATCH 04/66] fix issues in onion forward

---
 src/fiber/channel.rs | 34 ++++++++++++++++------------------
 src/fiber/network.rs |  3 ++-
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 6f056766..b794d072 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -450,30 +450,28 @@ where
             FiberChannelMessage::AddTlc(add_tlc) => {
                 state.check_for_tlc_update(Some(add_tlc.amount))?;
 
-                // check the onion_packet is valid or not, if not, we should return an error.
+                // check the onion_packet is valid or not, check the payment hash and amount.
+                // check time lock is valid or not.
                 // If there is a next hop, we should send the AddTlc message to the next hop.
-                // If this is the last hop, we should check the payment hash and amount and then
-                // try to fulfill the payment, find the corresponding payment preimage from payment hash.
+                // If this is the last hop, try to fulfill the payment,
+                // find the corresponding payment preimage from payment hash, this is done in try_to_settle_down_tlc.
                 let mut forward_to_next_hop = false;
                 let mut preimage = None;
                 if let Ok(onion_packet) = OnionPacket::deserialize(&add_tlc.onion_packet) {
                     if let Some(onion_info) = onion_packet.peek() {
-                        if onion_info.next_hop.is_some() {
-                            forward_to_next_hop = true;
-                        } else {
-                            // check the payment hash and amount
-                            if onion_info.payment_hash != add_tlc.payment_hash
-                                || onion_info.amount != add_tlc.amount
-                            {
-                                return Err(ProcessingChannelError::InvalidParameter(
-                                    "Payment hash or amount mismatch".to_string(),
-                                ));
-                            }
-                            // TODO: check the expiry time, if expired, we should return an error.
-
-                            // if this is the last hop, store the preimage.
-                            preimage = onion_info.preimage;
+                        // check the payment hash and amount
+                        if onion_info.payment_hash != add_tlc.payment_hash
+                            || onion_info.amount != add_tlc.amount
+                        {
+                            return Err(ProcessingChannelError::InvalidParameter(
+                                "Payment hash or amount mismatch".to_string(),
+                            ));
                         }
+                        // TODO: check the expiry time, if expired, we should return an error.
+                        // if this is the last hop, store the preimage.
+                        preimage = onion_info.preimage;
+
+                        forward_to_next_hop = onion_info.next_hop.is_some();
                     }
                 }
 
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 94a2b378..2f10d28b 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1955,7 +1955,8 @@ where
                         payment_data.payment_hash, onion_packet
                     );
                     payment_session.set_status(PaymentSessionStatus::Inflight);
-                    return Ok(payment_session.payment_hash());
+                    self.store.insert_payment_session(payment_session);
+                    return Ok(payment_data.payment_hash);
                 }
             }
         }

From 8f1d52a06df3706163196c444baeccbf6ba2c701 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 07:27:22 +0800
Subject: [PATCH 05/66] fix test case for duplicate payment hash in add_tlc

---
 src/fiber/channel.rs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index b794d072..b535867e 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -3074,14 +3074,14 @@ impl ChannelActorState {
 
     pub fn insert_tlc(&mut self, tlc: TLC) -> Result<DetailedTLCInfo, ProcessingChannelError> {
         let payment_hash = tlc.payment_hash;
-        if self
+        if let Some(tlc) = self
             .tlcs
             .values()
-            .any(|tlc| tlc.tlc.payment_hash == payment_hash)
+            .find(|tlc| tlc.tlc.payment_hash == payment_hash && tlc.removed_at.is_none())
         {
             return Err(ProcessingChannelError::InvalidParameter(format!(
-                "Trying to insert tlc with duplicate payment hash {:?}",
-                payment_hash
+                "Trying to insert tlc with duplicate payment hash {:?} with tlc {:?}",
+                payment_hash, tlc
             )));
         }
         if let Some(current) = self.tlcs.get(&tlc.id) {
@@ -6124,6 +6124,9 @@ mod tests {
         .expect("node_b alive")
         .expect("successfully removed tlc");
 
+        dbg!("Sleeping for some time to wait for the RemoveTlc processed by both party");
+        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+
         let add_tlc_result = call!(node_a.network_actor, |rpc_reply| {
             NetworkActorMessage::Command(NetworkActorCommand::ControlFiberChannel(
                 ChannelCommandWithId {

From dcbb5cba63ba6770f537d2b638e770eb27319e57 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 07:58:04 +0800
Subject: [PATCH 06/66] e2e test failure because of duplicated payment_hash
 checking in add_tlc

---
 .../e2e/open-use-close-a-channel/08-add-tlc-from-NODE1.bru  | 2 +-
 .../e2e/open-use-close-a-channel/09-add-tlc-from-NODE3.bru  | 2 +-
 .../10-add-tlc-from-NODE1-balance-err.bru                   | 2 +-
 .../e2e/open-use-close-a-channel/12-add-tlc-from-NODE3.bru  | 2 +-
 tests/nodes/start.sh                                        | 6 +++++-
 5 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/bruno/e2e/open-use-close-a-channel/08-add-tlc-from-NODE1.bru b/tests/bruno/e2e/open-use-close-a-channel/08-add-tlc-from-NODE1.bru
index 9bdff254..cdb4e663 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/08-add-tlc-from-NODE1.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/08-add-tlc-from-NODE1.bru
@@ -24,7 +24,7 @@ body:json {
       {
         "channel_id": "{{CHANNEL_ID}}",
         "amount": "0xbebc200",
-        "payment_hash": "0xcb7bce98a778f130d34da522623d7e56705bddfe0dc4781bd2331211134a19a5",
+        "payment_hash": "0xcb7bce98a778f130d34da522623d7e56705bddfe0dc4781bd2331211134a19a8",
         "expiry": 40
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/09-add-tlc-from-NODE3.bru b/tests/bruno/e2e/open-use-close-a-channel/09-add-tlc-from-NODE3.bru
index 094a3eb9..0b5323e6 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/09-add-tlc-from-NODE3.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/09-add-tlc-from-NODE3.bru
@@ -24,7 +24,7 @@ body:json {
       {
         "channel_id": "{{CHANNEL_ID}}",
         "amount": "0x11e1a300",
-        "payment_hash": "0x29449e2cc6f56a691253fe88e3a378171c81573b09247010b4f1cb8c806e1e38",
+        "payment_hash": "0x29449e2cc6f56a691253fe88e3a378171c81573b09247010b4f1cb8c806e1e09",
         "expiry": 40
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/10-add-tlc-from-NODE1-balance-err.bru b/tests/bruno/e2e/open-use-close-a-channel/10-add-tlc-from-NODE1-balance-err.bru
index 9f557b11..0fde456a 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/10-add-tlc-from-NODE1-balance-err.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/10-add-tlc-from-NODE1-balance-err.bru
@@ -24,7 +24,7 @@ body:json {
       {
         "channel_id": "{{CHANNEL_ID}}",
         "amount": "0xbebc200",
-        "payment_hash": "0xcb7bce98a778f130d34da522623d7e56705bddfe0dc4781bd2331211134a19a5",
+        "payment_hash": "0xcb7bce98a778f130d34da522623d7e56705bddfe0dc4781bd2331211134a19a6",
         "expiry": 40
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/12-add-tlc-from-NODE3.bru b/tests/bruno/e2e/open-use-close-a-channel/12-add-tlc-from-NODE3.bru
index 894c9318..66a31cd0 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/12-add-tlc-from-NODE3.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/12-add-tlc-from-NODE3.bru
@@ -24,7 +24,7 @@ body:json {
       {
         "channel_id": "{{CHANNEL_ID}}",
         "amount": "0x11e1a300",
-        "payment_hash": "0x29449e2cc6f56a691253fe88e3a378171c81573b09247010b4f1cb8c806e1e38",
+        "payment_hash": "0x29449e2cc6f56a691253fe88e3a378171c81573b09247010b4f1cb8c806e1e12",
         "expiry": 40
       }
     ]
diff --git a/tests/nodes/start.sh b/tests/nodes/start.sh
index af2d0117..f14fa358 100755
--- a/tests/nodes/start.sh
+++ b/tests/nodes/start.sh
@@ -4,6 +4,7 @@ export SHELLOPTS
 export RUST_BACKTRACE=full RUST_LOG=info,fnn=debug
 
 should_remove_old_state="${REMOVE_OLD_STATE:-}"
+should_clean_fiber_state="${REMOVE_OLD_FIBER:-}"
 should_start_bootnode="${START_BOOTNODE:-}"
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 nodes_dir="$(dirname "$script_dir")/nodes"
@@ -25,7 +26,10 @@ if [[ ! -f "$deploy_dir/.env" ]]; then
     echo "In case of issue pesisting, run $deploy_dir/init-dev-chain.sh -f to reintialize the devchain"
     exit 1
 else
-    if [ -n "$should_remove_old_state" ]; then
+    if [ -n "$should_clean_fiber_state" ]; then
+        echo "starting to clean fiber store ...."
+        rm -rf "$nodes_dir"/*/fiber/store
+    elif [ -n "$should_remove_old_state" ]; then
         echo "starting to reset ...."
         rm -rf "$nodes_dir"/*/fiber/store
         "$deploy_dir/init-dev-chain.sh" -f

From a7eb23b4380f48bbedf63bde9d16c305e144cf6d Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 09:58:36 +0800
Subject: [PATCH 07/66] report error message to tlc sender when error happened

---
 src/fiber/channel.rs | 179 ++++++++++++++++++++++++++++---------------
 1 file changed, 118 insertions(+), 61 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index b535867e..6270108c 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -5,7 +5,7 @@ use tracing::{debug, error, info, warn};
 use crate::{
     fiber::{
         network::{get_chain_hash, SendOnionPacketCommand},
-        types::{ChannelUpdate, OnionPacket},
+        types::{ChannelUpdate, OnionPacket, RemoveTlcFail},
     },
     invoice::InvoiceStore,
 };
@@ -448,68 +448,43 @@ where
                 Ok(())
             }
             FiberChannelMessage::AddTlc(add_tlc) => {
-                state.check_for_tlc_update(Some(add_tlc.amount))?;
-
-                // check the onion_packet is valid or not, check the payment hash and amount.
-                // check time lock is valid or not.
-                // If there is a next hop, we should send the AddTlc message to the next hop.
-                // If this is the last hop, try to fulfill the payment,
-                // find the corresponding payment preimage from payment hash, this is done in try_to_settle_down_tlc.
-                let mut forward_to_next_hop = false;
-                let mut preimage = None;
-                if let Ok(onion_packet) = OnionPacket::deserialize(&add_tlc.onion_packet) {
-                    if let Some(onion_info) = onion_packet.peek() {
-                        // check the payment hash and amount
-                        if onion_info.payment_hash != add_tlc.payment_hash
-                            || onion_info.amount != add_tlc.amount
-                        {
-                            return Err(ProcessingChannelError::InvalidParameter(
-                                "Payment hash or amount mismatch".to_string(),
-                            ));
+                let tlc_id = add_tlc.tlc_id;
+                let tlc_count = state.tlcs.len();
+                match self.handle_add_tlc_peer_message(state, add_tlc).await {
+                    Ok(_) => Ok(()),
+                    Err(e) => {
+                        // we assume that TLC was not inserted into our state,
+                        // so we can safely send RemoveTlc message to the peer
+                        // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
+                        // trying to add a duplicate tlc, so we use tlc count to make sure no new tlc was added
+                        // and don't send a wrong RemoveTlc message if there is already a tlc with the same id
+                        error!("Error handling AddTlc message: {:?}", e);
+                        assert!(tlc_count == state.tlcs.len());
+                        if state.get_received_tlc(tlc_id).is_none() {
+                            self.network
+                                .send_message(NetworkActorMessage::new_command(
+                                    NetworkActorCommand::SendFiberMessage(
+                                        FiberMessageWithPeerId::new(
+                                            state.get_remote_peer_id(),
+                                            FiberMessage::remove_tlc(RemoveTlc {
+                                                channel_id: state.get_id(),
+                                                tlc_id: tlc_id,
+                                                reason: RemoveTlcReason::RemoveTlcFail(
+                                                    RemoveTlcFail {
+                                                        //FIXME: we should define the error code carefully according
+                                                        //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
+                                                        error_code: 1,
+                                                    },
+                                                ),
+                                            }),
+                                        ),
+                                    ),
+                                ))
+                                .expect(ASSUME_NETWORK_ACTOR_ALIVE);
                         }
-                        // TODO: check the expiry time, if expired, we should return an error.
-                        // if this is the last hop, store the preimage.
-                        preimage = onion_info.preimage;
-
-                        forward_to_next_hop = onion_info.next_hop.is_some();
+                        Err(e)
                     }
                 }
-
-                let tlc = state.create_inbounding_tlc(add_tlc.clone(), preimage)?;
-                state.insert_tlc(tlc.clone())?;
-                if let Some(ref udt_type_script) = state.funding_udt_type_script {
-                    self.subscribers
-                        .pending_received_tlcs_subscribers
-                        .send(TlcNotification {
-                            tlc: tlc.clone(),
-                            channel_id: state.get_id(),
-                            script: udt_type_script.clone(),
-                        });
-                }
-                warn!("created tlc: {:?}", &tlc);
-                // TODO: here we didn't send any ack message to the peer.
-                // The peer may falsely believe that we have already processed this message,
-                // while we have crashed. We need a way to make sure that the peer will resend
-                // this message, and our processing of this message is idempotent.
-
-                if forward_to_next_hop {
-                    let (send, recv) = oneshot::channel::<Result<u64, String>>();
-                    let rpc_reply = RpcReplyPort::from(send);
-                    self.network
-                        .send_message(NetworkActorMessage::Command(
-                            NetworkActorCommand::SendOnionPacket(
-                                SendOnionPacketCommand {
-                                    packet: add_tlc.onion_packet.clone(),
-                                    previous_tlc: Some((state.get_id(), tlc.get_id())),
-                                },
-                                rpc_reply,
-                            ),
-                        ))
-                        .expect("network actor is alive");
-                    let _res = recv.await;
-                    // TODO: if the result is an error, we should handle it. we need to begin to remove the tlc.
-                }
-                Ok(())
             }
             FiberChannelMessage::RemoveTlc(remove_tlc) => {
                 state.check_for_tlc_update(None)?;
@@ -676,6 +651,75 @@ where
         }
     }
 
+    async fn handle_add_tlc_peer_message(
+        &self,
+        state: &mut ChannelActorState,
+        add_tlc: AddTlc,
+    ) -> Result<(), ProcessingChannelError> {
+        state.check_for_tlc_update(Some(add_tlc.amount))?;
+
+        // check the onion_packet is valid or not, check the payment hash and amount.
+        // check time lock is valid or not.
+        // If there is a next hop, we should send the AddTlc message to the next hop.
+        // If this is the last hop, try to fulfill the payment,
+        // find the corresponding payment preimage from payment hash, this is done in try_to_settle_down_tlc.
+        let mut forward_to_next_hop = false;
+        let mut preimage = None;
+        if let Ok(onion_packet) = OnionPacket::deserialize(&add_tlc.onion_packet) {
+            if let Some(onion_info) = onion_packet.peek() {
+                // check the payment hash and amount
+                if onion_info.payment_hash != add_tlc.payment_hash
+                    || onion_info.amount != add_tlc.amount
+                {
+                    return Err(ProcessingChannelError::InvalidParameter(
+                        "Payment hash or amount mismatch".to_string(),
+                    ));
+                }
+                // TODO: check the expiry time, if expired, we should return an error.
+                // if this is the last hop, store the preimage.
+                preimage = onion_info.preimage;
+
+                forward_to_next_hop = onion_info.next_hop.is_some();
+            }
+        }
+
+        let tlc = state.create_inbounding_tlc(add_tlc.clone(), preimage)?;
+        state.insert_tlc(tlc.clone())?;
+        if let Some(ref udt_type_script) = state.funding_udt_type_script {
+            self.subscribers
+                .pending_received_tlcs_subscribers
+                .send(TlcNotification {
+                    tlc: tlc.clone(),
+                    channel_id: state.get_id(),
+                    script: udt_type_script.clone(),
+                });
+        }
+        warn!("created tlc: {:?}", &tlc);
+        // TODO: here we didn't send any ack message to the peer.
+        // The peer may falsely believe that we have already processed this message,
+        // while we have crashed. We need a way to make sure that the peer will resend
+        // this message, and our processing of this message is idempotent.
+
+        if forward_to_next_hop {
+            let (send, recv) = oneshot::channel::<Result<u64, String>>();
+            let rpc_reply = RpcReplyPort::from(send);
+            self.network
+                .send_message(NetworkActorMessage::Command(
+                    NetworkActorCommand::SendOnionPacket(
+                        SendOnionPacketCommand {
+                            packet: add_tlc.onion_packet.clone(),
+                            previous_tlc: Some((state.get_id(), tlc.get_id())),
+                        },
+                        rpc_reply,
+                    ),
+                ))
+                .expect("network actor is alive");
+            let _res = recv.await;
+            // TODO: if the result is an error, we should handle it. we need to begin to remove the tlc.
+        }
+        Ok(())
+    }
+
     pub fn handle_commitment_signed_command(
         &self,
         state: &mut ChannelActorState,
@@ -3404,7 +3448,12 @@ impl ChannelActorState {
                     return am_i_sending_remove_tlc_message;
                 }
                 (_, Some(_), None) => {
-                    panic!("TLC {:?} is removed but not confirmed yet", info);
+                    if info.is_fullfill_removed() {
+                        panic!("TLC {:?} is fullfilled but not confirmed yet", info);
+                    }
+                    // This is a failed tlc, there is not `creation_confirmed_at` field means our peer does not inserted this tlc,
+                    // and we should not include it in the commitment transaction.
+                    return false;
                 }
                 (_, _, Some(n)) => {
                     debug!("Including TLC {:?} to commitment transaction because tlc confirmed at {:?}", tlc.id, n);
@@ -5404,6 +5453,14 @@ impl DetailedTLCInfo {
                 .expect("Commitment number is present")
         }
     }
+
+    fn is_fullfill_removed(&self) -> bool {
+        if let Some((_, removed_reason)) = self.removed_at {
+            matches!(removed_reason, RemoveTlcReason::RemoveTlcFulfill(_))
+        } else {
+            false
+        }
+    }
 }
 
 pub fn get_tweak_by_commitment_point(commitment_point: &Pubkey) -> [u8; 32] {

From d27d7e7afecf7e9e708a2cc8c2a51890d85c0888 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 15:25:05 +0800
Subject: [PATCH 08/66] if forward onion packet failed, remove the tlc

---
 src/fiber/channel.rs | 84 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 67 insertions(+), 17 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 6270108c..5c67b4c7 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -450,8 +450,22 @@ where
             FiberChannelMessage::AddTlc(add_tlc) => {
                 let tlc_id = add_tlc.tlc_id;
                 let tlc_count = state.tlcs.len();
-                match self.handle_add_tlc_peer_message(state, add_tlc).await {
-                    Ok(_) => Ok(()),
+                match self
+                    .handle_add_tlc_peer_message(state, add_tlc.clone())
+                    .await
+                {
+                    Ok((added_tlc_id, forward_to_next_hop)) => {
+                        if forward_to_next_hop {
+                            let _ = self
+                                .handle_forward_onion_packet(
+                                    state,
+                                    add_tlc.onion_packet,
+                                    added_tlc_id.into(),
+                                )
+                                .await?;
+                        }
+                        Ok(())
+                    }
                     Err(e) => {
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
@@ -655,7 +669,7 @@ where
         &self,
         state: &mut ChannelActorState,
         add_tlc: AddTlc,
-    ) -> Result<(), ProcessingChannelError> {
+    ) -> Result<(TLCId, bool), ProcessingChannelError> {
         state.check_for_tlc_update(Some(add_tlc.amount))?;
 
         // check the onion_packet is valid or not, check the payment hash and amount.
@@ -676,15 +690,16 @@ where
                     ));
                 }
                 // TODO: check the expiry time, if expired, we should return an error.
+
                 // if this is the last hop, store the preimage.
                 preimage = onion_info.preimage;
-
                 forward_to_next_hop = onion_info.next_hop.is_some();
             }
         }
 
         let tlc = state.create_inbounding_tlc(add_tlc.clone(), preimage)?;
         state.insert_tlc(tlc.clone())?;
+
         if let Some(ref udt_type_script) = state.funding_udt_type_script {
             self.subscribers
                 .pending_received_tlcs_subscribers
@@ -695,27 +710,62 @@ where
                 });
         }
         warn!("created tlc: {:?}", &tlc);
+
         // TODO: here we didn't send any ack message to the peer.
         // The peer may falsely believe that we have already processed this message,
         // while we have crashed. We need a way to make sure that the peer will resend
         // this message, and our processing of this message is idempotent.
+        Ok((tlc.id, forward_to_next_hop))
+    }
 
-        if forward_to_next_hop {
-            let (send, recv) = oneshot::channel::<Result<u64, String>>();
-            let rpc_reply = RpcReplyPort::from(send);
+    async fn handle_forward_onion_packet(
+        &self,
+        state: &mut ChannelActorState,
+        onion_packet: Vec<u8>,
+        added_tlc_id: u64,
+    ) -> Result<(), ProcessingChannelError> {
+        let (send, recv) = oneshot::channel::<Result<u64, String>>();
+        let rpc_reply = RpcReplyPort::from(send);
+        self.network
+            .send_message(NetworkActorMessage::Command(
+                NetworkActorCommand::SendOnionPacket(
+                    SendOnionPacketCommand {
+                        packet: onion_packet,
+                        previous_tlc: Some((state.get_id(), added_tlc_id.into())),
+                    },
+                    rpc_reply,
+                ),
+            ))
+            .expect("network actor is alive");
+        let res = match recv.await {
+            Ok(Ok(tlc_id)) => Ok(tlc_id),
+            Ok(Err(err)) => Err(err),
+            Err(e) => Err(e.to_string()),
+        };
+        // If we failed to forward the onion packet, we should remove the tlc.
+        if let Err(res) = res {
+            error!("Error forwarding onion packet: {:?}", res);
+            let (send, recv) = oneshot::channel::<Result<(), String>>();
+            let port = RpcReplyPort::from(send);
             self.network
-                .send_message(NetworkActorMessage::Command(
-                    NetworkActorCommand::SendOnionPacket(
-                        SendOnionPacketCommand {
-                            packet: add_tlc.onion_packet.clone(),
-                            previous_tlc: Some((state.get_id(), tlc.get_id())),
-                        },
-                        rpc_reply,
-                    ),
+                .send_message(NetworkActorMessage::new_command(
+                    NetworkActorCommand::ControlFiberChannel(ChannelCommandWithId {
+                        channel_id: state.get_id(),
+                        command: ChannelCommand::RemoveTlc(
+                            RemoveTlcCommand {
+                                id: added_tlc_id,
+                                reason: RemoveTlcReason::RemoveTlcFail(RemoveTlcFail {
+                                    //FIXME: we should define the error code carefully according
+                                    //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
+                                    error_code: 1,
+                                }),
+                            },
+                            port,
+                        ),
+                    }),
                 ))
                 .expect("network actor is alive");
-            let _res = recv.await;
-            // TODO: if the result is an error, we should handle it. we need to begin to remove the tlc.
+            let _ = recv.await.expect("RemoveTlc command replied");
         }
         Ok(())
     }

From bb19defd2e95a4544c93c91de67b8e1b5803cee2 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 15:43:41 +0800
Subject: [PATCH 09/66] clean up

---
 src/errors.rs        | 2 --
 src/fiber/network.rs | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/errors.rs b/src/errors.rs
index 34ba4b73..7cb16fe0 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -40,8 +40,6 @@ pub enum Error {
     SendPaymentError(String),
     #[error("InvalidParameter: {0}")]
     InvalidParameter(String),
-    #[error("Payment error: {0}")]
-    PaymentError(String),
     #[error("Network Graph error: {0}")]
     NetworkGraphError(#[from] GraphError),
     #[error("Invalid peer message: {0}")]
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 2f10d28b..1e766150 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1900,7 +1900,7 @@ where
                 Err(e) => {
                     error!("Failed to build route: {:?}", e);
                     payment_session.set_status(PaymentSessionStatus::Failed);
-                    error = Some(Err(Error::PaymentError(format!(
+                    error = Some(Err(Error::SendPaymentError(format!(
                         "Failed to build route: {:?}",
                         payment_data.payment_hash
                     ))));
@@ -1946,7 +1946,7 @@ where
                         payment_data.payment_hash, e
                     );
                     payment_session.last_error = Some(err.clone());
-                    error = Some(Err(Error::PaymentError(err)));
+                    error = Some(Err(Error::SendPaymentError(err)));
                     continue;
                 }
                 Ok(_) => {

From 6d1621964092125a7b31d93ce09466d82abd43a5 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 16:46:57 +0800
Subject: [PATCH 10/66] add on_tlc_remove_received to update payment_session
 status

---
 src/fiber/channel.rs |  9 ++++++++
 src/fiber/graph.rs   | 14 ++++++++++++-
 src/fiber/network.rs | 50 ++++++++++++++++++++++++++++++++++----------
 src/rpc/channel.rs   |  9 ++++++++
 4 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 5c67b4c7..236dc6b1 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -546,6 +546,15 @@ where
                     let res = recv.await.expect("network actor is alive");
                     info!("remove tlc from previous channel: {:?}", &res);
                 }
+
+                self.network
+                    .send_message(NetworkActorMessage::new_event(
+                        NetworkActorEvent::TlcRemoveReceived(
+                            tlc_details.tlc.payment_hash,
+                            remove_tlc,
+                        ),
+                    ))
+                    .expect("myself alive");
                 Ok(())
             }
             FiberChannelMessage::Shutdown(shutdown) => {
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 66463370..b6f00109 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -772,9 +772,13 @@ pub trait NetworkGraphStateStore {
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum PaymentSessionStatus {
+    // initial status, payment session is created, no HTLC is sent
     Created,
+    // related HTLC is send and waiting for the response
     Inflight,
+    // related HTLC is successfully settled
     Success,
+    // related HTLC is failed
     Failed,
 }
 
@@ -786,17 +790,20 @@ pub struct PaymentSession {
     pub try_limit: u32,
     pub status: PaymentSessionStatus,
     pub created_time: u128,
+    pub last_updated_time: u128,
 }
 
 impl PaymentSession {
     pub fn new(request: SendPaymentData, try_limit: u32) -> Self {
+        let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_millis();
         Self {
             request,
             retried_times: 0,
             last_error: None,
             try_limit,
             status: PaymentSessionStatus::Created,
-            created_time: std::time::UNIX_EPOCH.elapsed().unwrap().as_millis(),
+            created_time: now,
+            last_updated_time: now,
         }
     }
 
@@ -807,6 +814,11 @@ impl PaymentSession {
     pub fn set_status(&mut self, status: PaymentSessionStatus) {
         self.status = status;
     }
+
+    pub fn set_failed_status(&mut self, error: String) {
+        self.status = PaymentSessionStatus::Failed;
+        self.last_error = Some(error);
+    }
 }
 
 #[cfg(test)]
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 1e766150..a24db616 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -60,7 +60,7 @@ use super::types::{
     GetBroadcastMessagesResult, Hash256, NodeAnnouncement, NodeAnnouncementQuery, OpenChannel,
     Privkey, Pubkey, QueryBroadcastMessagesWithinTimeRange,
     QueryBroadcastMessagesWithinTimeRangeResult, QueryChannelsWithinBlockRange,
-    QueryChannelsWithinBlockRangeResult,
+    QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcReason,
 };
 use super::FiberConfig;
 
@@ -105,6 +105,9 @@ pub struct AcceptChannelResponse {
 #[derive(Debug)]
 pub struct SendPaymentResponse {
     pub payment_hash: Hash256,
+    pub status: PaymentSessionStatus,
+    pub last_update_time: u128,
+    pub failed_error: Option<String>,
 }
 
 /// What kind of local information should be broadcasted to the network.
@@ -454,6 +457,9 @@ pub enum NetworkActorEvent {
     // The graph syncer to the peer has exited with some reason.
     GraphSyncerExited(PeerId, GraphSyncerExitStatus),
 
+    // A tlc remove message is received. (payment_hash, remove_tlc)
+    TlcRemoveReceived(Hash256, RemoveTlc),
+
     /// Network service events to be sent to outside observers.
     /// These events may be both present at `NetworkActorEvent` and
     /// this branch of `NetworkActorEvent`. This is because some events
@@ -1153,6 +1159,9 @@ where
                 }
                 state.maybe_finish_sync();
             }
+            NetworkActorEvent::TlcRemoveReceived(payment_hash, remove_tlc) => {
+                self.on_tlc_remove_received(payment_hash, remove_tlc);
+            }
         }
         Ok(())
     }
@@ -1424,8 +1433,8 @@ where
             }
             NetworkActorCommand::SendPayment(payment_request, reply) => {
                 match self.on_send_payment(state, myself, payment_request).await {
-                    Ok(payment_hash) => {
-                        let _ = reply.send(Ok(SendPaymentResponse { payment_hash }));
+                    Ok(payment_res) => {
+                        let _ = reply.send(Ok(payment_res));
                     }
                     Err(e) => {
                         error!("Failed to send payment: {:?}", e);
@@ -1861,10 +1870,21 @@ where
         }
     }
 
-    fn prepare_payment_session(&self, payment_data: SendPaymentData) -> PaymentSession {
-        let payment_session = PaymentSession::new(payment_data.clone(), 5);
-        self.store.insert_payment_session(payment_session.clone());
-        payment_session
+    fn on_tlc_remove_received(&self, payment_hash: Hash256, remove_tlc: RemoveTlc) {
+        let payment_session = self.store.get_payment_session(payment_hash);
+        if let Some(mut payment_session) = payment_session {
+            if payment_session.status == PaymentSessionStatus::Inflight {
+                if matches!(remove_tlc.reason, RemoveTlcReason::RemoveTlcFulfill(_)) {
+                    payment_session.set_status(PaymentSessionStatus::Success);
+                } else {
+                    // FIXME(yukang): we should handle tlc fail reason here
+                    payment_session.set_failed_status(format!("{:?}", remove_tlc.reason));
+                }
+                payment_session.last_updated_time =
+                    std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
+            }
+            self.store.insert_payment_session(payment_session);
+        }
     }
 
     async fn on_send_payment(
@@ -1872,7 +1892,7 @@ where
         state: &mut NetworkActorState<S>,
         _my_self: ActorRef<NetworkActorMessage>,
         payment_request: SendPaymentCommand,
-    ) -> Result<Hash256, Error> {
+    ) -> Result<SendPaymentResponse, Error> {
         let graph = self.network_graph.read().await;
         let payment_data = SendPaymentData::new(payment_request.clone()).map_err(|e| {
             error!("Failed to validate payment request: {:?}", e);
@@ -1890,7 +1910,9 @@ where
             }
         }
 
-        let mut payment_session = self.prepare_payment_session(payment_data.clone());
+        let mut payment_session = PaymentSession::new(payment_data.clone(), 5);
+        self.store.insert_payment_session(payment_session.clone());
+
         let mut error = None;
 
         for _i in 0..payment_session.try_limit {
@@ -1955,8 +1977,14 @@ where
                         payment_data.payment_hash, onion_packet
                     );
                     payment_session.set_status(PaymentSessionStatus::Inflight);
-                    self.store.insert_payment_session(payment_session);
-                    return Ok(payment_data.payment_hash);
+                    self.store.insert_payment_session(payment_session.clone());
+                    let response = SendPaymentResponse {
+                        payment_hash: payment_data.payment_hash,
+                        last_update_time: payment_session.last_updated_time,
+                        status: PaymentSessionStatus::Inflight,
+                        failed_error: None,
+                    };
+                    return Ok(response);
                 }
             }
         }
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index 32d86de0..b72244b8 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -3,6 +3,7 @@ use crate::fiber::{
         AddTlcCommand, ChannelActorStateStore, ChannelCommand, ChannelCommandWithId, ChannelState,
         RemoveTlcCommand, ShutdownCommand, UpdateCommand,
     },
+    graph::PaymentSessionStatus,
     hash_algorithm::HashAlgorithm,
     network::{AcceptChannelCommand, OpenChannelCommand, SendPaymentCommand},
     serde_utils::{U128Hex, U32Hex, U64Hex},
@@ -210,9 +211,14 @@ pub struct SendPaymentCommandParams {
     pub udt_type_script: Option<Script>,
 }
 
+#[serde_as]
 #[derive(Clone, Serialize)]
 pub struct SendPaymentResult {
     pub payment_hash: Hash256,
+    pub status: PaymentSessionStatus,
+    #[serde_as(as = "U128Hex")]
+    pub last_update_time: u128,
+    pub failed_error: Option<String>,
 }
 #[rpc(server)]
 pub trait ChannelRpc {
@@ -491,6 +497,9 @@ where
         };
         handle_actor_call!(self.actor, message, params).map(|response| SendPaymentResult {
             payment_hash: response.payment_hash,
+            status: response.status,
+            last_update_time: response.last_update_time,
+            failed_error: response.failed_error,
         })
     }
 }

From 86657220b1c6bd7a4cb6821fe2a0626fa4cd3ef9 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 26 Sep 2024 17:15:26 +0800
Subject: [PATCH 11/66] fix tlc amount check in onion packet forwarding

---
 src/fiber/channel.rs                          | 21 +++++++++++++++----
 src/fiber/network.rs                          |  2 +-
 .../13-node-send-duplicate-payment-err.bru    |  4 ++++
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 236dc6b1..c5561b56 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -690,19 +690,32 @@ where
         let mut preimage = None;
         if let Ok(onion_packet) = OnionPacket::deserialize(&add_tlc.onion_packet) {
             if let Some(onion_info) = onion_packet.peek() {
+                forward_to_next_hop = onion_info.next_hop.is_some();
+
                 // check the payment hash and amount
-                if onion_info.payment_hash != add_tlc.payment_hash
-                    || onion_info.amount != add_tlc.amount
-                {
+                if onion_info.payment_hash != add_tlc.payment_hash {
                     return Err(ProcessingChannelError::InvalidParameter(
                         "Payment hash or amount mismatch".to_string(),
                     ));
                 }
+                if forward_to_next_hop && onion_info.amount > add_tlc.amount {
+                    return Err(ProcessingChannelError::InvalidParameter(
+                        format!("Payment amount is too large, expect next hop amount [{}] less than received amount [{}]",
+                            onion_info.amount, add_tlc.amount)
+                    ));
+                }
+                if !forward_to_next_hop {
+                    if onion_info.amount != add_tlc.amount {
+                        return Err(ProcessingChannelError::InvalidParameter(
+                            "Payment amount mismatch".to_string(),
+                        ));
+                    }
+                }
+
                 // TODO: check the expiry time, if expired, we should return an error.
 
                 // if this is the last hop, store the preimage.
                 preimage = onion_info.preimage;
-                forward_to_next_hop = onion_info.next_hop.is_some();
             }
         }
 
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index a24db616..c06fa563 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1902,6 +1902,7 @@ where
         // handle the payment process
         if let Some(payment_session) = self.store.get_payment_session(payment_data.payment_hash) {
             // we only allow retrying payment session with status failed
+            debug!("Payment session already exists: {:?}", payment_session);
             if payment_session.status != PaymentSessionStatus::Failed {
                 return Err(Error::InvalidParameter(format!(
                     "Payment session already exists: {} with payment session status: {:?}",
@@ -1914,7 +1915,6 @@ where
         self.store.insert_payment_session(payment_session.clone());
 
         let mut error = None;
-
         for _i in 0..payment_session.try_limit {
             payment_session.retried_times += 1;
             let onion_path = graph.build_route(payment_data.clone());
diff --git a/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru b/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
index 4429e1ba..07daed49 100644
--- a/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
+++ b/tests/bruno/e2e/router-pay/13-node-send-duplicate-payment-err.bru
@@ -34,6 +34,10 @@ assert {
   res.body.error: isDefined
 }
 
+script:pre-request {
+  await new Promise(r => setTimeout(r, 1000));
+}
+
 script:post-response {
   // Sleep for sometime to make sure current operation finishes before next request starts.
   await new Promise(r => setTimeout(r, 100));

From 23c41c35c6a9972dda9bff0a67be3a476de883b1 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 29 Sep 2024 11:40:46 +0800
Subject: [PATCH 12/66] clippy bless

---
 src/fiber/channel.rs | 33 ++++++++++++++-------------------
 src/fiber/network.rs |  9 ++++-----
 src/store.rs         |  6 +++---
 3 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index c5561b56..c6f42854 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -456,13 +456,12 @@ where
                 {
                     Ok((added_tlc_id, forward_to_next_hop)) => {
                         if forward_to_next_hop {
-                            let _ = self
-                                .handle_forward_onion_packet(
-                                    state,
-                                    add_tlc.onion_packet,
-                                    added_tlc_id.into(),
-                                )
-                                .await?;
+                            self.handle_forward_onion_packet(
+                                state,
+                                add_tlc.onion_packet,
+                                added_tlc_id.into(),
+                            )
+                            .await?;
                         }
                         Ok(())
                     }
@@ -482,7 +481,7 @@ where
                                             state.get_remote_peer_id(),
                                             FiberMessage::remove_tlc(RemoveTlc {
                                                 channel_id: state.get_id(),
-                                                tlc_id: tlc_id,
+                                                tlc_id,
                                                 reason: RemoveTlcReason::RemoveTlcFail(
                                                     RemoveTlcFail {
                                                         //FIXME: we should define the error code carefully according
@@ -704,12 +703,10 @@ where
                             onion_info.amount, add_tlc.amount)
                     ));
                 }
-                if !forward_to_next_hop {
-                    if onion_info.amount != add_tlc.amount {
-                        return Err(ProcessingChannelError::InvalidParameter(
-                            "Payment amount mismatch".to_string(),
-                        ));
-                    }
+                if !forward_to_next_hop && onion_info.amount != add_tlc.amount {
+                    return Err(ProcessingChannelError::InvalidParameter(
+                        "Payment amount mismatch".to_string(),
+                    ));
                 }
 
                 // TODO: check the expiry time, if expired, we should return an error.
@@ -753,7 +750,7 @@ where
                 NetworkActorCommand::SendOnionPacket(
                     SendOnionPacketCommand {
                         packet: onion_packet,
-                        previous_tlc: Some((state.get_id(), added_tlc_id.into())),
+                        previous_tlc: Some((state.get_id(), added_tlc_id)),
                     },
                     rpc_reply,
                 ),
@@ -6040,7 +6037,7 @@ mod tests {
             })
             .await;
 
-        let remove_tlc_result = call!(node_b.network_actor, |rpc_reply| {
+        call!(node_b.network_actor, |rpc_reply| {
             NetworkActorMessage::Command(NetworkActorCommand::ControlFiberChannel(
                 ChannelCommandWithId {
                     channel_id: new_channel_id,
@@ -6059,8 +6056,6 @@ mod tests {
         .expect("node_b alive")
         .expect("successfully removed tlc");
 
-        dbg!(&remove_tlc_result);
-
         // Since we currently automatically send a `CommitmentSigned` message
         // after sending a `RemoveTlc` message, we can expect the `RemoteCommitmentSigned`
         // to be received by node a.
@@ -6547,7 +6542,7 @@ mod tests {
             })
             .await;
 
-        let _ = node_a
+        node_a
             .network_actor
             .send_message(NetworkActorMessage::Command(
                 NetworkActorCommand::ControlFiberChannel(ChannelCommandWithId {
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index c06fa563..80f544d6 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -560,7 +560,7 @@ where
                         let auto_accept = if let Some(udt_type_script) =
                             open_channel.funding_udt_type_script.as_ref()
                         {
-                            is_udt_type_auto_accept(&udt_type_script, open_channel.funding_amount)
+                            is_udt_type_auto_accept(udt_type_script, open_channel.funding_amount)
                         } else {
                             state.auto_accept_channel_ckb_funding_amount > 0
                                 && open_channel.all_ckb_amount()
@@ -1718,7 +1718,7 @@ where
                 };
 
                 if let Err(err) = secp256k1_instance().verify_schnorr(
-                    &ckb_signature,
+                    ckb_signature,
                     &Message::from_digest(message),
                     &channel_announcement.ckb_key,
                 ) {
@@ -2519,14 +2519,13 @@ where
     }
 
     async fn get_funding_lock_script(&self) -> Script {
-        let funding_lock_script = call!(
+        call!(
             self.chain_actor,
             CkbChainMessage::GetFundingSourceScript,
             ()
         )
         .expect(ASSUME_CHAIN_ACTOR_ALWAYS_ALIVE_FOR_NOW)
-        .expect("Get funding source script from chain");
-        funding_lock_script
+        .expect("Get funding source script from chain")
     }
 
     fn get_peer_session(&self, peer_id: &PeerId) -> Option<SessionId> {
diff --git a/src/store.rs b/src/store.rs
index c4a31bf2..1f47441c 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -379,7 +379,7 @@ impl NetworkGraphStateStore for Store {
             .take_while(|(key, _)| key.starts_with(&channel_prefix))
             .filter_map(|(col_key, value)| {
                 if let Some(key) = &outpoint_key {
-                    if !col_key.starts_with(&key) {
+                    if !col_key.starts_with(key) {
                         return None;
                     }
                 }
@@ -427,7 +427,7 @@ impl NetworkGraphStateStore for Store {
             .take_while(|(key, _)| key.starts_with(&node_prefix))
             .filter_map(|(col_key, value)| {
                 if let Some(key) = &node_key {
-                    if !col_key.starts_with(&key) {
+                    if !col_key.starts_with(key) {
                         return None;
                     }
                 }
@@ -667,7 +667,7 @@ mod tests {
         }
 
         // sort by out_point
-        channels.sort_by(|a, b| a.out_point().cmp(&b.out_point()));
+        channels.sort_by_key(|a| a.out_point());
 
         let outpoint_0 = channels[0].out_point();
         assert_eq!(

From 47570049d17f44cddf9799be0663d1c7f08d6eff Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 29 Sep 2024 16:22:19 +0800
Subject: [PATCH 13/66] add get_payment rpc

---
 src/fiber/network.rs                          | 35 +++++++++++++-
 src/rpc/channel.rs                            | 41 ++++++++++++++++
 .../17-node1-get-payment-status.bru           | 48 +++++++++++++++++++
 ...channels.bru => 18-node1-get-channels.bru} |  2 +-
 ...s-page.bru => 19-node1-get-nodes-page.bru} |  2 +-
 ...ge-2.bru => 20-node1-get-nodes-page-2.bru} |  2 +-
 ...ge-3.bru => 21-node1-get-nodes-page-3.bru} |  2 +-
 7 files changed, 126 insertions(+), 6 deletions(-)
 create mode 100644 tests/bruno/e2e/router-pay/17-node1-get-payment-status.bru
 rename tests/bruno/e2e/router-pay/{17-node1-get-channels.bru => 18-node1-get-channels.bru} (98%)
 rename tests/bruno/e2e/router-pay/{18-node1-get-nodes-page.bru => 19-node1-get-nodes-page.bru} (98%)
 rename tests/bruno/e2e/router-pay/{19-node1-get-nodes-page-2.bru => 20-node1-get-nodes-page-2.bru} (98%)
 rename tests/bruno/e2e/router-pay/{20-node1-get-nodes-page-3.bru => 21-node1-get-nodes-page-3.bru} (98%)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 80f544d6..4259bc33 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -159,6 +159,8 @@ pub enum NetworkActorCommand {
         SendPaymentCommand,
         RpcReplyPort<Result<SendPaymentResponse, String>>,
     ),
+    // Get Payment Session for query payment status and errors
+    GetPayment(Hash256, RpcReplyPort<Result<SendPaymentResponse, String>>),
     GetAndProcessChannelsWithinBlockRangeFromPeer(
         (PeerId, u64, u64),
         RpcReplyPort<Result<(u64, bool), Error>>,
@@ -1433,8 +1435,8 @@ where
             }
             NetworkActorCommand::SendPayment(payment_request, reply) => {
                 match self.on_send_payment(state, myself, payment_request).await {
-                    Ok(payment_res) => {
-                        let _ = reply.send(Ok(payment_res));
+                    Ok(payment) => {
+                        let _ = reply.send(Ok(payment));
                     }
                     Err(e) => {
                         error!("Failed to send payment: {:?}", e);
@@ -1442,6 +1444,16 @@ where
                     }
                 }
             }
+            NetworkActorCommand::GetPayment(payment_hash, reply) => {
+                match self.on_get_payment(&payment_hash) {
+                    Ok(payment) => {
+                        let _ = reply.send(Ok(payment));
+                    }
+                    Err(e) => {
+                        let _ = reply.send(Err(e.to_string()));
+                    }
+                }
+            }
             NetworkActorCommand::BroadcastLocalInfo(kind) => match kind {
                 LocalInfoKind::NodeAnnouncement => {
                     let message = state.get_or_create_new_node_announcement_message();
@@ -1887,6 +1899,25 @@ where
         }
     }
 
+    fn on_get_payment(&self, payment_hash: &Hash256) -> Result<SendPaymentResponse, Error> {
+        let payment_session = self.store.get_payment_session(*payment_hash);
+        match payment_session {
+            Some(payment_session) => {
+                let response = SendPaymentResponse {
+                    payment_hash: payment_session.payment_hash(),
+                    last_update_time: payment_session.last_updated_time,
+                    status: payment_session.status,
+                    failed_error: payment_session.last_error.clone(),
+                };
+                Ok(response)
+            }
+            None => Err(Error::InvalidParameter(format!(
+                "Payment session not found: {:?}",
+                payment_hash
+            ))),
+        }
+    }
+
     async fn on_send_payment(
         &self,
         state: &mut NetworkActorState<S>,
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index b72244b8..9ffe7d6c 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -171,6 +171,22 @@ pub struct UpdateChannelParams {
     pub tlc_fee_proportional_millionths: Option<u128>,
 }
 
+#[serde_as]
+#[derive(Serialize, Deserialize, Debug)]
+pub struct GetPaymentCommandParams {
+    pub payment_hash: Hash256,
+}
+
+#[serde_as]
+#[derive(Serialize, Deserialize, Clone)]
+pub struct GetPaymentCommandResult {
+    pub payment_hash: Hash256,
+    pub status: PaymentSessionStatus,
+    #[serde_as(as = "U128Hex")]
+    pub last_update_time: u128,
+    pub failed_error: Option<String>,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct SendPaymentCommandParams {
@@ -220,6 +236,7 @@ pub struct SendPaymentResult {
     pub last_update_time: u128,
     pub failed_error: Option<String>,
 }
+
 #[rpc(server)]
 pub trait ChannelRpc {
     #[method(name = "open_channel")]
@@ -264,6 +281,12 @@ pub trait ChannelRpc {
         &self,
         params: SendPaymentCommandParams,
     ) -> Result<SendPaymentResult, ErrorObjectOwned>;
+
+    #[method(name = "get_payment")]
+    async fn get_payment(
+        &self,
+        params: GetPaymentCommandParams,
+    ) -> Result<GetPaymentCommandResult, ErrorObjectOwned>;
 }
 
 pub struct ChannelRpcServerImpl<S> {
@@ -502,4 +525,22 @@ where
             failed_error: response.failed_error,
         })
     }
+
+    async fn get_payment(
+        &self,
+        params: GetPaymentCommandParams,
+    ) -> Result<GetPaymentCommandResult, ErrorObjectOwned> {
+        let message = |rpc_reply| -> NetworkActorMessage {
+            NetworkActorMessage::Command(NetworkActorCommand::GetPayment(
+                params.payment_hash,
+                rpc_reply,
+            ))
+        };
+        handle_actor_call!(self.actor, message, params).map(|response| GetPaymentCommandResult {
+            payment_hash: response.payment_hash,
+            status: response.status,
+            last_update_time: response.last_update_time,
+            failed_error: response.failed_error,
+        })
+    }
 }
diff --git a/tests/bruno/e2e/router-pay/17-node1-get-payment-status.bru b/tests/bruno/e2e/router-pay/17-node1-get-payment-status.bru
new file mode 100644
index 00000000..e893f271
--- /dev/null
+++ b/tests/bruno/e2e/router-pay/17-node1-get-payment-status.bru
@@ -0,0 +1,48 @@
+meta {
+  name: Node1 send get_payment for query status
+  type: http
+  seq: 17
+}
+
+post {
+  url: {{NODE1_RPC_URL}}
+  body: json
+  auth: none
+}
+
+headers {
+  Content-Type: application/json
+  Accept: application/json
+}
+
+body:json {
+  {
+    "id": "42",
+    "jsonrpc": "2.0",
+    "method": "get_payment",
+    "params": [
+      {
+        "payment_hash": "{{payment_hash}}"
+      }
+    ]
+  }
+}
+
+assert {
+  res.body.error: isUndefined
+}
+
+script:pre-request {
+  // sleep for a while
+  await new Promise(r => setTimeout(r, 1000));
+}
+
+script:post-response {
+  // Sleep for sometime to make sure current operation finishes before next request starts.
+  await new Promise(r => setTimeout(r, 100));
+  console.log("get result: ", res.body.result);
+  if (res.body.result.status != "Success") {
+    throw new Error("Assertion failed: payment session status expected to be Success");
+  }
+
+}
diff --git a/tests/bruno/e2e/router-pay/17-node1-get-channels.bru b/tests/bruno/e2e/router-pay/18-node1-get-channels.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/17-node1-get-channels.bru
rename to tests/bruno/e2e/router-pay/18-node1-get-channels.bru
index 718b3b39..f6668492 100644
--- a/tests/bruno/e2e/router-pay/17-node1-get-channels.bru
+++ b/tests/bruno/e2e/router-pay/18-node1-get-channels.bru
@@ -1,7 +1,7 @@
 meta {
   name: Node1 send get_channels rpc request
   type: http
-  seq: 17
+  seq: 18
 }
 
 post {
diff --git a/tests/bruno/e2e/router-pay/18-node1-get-nodes-page.bru b/tests/bruno/e2e/router-pay/19-node1-get-nodes-page.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/18-node1-get-nodes-page.bru
rename to tests/bruno/e2e/router-pay/19-node1-get-nodes-page.bru
index 518765a8..6ef2b939 100644
--- a/tests/bruno/e2e/router-pay/18-node1-get-nodes-page.bru
+++ b/tests/bruno/e2e/router-pay/19-node1-get-nodes-page.bru
@@ -1,7 +1,7 @@
 meta {
   name: Node1 send get_nodes rpc request
   type: http
-  seq: 18
+  seq: 19
 }
 
 post {
diff --git a/tests/bruno/e2e/router-pay/19-node1-get-nodes-page-2.bru b/tests/bruno/e2e/router-pay/20-node1-get-nodes-page-2.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/19-node1-get-nodes-page-2.bru
rename to tests/bruno/e2e/router-pay/20-node1-get-nodes-page-2.bru
index 884250cf..1b9ac817 100644
--- a/tests/bruno/e2e/router-pay/19-node1-get-nodes-page-2.bru
+++ b/tests/bruno/e2e/router-pay/20-node1-get-nodes-page-2.bru
@@ -2,7 +2,7 @@
 meta {
   name: Node1 send get_nodes rpc request
   type: http
-  seq: 19
+  seq: 20
 }
 
 post {
diff --git a/tests/bruno/e2e/router-pay/20-node1-get-nodes-page-3.bru b/tests/bruno/e2e/router-pay/21-node1-get-nodes-page-3.bru
similarity index 98%
rename from tests/bruno/e2e/router-pay/20-node1-get-nodes-page-3.bru
rename to tests/bruno/e2e/router-pay/21-node1-get-nodes-page-3.bru
index 327a5119..16ee1a99 100644
--- a/tests/bruno/e2e/router-pay/20-node1-get-nodes-page-3.bru
+++ b/tests/bruno/e2e/router-pay/21-node1-get-nodes-page-3.bru
@@ -2,7 +2,7 @@
 meta {
   name: Node1 send get_nodes rpc request
   type: http
-  seq: 20
+  seq: 21
 }
 
 post {

From 602c2e06b5762378fd2b13469e08cd48f490061d Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 29 Sep 2024 16:30:39 +0800
Subject: [PATCH 14/66] update payment related rpc document

---
 src/rpc/README.md | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/rpc/README.md b/src/rpc/README.md
index 9a695476..b7357c0f 100644
--- a/src/rpc/README.md
+++ b/src/rpc/README.md
@@ -21,6 +21,7 @@ You may refer to the e2e test cases in the `tests/bruno/e2e` directory for examp
         * [Method `remove_tlc`](#remove_tlc)
         * [Method `shutdown_channel`](#shutdown_channel)
         * [Method `send_payment`](#send_payment)
+        * [Method `get_payment`](#get_payment)
 
     * [Module Invoice](#module-invoice)
         * [Method `new_invoice`](#new_invoice)
@@ -183,7 +184,30 @@ If `invoice` is not provided, the `target_pubkey`, `amount` must be provided, if
 
 ###### Returns
 
-Returns the `payment_hash` when the request is successful. Otherwise, returns an error message.
+Return a `SendPaymentResult` object with the following fields:
+- `payment_hash` (type: `Hash256`): The identifier of the payment, should be the same as the `payment_hash` in the request.
+- `status` (type: `String`): The status of the payment, possible values are `created`, `inflight`, `success`, `failed`.
+- `last_update_time` (type: `u128`): The last update time of the payment.
+- `failed_error` (type: `Option<String>`): The error message if the payment failed.
+
+<a id="get_payment"></a>
+#### Method `get_payment`
+
+Get the payment by the payment hash.
+
+###### Params
+
+- `payment_hash` (type: `Hash256`): The hash to use within the payment's HTLC.
+
+###### Returns
+
+If success, return a `SendPaymentResult` object with the following fields:
+- `payment_hash` (type: `Hash256`): The identifier of the payment, should be the same as the `payment_hash` in the request.
+- `status` (type: `String`): The status of the payment, possible values are `created`, `inflight`, `success`, `failed`.
+- `last_update_time` (type: `u128`): The last update time of the payment.
+- `failed_error` (type: `Option<String>`): The error message if the payment failed.
+
+If the payment is not found, return error message.
 
 ### Module `Invoice`
 

From abd1e465024e070e4ea5f996a22e51d570ebd806 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Tue, 1 Oct 2024 08:36:37 +0800
Subject: [PATCH 15/66] more on tlc errors

---
 src/fiber/channel.rs       |  35 +++++----
 src/fiber/gen/fiber.rs     | 148 ++++++++++++++++++++++++++++++++-----
 src/fiber/schema/fiber.mol |   3 +-
 src/fiber/types.rs         |   6 +-
 src/rpc/channel.rs         |   5 +-
 5 files changed, 162 insertions(+), 35 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 3f525f59..b205fd6a 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -487,6 +487,7 @@ where
                                                         //FIXME: we should define the error code carefully according
                                                         //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
                                                         error_code: 1,
+                                                        packet_data: vec![],
                                                     },
                                                 ),
                                             }),
@@ -503,13 +504,17 @@ where
                 state.check_for_tlc_update(None)?;
                 let channel_id = state.get_id();
 
-                let tlc_details = state
-                    .remove_tlc_with_reason(TLCId::Offered(remove_tlc.tlc_id), remove_tlc.reason)?;
+                let tlc_details = state.remove_tlc_with_reason(
+                    TLCId::Offered(remove_tlc.tlc_id),
+                    &remove_tlc.reason,
+                )?;
                 if let (
                     Some(ref udt_type_script),
                     RemoveTlcReason::RemoveTlcFulfill(RemoveTlcFulfill { payment_preimage }),
-                ) = (state.funding_udt_type_script.clone(), remove_tlc.reason)
-                {
+                ) = (
+                    state.funding_udt_type_script.clone(),
+                    remove_tlc.reason.clone(),
+                ) {
                     let mut tlc = tlc_details.tlc.clone();
                     tlc.payment_preimage = Some(payment_preimage);
                     self.subscribers
@@ -535,7 +540,7 @@ where
                                 command: ChannelCommand::RemoveTlc(
                                     RemoveTlcCommand {
                                         id: previous_tlc.into(),
-                                        reason: remove_tlc.reason,
+                                        reason: remove_tlc.reason.clone(),
                                     },
                                     port,
                                 ),
@@ -546,6 +551,7 @@ where
                     info!("remove tlc from previous channel: {:?}", &res);
                 }
 
+                // TODO: only the original sender of the TLC should send `TlcRemoveReceived` event
                 self.network
                     .send_message(NetworkActorMessage::new_event(
                         NetworkActorEvent::TlcRemoveReceived(
@@ -790,6 +796,7 @@ where
                                     //FIXME: we should define the error code carefully according
                                     //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
                                     error_code: 1,
+                                    packet_data: vec![],
                                 }),
                             },
                             port,
@@ -966,7 +973,7 @@ where
         command: RemoveTlcCommand,
     ) -> ProcessingChannelResult {
         state.check_for_tlc_update(None)?;
-        let tlc = state.remove_tlc_with_reason(TLCId::Received(command.id), command.reason)?;
+        let tlc = state.remove_tlc_with_reason(TLCId::Received(command.id), &command.reason)?;
         let msg = FiberMessageWithPeerId::new(
             state.get_remote_peer_id(),
             FiberMessage::remove_tlc(RemoveTlc {
@@ -3046,7 +3053,7 @@ impl ChannelActorState {
                 );
                 tlc.creation_confirmed_at = Some(commitment_numbers);
             }
-            match (tlc.removed_at, tlc.removal_confirmed_at) {
+            match (tlc.removed_at.clone(), tlc.removal_confirmed_at) {
                 (Some((_removed_at, reason)), None) => {
                     tlc.removal_confirmed_at = Some(commitment_numbers);
                      match reason {
@@ -3292,7 +3299,7 @@ impl ChannelActorState {
     pub fn remove_tlc_with_reason(
         &mut self,
         tlc_id: TLCId,
-        reason: RemoveTlcReason,
+        reason: &RemoveTlcReason,
     ) -> Result<DetailedTLCInfo, ProcessingChannelError> {
         let removed_at = self.get_current_commitment_numbers();
 
@@ -3304,9 +3311,9 @@ impl ChannelActorState {
                 )))
             }
             Some(current) => {
-                match current.removed_at {
+                match &current.removed_at {
                     Some((current_removed_at, current_remove_reason))
-                        if current_remove_reason == reason && removed_at == current_removed_at =>
+                        if current_remove_reason == reason && removed_at == *current_removed_at =>
                     {
                         debug!(
                             "Skipping removing of tlc {:?} as it is already removed at {:?} with the same reason {:?}", tlc_id, removed_at, reason
@@ -3338,7 +3345,7 @@ impl ChannelActorState {
                         }
                     }
                 };
-                current.removed_at = Some((removed_at, reason));
+                current.removed_at = Some((removed_at, reason.clone()));
                 current
             }
         };
@@ -4561,7 +4568,7 @@ impl ChannelActorState {
 
                                 need_resend_commitment_signed = true;
                             }
-                        } else if let Some((commitment_number, remove_reason)) = info.removed_at {
+                        } else if let Some((commitment_number, remove_reason)) = &info.removed_at {
                             if commitment_number.get_local() >= acutal_local_commitment_number {
                                 // resend RemoveTlc message
                                 network
@@ -4572,7 +4579,7 @@ impl ChannelActorState {
                                                 FiberMessage::remove_tlc(RemoveTlc {
                                                     channel_id: self.get_id(),
                                                     tlc_id: info.tlc.get_id(),
-                                                    reason: remove_reason,
+                                                    reason: remove_reason.clone(),
                                                 }),
                                             ),
                                         ),
@@ -5539,7 +5546,7 @@ impl DetailedTLCInfo {
     }
 
     fn is_fullfill_removed(&self) -> bool {
-        if let Some((_, removed_reason)) = self.removed_at {
+        if let Some((_, removed_reason)) = &self.removed_at {
             matches!(removed_reason, RemoveTlcReason::RemoveTlcFulfill(_))
         } else {
             false
diff --git a/src/fiber/gen/fiber.rs b/src/fiber/gen/fiber.rs
index 8232c0e5..7397b69b 100644
--- a/src/fiber/gen/fiber.rs
+++ b/src/fiber/gen/fiber.rs
@@ -9366,6 +9366,11 @@ impl ::core::fmt::Display for RemoveTlcFail {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
         write!(f, "{}: {}", "error_code", self.error_code())?;
+        write!(f, ", {}: {}", "packet_data", self.packet_data())?;
+        let extra_count = self.count_extra_fields();
+        if extra_count != 0 {
+            write!(f, ", .. ({} fields)", extra_count)?;
+        }
         write!(f, " }}")
     }
 }
@@ -9376,12 +9381,41 @@ impl ::core::default::Default for RemoveTlcFail {
     }
 }
 impl RemoveTlcFail {
-    const DEFAULT_VALUE: [u8; 4] = [0, 0, 0, 0];
-    pub const TOTAL_SIZE: usize = 4;
-    pub const FIELD_SIZES: [usize; 1] = [4];
-    pub const FIELD_COUNT: usize = 1;
+    const DEFAULT_VALUE: [u8; 20] = [
+        20, 0, 0, 0, 12, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    ];
+    pub const FIELD_COUNT: usize = 2;
+    pub fn total_size(&self) -> usize {
+        molecule::unpack_number(self.as_slice()) as usize
+    }
+    pub fn field_count(&self) -> usize {
+        if self.total_size() == molecule::NUMBER_SIZE {
+            0
+        } else {
+            (molecule::unpack_number(&self.as_slice()[molecule::NUMBER_SIZE..]) as usize / 4) - 1
+        }
+    }
+    pub fn count_extra_fields(&self) -> usize {
+        self.field_count() - Self::FIELD_COUNT
+    }
+    pub fn has_extra_fields(&self) -> bool {
+        Self::FIELD_COUNT != self.field_count()
+    }
     pub fn error_code(&self) -> Uint32 {
-        Uint32::new_unchecked(self.0.slice(0..4))
+        let slice = self.as_slice();
+        let start = molecule::unpack_number(&slice[4..]) as usize;
+        let end = molecule::unpack_number(&slice[8..]) as usize;
+        Uint32::new_unchecked(self.0.slice(start..end))
+    }
+    pub fn packet_data(&self) -> Bytes {
+        let slice = self.as_slice();
+        let start = molecule::unpack_number(&slice[8..]) as usize;
+        if self.has_extra_fields() {
+            let end = molecule::unpack_number(&slice[12..]) as usize;
+            Bytes::new_unchecked(self.0.slice(start..end))
+        } else {
+            Bytes::new_unchecked(self.0.slice(start..))
+        }
     }
     pub fn as_reader<'r>(&'r self) -> RemoveTlcFailReader<'r> {
         RemoveTlcFailReader::new_unchecked(self.as_slice())
@@ -9409,7 +9443,9 @@ impl molecule::prelude::Entity for RemoveTlcFail {
         ::core::default::Default::default()
     }
     fn as_builder(self) -> Self::Builder {
-        Self::new_builder().error_code(self.error_code())
+        Self::new_builder()
+            .error_code(self.error_code())
+            .packet_data(self.packet_data())
     }
 }
 #[derive(Clone, Copy)]
@@ -9432,15 +9468,47 @@ impl<'r> ::core::fmt::Display for RemoveTlcFailReader<'r> {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
         write!(f, "{}: {}", "error_code", self.error_code())?;
+        write!(f, ", {}: {}", "packet_data", self.packet_data())?;
+        let extra_count = self.count_extra_fields();
+        if extra_count != 0 {
+            write!(f, ", .. ({} fields)", extra_count)?;
+        }
         write!(f, " }}")
     }
 }
 impl<'r> RemoveTlcFailReader<'r> {
-    pub const TOTAL_SIZE: usize = 4;
-    pub const FIELD_SIZES: [usize; 1] = [4];
-    pub const FIELD_COUNT: usize = 1;
+    pub const FIELD_COUNT: usize = 2;
+    pub fn total_size(&self) -> usize {
+        molecule::unpack_number(self.as_slice()) as usize
+    }
+    pub fn field_count(&self) -> usize {
+        if self.total_size() == molecule::NUMBER_SIZE {
+            0
+        } else {
+            (molecule::unpack_number(&self.as_slice()[molecule::NUMBER_SIZE..]) as usize / 4) - 1
+        }
+    }
+    pub fn count_extra_fields(&self) -> usize {
+        self.field_count() - Self::FIELD_COUNT
+    }
+    pub fn has_extra_fields(&self) -> bool {
+        Self::FIELD_COUNT != self.field_count()
+    }
     pub fn error_code(&self) -> Uint32Reader<'r> {
-        Uint32Reader::new_unchecked(&self.as_slice()[0..4])
+        let slice = self.as_slice();
+        let start = molecule::unpack_number(&slice[4..]) as usize;
+        let end = molecule::unpack_number(&slice[8..]) as usize;
+        Uint32Reader::new_unchecked(&self.as_slice()[start..end])
+    }
+    pub fn packet_data(&self) -> BytesReader<'r> {
+        let slice = self.as_slice();
+        let start = molecule::unpack_number(&slice[8..]) as usize;
+        if self.has_extra_fields() {
+            let end = molecule::unpack_number(&slice[12..]) as usize;
+            BytesReader::new_unchecked(&self.as_slice()[start..end])
+        } else {
+            BytesReader::new_unchecked(&self.as_slice()[start..])
+        }
     }
 }
 impl<'r> molecule::prelude::Reader<'r> for RemoveTlcFailReader<'r> {
@@ -9455,36 +9523,82 @@ impl<'r> molecule::prelude::Reader<'r> for RemoveTlcFailReader<'r> {
     fn as_slice(&self) -> &'r [u8] {
         self.0
     }
-    fn verify(slice: &[u8], _compatible: bool) -> molecule::error::VerificationResult<()> {
+    fn verify(slice: &[u8], compatible: bool) -> molecule::error::VerificationResult<()> {
         use molecule::verification_error as ve;
         let slice_len = slice.len();
-        if slice_len != Self::TOTAL_SIZE {
-            return ve!(Self, TotalSizeNotMatch, Self::TOTAL_SIZE, slice_len);
+        if slice_len < molecule::NUMBER_SIZE {
+            return ve!(Self, HeaderIsBroken, molecule::NUMBER_SIZE, slice_len);
+        }
+        let total_size = molecule::unpack_number(slice) as usize;
+        if slice_len != total_size {
+            return ve!(Self, TotalSizeNotMatch, total_size, slice_len);
         }
+        if slice_len < molecule::NUMBER_SIZE * 2 {
+            return ve!(Self, HeaderIsBroken, molecule::NUMBER_SIZE * 2, slice_len);
+        }
+        let offset_first = molecule::unpack_number(&slice[molecule::NUMBER_SIZE..]) as usize;
+        if offset_first % molecule::NUMBER_SIZE != 0 || offset_first < molecule::NUMBER_SIZE * 2 {
+            return ve!(Self, OffsetsNotMatch);
+        }
+        if slice_len < offset_first {
+            return ve!(Self, HeaderIsBroken, offset_first, slice_len);
+        }
+        let field_count = offset_first / molecule::NUMBER_SIZE - 1;
+        if field_count < Self::FIELD_COUNT {
+            return ve!(Self, FieldCountNotMatch, Self::FIELD_COUNT, field_count);
+        } else if !compatible && field_count > Self::FIELD_COUNT {
+            return ve!(Self, FieldCountNotMatch, Self::FIELD_COUNT, field_count);
+        };
+        let mut offsets: Vec<usize> = slice[molecule::NUMBER_SIZE..offset_first]
+            .chunks_exact(molecule::NUMBER_SIZE)
+            .map(|x| molecule::unpack_number(x) as usize)
+            .collect();
+        offsets.push(total_size);
+        if offsets.windows(2).any(|i| i[0] > i[1]) {
+            return ve!(Self, OffsetsNotMatch);
+        }
+        Uint32Reader::verify(&slice[offsets[0]..offsets[1]], compatible)?;
+        BytesReader::verify(&slice[offsets[1]..offsets[2]], compatible)?;
         Ok(())
     }
 }
 #[derive(Clone, Debug, Default)]
 pub struct RemoveTlcFailBuilder {
     pub(crate) error_code: Uint32,
+    pub(crate) packet_data: Bytes,
 }
 impl RemoveTlcFailBuilder {
-    pub const TOTAL_SIZE: usize = 4;
-    pub const FIELD_SIZES: [usize; 1] = [4];
-    pub const FIELD_COUNT: usize = 1;
+    pub const FIELD_COUNT: usize = 2;
     pub fn error_code(mut self, v: Uint32) -> Self {
         self.error_code = v;
         self
     }
+    pub fn packet_data(mut self, v: Bytes) -> Self {
+        self.packet_data = v;
+        self
+    }
 }
 impl molecule::prelude::Builder for RemoveTlcFailBuilder {
     type Entity = RemoveTlcFail;
     const NAME: &'static str = "RemoveTlcFailBuilder";
     fn expected_length(&self) -> usize {
-        Self::TOTAL_SIZE
+        molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1)
+            + self.error_code.as_slice().len()
+            + self.packet_data.as_slice().len()
     }
     fn write<W: molecule::io::Write>(&self, writer: &mut W) -> molecule::io::Result<()> {
+        let mut total_size = molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1);
+        let mut offsets = Vec::with_capacity(Self::FIELD_COUNT);
+        offsets.push(total_size);
+        total_size += self.error_code.as_slice().len();
+        offsets.push(total_size);
+        total_size += self.packet_data.as_slice().len();
+        writer.write_all(&molecule::pack_number(total_size as molecule::Number))?;
+        for offset in offsets.into_iter() {
+            writer.write_all(&molecule::pack_number(offset as molecule::Number))?;
+        }
         writer.write_all(self.error_code.as_slice())?;
+        writer.write_all(self.packet_data.as_slice())?;
         Ok(())
     }
     fn build(&self) -> Self::Entity {
diff --git a/src/fiber/schema/fiber.mol b/src/fiber/schema/fiber.mol
index ab1e37e3..7513f8d7 100644
--- a/src/fiber/schema/fiber.mol
+++ b/src/fiber/schema/fiber.mol
@@ -127,8 +127,9 @@ struct RemoveTlcFulfill {
     payment_preimage:   Byte32,
 }
 
-struct RemoveTlcFail {
+table RemoveTlcFail {
     error_code:         Uint32,
+    packet_data:        Bytes,
 }
 
 union RemoveTlcReason {
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 9e5475a1..88483451 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1221,9 +1221,10 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
     }
 }
 
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct RemoveTlcFail {
     pub error_code: u32,
+    pub packet_data: Vec<u8>,
 }
 
 impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {
@@ -1240,11 +1241,12 @@ impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
     fn try_from(remove_tlc_fail: molecule_fiber::RemoveTlcFail) -> Result<Self, Self::Error> {
         Ok(RemoveTlcFail {
             error_code: remove_tlc_fail.error_code().unpack(),
+            packet_data: remove_tlc_fail.packet_data().unpack(),
         })
     }
 }
 
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),
     RemoveTlcFail(RemoveTlcFail),
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index 9ffe7d6c..bcffee0a 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -441,7 +441,10 @@ where
                                 }
                                 RemoveTlcReason::RemoveTlcFail { error_code } => {
                                     crate::fiber::types::RemoveTlcReason::RemoveTlcFail(
-                                        RemoveTlcFail { error_code },
+                                        RemoveTlcFail {
+                                            error_code,
+                                            packet_data: vec![],
+                                        },
                                     )
                                 }
                             },

From 8fa3c932218075968530d7255b17dc60ba7086ba Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Tue, 1 Oct 2024 08:42:48 +0800
Subject: [PATCH 16/66] fix TLCRemoveReceived events

---
 src/fiber/channel.rs | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index b205fd6a..05587d52 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -511,12 +511,10 @@ where
                 if let (
                     Some(ref udt_type_script),
                     RemoveTlcReason::RemoveTlcFulfill(RemoveTlcFulfill { payment_preimage }),
-                ) = (
-                    state.funding_udt_type_script.clone(),
-                    remove_tlc.reason.clone(),
-                ) {
+                ) = (state.funding_udt_type_script.clone(), &remove_tlc.reason)
+                {
                     let mut tlc = tlc_details.tlc.clone();
-                    tlc.payment_preimage = Some(payment_preimage);
+                    tlc.payment_preimage = Some(*payment_preimage);
                     self.subscribers
                         .settled_tlcs_subscribers
                         .send(TlcNotification {
@@ -549,17 +547,18 @@ where
                         .expect(ASSUME_NETWORK_ACTOR_ALIVE);
                     let res = recv.await.expect("network actor is alive");
                     info!("remove tlc from previous channel: {:?}", &res);
+                } else {
+                    // only the original sender of the TLC should send `TlcRemoveReceived` event
+                    // because only the original sender cares about the TLC event to settle the payment
+                    self.network
+                        .send_message(NetworkActorMessage::new_event(
+                            NetworkActorEvent::TlcRemoveReceived(
+                                tlc_details.tlc.payment_hash,
+                                remove_tlc,
+                            ),
+                        ))
+                        .expect("myself alive");
                 }
-
-                // TODO: only the original sender of the TLC should send `TlcRemoveReceived` event
-                self.network
-                    .send_message(NetworkActorMessage::new_event(
-                        NetworkActorEvent::TlcRemoveReceived(
-                            tlc_details.tlc.payment_hash,
-                            remove_tlc,
-                        ),
-                    ))
-                    .expect("myself alive");
                 Ok(())
             }
             FiberChannelMessage::Shutdown(shutdown) => {

From 9db6db1b6940e0a84308adaa794c2a6f9229cbda Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Wed, 2 Oct 2024 20:20:02 +0800
Subject: [PATCH 17/66] add TlcFailErrorCode

---
 src/fiber/serde_utils.rs |  9 +++-
 src/fiber/types.rs       | 89 ++++++++++++++++++++++++++++++++++++++--
 src/rpc/channel.rs       |  6 +--
 3 files changed, 96 insertions(+), 8 deletions(-)

diff --git a/src/fiber/serde_utils.rs b/src/fiber/serde_utils.rs
index 7592d8c2..1b872e32 100644
--- a/src/fiber/serde_utils.rs
+++ b/src/fiber/serde_utils.rs
@@ -109,6 +109,7 @@ macro_rules! uint_as_hex {
 uint_as_hex!(U128Hex, u128);
 uint_as_hex!(U64Hex, u64);
 uint_as_hex!(U32Hex, u32);
+uint_as_hex!(U16Hex, u16);
 
 #[cfg(test)]
 mod tests {
@@ -129,6 +130,10 @@ mod tests {
         bar_128: u128,
         #[serde_as(as = "U64Hex")]
         bar_64: u64,
+        #[serde_as(as = "U32Hex")]
+        bar_32: u32,
+        #[serde_as(as = "U16Hex")]
+        bar_16: u16,
     }
 
     #[test]
@@ -138,9 +143,11 @@ mod tests {
             enity: Some(Script::new_builder().build()),
             bar_128: 0xdeadbeef,
             bar_64: 0x123,
+            bar_32: 0x10,
+            bar_16: 0xa,
         };
 
-        let json = r#"{"slice":"0x01020304","enity":"0x3500000010000000300000003100000000000000000000000000000000000000000000000000000000000000000000000000000000","bar_128":"0xdeadbeef","bar_64":"0x123"}"#;
+        let json = r#"{"slice":"0x01020304","enity":"0x3500000010000000300000003100000000000000000000000000000000000000000000000000000000000000000000000000000000","bar_128":"0xdeadbeef","bar_64":"0x123","bar_32":"0x10","bar_16":"0xa"}"#;
         assert_eq!(serde_json::to_string(&foo).unwrap(), json);
         assert_eq!(serde_json::from_str::<Foo>(json).unwrap(), foo);
     }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 88483451..eb594330 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1223,14 +1223,15 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct RemoveTlcFail {
-    pub error_code: u32,
+    pub error_code: u16,
     pub packet_data: Vec<u8>,
 }
 
 impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {
     fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
         molecule_fiber::RemoveTlcFail::new_builder()
-            .error_code(remove_tlc_fail.error_code.pack())
+            .error_code((remove_tlc_fail.error_code as u32).pack())
+            .packet_data(remove_tlc_fail.packet_data.pack())
             .build()
     }
 }
@@ -1239,13 +1240,80 @@ impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
     type Error = Error;
 
     fn try_from(remove_tlc_fail: molecule_fiber::RemoveTlcFail) -> Result<Self, Self::Error> {
+        let error_code: u32 = remove_tlc_fail.error_code().unpack();
         Ok(RemoveTlcFail {
-            error_code: remove_tlc_fail.error_code().unpack(),
+            error_code: error_code as u16,
             packet_data: remove_tlc_fail.packet_data().unpack(),
         })
     }
 }
 
+// The onion packet is invalid
+const BADONION: u16 = 0x8000;
+// Permanent errors (otherwise transient)
+const PERM: u16 = 0x4000;
+// Node releated errors (otherwise channels)
+const NODE: u16 = 0x2000;
+//  Channel forwarding parameter was violated
+const UPDATE: u16 = 0x1000;
+
+impl RemoveTlcFail {
+    pub fn is_node(&self) -> bool {
+        self.error_code & NODE != 0
+    }
+
+    pub fn is_bad_onion(&self) -> bool {
+        self.error_code & BADONION != 0
+    }
+
+    pub fn is_perm(&self) -> bool {
+        self.error_code & PERM != 0
+    }
+
+    pub fn is_update(&self) -> bool {
+        self.error_code & UPDATE != 0
+    }
+
+    pub fn new(error_code: TlcFailErrorCode, packet_data: Vec<u8>) -> Self {
+        RemoveTlcFail {
+            error_code: error_code.into(),
+            packet_data,
+        }
+    }
+}
+
+#[repr(u16)]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub enum TlcFailErrorCode {
+    TemporaryNodeFailure = NODE | 2,
+    PermanentNodeFailure = PERM | NODE | 2,
+    RequiredNodeFeatureMissing = PERM | NODE | 3,
+    InvalidOnionVersion = BADONION | PERM | 4,
+    InvalidOnionHmac = BADONION | PERM | 5,
+    InvalidOnionKey = BADONION | PERM | 6,
+    TemporaryChannelFailure = UPDATE | 7,
+    PermanentChannelFailure = PERM | 8,
+    RequiredChannelFeatureMissing = PERM | 9,
+    UnknownNextPeer = PERM | 10,
+    AmountBelowMinimum = UPDATE | 11,
+    FeeInsufficient = UPDATE | 12,
+    IncorrectCltvExpiry = UPDATE | 13,
+    ExpiryTooSoon = UPDATE | 14,
+    IncorrectOrUnknownPaymentDetails = PERM | 15,
+    FinalIncorrectCltvExpiry = 18,
+    FinalIncorrectHtlcAmount = 19,
+    ChannelDisabled = UPDATE | 20,
+    ExpiryTooFar = 21,
+    InvalidOnionPayload = PERM | 22,
+    MppTimeout = 23,
+    InvalidOnionBlinding = BADONION | PERM | 24,
+}
+impl From<TlcFailErrorCode> for u16 {
+    fn from(error_code: TlcFailErrorCode) -> Self {
+        error_code as u16
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),
@@ -2947,7 +3015,7 @@ fn get_hop_data_len(buf: &[u8]) -> Option<usize> {
 
 #[cfg(test)]
 mod tests {
-    use super::{secp256k1_instance, Pubkey};
+    use super::{secp256k1_instance, Pubkey, RemoveTlcFail, TlcFailErrorCode};
     use crate::fiber::test_utils::generate_seckey;
     use crate::fiber::types::Privkey;
     use ckb_types::packed::OutPointBuilder;
@@ -3039,4 +3107,17 @@ mod tests {
         assert_eq!(packet.current, hops_infos[2]);
         assert!(packet.is_last());
     }
+
+    #[test]
+    fn test_tlc_fail_error() {
+        let packet_data: Vec<u8> = vec![42; 32];
+        let error = RemoveTlcFail::new(TlcFailErrorCode::InvalidOnionVersion, packet_data);
+        assert!(!error.is_node());
+        assert!(error.is_bad_onion());
+        assert!(error.is_perm());
+        let error_mol: super::molecule_fiber::RemoveTlcFail = error.clone().into();
+
+        let decoded: RemoveTlcFail = error_mol.try_into().expect("decode");
+        assert_eq!(error, decoded);
+    }
 }
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index bcffee0a..15c35c8a 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -6,7 +6,7 @@ use crate::fiber::{
     graph::PaymentSessionStatus,
     hash_algorithm::HashAlgorithm,
     network::{AcceptChannelCommand, OpenChannelCommand, SendPaymentCommand},
-    serde_utils::{U128Hex, U32Hex, U64Hex},
+    serde_utils::{U128Hex, U16Hex, U64Hex},
     types::{Hash256, LockTime, Pubkey, RemoveTlcFail, RemoveTlcFulfill},
     NetworkActorCommand, NetworkActorMessage,
 };
@@ -142,8 +142,8 @@ pub enum RemoveTlcReason {
         payment_preimage: Hash256,
     },
     RemoveTlcFail {
-        #[serde_as(as = "U32Hex")]
-        error_code: u32,
+        #[serde_as(as = "U16Hex")]
+        error_code: u16,
     },
 }
 

From 62e9fac3720847168cb4eed2079216447c66fa65 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Tue, 8 Oct 2024 10:14:32 +0800
Subject: [PATCH 18/66] more on error code of tlcs

---
 src/fiber/channel.rs | 70 +++++++++++++++++++++++++++++---------------
 1 file changed, 47 insertions(+), 23 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 1f150bff..0c6a79ce 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -5,7 +5,7 @@ use tracing::{debug, error, info, warn};
 use crate::{
     fiber::{
         network::{get_chain_hash, SendOnionPacketCommand},
-        types::{ChannelUpdate, RemoveTlcFail},
+        types::{ChannelUpdate, RemoveTlcFail, TlcFailErrorCode},
     },
     invoice::InvoiceStore,
 };
@@ -466,6 +466,37 @@ where
                         Ok(())
                     }
                     Err(e) => {
+                        let error_code = match e {
+                            ProcessingChannelError::PeelingOnionPacketError(_) => {
+                                TlcFailErrorCode::InvalidOnionPayload
+                            }
+                            ProcessingChannelError::InvalidParameter(_) => {
+                                TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
+                            }
+                            ProcessingChannelError::FinalIncorrectHTLCAmount => {
+                                TlcFailErrorCode::FinalIncorrectHtlcAmount
+                            }
+                            ProcessingChannelError::TlcAmountIsTooLow => {
+                                TlcFailErrorCode::AmountBelowMinimum
+                            }
+                            ProcessingChannelError::TlcNumberExceedLimit
+                            | ProcessingChannelError::TlcValueInflightExceedLimit => {
+                                TlcFailErrorCode::TemporaryChannelFailure
+                            }
+                            ProcessingChannelError::InvalidState(_) => match state.state {
+                                ChannelState::Closed(_) => TlcFailErrorCode::ChannelDisabled,
+                                ChannelState::ShuttingDown(_) | ChannelState::ChannelReady() => {
+                                    // we expect `ShuttingDown` and `ChannelReady` are both OK for tlc forwarding,
+                                    // so here are the unreachable point in normal workflow,
+                                    // set `TemporaryNodeFailure` for general temporary failure of the processing node here
+                                    TlcFailErrorCode::TemporaryNodeFailure
+                                }
+                                // otherwise, channel maybe not ready
+                                _ => TlcFailErrorCode::TemporaryChannelFailure,
+                            },
+                            // TODO: there maybe more error types here
+                            _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
+                        };
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
@@ -483,12 +514,7 @@ where
                                                 channel_id: state.get_id(),
                                                 tlc_id,
                                                 reason: RemoveTlcReason::RemoveTlcFail(
-                                                    RemoveTlcFail {
-                                                        //FIXME: we should define the error code carefully according
-                                                        //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
-                                                        error_code: 1,
-                                                        packet_data: vec![],
-                                                    },
+                                                    RemoveTlcFail::new(error_code, vec![]),
                                                 ),
                                             }),
                                         ),
@@ -734,15 +760,14 @@ where
             }
 
             if forward_to_next_hop && peeled_packet.current.amount > add_tlc.amount {
-                return Err(ProcessingChannelError::InvalidParameter(
+                return Err(ProcessingChannelError::InvalidParameter
+                    (
                     format!("Payment amount is too large, expect next hop amount [{}] less than received amount [{}]",
                     peeled_packet.current.amount , add_tlc.amount)
                 ));
             }
             if !forward_to_next_hop && peeled_packet.current.amount != add_tlc.amount {
-                return Err(ProcessingChannelError::InvalidParameter(
-                    "Payment amount mismatch".to_string(),
-                ));
+                return Err(ProcessingChannelError::FinalIncorrectHTLCAmount);
             }
         }
 
@@ -2050,6 +2075,14 @@ pub enum ProcessingChannelError {
     Musig2SigningError(#[from] SigningError),
     #[error("Failed to peel onion packet: {0}")]
     PeelingOnionPacketError(String),
+    #[error("The amount in the HTLC is not expected")]
+    FinalIncorrectHTLCAmount,
+    #[error("The tlc number exceed limit of this channel")]
+    TlcNumberExceedLimit,
+    #[error("The tlc flight value exceed limit of this channel")]
+    TlcValueInflightExceedLimit,
+    #[error("The tlc amount below minimal")]
+    TlcAmountIsTooLow,
 }
 
 bitflags! {
@@ -3243,10 +3276,7 @@ impl ChannelActorState {
             }
         };
         if tlc.amount == 0 {
-            return Err(ProcessingChannelError::InvalidParameter(format!(
-                "Expect the amount of tlc with id {:?} is larger than zero",
-                tlc.id
-            )));
+            return Err(ProcessingChannelError::TlcAmountIsTooLow);
         }
         if tlc.is_offered() {
             // TODO: We should actually also consider all our fulfilled tlcs here.
@@ -3728,10 +3758,7 @@ impl ChannelActorState {
                 + self.get_active_received_tlcs(true).count();
 
             if active_tls_number as u64 + 1 > self.max_num_of_accept_tlcs {
-                return Err(ProcessingChannelError::InvalidParameter(format!(
-                    "Exceeding the maximum number of tlcs: {}",
-                    self.max_num_of_accept_tlcs
-                )));
+                return Err(ProcessingChannelError::TlcNumberExceedLimit);
             }
 
             if self
@@ -3741,10 +3768,7 @@ impl ChannelActorState {
                 + add_amount
                 > self.max_tlc_value_in_flight
             {
-                return Err(ProcessingChannelError::InvalidParameter(format!(
-                    "Exceeding the maximum amount of tlcs: {}",
-                    self.max_tlc_value_in_flight
-                )));
+                return Err(ProcessingChannelError::TlcValueInflightExceedLimit);
             }
         }
         Ok(())

From 0ec104f1a2da05bef59ca38a48e8a396f59e1022 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 10:34:16 +0800
Subject: [PATCH 19/66] add TlcFailDetail for tlc error handling

---
 Cargo.lock                 |  35 ++++++++-
 Cargo.toml                 |   1 +
 src/fiber/channel.rs       |  11 ++-
 src/fiber/gen/fiber.rs     |  65 +++++------------
 src/fiber/graph.rs         |   4 +-
 src/fiber/network.rs       |  20 +++---
 src/fiber/schema/fiber.mol |   3 +-
 src/fiber/types.rs         | 142 +++++++++++++++++++++++++------------
 src/rpc/channel.rs         |   6 +-
 9 files changed, 172 insertions(+), 115 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 04dbe341..1acf4dda 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1198,7 +1198,7 @@ version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47"
 dependencies = [
- "heck",
+ "heck 0.4.1",
  "proc-macro2",
  "quote",
  "syn 2.0.52",
@@ -1608,6 +1608,7 @@ dependencies = [
  "serde_with",
  "serde_yaml",
  "socket2",
+ "strum",
  "tempfile",
  "tentacle",
  "thiserror",
@@ -1893,6 +1894,12 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.9"
@@ -2307,7 +2314,7 @@ version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d0bb047e79a143b32ea03974a6bf59b62c2a4c5f5d42a381c907a8bbb3f75c0"
 dependencies = [
- "heck",
+ "heck 0.4.1",
  "proc-macro-crate",
  "proc-macro2",
  "quote",
@@ -3108,7 +3115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
 dependencies = [
  "bytes",
- "heck",
+ "heck 0.4.1",
  "itertools 0.12.1",
  "log",
  "multimap",
@@ -3895,6 +3902,28 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
 
+[[package]]
+name = "strum"
+version = "0.26.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+dependencies = [
+ "strum_macros",
+]
+
+[[package]]
+name = "strum_macros"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
diff --git a/Cargo.toml b/Cargo.toml
index f0c53edd..95b580a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -71,6 +71,7 @@ socket2 = "0.5.7"
 lnd-grpc-tonic-client = "0.3.0"
 git-version = "0.3.9"
 fiber-sphinx = "1.0.1"
+strum = { version = "0.26", features = ["derive"] }
 
 [profile.release]
 panic = "abort"
diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 0c6a79ce..bf79f0a5 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -5,7 +5,7 @@ use tracing::{debug, error, info, warn};
 use crate::{
     fiber::{
         network::{get_chain_hash, SendOnionPacketCommand},
-        types::{ChannelUpdate, RemoveTlcFail, TlcFailErrorCode},
+        types::{ChannelUpdate, RemoveTlcFail, TlcFailDetail, TlcFailErrorCode},
     },
     invoice::InvoiceStore,
 };
@@ -497,6 +497,8 @@ where
                             // TODO: there maybe more error types here
                             _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
                         };
+                        let error_detail =
+                            TlcFailDetail::new(state.get_id(), add_tlc.tlc_id, error_code);
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
@@ -514,7 +516,7 @@ where
                                                 channel_id: state.get_id(),
                                                 tlc_id,
                                                 reason: RemoveTlcReason::RemoveTlcFail(
-                                                    RemoveTlcFail::new(error_code, vec![]),
+                                                    error_detail.into(),
                                                 ),
                                             }),
                                         ),
@@ -827,10 +829,7 @@ where
                             RemoveTlcCommand {
                                 id: added_tlc_id,
                                 reason: RemoveTlcReason::RemoveTlcFail(RemoveTlcFail {
-                                    //FIXME: we should define the error code carefully according
-                                    //refer to https://github.com/lightning/bolts/blob/master/04-onion-routing.md#failure-messages
-                                    error_code: 1,
-                                    packet_data: vec![],
+                                    onion_packet: vec![],
                                 }),
                             },
                             port,
diff --git a/src/fiber/gen/fiber.rs b/src/fiber/gen/fiber.rs
index 0476a42d..c580fe86 100644
--- a/src/fiber/gen/fiber.rs
+++ b/src/fiber/gen/fiber.rs
@@ -9345,8 +9345,7 @@ impl ::core::fmt::Debug for RemoveTlcFail {
 impl ::core::fmt::Display for RemoveTlcFail {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
-        write!(f, "{}: {}", "error_code", self.error_code())?;
-        write!(f, ", {}: {}", "packet_data", self.packet_data())?;
+        write!(f, "{}: {}", "onion_packet", self.onion_packet())?;
         let extra_count = self.count_extra_fields();
         if extra_count != 0 {
             write!(f, ", .. ({} fields)", extra_count)?;
@@ -9361,10 +9360,8 @@ impl ::core::default::Default for RemoveTlcFail {
     }
 }
 impl RemoveTlcFail {
-    const DEFAULT_VALUE: [u8; 20] = [
-        20, 0, 0, 0, 12, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    ];
-    pub const FIELD_COUNT: usize = 2;
+    const DEFAULT_VALUE: [u8; 12] = [12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0];
+    pub const FIELD_COUNT: usize = 1;
     pub fn total_size(&self) -> usize {
         molecule::unpack_number(self.as_slice()) as usize
     }
@@ -9381,17 +9378,11 @@ impl RemoveTlcFail {
     pub fn has_extra_fields(&self) -> bool {
         Self::FIELD_COUNT != self.field_count()
     }
-    pub fn error_code(&self) -> Uint32 {
+    pub fn onion_packet(&self) -> Bytes {
         let slice = self.as_slice();
         let start = molecule::unpack_number(&slice[4..]) as usize;
-        let end = molecule::unpack_number(&slice[8..]) as usize;
-        Uint32::new_unchecked(self.0.slice(start..end))
-    }
-    pub fn packet_data(&self) -> Bytes {
-        let slice = self.as_slice();
-        let start = molecule::unpack_number(&slice[8..]) as usize;
         if self.has_extra_fields() {
-            let end = molecule::unpack_number(&slice[12..]) as usize;
+            let end = molecule::unpack_number(&slice[8..]) as usize;
             Bytes::new_unchecked(self.0.slice(start..end))
         } else {
             Bytes::new_unchecked(self.0.slice(start..))
@@ -9423,9 +9414,7 @@ impl molecule::prelude::Entity for RemoveTlcFail {
         ::core::default::Default::default()
     }
     fn as_builder(self) -> Self::Builder {
-        Self::new_builder()
-            .error_code(self.error_code())
-            .packet_data(self.packet_data())
+        Self::new_builder().onion_packet(self.onion_packet())
     }
 }
 #[derive(Clone, Copy)]
@@ -9447,8 +9436,7 @@ impl<'r> ::core::fmt::Debug for RemoveTlcFailReader<'r> {
 impl<'r> ::core::fmt::Display for RemoveTlcFailReader<'r> {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
-        write!(f, "{}: {}", "error_code", self.error_code())?;
-        write!(f, ", {}: {}", "packet_data", self.packet_data())?;
+        write!(f, "{}: {}", "onion_packet", self.onion_packet())?;
         let extra_count = self.count_extra_fields();
         if extra_count != 0 {
             write!(f, ", .. ({} fields)", extra_count)?;
@@ -9457,7 +9445,7 @@ impl<'r> ::core::fmt::Display for RemoveTlcFailReader<'r> {
     }
 }
 impl<'r> RemoveTlcFailReader<'r> {
-    pub const FIELD_COUNT: usize = 2;
+    pub const FIELD_COUNT: usize = 1;
     pub fn total_size(&self) -> usize {
         molecule::unpack_number(self.as_slice()) as usize
     }
@@ -9474,17 +9462,11 @@ impl<'r> RemoveTlcFailReader<'r> {
     pub fn has_extra_fields(&self) -> bool {
         Self::FIELD_COUNT != self.field_count()
     }
-    pub fn error_code(&self) -> Uint32Reader<'r> {
+    pub fn onion_packet(&self) -> BytesReader<'r> {
         let slice = self.as_slice();
         let start = molecule::unpack_number(&slice[4..]) as usize;
-        let end = molecule::unpack_number(&slice[8..]) as usize;
-        Uint32Reader::new_unchecked(&self.as_slice()[start..end])
-    }
-    pub fn packet_data(&self) -> BytesReader<'r> {
-        let slice = self.as_slice();
-        let start = molecule::unpack_number(&slice[8..]) as usize;
         if self.has_extra_fields() {
-            let end = molecule::unpack_number(&slice[12..]) as usize;
+            let end = molecule::unpack_number(&slice[8..]) as usize;
             BytesReader::new_unchecked(&self.as_slice()[start..end])
         } else {
             BytesReader::new_unchecked(&self.as_slice()[start..])
@@ -9537,24 +9519,18 @@ impl<'r> molecule::prelude::Reader<'r> for RemoveTlcFailReader<'r> {
         if offsets.windows(2).any(|i| i[0] > i[1]) {
             return ve!(Self, OffsetsNotMatch);
         }
-        Uint32Reader::verify(&slice[offsets[0]..offsets[1]], compatible)?;
-        BytesReader::verify(&slice[offsets[1]..offsets[2]], compatible)?;
+        BytesReader::verify(&slice[offsets[0]..offsets[1]], compatible)?;
         Ok(())
     }
 }
 #[derive(Clone, Debug, Default)]
 pub struct RemoveTlcFailBuilder {
-    pub(crate) error_code: Uint32,
-    pub(crate) packet_data: Bytes,
+    pub(crate) onion_packet: Bytes,
 }
 impl RemoveTlcFailBuilder {
-    pub const FIELD_COUNT: usize = 2;
-    pub fn error_code(mut self, v: Uint32) -> Self {
-        self.error_code = v;
-        self
-    }
-    pub fn packet_data(mut self, v: Bytes) -> Self {
-        self.packet_data = v;
+    pub const FIELD_COUNT: usize = 1;
+    pub fn onion_packet(mut self, v: Bytes) -> Self {
+        self.onion_packet = v;
         self
     }
 }
@@ -9562,23 +9538,18 @@ impl molecule::prelude::Builder for RemoveTlcFailBuilder {
     type Entity = RemoveTlcFail;
     const NAME: &'static str = "RemoveTlcFailBuilder";
     fn expected_length(&self) -> usize {
-        molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1)
-            + self.error_code.as_slice().len()
-            + self.packet_data.as_slice().len()
+        molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1) + self.onion_packet.as_slice().len()
     }
     fn write<W: molecule::io::Write>(&self, writer: &mut W) -> molecule::io::Result<()> {
         let mut total_size = molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1);
         let mut offsets = Vec::with_capacity(Self::FIELD_COUNT);
         offsets.push(total_size);
-        total_size += self.error_code.as_slice().len();
-        offsets.push(total_size);
-        total_size += self.packet_data.as_slice().len();
+        total_size += self.onion_packet.as_slice().len();
         writer.write_all(&molecule::pack_number(total_size as molecule::Number))?;
         for offset in offsets.into_iter() {
             writer.write_all(&molecule::pack_number(offset as molecule::Number))?;
         }
-        writer.write_all(self.error_code.as_slice())?;
-        writer.write_all(self.packet_data.as_slice())?;
+        writer.write_all(self.onion_packet.as_slice())?;
         Ok(())
     }
     fn build(&self) -> Self::Entity {
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index b50d6d58..101c2cd3 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -819,9 +819,9 @@ impl PaymentSession {
         self.status = status;
     }
 
-    pub fn set_failed_status(&mut self, error: String) {
+    pub fn set_failed_status(&mut self, error: &str) {
         self.status = PaymentSessionStatus::Failed;
-        self.last_error = Some(error);
+        self.last_error = Some(error.to_string());
     }
 }
 
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index e5a7ed34..7a3ae7e5 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -60,7 +60,8 @@ use super::types::{
     GetBroadcastMessagesResult, Hash256, NodeAnnouncement, NodeAnnouncementQuery, OpenChannel,
     Privkey, Pubkey, QueryBroadcastMessagesWithinTimeRange,
     QueryBroadcastMessagesWithinTimeRangeResult, QueryChannelsWithinBlockRange,
-    QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcReason,
+    QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcReason, TlcFailDetail,
+    TlcFailErrorCode,
 };
 use super::FiberConfig;
 
@@ -1959,11 +1960,15 @@ where
     fn on_tlc_remove_received(&self, payment_hash: Hash256, remove_tlc: RemoveTlc) {
         if let Some(mut payment_session) = self.store.get_payment_session(payment_hash) {
             if payment_session.status == PaymentSessionStatus::Inflight {
-                if matches!(remove_tlc.reason, RemoveTlcReason::RemoveTlcFulfill(_)) {
-                    payment_session.set_status(PaymentSessionStatus::Success);
-                } else {
-                    // FIXME(yukang): we should handle tlc fail reason here
-                    payment_session.set_failed_status(format!("{:?}", remove_tlc.reason));
+                match remove_tlc.reason {
+                    RemoveTlcReason::RemoveTlcFulfill(_) => {
+                        payment_session.set_status(PaymentSessionStatus::Success);
+                    }
+                    RemoveTlcReason::RemoveTlcFail(reason) => {
+                        let detail_error: TlcFailDetail = reason.into();
+                        let error_code: TlcFailErrorCode = detail_error.error_code.into();
+                        payment_session.set_failed_status(error_code.as_ref());
+                    }
                 }
                 payment_session.last_updated_time =
                     std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
@@ -2072,8 +2077,7 @@ where
             match res {
                 Err(e) => {
                     error!("Failed to send onion packet: {:?}", e);
-                    // for some error we may need to update the graph and then retry
-                    // TODO: update the graph here according to the error
+                    // This is the error implies we send payment request to the first hop failed
                     let err = format!(
                         "Failed to send onion packet: {:?} with error: {:?}",
                         payment_data.payment_hash, e
diff --git a/src/fiber/schema/fiber.mol b/src/fiber/schema/fiber.mol
index 33116d91..3bd0caf1 100644
--- a/src/fiber/schema/fiber.mol
+++ b/src/fiber/schema/fiber.mol
@@ -128,8 +128,7 @@ struct RemoveTlcFulfill {
 }
 
 table RemoveTlcFail {
-    error_code:         Uint32,
-    packet_data:        Bytes,
+    onion_packet:        Bytes,
 }
 
 union RemoveTlcReason {
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 4feb9e99..9fb92f54 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -28,6 +28,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use std::marker::PhantomData;
 use std::str::FromStr;
+use strum::{AsRefStr, IntoStaticStr};
 use tentacle::multiaddr::MultiAddr;
 use tentacle::secio::PeerId;
 use thiserror::Error;
@@ -1245,42 +1246,30 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct RemoveTlcFail {
+pub enum TlcFailDetailData {}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TlcFailDetail {
+    pub channel_id: Hash256,
+    pub tlc_id: u64,
     pub error_code: u16,
-    pub packet_data: Vec<u8>,
+    pub extra_data: Option<TlcFailDetailData>,
 }
 
-impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {
-    fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
-        molecule_fiber::RemoveTlcFail::new_builder()
-            .error_code((remove_tlc_fail.error_code as u32).pack())
-            .packet_data(remove_tlc_fail.packet_data.pack())
-            .build()
+impl TlcFailDetail {
+    pub fn new(channel_id: Hash256, tlc_id: u64, error_code: TlcFailErrorCode) -> Self {
+        TlcFailDetail {
+            channel_id,
+            tlc_id,
+            error_code: error_code.into(),
+            extra_data: None,
+        }
     }
-}
 
-impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
-    type Error = Error;
-
-    fn try_from(remove_tlc_fail: molecule_fiber::RemoveTlcFail) -> Result<Self, Self::Error> {
-        let error_code: u32 = remove_tlc_fail.error_code().unpack();
-        Ok(RemoveTlcFail {
-            error_code: error_code as u16,
-            packet_data: remove_tlc_fail.packet_data().unpack(),
-        })
+    pub fn set_extra_data(&mut self, extra_data: TlcFailDetailData) {
+        self.extra_data = Some(extra_data);
     }
-}
 
-// The onion packet is invalid
-const BADONION: u16 = 0x8000;
-// Permanent errors (otherwise transient)
-const PERM: u16 = 0x4000;
-// Node releated errors (otherwise channels)
-const NODE: u16 = 0x2000;
-//  Channel forwarding parameter was violated
-const UPDATE: u16 = 0x1000;
-
-impl RemoveTlcFail {
     pub fn is_node(&self) -> bool {
         self.error_code & NODE != 0
     }
@@ -1297,16 +1286,73 @@ impl RemoveTlcFail {
         self.error_code & UPDATE != 0
     }
 
-    pub fn new(error_code: TlcFailErrorCode, packet_data: Vec<u8>) -> Self {
+    fn serialize(&self) -> Vec<u8> {
+        deterministically_serialize(self)
+    }
+
+    fn deserialize(data: &[u8]) -> Option<Self> {
+        serde_json::from_slice(data).ok()
+    }
+}
+
+impl From<TlcFailDetail> for RemoveTlcFail {
+    fn from(tlc_fail: TlcFailDetail) -> Self {
+        RemoveTlcFail::new(tlc_fail.serialize())
+    }
+}
+
+impl From<RemoveTlcFail> for TlcFailDetail {
+    fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
+        TlcFailDetail::deserialize(&remove_tlc_fail.onion_packet).expect("deserialize fail")
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RemoveTlcFail {
+    // TODO: replace this with the real onion packet
+    // This is the onion packet we need to encode and send back to the sender
+    // currently it's the raw TlcFailDetail serialized data
+    // sender should decode it and get the real TlcFailDetail
+    pub onion_packet: Vec<u8>,
+}
+
+impl RemoveTlcFail {
+    pub fn new(packet_data: Vec<u8>) -> Self {
         RemoveTlcFail {
-            error_code: error_code.into(),
-            packet_data,
+            onion_packet: packet_data,
         }
     }
 }
 
+impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {
+    fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
+        molecule_fiber::RemoveTlcFail::new_builder()
+            .onion_packet(remove_tlc_fail.onion_packet.pack())
+            .build()
+    }
+}
+
+impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
+    type Error = Error;
+
+    fn try_from(remove_tlc_fail: molecule_fiber::RemoveTlcFail) -> Result<Self, Self::Error> {
+        Ok(RemoveTlcFail {
+            onion_packet: remove_tlc_fail.onion_packet().unpack(),
+        })
+    }
+}
+
+// The onion packet is invalid
+const BADONION: u16 = 0x8000;
+// Permanent errors (otherwise transient)
+const PERM: u16 = 0x4000;
+// Node releated errors (otherwise channels)
+const NODE: u16 = 0x2000;
+//  Channel forwarding parameter was violated
+const UPDATE: u16 = 0x1000;
+
 #[repr(u16)]
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, AsRefStr, IntoStaticStr)]
 pub enum TlcFailErrorCode {
     TemporaryNodeFailure = NODE | 2,
     PermanentNodeFailure = PERM | NODE | 2,
@@ -1331,12 +1377,20 @@ pub enum TlcFailErrorCode {
     MppTimeout = 23,
     InvalidOnionBlinding = BADONION | PERM | 24,
 }
+
 impl From<TlcFailErrorCode> for u16 {
     fn from(error_code: TlcFailErrorCode) -> Self {
         error_code as u16
     }
 }
 
+impl From<u16> for TlcFailErrorCode {
+    fn from(value: u16) -> Self {
+        TlcFailErrorCode::try_from(value)
+            .unwrap_or_else(|_| panic!("Invalid value for TlcFailErrorCode: {}", value))
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),
@@ -3039,8 +3093,8 @@ fn get_hop_data_len(buf: &[u8]) -> Option<usize> {
 #[cfg(test)]
 mod tests {
     use super::{secp256k1_instance, Pubkey, RemoveTlcFail, TlcFailErrorCode};
-    use crate::fiber::test_utils::generate_seckey;
-    use crate::fiber::types::Privkey;
+    use crate::fiber::types::{Hash256, Privkey};
+    use crate::fiber::{test_utils::generate_seckey, types::TlcFailDetail};
     use ckb_types::packed::OutPointBuilder;
     use ckb_types::prelude::Builder;
     use secp256k1::{Secp256k1, SecretKey};
@@ -3133,14 +3187,14 @@ mod tests {
 
     #[test]
     fn test_tlc_fail_error() {
-        let packet_data: Vec<u8> = vec![42; 32];
-        let error = RemoveTlcFail::new(TlcFailErrorCode::InvalidOnionVersion, packet_data);
-        assert!(!error.is_node());
-        assert!(error.is_bad_onion());
-        assert!(error.is_perm());
-        let error_mol: super::molecule_fiber::RemoveTlcFail = error.clone().into();
-
-        let decoded: RemoveTlcFail = error_mol.try_into().expect("decode");
-        assert_eq!(error, decoded);
+        let tlc_fail_detail =
+            TlcFailDetail::new(Hash256::default(), 0, TlcFailErrorCode::InvalidOnionVersion);
+        assert!(!tlc_fail_detail.is_node());
+        assert!(tlc_fail_detail.is_bad_onion());
+        assert!(tlc_fail_detail.is_perm());
+        let tlc_fail: RemoveTlcFail = tlc_fail_detail.clone().into();
+
+        let convert_back: TlcFailDetail = tlc_fail.into();
+        assert_eq!(tlc_fail_detail, convert_back);
     }
 }
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index 9afea5a9..588d26f6 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -443,11 +443,11 @@ where
                                         RemoveTlcFulfill { payment_preimage },
                                     )
                                 }
-                                RemoveTlcReason::RemoveTlcFail { error_code } => {
+                                RemoveTlcReason::RemoveTlcFail { error_code: _ } => {
+                                    // TODO: maybe we should remove this PRC or move add_tlc and remove_tlc to `test` module?
                                     crate::fiber::types::RemoveTlcReason::RemoveTlcFail(
                                         RemoveTlcFail {
-                                            error_code,
-                                            packet_data: vec![],
+                                            onion_packet: vec![],
                                         },
                                     )
                                 }

From 13b669dfe689f4394722b00bf4d1f6bef69598ae Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 11:52:39 +0800
Subject: [PATCH 20/66] fix comments

---
 src/fiber/channel.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index bf79f0a5..3556848f 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -503,7 +503,7 @@ where
                         // so we can safely send RemoveTlc message to the peer
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
                         // trying to add a duplicate tlc, so we use tlc count to make sure no new tlc was added
-                        // and don't send a wrong RemoveTlc message if there is already a tlc with the same id
+                        // and only send RemoveTlc message to peer if the TLC is not in our state
                         error!("Error handling AddTlc message: {:?}", e);
                         assert!(tlc_count == state.tlcs.len());
                         if state.get_received_tlc(tlc_id).is_none() {

From 53cca94c3d96dc1f8857ef046d9f374c28a8d285 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 14:47:44 +0800
Subject: [PATCH 21/66] fix fmt

---
 src/fiber/tests/types.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 3ecf49e6..89d6f8c8 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -3,7 +3,7 @@ use crate::fiber::{
     hash_algorithm::HashAlgorithm,
     types::{
         secp256k1_instance, AddTlc, Hash256, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey,
-        TlcFailDetail, TlcFailErrorCode, RemoveTlcFail
+        RemoveTlcFail, TlcFailDetail, TlcFailErrorCode,
     },
 };
 use ckb_types::packed::OutPointBuilder;

From ab8541f7efdd586d543c90e519ee283e2d439fe4 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 15:04:47 +0800
Subject: [PATCH 22/66] code refactor

---
 src/fiber/graph.rs   |  8 +++++++-
 src/fiber/network.rs | 10 +++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 5b46da29..674cc33b 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -817,10 +817,16 @@ impl PaymentSession {
 
     pub fn set_status(&mut self, status: PaymentSessionStatus) {
         self.status = status;
+        self.last_updated_time = std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
+    }
+
+    pub fn set_success_status(&mut self) {
+        self.set_status(PaymentSessionStatus::Success);
+        self.last_error = None;
     }
 
     pub fn set_failed_status(&mut self, error: &str) {
-        self.status = PaymentSessionStatus::Failed;
+        self.set_status(PaymentSessionStatus::Failed);
         self.last_error = Some(error.to_string());
     }
 }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 182a63c0..be1497f0 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1961,17 +1961,13 @@ where
         if let Some(mut payment_session) = self.store.get_payment_session(payment_hash) {
             if payment_session.status == PaymentSessionStatus::Inflight {
                 match remove_tlc.reason {
-                    RemoveTlcReason::RemoveTlcFulfill(_) => {
-                        payment_session.set_status(PaymentSessionStatus::Success);
-                    }
+                    RemoveTlcReason::RemoveTlcFulfill(_) => payment_session.set_success_status(),
                     RemoveTlcReason::RemoveTlcFail(reason) => {
                         let detail_error: TlcFailDetail = reason.into();
                         let error_code: TlcFailErrorCode = detail_error.error_code.into();
                         payment_session.set_failed_status(error_code.as_ref());
                     }
                 }
-                payment_session.last_updated_time =
-                    std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
             }
             self.store.insert_payment_session(payment_session);
         }
@@ -2006,8 +2002,8 @@ where
             error!("Failed to validate payment request: {:?}", e);
             Error::InvalidParameter(format!("Failed to validate payment request: {:?}", e))
         })?;
-        // initialize the payment session in db and begin the payment process in a statemachine to
-        // handle the payment process
+
+        // initialize the payment session in db and begin the payment process lifecycle
         if let Some(payment_session) = self.store.get_payment_session(payment_data.payment_hash) {
             // we only allow retrying payment session with status failed
             debug!("Payment session already exists: {:?}", payment_session);

From c6fc57be017307c78a5c7d0fb86c049337dff2e3 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 16:20:51 +0800
Subject: [PATCH 23/66] add node and channel fail logic for graph

---
 src/fiber/channel.rs     |  7 ++---
 src/fiber/graph.rs       | 34 +++++++++++++++++++++++
 src/fiber/network.rs     | 60 +++++++++++++++++++++++++++++-----------
 src/fiber/tests/types.rs | 15 ++++++++--
 src/fiber/types.rs       | 57 ++++++++++++++++++++++++++++++++++----
 5 files changed, 144 insertions(+), 29 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 60269b32..8d85d5da 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -497,8 +497,7 @@ where
                             // TODO: there maybe more error types here
                             _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
                         };
-                        let error_detail =
-                            TlcFailDetail::new(state.get_id(), add_tlc.tlc_id, error_code);
+                        let error_detail = TlcFailDetail::new(error_code);
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
@@ -799,7 +798,7 @@ where
         onion_packet: Vec<u8>,
         added_tlc_id: u64,
     ) -> Result<(), ProcessingChannelError> {
-        let (send, recv) = oneshot::channel::<Result<u64, String>>();
+        let (send, recv) = oneshot::channel::<Result<u64, RemoveTlcFail>>();
         let rpc_reply = RpcReplyPort::from(send);
         self.network
             .send_message(NetworkActorMessage::Command(
@@ -814,7 +813,7 @@ where
             .expect("network actor is alive");
         let res = match recv.await.expect("expect command replied") {
             Ok(tlc_id) => Ok(tlc_id),
-            Err(e) => Err(e.to_string()),
+            Err(e) => Err(e),
         };
         // If we failed to forward the onion packet, we should remove the tlc.
         if let Err(res) = res {
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 674cc33b..28128e4b 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -326,6 +326,15 @@ where
             .filter(move |channel| channel.node1() == node_id || channel.node2() == node_id)
     }
 
+    pub fn get_mut_channels_by_peer(
+        &mut self,
+        node_id: Pubkey,
+    ) -> impl Iterator<Item = &mut ChannelInfo> {
+        self.channels
+            .values_mut()
+            .filter(move |channel| channel.node1() == node_id || channel.node2() == node_id)
+    }
+
     pub fn get_channels_within_block_range(
         &self,
         start_block: u64,
@@ -451,6 +460,31 @@ where
         }
     }
 
+    pub(crate) fn mark_channel_failed(&mut self, channel_outpoint: &OutPoint) {
+        if let Some(channel) = self.channels.get_mut(channel_outpoint) {
+            if let Some(info) = channel.node1_to_node2.as_mut() {
+                info.enabled = false;
+            }
+            if let Some(info) = channel.node2_to_node1.as_mut() {
+                info.enabled = false;
+            }
+        }
+    }
+
+    pub(crate) fn mark_node_failed(&mut self, node_id: Pubkey) {
+        for channel in self.get_mut_channels_by_peer(node_id) {
+            if channel.node1() == node_id {
+                if let Some(info) = channel.node1_to_node2.as_mut() {
+                    info.enabled = false;
+                }
+            } else {
+                if let Some(info) = channel.node2_to_node1.as_mut() {
+                    info.enabled = false;
+                }
+            }
+        }
+    }
+
     #[cfg(test)]
     pub fn reset(&mut self) {
         self.channels.clear();
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index be1497f0..d71f046c 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -60,7 +60,7 @@ use super::types::{
     GetBroadcastMessagesResult, Hash256, NodeAnnouncement, NodeAnnouncementQuery, OpenChannel,
     Privkey, Pubkey, QueryBroadcastMessagesWithinTimeRange,
     QueryBroadcastMessagesWithinTimeRangeResult, QueryChannelsWithinBlockRange,
-    QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcReason, TlcFailDetail,
+    QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcFail, RemoveTlcReason, TlcFailDetail,
     TlcFailErrorCode,
 };
 use super::FiberConfig;
@@ -147,7 +147,10 @@ pub enum NetworkActorCommand {
     ControlFiberChannel(ChannelCommandWithId),
     // The first parameter is the peeled onion in binary via `PeeledOnionPacket::serialize`. `PeeledOnionPacket::current`
     // is for the current node.
-    SendPaymentOnionPacket(SendOnionPacketCommand, RpcReplyPort<Result<u64, String>>),
+    SendPaymentOnionPacket(
+        SendOnionPacketCommand,
+        RpcReplyPort<Result<u64, RemoveTlcFail>>,
+    ),
     PeelPaymentOnionPacket(
         Vec<u8>, // onion_packet
         Hash256, // payment_hash
@@ -1898,7 +1901,7 @@ where
         &self,
         state: &mut NetworkActorState<S>,
         command: SendOnionPacketCommand,
-        reply: RpcReplyPort<Result<u64, String>>,
+        reply: RpcReplyPort<Result<u64, RemoveTlcFail>>,
     ) {
         let SendOnionPacketCommand {
             packet,
@@ -1919,8 +1922,12 @@ where
                             Are we connected to the peer?",
                             channel_outpoint
                         );
+                    let error_detail = TlcFailDetail::new_channel_fail(
+                        TlcFailErrorCode::PermanentChannelFailure,
+                        channel_outpoint.clone(),
+                    );
                     reply
-                        .send(Err("channel id not found".to_string()))
+                        .send(Err(error_detail.into()))
                         .expect("send add tlc response");
                     return;
                 }
@@ -1948,11 +1955,16 @@ where
 
             match recv.await.expect("recv error") {
                 Ok(res) => Ok(res.tlc_id),
-                Err(e) => Err(e.to_string()),
+                Err(_err) => {
+                    // FIXME: rework the error handling
+                    let error_detail = TlcFailDetail::new(TlcFailErrorCode::TemporaryNodeFailure);
+                    Err(error_detail.into())
+                }
             }
         } else {
             info!("onion packet is empty, ignore it");
-            Err("onion packet is empty".to_string())
+            let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
+            Err(error_detail.into())
         };
         reply.send(res).expect("send error");
     }
@@ -1973,6 +1985,25 @@ where
         }
     }
 
+    async fn update_with_tcl_fail_detail(&self, tcl_error_detail: TlcFailDetail) {
+        let error_code = tcl_error_detail.error_code();
+        match error_code {
+            TlcFailErrorCode::PermanentChannelFailure => {
+                let channel_outpoint = tcl_error_detail
+                    .error_channel_outpoint()
+                    .expect("expect channel outpoint");
+                let mut graph = self.network_graph.write().await;
+                graph.mark_channel_failed(&channel_outpoint);
+            }
+            TlcFailErrorCode::PermanentNodeFailure => {
+                let node_id = tcl_error_detail.error_node_id().expect("expect node id");
+                let mut graph = self.network_graph.write().await;
+                graph.mark_node_failed(node_id);
+            }
+            _ => {}
+        }
+    }
+
     fn on_get_payment(&self, payment_hash: &Hash256) -> Result<SendPaymentResponse, Error> {
         let payment_session = self.store.get_payment_session(*payment_hash);
         match payment_session {
@@ -2047,7 +2078,7 @@ where
             )
             .map_err(|err| Error::InvalidOnionPacket(err))?;
 
-            let (send, recv) = oneshot::channel::<Result<u64, String>>();
+            let (send, recv) = oneshot::channel::<Result<u64, RemoveTlcFail>>();
             let rpc_reply = RpcReplyPort::from(send);
             let command = SendOnionPacketCommand {
                 packet: peeled_packet.serialize(),
@@ -2055,31 +2086,28 @@ where
             };
             self.handle_onion_packet_send_command(state, command, rpc_reply)
                 .await;
-            let res = recv.await;
+            let res = recv.await.expect("msg recv error");
             info!(
                 "send_payment: {:?} => onion packet sent: {:?}",
                 payment_data.payment_hash, res,
             );
-            let res = match res {
-                Ok(Ok(tlc_id)) => Ok(tlc_id),
-                Ok(Err(err)) => Err(err),
-                Err(e) => Err(e.to_string()),
-            }
-            .map_err(|err| Error::SendPaymentError(err.to_string()));
             info!(
                 "send_payment: {:?} => result: {:?}",
                 payment_data.payment_hash, res
             );
             match res {
                 Err(e) => {
-                    error!("Failed to send onion packet: {:?}", e);
+                    let error_detail: TlcFailDetail = e.clone().into();
+                    error!("Failed to send onion packet with error: {:?}", e);
                     // This is the error implies we send payment request to the first hop failed
                     let err = format!(
                         "Failed to send onion packet: {:?} with error: {:?}",
-                        payment_data.payment_hash, e
+                        payment_data.payment_hash,
+                        error_detail.error_code_as_str()
                     );
                     payment_session.last_error = Some(err.clone());
                     error = Some(Err(Error::SendPaymentError(err)));
+                    self.update_with_tcl_fail_detail(error_detail).await;
                     continue;
                 }
                 Ok(_) => {
diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 89d6f8c8..829b9b1c 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -1,8 +1,9 @@
 use crate::fiber::{
     gen::fiber as molecule_fiber,
     hash_algorithm::HashAlgorithm,
+    tests::test_utils::generate_pubkey,
     types::{
-        secp256k1_instance, AddTlc, Hash256, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey,
+        secp256k1_instance, AddTlc, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey,
         RemoveTlcFail, TlcFailDetail, TlcFailErrorCode,
     },
 };
@@ -99,8 +100,7 @@ fn test_peeled_onion_packet() {
 
 #[test]
 fn test_tlc_fail_error() {
-    let tlc_fail_detail =
-        TlcFailDetail::new(Hash256::default(), 0, TlcFailErrorCode::InvalidOnionVersion);
+    let tlc_fail_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionVersion);
     assert!(!tlc_fail_detail.is_node());
     assert!(tlc_fail_detail.is_bad_onion());
     assert!(tlc_fail_detail.is_perm());
@@ -108,4 +108,13 @@ fn test_tlc_fail_error() {
 
     let convert_back: TlcFailDetail = tlc_fail.into();
     assert_eq!(tlc_fail_detail, convert_back);
+
+    let node_fail = TlcFailDetail::new_node_fail(
+        TlcFailErrorCode::PermanentNodeFailure,
+        generate_pubkey().into(),
+    );
+    assert!(node_fail.is_node());
+    let tlc_fail = RemoveTlcFail::from(node_fail.clone());
+    let convert_back: TlcFailDetail = tlc_fail.into();
+    assert_eq!(node_fail, convert_back);
 }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 7f6e8f5f..b6094c6b 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1250,27 +1250,72 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
     }
 }
 
+#[serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum TlcFailDetailData {}
+pub enum TlcFailDetailData {
+    ChannelFailed {
+        #[serde_as(as = "EntityHex")]
+        channel_outpoint: OutPoint,
+    },
+    NodeFailed {
+        node_id: Pubkey,
+    },
+}
 
+#[serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct TlcFailDetail {
-    pub channel_id: Hash256,
-    pub tlc_id: u64,
     pub error_code: u16,
     pub extra_data: Option<TlcFailDetailData>,
 }
 
 impl TlcFailDetail {
-    pub fn new(channel_id: Hash256, tlc_id: u64, error_code: TlcFailErrorCode) -> Self {
+    pub fn new(error_code: TlcFailErrorCode) -> Self {
         TlcFailDetail {
-            channel_id,
-            tlc_id,
             error_code: error_code.into(),
             extra_data: None,
         }
     }
 
+    pub fn new_node_fail(error_code: TlcFailErrorCode, node_id: Pubkey) -> Self {
+        TlcFailDetail {
+            error_code: error_code.into(),
+            extra_data: Some(TlcFailDetailData::NodeFailed { node_id }),
+        }
+    }
+
+    pub fn new_channel_fail(error_code: TlcFailErrorCode, channel_outpoint: OutPoint) -> Self {
+        TlcFailDetail {
+            error_code: error_code.into(),
+            extra_data: Some(TlcFailDetailData::ChannelFailed { channel_outpoint }),
+        }
+    }
+
+    pub fn error_node_id(&self) -> Option<Pubkey> {
+        match &self.extra_data {
+            Some(TlcFailDetailData::NodeFailed { node_id }) => Some(*node_id),
+            _ => None,
+        }
+    }
+
+    pub fn error_channel_outpoint(&self) -> Option<OutPoint> {
+        match &self.extra_data {
+            Some(TlcFailDetailData::ChannelFailed { channel_outpoint }) => {
+                Some(channel_outpoint.clone())
+            }
+            _ => None,
+        }
+    }
+
+    pub fn error_code(&self) -> TlcFailErrorCode {
+        self.error_code.into()
+    }
+
+    pub fn error_code_as_str(&self) -> String {
+        let error_code: TlcFailErrorCode = self.error_code.into();
+        error_code.as_ref().to_string()
+    }
+
     pub fn set_extra_data(&mut self, extra_data: TlcFailDetailData) {
         self.extra_data = Some(extra_data);
     }

From 9b10457708bad21cdc86633a0f645d86854d190b Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Thu, 10 Oct 2024 17:08:59 +0800
Subject: [PATCH 24/66] add mark node/channel failed tests

---
 src/fiber/tests/graph.rs | 141 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)

diff --git a/src/fiber/tests/graph.rs b/src/fiber/tests/graph.rs
index c4565933..c2457e39 100644
--- a/src/fiber/tests/graph.rs
+++ b/src/fiber/tests/graph.rs
@@ -77,6 +77,20 @@ impl MockNetworkGraph {
         }
     }
 
+    pub fn mark_node_failed(&mut self, node: usize) {
+        self.graph.mark_node_failed(self.keys[node].into());
+    }
+
+    pub fn mark_channel_failed(&mut self, node_a: usize, node_b: usize) {
+        let outpoint = self
+            .edges
+            .iter()
+            .find(|(a, b, _)| (*a == node_a && *b == node_b) || (*a == node_b && *b == node_a));
+        if let Some((_, _, outpoint)) = outpoint {
+            self.graph.mark_channel_failed(&outpoint);
+        }
+    }
+
     pub fn add_edge_with_config(
         &mut self,
         node_a: usize,
@@ -614,3 +628,130 @@ fn test_graph_find_path_udt() {
     let route = network.find_route(1, 3, 10, 100);
     assert!(route.is_err());
 }
+
+#[test]
+fn test_graph_mark_failed_channel() {
+    let mut network = MockNetworkGraph::new(5);
+    network.add_edge(0, 2, Some(500), Some(2));
+    network.add_edge(2, 3, Some(500), Some(2));
+    let node3 = network.keys[3];
+
+    network.mark_channel_failed(2, 3);
+    // Test build route from node1 to node3
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node3.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_err());
+
+    network.add_edge(0, 5, Some(500), Some(2));
+    network.add_edge(5, 3, Some(500), Some(2));
+
+    // Test build route from node1 to node3
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node3.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_ok());
+}
+
+#[test]
+fn test_graph_mark_failed_node() {
+    let mut network = MockNetworkGraph::new(5);
+    network.add_edge(0, 2, Some(500), Some(2));
+    network.add_edge(2, 3, Some(500), Some(2));
+    network.add_edge(2, 4, Some(500), Some(2));
+
+    let node3 = network.keys[3];
+    let node4 = network.keys[4];
+
+    // Test build route from node1 to node3
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node3.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_ok());
+
+    // Test build route from node1 to node4 should be Ok
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node4.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_ok());
+
+    network.mark_node_failed(2);
+
+    // Test build route from node1 to node3
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node3.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_err());
+
+    // Test build route from node1 to node4
+    let route = network.graph.build_route(SendPaymentData {
+        target_pubkey: node4.into(),
+        amount: 100,
+        payment_hash: Hash256::default(),
+        invoice: None,
+        final_cltv_delta: Some(100),
+        timeout: Some(10),
+        max_fee_amount: Some(1000),
+        max_parts: None,
+        keysend: false,
+        udt_type_script: None,
+        preimage: None,
+    });
+    eprintln!("return {:?}", route);
+    assert!(route.is_err());
+}

From 8ff53c837a7c15698cb2c8679db9dabf44e6704a Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 01:21:06 +0800
Subject: [PATCH 25/66] refactor payment session and add update_payment_session

---
 src/fiber/channel.rs          |  19 ++++
 src/fiber/graph.rs            |  18 ++++
 src/fiber/network.rs          | 167 ++++++++++++++++++++++++----------
 src/fiber/tests/test_utils.rs |  11 +++
 src/store.rs                  |  22 ++++-
 5 files changed, 189 insertions(+), 48 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 8d85d5da..2d3febdc 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -117,6 +117,7 @@ pub enum ChannelCommand {
     RemoveTlc(RemoveTlcCommand, RpcReplyPort<Result<(), String>>),
     Shutdown(ShutdownCommand, RpcReplyPort<Result<(), String>>),
     Update(UpdateCommand, RpcReplyPort<Result<(), String>>),
+    GetTlcStatus(u64, RpcReplyPort<Result<RemoveTlcReason, String>>),
 }
 
 #[derive(Debug)]
@@ -1324,6 +1325,24 @@ where
                     }
                 }
             }
+            ChannelCommand::GetTlcStatus(tlc_id, reply) => {
+                let status = match state
+                    .tlcs
+                    .iter()
+                    .find(|&(id, _details)| *id == TLCId::Offered(tlc_id))
+                {
+                    Some((_, details)) => {
+                        if let Some((_, remove_at)) = &details.removed_at {
+                            Ok(remove_at.clone())
+                        } else {
+                            Err("TLC is still active".to_string())
+                        }
+                    }
+                    None => Err("TLC not found".to_string()),
+                };
+                let _ = reply.send(status);
+                Ok(())
+            }
         }
     }
 
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 28128e4b..7d4f2f03 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -3,6 +3,7 @@ use super::path::NodeHeap;
 use super::types::Pubkey;
 use super::types::{ChannelAnnouncement, ChannelUpdate, Hash256, NodeAnnouncement};
 use crate::fiber::path::{NodeHeapElement, ProbabilityEvaluator};
+use crate::fiber::serde_utils::EntityHex;
 use crate::fiber::types::PaymentHopData;
 use crate::invoice::CkbInvoice;
 use ckb_jsonrpc_types::JsonBytes;
@@ -806,6 +807,7 @@ pub trait NetworkGraphStateStore {
     fn remove_connected_peer(&self, peer_id: &PeerId);
     fn get_payment_session(&self, payment_hash: Hash256) -> Option<PaymentSession>;
     fn insert_payment_session(&self, session: PaymentSession);
+    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession>;
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -820,6 +822,7 @@ pub enum PaymentSessionStatus {
     Failed,
 }
 
+#[serde_as]
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct PaymentSession {
     pub request: SendPaymentData,
@@ -829,6 +832,10 @@ pub struct PaymentSession {
     pub status: PaymentSessionStatus,
     pub created_time: u128,
     pub last_updated_time: u128,
+    // The channel_outpoint and the tlc_id of the first hop
+    #[serde_as(as = "Option<EntityHex>")]
+    pub first_hop_channel_outpoint: Option<OutPoint>,
+    pub first_hop_tlc_id: Option<u64>,
 }
 
 impl PaymentSession {
@@ -842,6 +849,8 @@ impl PaymentSession {
             status: PaymentSessionStatus::Created,
             created_time: now,
             last_updated_time: now,
+            first_hop_channel_outpoint: None,
+            first_hop_tlc_id: None,
         }
     }
 
@@ -854,6 +863,11 @@ impl PaymentSession {
         self.last_updated_time = std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
     }
 
+    pub fn set_first_hop_info(&mut self, channel_outpoint: OutPoint, tlc_id: u64) {
+        self.first_hop_channel_outpoint = Some(channel_outpoint);
+        self.first_hop_tlc_id = Some(tlc_id);
+    }
+
     pub fn set_success_status(&mut self) {
         self.set_status(PaymentSessionStatus::Success);
         self.last_error = None;
@@ -863,4 +877,8 @@ impl PaymentSession {
         self.set_status(PaymentSessionStatus::Failed);
         self.last_error = Some(error.to_string());
     }
+
+    pub fn can_retry(&self) -> bool {
+        self.retried_times < self.try_limit
+    }
 }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index d71f046c..c6799673 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -172,6 +172,8 @@ pub enum NetworkActorCommand {
         SendPaymentCommand,
         RpcReplyPort<Result<SendPaymentResponse, String>>,
     ),
+    // Update the payment status of a payment session.
+    UpdatePaymentSession,
     // Get Payment Session for query payment status and errors
     GetPayment(Hash256, RpcReplyPort<Result<SendPaymentResponse, String>>),
     GetAndProcessChannelsWithinBlockRangeFromPeer(
@@ -1183,7 +1185,8 @@ where
                 state.maybe_finish_sync();
             }
             NetworkActorEvent::TlcRemoveReceived(payment_hash, remove_tlc) => {
-                self.on_tlc_remove_received(payment_hash, remove_tlc);
+                self.on_tlc_remove_received(state, payment_hash, remove_tlc.reason)
+                    .await;
             }
         }
         Ok(())
@@ -1606,6 +1609,9 @@ where
                     );
                 }
             },
+            NetworkActorCommand::UpdatePaymentSession => {
+                self.update_payment_session(state).await;
+            }
         };
         Ok(())
     }
@@ -1969,15 +1975,78 @@ where
         reply.send(res).expect("send error");
     }
 
-    fn on_tlc_remove_received(&self, payment_hash: Hash256, remove_tlc: RemoveTlc) {
+    async fn update_payment_session(&self, state: &mut NetworkActorState<S>) {
+        for mut payment_session in self
+            .store
+            .get_payment_sessions_by_status(PaymentSessionStatus::Inflight)
+        {
+            if payment_session.status == PaymentSessionStatus::Inflight {
+                match (
+                    &payment_session.first_hop_channel_outpoint,
+                    &payment_session.first_hop_tlc_id,
+                ) {
+                    (Some(outpoint), Some(tlc_id)) => {
+                        if let Some(channel_id) = state.outpoint_channel_map.get(&outpoint) {
+                            let (send, recv) =
+                                oneshot::channel::<Result<RemoveTlcReason, String>>();
+                            let rpc_reply = RpcReplyPort::from(send);
+                            let command = ChannelCommand::GetTlcStatus(*tlc_id, rpc_reply);
+                            let _ = state.send_command_to_channel(*channel_id, command).await;
+                            let res = recv.await.expect("recv error");
+                            match res {
+                                Ok(reason) => match reason {
+                                    RemoveTlcReason::RemoveTlcFulfill(_) => {
+                                        payment_session.set_success_status();
+                                    }
+                                    RemoveTlcReason::RemoveTlcFail(_) => {
+                                        self.on_tlc_remove_received(
+                                            state,
+                                            payment_session.payment_hash(),
+                                            reason,
+                                        )
+                                        .await;
+                                    }
+                                },
+                                Err(_) => {}
+                            }
+                        }
+                    }
+                    (_, _) => {}
+                }
+
+                if payment_session.can_retry() {
+                    let _ = self
+                        .try_payment_session(state, payment_session.clone())
+                        .await;
+                } else {
+                    payment_session.set_failed_status("Retry limit reached");
+                }
+                self.store.insert_payment_session(payment_session);
+            }
+        }
+    }
+
+    async fn on_tlc_remove_received(
+        &self,
+        state: &mut NetworkActorState<S>,
+        payment_hash: Hash256,
+        reason: RemoveTlcReason,
+    ) {
         if let Some(mut payment_session) = self.store.get_payment_session(payment_hash) {
             if payment_session.status == PaymentSessionStatus::Inflight {
-                match remove_tlc.reason {
+                match reason {
                     RemoveTlcReason::RemoveTlcFulfill(_) => payment_session.set_success_status(),
                     RemoveTlcReason::RemoveTlcFail(reason) => {
                         let detail_error: TlcFailDetail = reason.into();
-                        let error_code: TlcFailErrorCode = detail_error.error_code.into();
-                        payment_session.set_failed_status(error_code.as_ref());
+                        self.update_with_tcl_fail_detail(detail_error.clone()).await;
+                        if payment_session.can_retry() {
+                            let _ = self
+                                .try_payment_session(state, payment_session.clone())
+                                .await;
+                        } else {
+                            let error_code: TlcFailErrorCode = detail_error.error_code.into();
+                            payment_session.set_failed_status(error_code.as_ref());
+                        }
                     }
                 }
             }
@@ -2023,44 +2092,25 @@ where
         }
     }
 
-    async fn on_send_payment(
+    async fn try_payment_session(
         &self,
         state: &mut NetworkActorState<S>,
-        payment_request: SendPaymentCommand,
+        mut payment_session: PaymentSession,
     ) -> Result<SendPaymentResponse, Error> {
         let graph = self.network_graph.read().await;
-        let payment_data = SendPaymentData::new(payment_request.clone()).map_err(|e| {
-            error!("Failed to validate payment request: {:?}", e);
-            Error::InvalidParameter(format!("Failed to validate payment request: {:?}", e))
-        })?;
-
-        // initialize the payment session in db and begin the payment process lifecycle
-        if let Some(payment_session) = self.store.get_payment_session(payment_data.payment_hash) {
-            // we only allow retrying payment session with status failed
-            debug!("Payment session already exists: {:?}", payment_session);
-            if payment_session.status != PaymentSessionStatus::Failed {
-                return Err(Error::InvalidParameter(format!(
-                    "Payment session already exists: {} with payment session status: {:?}",
-                    payment_data.payment_hash, payment_session.status
-                )));
-            }
-        }
-
-        let mut payment_session = PaymentSession::new(payment_data.clone(), 5);
-        self.store.insert_payment_session(payment_session.clone());
-
+        let payment_data = payment_session.request.clone();
         let mut error = None;
-        for _i in 0..payment_session.try_limit {
+        while payment_session.retried_times < payment_session.try_limit {
             payment_session.retried_times += 1;
             let hops_infos = graph.build_route(payment_data.clone());
             let hops_infos = match hops_infos {
                 Err(e) => {
                     error!("Failed to build route: {:?}", e);
                     payment_session.set_status(PaymentSessionStatus::Failed);
-                    error = Some(Err(Error::SendPaymentError(format!(
+                    error = Some(format!(
                         "Failed to build route: {:?}",
                         payment_data.payment_hash
-                    ))));
+                    ));
                     break;
                 }
                 Ok(onion_path) => {
@@ -2068,6 +2118,10 @@ where
                     onion_path
                 }
             };
+            let first_channel_outpoint = hops_infos[0]
+                .channel_outpoint
+                .clone()
+                .expect("first hop channel outpoint");
 
             // generate session key
             let session_key = Privkey::from_slice(KeyPair::generate_random_key().as_ref());
@@ -2086,16 +2140,7 @@ where
             };
             self.handle_onion_packet_send_command(state, command, rpc_reply)
                 .await;
-            let res = recv.await.expect("msg recv error");
-            info!(
-                "send_payment: {:?} => onion packet sent: {:?}",
-                payment_data.payment_hash, res,
-            );
-            info!(
-                "send_payment: {:?} => result: {:?}",
-                payment_data.payment_hash, res
-            );
-            match res {
+            match recv.await.expect("msg recv error") {
                 Err(e) => {
                     let error_detail: TlcFailDetail = e.clone().into();
                     error!("Failed to send onion packet with error: {:?}", e);
@@ -2105,27 +2150,55 @@ where
                         payment_data.payment_hash,
                         error_detail.error_code_as_str()
                     );
-                    payment_session.last_error = Some(err.clone());
-                    error = Some(Err(Error::SendPaymentError(err)));
+                    error = Some(err);
                     self.update_with_tcl_fail_detail(error_detail).await;
                     continue;
                 }
-                Ok(_) => {
+                Ok(tlc_id) => {
                     payment_session.set_status(PaymentSessionStatus::Inflight);
+                    payment_session.set_first_hop_info(first_channel_outpoint, tlc_id);
                     self.store.insert_payment_session(payment_session.clone());
-                    let response = SendPaymentResponse {
+                    return Ok(SendPaymentResponse {
                         payment_hash: payment_data.payment_hash,
                         last_update_time: payment_session.last_updated_time,
                         status: PaymentSessionStatus::Inflight,
                         failed_error: None,
-                    };
-                    return Ok(response);
+                    });
                 }
             }
         }
         payment_session.set_status(PaymentSessionStatus::Failed);
+        let final_error = error.expect("expect error details");
+        payment_session.set_failed_status(&final_error);
         self.store.insert_payment_session(payment_session);
-        return error.expect("expect error details");
+        return Err(Error::SendPaymentError(final_error));
+    }
+
+    async fn on_send_payment(
+        &self,
+        state: &mut NetworkActorState<S>,
+        payment_request: SendPaymentCommand,
+    ) -> Result<SendPaymentResponse, Error> {
+        let payment_data = SendPaymentData::new(payment_request.clone()).map_err(|e| {
+            error!("Failed to validate payment request: {:?}", e);
+            Error::InvalidParameter(format!("Failed to validate payment request: {:?}", e))
+        })?;
+
+        // initialize the payment session in db and begin the payment process lifecycle
+        if let Some(payment_session) = self.store.get_payment_session(payment_data.payment_hash) {
+            // we only allow retrying payment session with status failed
+            debug!("Payment session already exists: {:?}", payment_session);
+            if payment_session.status != PaymentSessionStatus::Failed {
+                return Err(Error::InvalidParameter(format!(
+                    "Payment session already exists: {} with payment session status: {:?}",
+                    payment_data.payment_hash, payment_session.status
+                )));
+            }
+        }
+
+        let payment_session = PaymentSession::new(payment_data.clone(), 5);
+        self.store.insert_payment_session(payment_session.clone());
+        self.try_payment_session(state, payment_session).await
     }
 }
 
diff --git a/src/fiber/tests/test_utils.rs b/src/fiber/tests/test_utils.rs
index 2bd05d99..48a879b4 100644
--- a/src/fiber/tests/test_utils.rs
+++ b/src/fiber/tests/test_utils.rs
@@ -1,3 +1,4 @@
+use crate::fiber::graph::PaymentSessionStatus;
 use crate::fiber::graph::{ChannelInfo, NetworkGraph, NodeInfo};
 use crate::fiber::types::Pubkey;
 use crate::invoice::{CkbInvoice, InvoiceError, InvoiceStore};
@@ -579,6 +580,16 @@ impl NetworkGraphStateStore for MemoryStore {
             .unwrap()
             .insert(session.payment_hash(), session);
     }
+
+    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession> {
+        self.payment_sessions
+            .read()
+            .unwrap()
+            .values()
+            .filter(|session| session.status == status)
+            .cloned()
+            .collect()
+    }
 }
 
 impl ChannelActorStateStore for MemoryStore {
diff --git a/src/store.rs b/src/store.rs
index 5cc52d03..e4fa1111 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -1,7 +1,9 @@
 use crate::{
     fiber::{
         channel::{ChannelActorState, ChannelActorStateStore, ChannelState},
-        graph::{ChannelInfo, NetworkGraphStateStore, NodeInfo, PaymentSession},
+        graph::{
+            ChannelInfo, NetworkGraphStateStore, NodeInfo, PaymentSession, PaymentSessionStatus,
+        },
         types::{Hash256, Pubkey},
     },
     invoice::{CkbInvoice, InvoiceError, InvoiceStore},
@@ -510,6 +512,24 @@ impl NetworkGraphStateStore for Store {
         batch.put_kv(KeyValue::PaymentSession(session.payment_hash(), session));
         batch.commit();
     }
+
+    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession> {
+        let prefix = vec![PAYMENT_SESSION_PREFIX];
+        let iter = self
+            .db
+            .prefix_iterator(prefix.as_ref())
+            .take_while(|(key, _)| key.starts_with(&prefix));
+        iter.filter_map(|(_key, value)| {
+            let payment_session: PaymentSession = serde_json::from_slice(value.as_ref())
+                .expect("deserialize ChannelState should be OK");
+            if payment_session.status == status {
+                Some(payment_session)
+            } else {
+                None
+            }
+        })
+        .collect()
+    }
 }
 
 impl WatchtowerStore for Store {

From 6e5bda9a3ce1e5c82004446abaab63c9c71e192f Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 08:52:18 +0800
Subject: [PATCH 26/66] remove update_payment_session

---
 src/fiber/channel.rs          | 19 ------------
 src/fiber/graph.rs            |  1 -
 src/fiber/network.rs          | 56 -----------------------------------
 src/fiber/tests/test_utils.rs | 11 -------
 src/store.rs                  | 22 +-------------
 5 files changed, 1 insertion(+), 108 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 2d3febdc..8d85d5da 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -117,7 +117,6 @@ pub enum ChannelCommand {
     RemoveTlc(RemoveTlcCommand, RpcReplyPort<Result<(), String>>),
     Shutdown(ShutdownCommand, RpcReplyPort<Result<(), String>>),
     Update(UpdateCommand, RpcReplyPort<Result<(), String>>),
-    GetTlcStatus(u64, RpcReplyPort<Result<RemoveTlcReason, String>>),
 }
 
 #[derive(Debug)]
@@ -1325,24 +1324,6 @@ where
                     }
                 }
             }
-            ChannelCommand::GetTlcStatus(tlc_id, reply) => {
-                let status = match state
-                    .tlcs
-                    .iter()
-                    .find(|&(id, _details)| *id == TLCId::Offered(tlc_id))
-                {
-                    Some((_, details)) => {
-                        if let Some((_, remove_at)) = &details.removed_at {
-                            Ok(remove_at.clone())
-                        } else {
-                            Err("TLC is still active".to_string())
-                        }
-                    }
-                    None => Err("TLC not found".to_string()),
-                };
-                let _ = reply.send(status);
-                Ok(())
-            }
         }
     }
 
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 7d4f2f03..4eeffc5d 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -807,7 +807,6 @@ pub trait NetworkGraphStateStore {
     fn remove_connected_peer(&self, peer_id: &PeerId);
     fn get_payment_session(&self, payment_hash: Hash256) -> Option<PaymentSession>;
     fn insert_payment_session(&self, session: PaymentSession);
-    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession>;
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index c6799673..ad10738b 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -172,8 +172,6 @@ pub enum NetworkActorCommand {
         SendPaymentCommand,
         RpcReplyPort<Result<SendPaymentResponse, String>>,
     ),
-    // Update the payment status of a payment session.
-    UpdatePaymentSession,
     // Get Payment Session for query payment status and errors
     GetPayment(Hash256, RpcReplyPort<Result<SendPaymentResponse, String>>),
     GetAndProcessChannelsWithinBlockRangeFromPeer(
@@ -1609,9 +1607,6 @@ where
                     );
                 }
             },
-            NetworkActorCommand::UpdatePaymentSession => {
-                self.update_payment_session(state).await;
-            }
         };
         Ok(())
     }
@@ -1975,57 +1970,6 @@ where
         reply.send(res).expect("send error");
     }
 
-    async fn update_payment_session(&self, state: &mut NetworkActorState<S>) {
-        for mut payment_session in self
-            .store
-            .get_payment_sessions_by_status(PaymentSessionStatus::Inflight)
-        {
-            if payment_session.status == PaymentSessionStatus::Inflight {
-                match (
-                    &payment_session.first_hop_channel_outpoint,
-                    &payment_session.first_hop_tlc_id,
-                ) {
-                    (Some(outpoint), Some(tlc_id)) => {
-                        if let Some(channel_id) = state.outpoint_channel_map.get(&outpoint) {
-                            let (send, recv) =
-                                oneshot::channel::<Result<RemoveTlcReason, String>>();
-                            let rpc_reply = RpcReplyPort::from(send);
-                            let command = ChannelCommand::GetTlcStatus(*tlc_id, rpc_reply);
-                            let _ = state.send_command_to_channel(*channel_id, command).await;
-                            let res = recv.await.expect("recv error");
-                            match res {
-                                Ok(reason) => match reason {
-                                    RemoveTlcReason::RemoveTlcFulfill(_) => {
-                                        payment_session.set_success_status();
-                                    }
-                                    RemoveTlcReason::RemoveTlcFail(_) => {
-                                        self.on_tlc_remove_received(
-                                            state,
-                                            payment_session.payment_hash(),
-                                            reason,
-                                        )
-                                        .await;
-                                    }
-                                },
-                                Err(_) => {}
-                            }
-                        }
-                    }
-                    (_, _) => {}
-                }
-
-                if payment_session.can_retry() {
-                    let _ = self
-                        .try_payment_session(state, payment_session.clone())
-                        .await;
-                } else {
-                    payment_session.set_failed_status("Retry limit reached");
-                }
-                self.store.insert_payment_session(payment_session);
-            }
-        }
-    }
-
     async fn on_tlc_remove_received(
         &self,
         state: &mut NetworkActorState<S>,
diff --git a/src/fiber/tests/test_utils.rs b/src/fiber/tests/test_utils.rs
index 48a879b4..2bd05d99 100644
--- a/src/fiber/tests/test_utils.rs
+++ b/src/fiber/tests/test_utils.rs
@@ -1,4 +1,3 @@
-use crate::fiber::graph::PaymentSessionStatus;
 use crate::fiber::graph::{ChannelInfo, NetworkGraph, NodeInfo};
 use crate::fiber::types::Pubkey;
 use crate::invoice::{CkbInvoice, InvoiceError, InvoiceStore};
@@ -580,16 +579,6 @@ impl NetworkGraphStateStore for MemoryStore {
             .unwrap()
             .insert(session.payment_hash(), session);
     }
-
-    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession> {
-        self.payment_sessions
-            .read()
-            .unwrap()
-            .values()
-            .filter(|session| session.status == status)
-            .cloned()
-            .collect()
-    }
 }
 
 impl ChannelActorStateStore for MemoryStore {
diff --git a/src/store.rs b/src/store.rs
index e4fa1111..5cc52d03 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -1,9 +1,7 @@
 use crate::{
     fiber::{
         channel::{ChannelActorState, ChannelActorStateStore, ChannelState},
-        graph::{
-            ChannelInfo, NetworkGraphStateStore, NodeInfo, PaymentSession, PaymentSessionStatus,
-        },
+        graph::{ChannelInfo, NetworkGraphStateStore, NodeInfo, PaymentSession},
         types::{Hash256, Pubkey},
     },
     invoice::{CkbInvoice, InvoiceError, InvoiceStore},
@@ -512,24 +510,6 @@ impl NetworkGraphStateStore for Store {
         batch.put_kv(KeyValue::PaymentSession(session.payment_hash(), session));
         batch.commit();
     }
-
-    fn get_payment_sessions_by_status(&self, status: PaymentSessionStatus) -> Vec<PaymentSession> {
-        let prefix = vec![PAYMENT_SESSION_PREFIX];
-        let iter = self
-            .db
-            .prefix_iterator(prefix.as_ref())
-            .take_while(|(key, _)| key.starts_with(&prefix));
-        iter.filter_map(|(_key, value)| {
-            let payment_session: PaymentSession = serde_json::from_slice(value.as_ref())
-                .expect("deserialize ChannelState should be OK");
-            if payment_session.status == status {
-                Some(payment_session)
-            } else {
-                None
-            }
-        })
-        .collect()
-    }
 }
 
 impl WatchtowerStore for Store {

From e58364c49cfe66613d0053ba63c4646d1d23ce67 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 09:05:27 +0800
Subject: [PATCH 27/66] code cleanup on payment session

---
 src/fiber/graph.rs   |  7 +------
 src/fiber/network.rs | 34 ++++++++++++++++------------------
 2 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 4eeffc5d..eef59268 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -572,15 +572,10 @@ where
             current_expiry += expiry;
         }
         // Add the first hop as the instruction for the current node, so the logic for send HTLC can be reused.
-        let next_hop = if !route.is_empty() {
-            Some(route[0].target)
-        } else {
-            None
-        };
         onion_infos.push(PaymentHopData {
             amount: current_amount,
             payment_hash,
-            next_hop,
+            next_hop: Some(route[0].target),
             tlc_hash_algorithm: hash_algorithm,
             expiry: current_expiry,
             channel_outpoint: Some(route[0].channel_outpoint.clone()),
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index ad10738b..02ca804e 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -15,7 +15,6 @@ use secp256k1::{Message, Secp256k1};
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use std::borrow::Cow;
-
 use std::collections::{HashMap, HashSet};
 use std::str::FromStr;
 use std::sync::atomic::{AtomicU64, Ordering};
@@ -1984,9 +1983,12 @@ where
                         let detail_error: TlcFailDetail = reason.into();
                         self.update_with_tcl_fail_detail(detail_error.clone()).await;
                         if payment_session.can_retry() {
-                            let _ = self
+                            let res = self
                                 .try_payment_session(state, payment_session.clone())
                                 .await;
+                            if res.is_err() {
+                                debug!("Failed to retry payment session: {:?}", res);
+                            }
                         } else {
                             let error_code: TlcFailErrorCode = detail_error.error_code.into();
                             payment_session.set_failed_status(error_code.as_ref());
@@ -2040,27 +2042,22 @@ where
         &self,
         state: &mut NetworkActorState<S>,
         mut payment_session: PaymentSession,
-    ) -> Result<SendPaymentResponse, Error> {
+    ) -> Result<PaymentSession, Error> {
         let graph = self.network_graph.read().await;
         let payment_data = payment_session.request.clone();
         let mut error = None;
-        while payment_session.retried_times < payment_session.try_limit {
+        while payment_session.can_retry() {
             payment_session.retried_times += 1;
-            let hops_infos = graph.build_route(payment_data.clone());
-            let hops_infos = match hops_infos {
+            let hops_infos = match graph.build_route(payment_data.clone()) {
                 Err(e) => {
                     error!("Failed to build route: {:?}", e);
-                    payment_session.set_status(PaymentSessionStatus::Failed);
                     error = Some(format!(
                         "Failed to build route: {:?}",
                         payment_data.payment_hash
                     ));
                     break;
                 }
-                Ok(onion_path) => {
-                    assert!(!onion_path.is_empty());
-                    onion_path
-                }
+                Ok(onion_path) => onion_path,
             };
             let first_channel_outpoint = hops_infos[0]
                 .channel_outpoint
@@ -2102,12 +2099,7 @@ where
                     payment_session.set_status(PaymentSessionStatus::Inflight);
                     payment_session.set_first_hop_info(first_channel_outpoint, tlc_id);
                     self.store.insert_payment_session(payment_session.clone());
-                    return Ok(SendPaymentResponse {
-                        payment_hash: payment_data.payment_hash,
-                        last_update_time: payment_session.last_updated_time,
-                        status: PaymentSessionStatus::Inflight,
-                        failed_error: None,
-                    });
+                    return Ok(payment_session);
                 }
             }
         }
@@ -2142,7 +2134,13 @@ where
 
         let payment_session = PaymentSession::new(payment_data.clone(), 5);
         self.store.insert_payment_session(payment_session.clone());
-        self.try_payment_session(state, payment_session).await
+        let session = self.try_payment_session(state, payment_session).await?;
+        return Ok(SendPaymentResponse {
+            payment_hash: payment_data.payment_hash,
+            last_update_time: session.last_updated_time,
+            status: session.status,
+            failed_error: session.last_error.clone(),
+        });
     }
 }
 

From f3494a1fc1ec75ec5dc0587fd3f28dd228a4dfd8 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 15:46:42 +0800
Subject: [PATCH 28/66] add created_at for payment_session

---
 src/fiber/graph.rs   | 24 ++++++++++++++++++------
 src/fiber/network.rs | 27 ++++++++-------------------
 src/rpc/channel.rs   | 26 ++++++++++----------------
 3 files changed, 36 insertions(+), 41 deletions(-)

diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index eef59268..d3ab99ad 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -1,4 +1,4 @@
-use super::network::{get_chain_hash, SendPaymentData};
+use super::network::{get_chain_hash, SendPaymentData, SendPaymentResponse};
 use super::path::NodeHeap;
 use super::types::Pubkey;
 use super::types::{ChannelAnnouncement, ChannelUpdate, Hash256, NodeAnnouncement};
@@ -824,8 +824,8 @@ pub struct PaymentSession {
     pub last_error: Option<String>,
     pub try_limit: u32,
     pub status: PaymentSessionStatus,
-    pub created_time: u128,
-    pub last_updated_time: u128,
+    pub created_at: u128,
+    pub last_updated_at: u128,
     // The channel_outpoint and the tlc_id of the first hop
     #[serde_as(as = "Option<EntityHex>")]
     pub first_hop_channel_outpoint: Option<OutPoint>,
@@ -841,8 +841,8 @@ impl PaymentSession {
             last_error: None,
             try_limit,
             status: PaymentSessionStatus::Created,
-            created_time: now,
-            last_updated_time: now,
+            created_at: now,
+            last_updated_at: now,
             first_hop_channel_outpoint: None,
             first_hop_tlc_id: None,
         }
@@ -854,7 +854,7 @@ impl PaymentSession {
 
     pub fn set_status(&mut self, status: PaymentSessionStatus) {
         self.status = status;
-        self.last_updated_time = std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
+        self.last_updated_at = std::time::UNIX_EPOCH.elapsed().unwrap().as_micros();
     }
 
     pub fn set_first_hop_info(&mut self, channel_outpoint: OutPoint, tlc_id: u64) {
@@ -876,3 +876,15 @@ impl PaymentSession {
         self.retried_times < self.try_limit
     }
 }
+
+impl From<PaymentSession> for SendPaymentResponse {
+    fn from(session: PaymentSession) -> Self {
+        Self {
+            payment_hash: session.request.payment_hash,
+            status: session.status,
+            failed_error: session.last_error,
+            created_at: session.created_at,
+            last_updated_at: session.last_updated_at,
+        }
+    }
+}
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 02ca804e..7a407e5b 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -108,7 +108,8 @@ pub struct AcceptChannelResponse {
 pub struct SendPaymentResponse {
     pub payment_hash: Hash256,
     pub status: PaymentSessionStatus,
-    pub last_update_time: u128,
+    pub created_at: u128,
+    pub last_updated_at: u128,
     pub failed_error: Option<String>,
 }
 
@@ -1182,6 +1183,7 @@ where
                 state.maybe_finish_sync();
             }
             NetworkActorEvent::TlcRemoveReceived(payment_hash, remove_tlc) => {
+                // When a node is restarted, RemoveTLC will also be resent if necessary
                 self.on_tlc_remove_received(state, payment_hash, remove_tlc.reason)
                     .await;
             }
@@ -2020,17 +2022,8 @@ where
     }
 
     fn on_get_payment(&self, payment_hash: &Hash256) -> Result<SendPaymentResponse, Error> {
-        let payment_session = self.store.get_payment_session(*payment_hash);
-        match payment_session {
-            Some(payment_session) => {
-                let response = SendPaymentResponse {
-                    payment_hash: payment_session.payment_hash(),
-                    last_update_time: payment_session.last_updated_time,
-                    status: payment_session.status,
-                    failed_error: payment_session.last_error.clone(),
-                };
-                Ok(response)
-            }
+        match self.store.get_payment_session(*payment_hash) {
+            Some(payment_session) => Ok(payment_session.into()),
             None => Err(Error::InvalidParameter(format!(
                 "Payment session not found: {:?}",
                 payment_hash
@@ -2135,12 +2128,7 @@ where
         let payment_session = PaymentSession::new(payment_data.clone(), 5);
         self.store.insert_payment_session(payment_session.clone());
         let session = self.try_payment_session(state, payment_session).await?;
-        return Ok(SendPaymentResponse {
-            payment_hash: payment_data.payment_hash,
-            last_update_time: session.last_updated_time,
-            status: session.status,
-            failed_error: session.last_error.clone(),
-        });
+        return Ok(session.into());
     }
 }
 
@@ -2199,7 +2187,8 @@ impl NetworkSyncState {
                     // TODO: we may want more than one successful syncing.
                     false
                 } else {
-                    debug!("Adding peer to dynamic syncing peers list: peer {:?}, succeeded syncing {}, failed syncing {}, pinned syncing peers {}", peer_id, self.succeeded, self.failed, self.pinned_syncing_peers.len());
+                    debug!("Adding peer to dynamic syncing peers list: peer {:?}, succeeded syncing {}, failed syncing {}, pinned syncing peers {}",
+                            peer_id, self.succeeded, self.failed, self.pinned_syncing_peers.len());
                     true
                 }
             } else {
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index a690bca0..0a51c6d3 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -185,7 +185,9 @@ pub struct GetPaymentCommandResult {
     pub payment_hash: Hash256,
     pub status: PaymentSessionStatus,
     #[serde_as(as = "U128Hex")]
-    pub last_update_time: u128,
+    created_at: u128,
+    #[serde_as(as = "U128Hex")]
+    pub last_updated_at: u128,
     pub failed_error: Option<String>,
 }
 
@@ -229,16 +231,6 @@ pub(crate) struct SendPaymentCommandParams {
     udt_type_script: Option<Script>,
 }
 
-#[serde_as]
-#[derive(Clone, Serialize)]
-pub(crate) struct SendPaymentResult {
-    payment_hash: Hash256,
-    status: PaymentSessionStatus,
-    #[serde_as(as = "U128Hex")]
-    last_update_time: u128,
-    failed_error: Option<String>,
-}
-
 #[rpc(server)]
 trait ChannelRpc {
     #[method(name = "open_channel")]
@@ -282,7 +274,7 @@ trait ChannelRpc {
     async fn send_payment(
         &self,
         params: SendPaymentCommandParams,
-    ) -> Result<SendPaymentResult, ErrorObjectOwned>;
+    ) -> Result<GetPaymentCommandResult, ErrorObjectOwned>;
 
     #[method(name = "get_payment")]
     async fn get_payment(
@@ -507,7 +499,7 @@ where
     async fn send_payment(
         &self,
         params: SendPaymentCommandParams,
-    ) -> Result<SendPaymentResult, ErrorObjectOwned> {
+    ) -> Result<GetPaymentCommandResult, ErrorObjectOwned> {
         let message = |rpc_reply| -> NetworkActorMessage {
             NetworkActorMessage::Command(NetworkActorCommand::SendPayment(
                 SendPaymentCommand {
@@ -525,10 +517,11 @@ where
                 rpc_reply,
             ))
         };
-        handle_actor_call!(self.actor, message, params).map(|response| SendPaymentResult {
+        handle_actor_call!(self.actor, message, params).map(|response| GetPaymentCommandResult {
             payment_hash: response.payment_hash,
             status: response.status,
-            last_update_time: response.last_update_time,
+            created_at: response.created_at,
+            last_updated_at: response.last_updated_at,
             failed_error: response.failed_error,
         })
     }
@@ -546,7 +539,8 @@ where
         handle_actor_call!(self.actor, message, params).map(|response| GetPaymentCommandResult {
             payment_hash: response.payment_hash,
             status: response.status,
-            last_update_time: response.last_update_time,
+            last_updated_at: response.last_updated_at,
+            created_at: response.created_at,
             failed_error: response.failed_error,
         })
     }

From 3010926a847c6d0b7e83fc9007babf2a10d69eb6 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 16:00:06 +0800
Subject: [PATCH 29/66] make TlcFailErrorCode easy to use

---
 src/fiber/network.rs |  6 ++----
 src/fiber/types.rs   | 27 +++++++--------------------
 2 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 7a407e5b..8cb8009c 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1992,8 +1992,7 @@ where
                                 debug!("Failed to retry payment session: {:?}", res);
                             }
                         } else {
-                            let error_code: TlcFailErrorCode = detail_error.error_code.into();
-                            payment_session.set_failed_status(error_code.as_ref());
+                            payment_session.set_failed_status(detail_error.error_code.as_ref());
                         }
                     }
                 }
@@ -2003,8 +2002,7 @@ where
     }
 
     async fn update_with_tcl_fail_detail(&self, tcl_error_detail: TlcFailDetail) {
-        let error_code = tcl_error_detail.error_code();
-        match error_code {
+        match tcl_error_detail.error_code() {
             TlcFailErrorCode::PermanentChannelFailure => {
                 let channel_outpoint = tcl_error_detail
                     .error_channel_outpoint()
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index b6094c6b..5ddf851b 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1265,14 +1265,14 @@ pub enum TlcFailDetailData {
 #[serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct TlcFailDetail {
-    pub error_code: u16,
+    pub error_code: TlcFailErrorCode,
     pub extra_data: Option<TlcFailDetailData>,
 }
 
 impl TlcFailDetail {
     pub fn new(error_code: TlcFailErrorCode) -> Self {
         TlcFailDetail {
-            error_code: error_code.into(),
+            error_code: error_code,
             extra_data: None,
         }
     }
@@ -1308,7 +1308,7 @@ impl TlcFailDetail {
     }
 
     pub fn error_code(&self) -> TlcFailErrorCode {
-        self.error_code.into()
+        self.error_code
     }
 
     pub fn error_code_as_str(&self) -> String {
@@ -1321,19 +1321,19 @@ impl TlcFailDetail {
     }
 
     pub fn is_node(&self) -> bool {
-        self.error_code & NODE != 0
+        self.error_code as u16 & NODE != 0
     }
 
     pub fn is_bad_onion(&self) -> bool {
-        self.error_code & BADONION != 0
+        self.error_code as u16 & BADONION != 0
     }
 
     pub fn is_perm(&self) -> bool {
-        self.error_code & PERM != 0
+        self.error_code as u16 & PERM != 0
     }
 
     pub fn is_update(&self) -> bool {
-        self.error_code & UPDATE != 0
+        self.error_code as u16 & UPDATE != 0
     }
 
     fn serialize(&self) -> Vec<u8> {
@@ -1428,19 +1428,6 @@ pub enum TlcFailErrorCode {
     InvalidOnionBlinding = BADONION | PERM | 24,
 }
 
-impl From<TlcFailErrorCode> for u16 {
-    fn from(error_code: TlcFailErrorCode) -> Self {
-        error_code as u16
-    }
-}
-
-impl From<u16> for TlcFailErrorCode {
-    fn from(value: u16) -> Self {
-        TlcFailErrorCode::try_from(value)
-            .unwrap_or_else(|_| panic!("Invalid value for TlcFailErrorCode: {}", value))
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),

From 27583fffa14cdf1e4b18f05bd780f6496949928e Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 20:17:01 +0800
Subject: [PATCH 30/66] add channel_update for tlc error

---
 src/fiber/channel.rs | 42 ++++++++++++++++++++++++++++++------------
 src/fiber/network.rs |  9 +++------
 src/fiber/types.rs   | 31 +++++++++++++++++++++++--------
 src/rpc/utils.rs     |  2 +-
 4 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 8d85d5da..e9bd096d 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -113,7 +113,10 @@ pub enum ChannelCommand {
     // TODO: maybe we should automatically send commitment_signed message after receiving
     // tx_complete event.
     CommitmentSigned(),
-    AddTlc(AddTlcCommand, RpcReplyPort<Result<AddTlcResponse, String>>),
+    AddTlc(
+        AddTlcCommand,
+        RpcReplyPort<Result<AddTlcResponse, RemoveTlcFail>>,
+    ),
     RemoveTlc(RemoveTlcCommand, RpcReplyPort<Result<(), String>>),
     Shutdown(ShutdownCommand, RpcReplyPort<Result<(), String>>),
     Update(UpdateCommand, RpcReplyPort<Result<(), String>>),
@@ -1279,7 +1282,14 @@ where
                         Ok(())
                     }
                     Err(err) => {
-                        let _ = reply.send(Err(err.to_string()));
+                        let channel_update =
+                            state.get_signed_channel_update_message(&self.network).await;
+                        let detail_error = TlcFailDetail::new_channel_fail(
+                            TlcFailErrorCode::PermanentNodeFailure,
+                            state.get_funding_transaction_outpoint(),
+                            channel_update,
+                        );
+                        let _ = reply.send(Err(detail_error.into()));
                         Err(err)
                     }
                 }
@@ -4374,12 +4384,15 @@ impl ChannelActorState {
         }
     }
 
-    pub async fn broadcast_channel_update(&mut self, network: &ActorRef<NetworkActorMessage>) {
+    async fn get_signed_channel_update_message(
+        &mut self,
+        network: &ActorRef<NetworkActorMessage>,
+    ) -> Option<ChannelUpdate> {
         let mut channel_update = match self.get_unsigned_channel_update_message() {
             Some(message) => message,
             _ => {
                 warn!("Failed to generate channel update message");
-                return;
+                return None;
             }
         };
 
@@ -4396,15 +4409,20 @@ impl ChannelActorState {
             "Broadcasting channel update message to peers: {:?}",
             &channel_update
         );
+        Some(channel_update)
+    }
 
-        network
-            .send_message(NetworkActorMessage::new_command(
-                NetworkActorCommand::BroadcastMessage(
-                    vec![self.get_remote_peer_id()],
-                    FiberBroadcastMessage::ChannelUpdate(channel_update),
-                ),
-            ))
-            .expect(ASSUME_NETWORK_ACTOR_ALIVE);
+    pub async fn broadcast_channel_update(&mut self, network: &ActorRef<NetworkActorMessage>) {
+        if let Some(channel_update) = self.get_signed_channel_update_message(network).await {
+            network
+                .send_message(NetworkActorMessage::new_command(
+                    NetworkActorCommand::BroadcastMessage(
+                        vec![self.get_remote_peer_id()],
+                        FiberBroadcastMessage::ChannelUpdate(channel_update),
+                    ),
+                ))
+                .expect(ASSUME_NETWORK_ACTOR_ALIVE);
+        }
     }
 
     pub async fn on_channel_ready(&mut self, network: &ActorRef<NetworkActorMessage>) {
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 8cb8009c..28ece737 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1927,6 +1927,7 @@ where
                     let error_detail = TlcFailDetail::new_channel_fail(
                         TlcFailErrorCode::PermanentChannelFailure,
                         channel_outpoint.clone(),
+                        None,
                     );
                     reply
                         .send(Err(error_detail.into()))
@@ -1934,7 +1935,7 @@ where
                     return;
                 }
             };
-            let (send, recv) = oneshot::channel::<Result<AddTlcResponse, String>>();
+            let (send, recv) = oneshot::channel::<Result<AddTlcResponse, RemoveTlcFail>>();
             let rpc_reply = RpcReplyPort::from(send);
             let command = ChannelCommand::AddTlc(
                 AddTlcCommand {
@@ -1957,11 +1958,7 @@ where
 
             match recv.await.expect("recv error") {
                 Ok(res) => Ok(res.tlc_id),
-                Err(_err) => {
-                    // FIXME: rework the error handling
-                    let error_detail = TlcFailDetail::new(TlcFailErrorCode::TemporaryNodeFailure);
-                    Err(error_detail.into())
-                }
+                Err(err) => Err(err),
             }
         } else {
             info!("onion packet is empty, ignore it");
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 5ddf851b..483e51cc 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -18,6 +18,7 @@ use ckb_types::{
     packed::{Byte32 as MByte32, BytesVec, Script, Transaction},
     prelude::{Pack, Unpack},
 };
+use core::fmt::{self, Formatter};
 use fiber_sphinx::SphinxError;
 use molecule::prelude::{Builder, Byte, Entity};
 use musig2::errors::DecodeError;
@@ -1251,11 +1252,12 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
 }
 
 #[serde_as]
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub enum TlcFailDetailData {
     ChannelFailed {
         #[serde_as(as = "EntityHex")]
         channel_outpoint: OutPoint,
+        channel_update: Option<ChannelUpdate>,
     },
     NodeFailed {
         node_id: Pubkey,
@@ -1263,7 +1265,7 @@ pub enum TlcFailDetailData {
 }
 
 #[serde_as]
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct TlcFailDetail {
     pub error_code: TlcFailErrorCode,
     pub extra_data: Option<TlcFailDetailData>,
@@ -1284,10 +1286,17 @@ impl TlcFailDetail {
         }
     }
 
-    pub fn new_channel_fail(error_code: TlcFailErrorCode, channel_outpoint: OutPoint) -> Self {
+    pub fn new_channel_fail(
+        error_code: TlcFailErrorCode,
+        channel_outpoint: OutPoint,
+        channel_update: Option<ChannelUpdate>,
+    ) -> Self {
         TlcFailDetail {
             error_code: error_code.into(),
-            extra_data: Some(TlcFailDetailData::ChannelFailed { channel_outpoint }),
+            extra_data: Some(TlcFailDetailData::ChannelFailed {
+                channel_outpoint,
+                channel_update,
+            }),
         }
     }
 
@@ -1300,9 +1309,9 @@ impl TlcFailDetail {
 
     pub fn error_channel_outpoint(&self) -> Option<OutPoint> {
         match &self.extra_data {
-            Some(TlcFailDetailData::ChannelFailed { channel_outpoint }) => {
-                Some(channel_outpoint.clone())
-            }
+            Some(TlcFailDetailData::ChannelFailed {
+                channel_outpoint, ..
+            }) => Some(channel_outpoint.clone()),
             _ => None,
         }
     }
@@ -1392,13 +1401,19 @@ impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
     }
 }
 
+impl std::fmt::Display for RemoveTlcFail {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "RemoveTlcFail")
+    }
+}
+
 // The onion packet is invalid
 const BADONION: u16 = 0x8000;
 // Permanent errors (otherwise transient)
 const PERM: u16 = 0x4000;
 // Node releated errors (otherwise channels)
 const NODE: u16 = 0x2000;
-//  Channel forwarding parameter was violated
+// Channel forwarding parameter was violated
 const UPDATE: u16 = 0x1000;
 
 #[repr(u16)]
diff --git a/src/rpc/utils.rs b/src/rpc/utils.rs
index 9c77a34d..6db49d14 100644
--- a/src/rpc/utils.rs
+++ b/src/rpc/utils.rs
@@ -16,7 +16,7 @@ macro_rules! handle_actor_call {
         match call!($actor, $message) {
             Ok(result) => match result {
                 Ok(res) => Ok(res),
-                Err(e) => log_and_error!($params, e),
+                Err(e) => log_and_error!($params, e.to_string()),
             },
             Err(e) => log_and_error!($params, e.to_string()),
         }

From 36dd5f810b92cc0152a6b0081326450c6088980d Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 11 Oct 2024 21:40:38 +0800
Subject: [PATCH 31/66] replace from with constructor to make it explicit

---
 src/fiber/channel.rs     |  4 ++--
 src/fiber/network.rs     | 14 +++++++-------
 src/fiber/tests/types.rs |  8 ++++----
 src/fiber/types.rs       | 20 ++++++--------------
 4 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index e9bd096d..9410d967 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -518,7 +518,7 @@ where
                                                 channel_id: state.get_id(),
                                                 tlc_id,
                                                 reason: RemoveTlcReason::RemoveTlcFail(
-                                                    error_detail.into(),
+                                                    RemoveTlcFail::new(error_detail),
                                                 ),
                                             }),
                                         ),
@@ -1289,7 +1289,7 @@ where
                             state.get_funding_transaction_outpoint(),
                             channel_update,
                         );
-                        let _ = reply.send(Err(detail_error.into()));
+                        let _ = reply.send(Err(RemoveTlcFail::new(detail_error)));
                         Err(err)
                     }
                 }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 28ece737..6b417cd1 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1930,7 +1930,7 @@ where
                         None,
                     );
                     reply
-                        .send(Err(error_detail.into()))
+                        .send(Err(RemoveTlcFail::new(error_detail)))
                         .expect("send add tlc response");
                     return;
                 }
@@ -1963,7 +1963,7 @@ where
         } else {
             info!("onion packet is empty, ignore it");
             let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
-            Err(error_detail.into())
+            Err(RemoveTlcFail::new(error_detail))
         };
         reply.send(res).expect("send error");
     }
@@ -1979,8 +1979,8 @@ where
                 match reason {
                     RemoveTlcReason::RemoveTlcFulfill(_) => payment_session.set_success_status(),
                     RemoveTlcReason::RemoveTlcFail(reason) => {
-                        let detail_error: TlcFailDetail = reason.into();
-                        self.update_with_tcl_fail_detail(detail_error.clone()).await;
+                        let detail_error = reason.decode().expect("decoded error");
+                        self.update_with_tcl_fail_detail(&detail_error).await;
                         if payment_session.can_retry() {
                             let res = self
                                 .try_payment_session(state, payment_session.clone())
@@ -1998,7 +1998,7 @@ where
         }
     }
 
-    async fn update_with_tcl_fail_detail(&self, tcl_error_detail: TlcFailDetail) {
+    async fn update_with_tcl_fail_detail(&self, tcl_error_detail: &TlcFailDetail) {
         match tcl_error_detail.error_code() {
             TlcFailErrorCode::PermanentChannelFailure => {
                 let channel_outpoint = tcl_error_detail
@@ -2071,7 +2071,7 @@ where
                 .await;
             match recv.await.expect("msg recv error") {
                 Err(e) => {
-                    let error_detail: TlcFailDetail = e.clone().into();
+                    let error_detail = e.decode().expect("decoded error");
                     error!("Failed to send onion packet with error: {:?}", e);
                     // This is the error implies we send payment request to the first hop failed
                     let err = format!(
@@ -2080,7 +2080,7 @@ where
                         error_detail.error_code_as_str()
                     );
                     error = Some(err);
-                    self.update_with_tcl_fail_detail(error_detail).await;
+                    self.update_with_tcl_fail_detail(&error_detail).await;
                     continue;
                 }
                 Ok(tlc_id) => {
diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 829b9b1c..7a666ba5 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -104,9 +104,9 @@ fn test_tlc_fail_error() {
     assert!(!tlc_fail_detail.is_node());
     assert!(tlc_fail_detail.is_bad_onion());
     assert!(tlc_fail_detail.is_perm());
-    let tlc_fail: RemoveTlcFail = tlc_fail_detail.clone().into();
+    let tlc_fail = RemoveTlcFail::new(tlc_fail_detail.clone());
 
-    let convert_back: TlcFailDetail = tlc_fail.into();
+    let convert_back: TlcFailDetail = tlc_fail.decode().expect("decoded fail");
     assert_eq!(tlc_fail_detail, convert_back);
 
     let node_fail = TlcFailDetail::new_node_fail(
@@ -114,7 +114,7 @@ fn test_tlc_fail_error() {
         generate_pubkey().into(),
     );
     assert!(node_fail.is_node());
-    let tlc_fail = RemoveTlcFail::from(node_fail.clone());
-    let convert_back: TlcFailDetail = tlc_fail.into();
+    let tlc_fail = RemoveTlcFail::new(node_fail.clone());
+    let convert_back = tlc_fail.decode().expect("decoded fail");
     assert_eq!(node_fail, convert_back);
 }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 483e51cc..4f785231 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1354,18 +1354,6 @@ impl TlcFailDetail {
     }
 }
 
-impl From<TlcFailDetail> for RemoveTlcFail {
-    fn from(tlc_fail: TlcFailDetail) -> Self {
-        RemoveTlcFail::new(tlc_fail.serialize())
-    }
-}
-
-impl From<RemoveTlcFail> for TlcFailDetail {
-    fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
-        TlcFailDetail::deserialize(&remove_tlc_fail.onion_packet).expect("deserialize fail")
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct RemoveTlcFail {
     // TODO: replace this with the real onion packet
@@ -1376,11 +1364,15 @@ pub struct RemoveTlcFail {
 }
 
 impl RemoveTlcFail {
-    pub fn new(packet_data: Vec<u8>) -> Self {
+    pub fn new(tlc_fail: TlcFailDetail) -> Self {
         RemoveTlcFail {
-            onion_packet: packet_data,
+            onion_packet: tlc_fail.serialize(),
         }
     }
+
+    pub fn decode(&self) -> Option<TlcFailDetail> {
+        TlcFailDetail::deserialize(&self.onion_packet)
+    }
 }
 
 impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {

From cddf7a7569f002b98fdddad9ead5080c2d0986e7 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 06:49:53 +0800
Subject: [PATCH 32/66] add get_tlc_detail_error for channel to construct
 detail error

---
 src/fiber/channel.rs     | 90 ++++++++++++++++++++++------------------
 src/fiber/tests/types.rs |  8 ++--
 src/fiber/types.rs       | 34 ++++++++-------
 3 files changed, 72 insertions(+), 60 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 9410d967..7757d80b 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -469,44 +469,13 @@ where
                         Ok(())
                     }
                     Err(e) => {
-                        let error_code = match e {
-                            ProcessingChannelError::PeelingOnionPacketError(_) => {
-                                TlcFailErrorCode::InvalidOnionPayload
-                            }
-                            ProcessingChannelError::InvalidParameter(_) => {
-                                TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
-                            }
-                            ProcessingChannelError::FinalIncorrectHTLCAmount => {
-                                TlcFailErrorCode::FinalIncorrectHtlcAmount
-                            }
-                            ProcessingChannelError::TlcAmountIsTooLow => {
-                                TlcFailErrorCode::AmountBelowMinimum
-                            }
-                            ProcessingChannelError::TlcNumberExceedLimit
-                            | ProcessingChannelError::TlcValueInflightExceedLimit => {
-                                TlcFailErrorCode::TemporaryChannelFailure
-                            }
-                            ProcessingChannelError::InvalidState(_) => match state.state {
-                                ChannelState::Closed(_) => TlcFailErrorCode::ChannelDisabled,
-                                ChannelState::ShuttingDown(_) | ChannelState::ChannelReady() => {
-                                    // we expect `ShuttingDown` and `ChannelReady` are both OK for tlc forwarding,
-                                    // so here are the unreachable point in normal workflow,
-                                    // set `TemporaryNodeFailure` for general temporary failure of the processing node here
-                                    TlcFailErrorCode::TemporaryNodeFailure
-                                }
-                                // otherwise, channel maybe not ready
-                                _ => TlcFailErrorCode::TemporaryChannelFailure,
-                            },
-                            // TODO: there maybe more error types here
-                            _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
-                        };
-                        let error_detail = TlcFailDetail::new(error_code);
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
                         // trying to add a duplicate tlc, so we use tlc count to make sure no new tlc was added
                         // and only send RemoveTlc message to peer if the TLC is not in our state
                         error!("Error handling AddTlc message: {:?}", e);
+                        let error_detail = self.get_tlc_detail_error(state, &e).await;
                         assert!(tlc_count == state.tlcs.len());
                         if state.get_received_tlc(tlc_id).is_none() {
                             self.network
@@ -693,6 +662,53 @@ where
         }
     }
 
+    async fn get_tlc_detail_error(
+        &self,
+        state: &mut ChannelActorState,
+        error: &ProcessingChannelError,
+    ) -> TlcFailDetail {
+        let error_code = match error {
+            ProcessingChannelError::PeelingOnionPacketError(_) => {
+                TlcFailErrorCode::InvalidOnionPayload
+            }
+            ProcessingChannelError::InvalidParameter(_) => {
+                TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
+            }
+            ProcessingChannelError::FinalIncorrectHTLCAmount => {
+                TlcFailErrorCode::FinalIncorrectHtlcAmount
+            }
+            ProcessingChannelError::TlcAmountIsTooLow => TlcFailErrorCode::AmountBelowMinimum,
+            ProcessingChannelError::TlcNumberExceedLimit
+            | ProcessingChannelError::TlcValueInflightExceedLimit => {
+                TlcFailErrorCode::TemporaryChannelFailure
+            }
+            ProcessingChannelError::InvalidState(_) => match state.state {
+                ChannelState::Closed(_) => TlcFailErrorCode::ChannelDisabled,
+                ChannelState::ShuttingDown(_) | ChannelState::ChannelReady() => {
+                    // we expect `ShuttingDown` and `ChannelReady` are both OK for tlc forwarding,
+                    // so here are the unreachable point in normal workflow,
+                    // set `TemporaryNodeFailure` for general temporary failure of the processing node here
+                    TlcFailErrorCode::TemporaryNodeFailure
+                }
+                // otherwise, channel maybe not ready
+                _ => TlcFailErrorCode::TemporaryChannelFailure,
+            },
+            // TODO: there maybe more error types here
+            _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
+        };
+
+        let channel_update = if error_code.is_update() {
+            state.get_signed_channel_update_message(&self.network).await
+        } else {
+            None
+        };
+        TlcFailDetail::new_channel_fail(
+            error_code,
+            state.get_funding_transaction_outpoint(),
+            channel_update,
+        )
+    }
+
     fn try_to_settle_down_tlc(&self, state: &mut ChannelActorState) {
         let tlcs = state.get_tlcs_for_settle_down();
         info!("try_to_settle_down_tlc get tlcs: {:?}", &tlcs);
@@ -1282,14 +1298,8 @@ where
                         Ok(())
                     }
                     Err(err) => {
-                        let channel_update =
-                            state.get_signed_channel_update_message(&self.network).await;
-                        let detail_error = TlcFailDetail::new_channel_fail(
-                            TlcFailErrorCode::PermanentNodeFailure,
-                            state.get_funding_transaction_outpoint(),
-                            channel_update,
-                        );
-                        let _ = reply.send(Err(RemoveTlcFail::new(detail_error)));
+                        let error_detail = self.get_tlc_detail_error(state, &err).await;
+                        let _ = reply.send(Err(RemoveTlcFail::new(error_detail)));
                         Err(err)
                     }
                 }
diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 7a666ba5..0c6233c1 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -101,9 +101,9 @@ fn test_peeled_onion_packet() {
 #[test]
 fn test_tlc_fail_error() {
     let tlc_fail_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionVersion);
-    assert!(!tlc_fail_detail.is_node());
-    assert!(tlc_fail_detail.is_bad_onion());
-    assert!(tlc_fail_detail.is_perm());
+    assert!(!tlc_fail_detail.error_code.is_node());
+    assert!(tlc_fail_detail.error_code.is_bad_onion());
+    assert!(tlc_fail_detail.error_code.is_perm());
     let tlc_fail = RemoveTlcFail::new(tlc_fail_detail.clone());
 
     let convert_back: TlcFailDetail = tlc_fail.decode().expect("decoded fail");
@@ -113,7 +113,7 @@ fn test_tlc_fail_error() {
         TlcFailErrorCode::PermanentNodeFailure,
         generate_pubkey().into(),
     );
-    assert!(node_fail.is_node());
+    assert!(node_fail.error_code.is_node());
     let tlc_fail = RemoveTlcFail::new(node_fail.clone());
     let convert_back = tlc_fail.decode().expect("decoded fail");
     assert_eq!(node_fail, convert_back);
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 4f785231..5b34e125 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1329,22 +1329,6 @@ impl TlcFailDetail {
         self.extra_data = Some(extra_data);
     }
 
-    pub fn is_node(&self) -> bool {
-        self.error_code as u16 & NODE != 0
-    }
-
-    pub fn is_bad_onion(&self) -> bool {
-        self.error_code as u16 & BADONION != 0
-    }
-
-    pub fn is_perm(&self) -> bool {
-        self.error_code as u16 & PERM != 0
-    }
-
-    pub fn is_update(&self) -> bool {
-        self.error_code as u16 & UPDATE != 0
-    }
-
     fn serialize(&self) -> Vec<u8> {
         deterministically_serialize(self)
     }
@@ -1435,6 +1419,24 @@ pub enum TlcFailErrorCode {
     InvalidOnionBlinding = BADONION | PERM | 24,
 }
 
+impl TlcFailErrorCode {
+    pub fn is_node(&self) -> bool {
+        *self as u16 & NODE != 0
+    }
+
+    pub fn is_bad_onion(&self) -> bool {
+        *self as u16 & BADONION != 0
+    }
+
+    pub fn is_perm(&self) -> bool {
+        *self as u16 & PERM != 0
+    }
+
+    pub fn is_update(&self) -> bool {
+        *self as u16 & UPDATE != 0
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),

From 4e3fce8c42a70f358cfa13a4a88088f48d1ae541 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 08:31:36 +0800
Subject: [PATCH 33/66] fix coding style in visibility

---
 src/fiber/channel.rs | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 7757d80b..f1c9b459 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -3994,7 +3994,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub fn handle_accept_channel_message(
+    fn handle_accept_channel_message(
         &mut self,
         accept_channel: AcceptChannel,
     ) -> ProcessingChannelResult {
@@ -4151,7 +4151,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub fn handle_commitment_signed_message(
+    fn handle_commitment_signed_message(
         &mut self,
         commitment_signed: CommitmentSigned,
         network: &ActorRef<NetworkActorMessage>,
@@ -4275,7 +4275,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub fn maybe_transition_to_tx_signatures(
+    fn maybe_transition_to_tx_signatures(
         &mut self,
         flags: SigningCommitmentFlags,
         network: &ActorRef<NetworkActorMessage>,
@@ -4296,7 +4296,7 @@ impl ChannelActorState {
     // TODO: currently witnesses in the tx_signatures molecule message are a list of bytes.
     // It is unclear how can we compose two partial sets witnesses into a complete
     // set of witnesses.
-    pub fn handle_tx_signatures(
+    fn handle_tx_signatures(
         &mut self,
         network: &ActorRef<NetworkActorMessage>,
         // If partial_witnesses is given, then it is the counterparty that send a message
@@ -4367,7 +4367,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub async fn maybe_broadcast_announcement_signatures(
+    async fn maybe_broadcast_announcement_signatures(
         &mut self,
         network: &ActorRef<NetworkActorMessage>,
     ) {
@@ -4395,7 +4395,7 @@ impl ChannelActorState {
     }
 
     async fn get_signed_channel_update_message(
-        &mut self,
+        &self,
         network: &ActorRef<NetworkActorMessage>,
     ) -> Option<ChannelUpdate> {
         let mut channel_update = match self.get_unsigned_channel_update_message() {
@@ -4407,7 +4407,6 @@ impl ChannelActorState {
         };
 
         debug!("Generated channel update message: {:?}", &channel_update);
-
         let node_signature =
             sign_network_message(network.clone(), channel_update.message_to_sign())
                 .await
@@ -4422,7 +4421,7 @@ impl ChannelActorState {
         Some(channel_update)
     }
 
-    pub async fn broadcast_channel_update(&mut self, network: &ActorRef<NetworkActorMessage>) {
+    async fn broadcast_channel_update(&self, network: &ActorRef<NetworkActorMessage>) {
         if let Some(channel_update) = self.get_signed_channel_update_message(network).await {
             network
                 .send_message(NetworkActorMessage::new_command(
@@ -4435,7 +4434,7 @@ impl ChannelActorState {
         }
     }
 
-    pub async fn on_channel_ready(&mut self, network: &ActorRef<NetworkActorMessage>) {
+    async fn on_channel_ready(&mut self, network: &ActorRef<NetworkActorMessage>) {
         self.update_state(ChannelState::ChannelReady());
         self.increment_local_commitment_number();
         self.increment_remote_commitment_number();
@@ -4452,7 +4451,7 @@ impl ChannelActorState {
         self.maybe_broadcast_announcement_signatures(network).await;
     }
 
-    pub fn append_remote_commitment_point(&mut self, commitment_point: Pubkey) {
+    fn append_remote_commitment_point(&mut self, commitment_point: Pubkey) {
         debug!(
             "Setting remote commitment point #{} (counting from 0)): {:?}",
             self.remote_commitment_points.len(),
@@ -4465,7 +4464,7 @@ impl ChannelActorState {
         self.remote_commitment_points.push(commitment_point);
     }
 
-    pub fn handle_revoke_and_ack_message(
+    fn handle_revoke_and_ack_message(
         &mut self,
         network: &ActorRef<NetworkActorMessage>,
         revoke_and_ack: RevokeAndAck,
@@ -4696,7 +4695,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub fn is_tx_final(&self, tx: &Transaction) -> Result<bool, ProcessingChannelError> {
+    fn is_tx_final(&self, tx: &Transaction) -> Result<bool, ProcessingChannelError> {
         // TODO: check if the tx is valid
         let tx = tx.clone().into_view();
 
@@ -4750,7 +4749,7 @@ impl ChannelActorState {
         }
     }
 
-    pub fn maybe_complete_tx_collaboration(
+    fn maybe_complete_tx_collaboration(
         &mut self,
         tx: Transaction,
         network: &ActorRef<NetworkActorMessage>,
@@ -4817,7 +4816,7 @@ impl ChannelActorState {
         Ok(())
     }
 
-    pub fn fill_in_channel_id(&mut self) {
+    fn fill_in_channel_id(&mut self) {
         assert!(
             self.remote_channel_parameters.is_some(),
             "Counterparty pubkeys is required to derive actual channel id"
@@ -4869,7 +4868,7 @@ impl ChannelActorState {
                 && self.should_local_go_first_in_musig2()
     }
 
-    pub fn build_shutdown_tx(&self) -> Result<TransactionView, ProcessingChannelError> {
+    fn build_shutdown_tx(&self) -> Result<TransactionView, ProcessingChannelError> {
         let local_shutdown_info = self.local_shutdown_info.as_ref().unwrap();
         let remote_shutdown_info = self.remote_shutdown_info.as_ref().unwrap();
 
@@ -4982,7 +4981,7 @@ impl ChannelActorState {
     // and the second element is the message to be signed by the each party,
     // so as to consume the funding cell. The last element is the witnesses for the
     // commitment transaction.
-    pub fn build_commitment_and_settlement_tx(
+    fn build_commitment_and_settlement_tx(
         &self,
         local: bool,
     ) -> (TransactionView, TransactionView) {
@@ -5205,7 +5204,7 @@ impl ChannelActorState {
         })
     }
 
-    pub fn build_and_sign_commitment_tx(
+    fn build_and_sign_commitment_tx(
         &self,
     ) -> Result<PartiallySignedCommitmentTransaction, ProcessingChannelError> {
         let (commitment_tx, settlement_tx) = self.build_commitment_and_settlement_tx(true);
@@ -5247,7 +5246,7 @@ impl ChannelActorState {
 
     /// Verify the partial signature from the peer and create a complete transaction
     /// with valid witnesses.
-    pub fn verify_and_complete_tx(
+    fn verify_and_complete_tx(
         &self,
         funding_tx_partial_signature: PartialSignature,
         commitment_tx_partial_signature: PartialSignature,

From 33ca6283103824faf2dcd2063adc9787afddb906 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 09:18:35 +0800
Subject: [PATCH 34/66] go through all the error codes

---
 src/fiber/channel.rs | 12 +++++++-----
 src/fiber/network.rs |  7 ++-----
 src/fiber/types.rs   | 14 +++++++++++---
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index f1c9b459..c5828c48 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -621,7 +621,6 @@ where
                 }
                 state.update_state(ChannelState::ShuttingDown(flags));
                 state.maybe_transition_to_shutdown(&self.network)?;
-
                 Ok(())
             }
             FiberChannelMessage::ClosingSigned(closing) => {
@@ -683,11 +682,14 @@ where
                 TlcFailErrorCode::TemporaryChannelFailure
             }
             ProcessingChannelError::InvalidState(_) => match state.state {
-                ChannelState::Closed(_) => TlcFailErrorCode::ChannelDisabled,
-                ChannelState::ShuttingDown(_) | ChannelState::ChannelReady() => {
-                    // we expect `ShuttingDown` and `ChannelReady` are both OK for tlc forwarding,
+                ChannelState::Closed(_) => TlcFailErrorCode::PermanentChannelFailure,
+                // ShuttingDown is a temporary state
+                ChannelState::ShuttingDown(_) => TlcFailErrorCode::TemporaryChannelFailure,
+                ChannelState::ChannelReady() => {
+                    // we expect `ChannelReady` will be both OK for tlc forwarding,
                     // so here are the unreachable point in normal workflow,
                     // set `TemporaryNodeFailure` for general temporary failure of the processing node here
+                    assert!(false, "unreachable point in normal workflow");
                     TlcFailErrorCode::TemporaryNodeFailure
                 }
                 // otherwise, channel maybe not ready
@@ -3762,7 +3764,7 @@ impl ChannelActorState {
 
     pub fn check_for_tlc_update(&self, add_tlc_amount: Option<u128>) -> ProcessingChannelResult {
         match self.state {
-            ChannelState::ChannelReady() | ChannelState::ShuttingDown(_) => {}
+            ChannelState::ChannelReady() => {}
             _ => {
                 return Err(ProcessingChannelError::InvalidState(format!(
                     "Invalid state {:?} for adding tlc",
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 6b417cd1..adccb1fa 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1925,7 +1925,7 @@ where
                             channel_outpoint
                         );
                     let error_detail = TlcFailDetail::new_channel_fail(
-                        TlcFailErrorCode::PermanentChannelFailure,
+                        TlcFailErrorCode::UnknownNextPeer,
                         channel_outpoint.clone(),
                         None,
                     );
@@ -1956,10 +1956,7 @@ where
                 .await
                 .expect("send command failed");
 
-            match recv.await.expect("recv error") {
-                Ok(res) => Ok(res.tlc_id),
-                Err(err) => Err(err),
-            }
+            recv.await.expect("recv error").map(|res| res.tlc_id)
         } else {
             info!("onion packet is empty, ignore it");
             let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 5b34e125..10aea196 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1338,12 +1338,17 @@ impl TlcFailDetail {
     }
 }
 
+// This is the onion packet we need to encode and send back to the sender
+// currently it's the raw TlcFailDetail serialized data from the TlcFailDetail struct
+// sender should decode it and get the TlcFailDetail, then decide what to do
+// Note: this supposed to be only accessible by the sender, and it's not reliable since it
+// is not placed on-chain due to the possibility of hop failure.
+//
+// FIXME: a better name like `TlcFailOnionPacket`?, since it's not only caused by removing tlc now,
+//        maybe we will replace it later.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct RemoveTlcFail {
     // TODO: replace this with the real onion packet
-    // This is the onion packet we need to encode and send back to the sender
-    // currently it's the raw TlcFailDetail serialized data
-    // sender should decode it and get the real TlcFailDetail
     pub onion_packet: Vec<u8>,
 }
 
@@ -1397,11 +1402,14 @@ const UPDATE: u16 = 0x1000;
 pub enum TlcFailErrorCode {
     TemporaryNodeFailure = NODE | 2,
     PermanentNodeFailure = PERM | NODE | 2,
+    // unused right now
     RequiredNodeFeatureMissing = PERM | NODE | 3,
+    // unused right now, maybe need to add onion version in future?
     InvalidOnionVersion = BADONION | PERM | 4,
     InvalidOnionHmac = BADONION | PERM | 5,
     InvalidOnionKey = BADONION | PERM | 6,
     TemporaryChannelFailure = UPDATE | 7,
+    // used for shutting down the channel
     PermanentChannelFailure = PERM | 8,
     RequiredChannelFeatureMissing = PERM | 9,
     UnknownNextPeer = PERM | 10,

From 6052f8636911d8a2a2bb21187c9a5d8035f70b12 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 11:56:48 +0800
Subject: [PATCH 35/66] add more error code check for tlc forwarding

---
 src/fiber/channel.rs | 62 ++++++++++++++++++++++++++++++++------------
 src/fiber/fee.rs     | 11 ++++++++
 src/fiber/graph.rs   |  3 ++-
 src/fiber/network.rs | 23 +++++++++++++---
 src/fiber/types.rs   | 11 ++++++++
 5 files changed, 90 insertions(+), 20 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index c5828c48..dbce34fb 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -4,6 +4,7 @@ use tracing::{debug, error, info, warn};
 
 use crate::{
     fiber::{
+        fee::calculate_tlc_forward_fee,
         network::{get_chain_hash, SendOnionPacketCommand},
         types::{ChannelUpdate, RemoveTlcFail, TlcFailDetail, TlcFailErrorCode},
     },
@@ -670,7 +671,9 @@ where
             ProcessingChannelError::PeelingOnionPacketError(_) => {
                 TlcFailErrorCode::InvalidOnionPayload
             }
-            ProcessingChannelError::InvalidParameter(_) => {
+            ProcessingChannelError::TlcForwardFeeIsTooLow => TlcFailErrorCode::FeeInsufficient,
+            ProcessingChannelError::FinalIncorrectPreimage
+            | ProcessingChannelError::FinalIncorrectPaymentHash => {
                 TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
             }
             ProcessingChannelError::FinalIncorrectHTLCAmount => {
@@ -713,7 +716,6 @@ where
 
     fn try_to_settle_down_tlc(&self, state: &mut ChannelActorState) {
         let tlcs = state.get_tlcs_for_settle_down();
-        info!("try_to_settle_down_tlc get tlcs: {:?}", &tlcs);
         for tlc_info in tlcs {
             let tlc = tlc_info.tlc.clone();
             let preimage = if let Some(preimage) = tlc.payment_preimage {
@@ -721,6 +723,10 @@ where
             } else if let Some(preimage) = self.store.get_invoice_preimage(&tlc.payment_hash) {
                 preimage
             } else {
+                error!(
+                    "No preimage found for payment hash: {:?}",
+                    &tlc.payment_hash
+                );
                 continue;
             };
             let command = RemoveTlcCommand {
@@ -772,23 +778,45 @@ where
                 ));
             }
             // TODO: check the expiry time, if expired, we should return an error.
-
             if peeled_packet.is_last() {
                 // if this is the last hop, store the preimage.
-                preimage = peeled_packet.current.preimage;
+                // though we will RemoveTlcFulfill the TLC in try_to_settle_down_tlc function,
+                // here we can do error check early here for better error handling.
+                preimage = peeled_packet
+                    .current
+                    .preimage
+                    .or_else(|| self.store.get_invoice_preimage(&add_tlc.payment_hash));
+                if let Some(preimage) = preimage {
+                    let filled_payment_hash: Hash256 = add_tlc.hash_algorithm.hash(preimage).into();
+                    if add_tlc.payment_hash != filled_payment_hash {
+                        return Err(ProcessingChannelError::FinalIncorrectPreimage);
+                    }
+                } else {
+                    return Err(ProcessingChannelError::FinalIncorrectPaymentHash);
+                }
             } else {
                 peeled_packet_bytes = Some(peeled_packet.serialize());
                 forward_to_next_hop = true;
             }
 
-            if forward_to_next_hop && peeled_packet.current.amount > add_tlc.amount {
-                return Err(ProcessingChannelError::InvalidParameter
-                    (
-                    format!("Payment amount is too large, expect next hop amount [{}] less than received amount [{}]",
-                    peeled_packet.current.amount , add_tlc.amount)
-                ));
+            let forward_amount = add_tlc.amount;
+            if forward_to_next_hop {
+                let forward_fee = forward_amount - peeled_packet.current.amount;
+                let fee_rate: u128 = state
+                    .public_channel_info
+                    .as_ref()
+                    .map(|info| info.tlc_fee_proportional_millionths.unwrap_or_default())
+                    .unwrap_or_default();
+                let expected_fee = calculate_tlc_forward_fee(forward_amount, fee_rate);
+                if forward_fee < expected_fee {
+                    error!(
+                        "too low forward_fee: {}, expected_fee: {}",
+                        forward_fee, expected_fee
+                    );
+                    return Err(ProcessingChannelError::TlcForwardFeeIsTooLow);
+                }
             }
-            if !forward_to_next_hop && peeled_packet.current.amount != add_tlc.amount {
+            if !forward_to_next_hop && forward_amount != add_tlc.amount {
                 return Err(ProcessingChannelError::FinalIncorrectHTLCAmount);
             }
         }
@@ -1085,7 +1113,6 @@ where
         };
 
         state.check_shutdown_fee_rate(command.fee_rate, &command.close_script)?;
-
         if command.force {
             if let Some(transaction) = &state.latest_commitment_transaction {
                 self.network
@@ -2097,6 +2124,12 @@ pub enum ProcessingChannelError {
     PeelingOnionPacketError(String),
     #[error("The amount in the HTLC is not expected")]
     FinalIncorrectHTLCAmount,
+    #[error("The payment_hash is not expected for final hop")]
+    FinalIncorrectPaymentHash,
+    #[error("The payment_hash and preimage does not match for final hop")]
+    FinalIncorrectPreimage,
+    #[error("The tlc forward fee is tow low")]
+    TlcForwardFeeIsTooLow,
     #[error("The tlc number exceed limit of this channel")]
     TlcNumberExceedLimit,
     #[error("The tlc flight value exceed limit of this channel")]
@@ -3396,10 +3429,7 @@ impl ChannelActorState {
                                 .hash(fulfill.payment_preimage)
                                 .into();
                             if current.tlc.payment_hash != filled_payment_hash {
-                                return Err(ProcessingChannelError::InvalidParameter(format!(
-                                    "Preimage {:?} is hashed to {}, which does not match payment hash {:?}",
-                                    fulfill.payment_preimage, filled_payment_hash, current.tlc.payment_hash,
-                                )));
+                                return Err(ProcessingChannelError::FinalIncorrectPreimage);
                             }
                         }
                     }
diff --git a/src/fiber/fee.rs b/src/fiber/fee.rs
index 93e0e352..1f788867 100644
--- a/src/fiber/fee.rs
+++ b/src/fiber/fee.rs
@@ -118,3 +118,14 @@ pub(crate) fn calculate_shutdown_tx_fee(
     let tx_size = shutdown_tx_size(udt_type_script, shutdown_scripts) as u64;
     fee_rate.fee(tx_size).as_u64()
 }
+
+pub(crate) fn calculate_tlc_forward_fee(amount: u128, fee_proportational_millionths: u128) -> u128 {
+    let fee = fee_proportational_millionths * amount;
+    let base_fee = fee / 1_000_000;
+    let remainder = fee % 1_000_000;
+    if remainder > 0 {
+        base_fee + 1
+    } else {
+        base_fee
+    }
+}
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index d3ab99ad..f83e676d 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -2,6 +2,7 @@ use super::network::{get_chain_hash, SendPaymentData, SendPaymentResponse};
 use super::path::NodeHeap;
 use super::types::Pubkey;
 use super::types::{ChannelAnnouncement, ChannelUpdate, Hash256, NodeAnnouncement};
+use crate::fiber::fee::calculate_tlc_forward_fee;
 use crate::fiber::path::{NodeHeapElement, ProbabilityEvaluator};
 use crate::fiber::serde_utils::EntityHex;
 use crate::fiber::types::PaymentHopData;
@@ -552,7 +553,7 @@ where
                 }
                 .expect("channel_update is none");
                 let fee_rate = channel_update.fee_rate;
-                let fee = self.calculate_fee(current_amount, fee_rate as u128);
+                let fee = calculate_tlc_forward_fee(current_amount, fee_rate as u128);
                 let expiry = channel_update.cltv_expiry_delta;
                 (fee, expiry)
             };
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index adccb1fa..86e40343 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -60,7 +60,7 @@ use super::types::{
     Privkey, Pubkey, QueryBroadcastMessagesWithinTimeRange,
     QueryBroadcastMessagesWithinTimeRangeResult, QueryChannelsWithinBlockRange,
     QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcFail, RemoveTlcReason, TlcFailDetail,
-    TlcFailErrorCode,
+    TlcFailDetailData, TlcFailErrorCode,
 };
 use super::FiberConfig;
 
@@ -1978,7 +1978,8 @@ where
                     RemoveTlcReason::RemoveTlcFail(reason) => {
                         let detail_error = reason.decode().expect("decoded error");
                         self.update_with_tcl_fail_detail(&detail_error).await;
-                        if payment_session.can_retry() {
+                        if payment_session.can_retry() && !detail_error.error_code.payment_failed()
+                        {
                             let res = self
                                 .try_payment_session(state, payment_session.clone())
                                 .await;
@@ -1996,8 +1997,24 @@ where
     }
 
     async fn update_with_tcl_fail_detail(&self, tcl_error_detail: &TlcFailDetail) {
+        let error_code = tcl_error_detail.error_code();
+        // https://github.com/lightning/bolts/blob/master/04-onion-routing.md#rationale-6
+        // we now still update the graph, maybe we need to remove it later?
+        if error_code.is_update() {
+            if let Some(extra_data) = &tcl_error_detail.extra_data {
+                match extra_data {
+                    TlcFailDetailData::ChannelFailed { channel_update, .. } => {
+                        if let Some(channel_update) = channel_update {
+                            let mut graph = self.network_graph.write().await;
+                            let _ = graph.process_channel_update(channel_update.clone());
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
         match tcl_error_detail.error_code() {
-            TlcFailErrorCode::PermanentChannelFailure => {
+            TlcFailErrorCode::PermanentChannelFailure | TlcFailErrorCode::ChannelDisabled => {
                 let channel_outpoint = tcl_error_detail
                     .error_channel_outpoint()
                     .expect("expect channel outpoint");
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 10aea196..38c07418 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1415,6 +1415,7 @@ pub enum TlcFailErrorCode {
     UnknownNextPeer = PERM | 10,
     AmountBelowMinimum = UPDATE | 11,
     FeeInsufficient = UPDATE | 12,
+    // TODO: cltv expiry check
     IncorrectCltvExpiry = UPDATE | 13,
     ExpiryTooSoon = UPDATE | 14,
     IncorrectOrUnknownPaymentDetails = PERM | 15,
@@ -1443,6 +1444,16 @@ impl TlcFailErrorCode {
     pub fn is_update(&self) -> bool {
         *self as u16 & UPDATE != 0
     }
+
+    pub fn payment_failed(&self) -> bool {
+        match self {
+            TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
+            | TlcFailErrorCode::FinalIncorrectCltvExpiry
+            | TlcFailErrorCode::FinalIncorrectHtlcAmount
+            | TlcFailErrorCode::MppTimeout => true,
+            _ => false,
+        }
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]

From 3b203ebccdd9508102da139a36e39d66ff153805 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 12:33:41 +0800
Subject: [PATCH 36/66] fix remove_tlc check issue

---
 src/fiber/channel.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index dbce34fb..34efb10d 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -3795,10 +3795,16 @@ impl ChannelActorState {
     pub fn check_for_tlc_update(&self, add_tlc_amount: Option<u128>) -> ProcessingChannelResult {
         match self.state {
             ChannelState::ChannelReady() => {}
+            ChannelState::ShuttingDown(_) if add_tlc_amount.is_none() => {}
             _ => {
                 return Err(ProcessingChannelError::InvalidState(format!(
-                    "Invalid state {:?} for adding tlc",
-                    self.state
+                    "Invalid state {:?} for {} tlc",
+                    self.state,
+                    if add_tlc_amount.is_some() {
+                        "adding"
+                    } else {
+                        "removing"
+                    }
                 )))
             }
         }

From 9511e4447c80729fd554b7382b3076a6e07e6998 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 17:22:06 +0800
Subject: [PATCH 37/66] fix bugs in remove_tlc and fee check

---
 src/fiber/channel.rs | 17 +++++++++++------
 src/fiber/graph.rs   | 28 ++++++----------------------
 src/fiber/network.rs | 11 ++++++-----
 3 files changed, 23 insertions(+), 33 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 34efb10d..05a1271b 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -545,7 +545,7 @@ where
                             }),
                         ))
                         .expect(ASSUME_NETWORK_ACTOR_ALIVE);
-                    let res = recv.await.expect("network actor is alive");
+                    let res = recv.await.expect("remove tlc replied");
                     info!("remove tlc from previous channel: {:?}", &res);
                 } else {
                     // only the original sender of the TLC should send `TlcRemoveReceived` event
@@ -799,9 +799,14 @@ where
                 forward_to_next_hop = true;
             }
 
-            let forward_amount = add_tlc.amount;
+            let received_amount = add_tlc.amount;
+            let forward_amount = peeled_packet.current.amount;
+            debug!(
+                "received_amount: {} forward_amount: {}",
+                add_tlc.amount, forward_amount
+            );
             if forward_to_next_hop {
-                let forward_fee = forward_amount - peeled_packet.current.amount;
+                let forward_fee = received_amount.saturating_sub(forward_amount);
                 let fee_rate: u128 = state
                     .public_channel_info
                     .as_ref()
@@ -816,7 +821,7 @@ where
                     return Err(ProcessingChannelError::TlcForwardFeeIsTooLow);
                 }
             }
-            if !forward_to_next_hop && forward_amount != add_tlc.amount {
+            if !forward_to_next_hop && received_amount != add_tlc.amount {
                 return Err(ProcessingChannelError::FinalIncorrectHTLCAmount);
             }
         }
@@ -859,7 +864,7 @@ where
                     rpc_reply,
                 ),
             ))
-            .expect("network actor is alive");
+            .expect(ASSUME_NETWORK_ACTOR_ALIVE);
         let res = match recv.await.expect("expect command replied") {
             Ok(tlc_id) => Ok(tlc_id),
             Err(e) => Err(e),
@@ -884,7 +889,7 @@ where
                         ),
                     }),
                 ))
-                .expect("network actor is alive");
+                .expect(ASSUME_NETWORK_ACTOR_ALIVE);
             let _ = recv.await.expect("RemoveTlc command replied");
         }
         Ok(())
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index f83e676d..fc2b4b4f 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -451,17 +451,6 @@ where
         self.source
     }
 
-    pub fn calculate_fee(&self, amount: u128, fee_proportational_millionths: u128) -> u128 {
-        let fee = fee_proportational_millionths * amount;
-        let base_fee = fee / 1_000_000;
-        let remainder = fee % 1_000_000;
-        if remainder > 0 {
-            base_fee + 1
-        } else {
-            base_fee
-        }
-    }
-
     pub(crate) fn mark_channel_failed(&mut self, channel_outpoint: &OutPoint) {
         if let Some(channel) = self.channels.get_mut(channel_outpoint) {
             if let Some(info) = channel.node1_to_node2.as_mut() {
@@ -646,20 +635,15 @@ where
             for (from, channel_info, channel_update) in self.get_node_inbounds(cur_hop.node_id) {
                 edges_expanded += 1;
                 // if charge inbound fees for exit hop
-                let fee_rate = channel_update.fee_rate;
-                let next_hop_received_amount = cur_hop.amount_received;
-                let fee = self.calculate_fee(next_hop_received_amount, fee_rate as u128);
-                let amount_to_send = next_hop_received_amount + fee;
-
-                debug!(
-                    "fee_rate: {:?} next_hop_received_amount: {:?}, fee: {:?} amount_to_send: {:?} channel_capacity: {:?} htlc_max_value: {:?}",
-                    fee_rate, next_hop_received_amount, fee, amount_to_send, channel_info.capacity(), channel_update.htlc_maximum_value
-                );
-
                 if udt_type_script != channel_info.announcement_msg.udt_type_script {
                     continue;
                 }
 
+                let fee_rate = channel_update.fee_rate;
+                let next_hop_received_amount = cur_hop.amount_received;
+                let fee = calculate_tlc_forward_fee(next_hop_received_amount, fee_rate as u128);
+                let amount_to_send = next_hop_received_amount + fee;
+
                 // if the amount to send is greater than the amount we have, skip this edge
                 if let Some(max_fee_amount) = max_fee_amount {
                     if amount_to_send > amount + max_fee_amount {
@@ -722,7 +706,7 @@ where
                         continue;
                     }
                 }
-                // info!("weight: {:?} dist: {:?} fee: {:?}", weight, dist, fee);
+                // info!("weight: {:?} dist: {:?} fee: {:?}", weight, distance, fee);
                 // TODO: update weight and distance here
                 let node: NodeHeapElement = NodeHeapElement {
                     node_id: from,
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 86e40343..85979bab 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1974,25 +1974,26 @@ where
         if let Some(mut payment_session) = self.store.get_payment_session(payment_hash) {
             if payment_session.status == PaymentSessionStatus::Inflight {
                 match reason {
-                    RemoveTlcReason::RemoveTlcFulfill(_) => payment_session.set_success_status(),
+                    RemoveTlcReason::RemoveTlcFulfill(_) => {
+                        payment_session.set_success_status();
+                        self.store.insert_payment_session(payment_session);
+                    }
                     RemoveTlcReason::RemoveTlcFail(reason) => {
                         let detail_error = reason.decode().expect("decoded error");
                         self.update_with_tcl_fail_detail(&detail_error).await;
                         if payment_session.can_retry() && !detail_error.error_code.payment_failed()
                         {
-                            let res = self
-                                .try_payment_session(state, payment_session.clone())
-                                .await;
+                            let res = self.try_payment_session(state, payment_session).await;
                             if res.is_err() {
                                 debug!("Failed to retry payment session: {:?}", res);
                             }
                         } else {
                             payment_session.set_failed_status(detail_error.error_code.as_ref());
+                            self.store.insert_payment_session(payment_session);
                         }
                     }
                 }
             }
-            self.store.insert_payment_session(payment_session);
         }
     }
 

From 05b1cfd28df0972029b3b3341b1fe5a19ac44900 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sat, 12 Oct 2024 17:35:57 +0800
Subject: [PATCH 38/66] fix fmt

---
 src/fiber/channel.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 734324b1..26972908 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -69,9 +69,9 @@ use super::{
     serde_utils::EntityHex,
     types::{
         AcceptChannel, AddTlc, ChannelAnnouncement, ChannelReady, ClosingSigned, CommitmentSigned,
-        EcdsaSignature, FiberChannelMessage, FiberMessage, Hash256,
-        LockTime, OpenChannel, Privkey, Pubkey, ReestablishChannel, RemoveTlc, RemoveTlcFulfill,
-        RemoveTlcReason, RevokeAndAck, TxCollaborationMsg, TxComplete, TxUpdate,
+        EcdsaSignature, FiberChannelMessage, FiberMessage, Hash256, LockTime, OpenChannel, Privkey,
+        Pubkey, ReestablishChannel, RemoveTlc, RemoveTlcFulfill, RemoveTlcReason, RevokeAndAck,
+        TxCollaborationMsg, TxComplete, TxUpdate,
     },
     NetworkActorCommand, NetworkActorEvent, NetworkActorMessage, ASSUME_NETWORK_ACTOR_ALIVE,
 };

From b06ff72e7881456f64700d833be731de4ca58fdc Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 00:04:45 +0800
Subject: [PATCH 39/66] fix merge issue from channel update

---
 src/fiber/channel.rs |  5 +++--
 src/fiber/graph.rs   | 20 ++++++++++++++++++--
 src/fiber/network.rs | 37 +++++++++++++++++++++++++------------
 3 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 26972908..b2df6574 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -716,7 +716,7 @@ where
         };
 
         let channel_update = if error_code.is_update() {
-            Some(state.generate_channel_update(&self.network).await)
+            state.try_create_channel_update_message(&self.network).await
         } else {
             None
         };
@@ -790,7 +790,7 @@ where
                     "Payment hash mismatch".to_string(),
                 ));
             }
-            // TODO: check the expiry time, if expired, we should return an error.
+            // TODO: check the expiry time, if it's expired, we should return an error.
             if peeled_packet.is_last() {
                 // if this is the last hop, store the preimage.
                 // though we will RemoveTlcFulfill the TLC in try_to_settle_down_tlc function,
@@ -819,6 +819,7 @@ where
                 add_tlc.amount, forward_amount
             );
             if forward_to_next_hop {
+                assert!(received_amount >= forward_amount);
                 let forward_fee = received_amount.saturating_sub(forward_amount);
                 let fee_rate: u128 = state
                     .public_channel_info
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index d18ec2de..9fca5eaf 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -227,6 +227,16 @@ where
         self.last_update_timestamp
     }
 
+    pub(crate) fn process_node_announcement(&mut self, node_announcement: NodeAnnouncement) {
+        let node_id = node_announcement.node_id;
+        let node_info = NodeInfo {
+            node_id,
+            timestamp: std::time::UNIX_EPOCH.elapsed().unwrap().as_millis() as u64,
+            anouncement_msg: node_announcement,
+        };
+        self.add_node(node_info);
+    }
+
     pub fn add_node(&mut self, node_info: NodeInfo) {
         debug!("Adding node to network graph: {:?}", node_info);
 
@@ -620,10 +630,16 @@ where
             ));
         }
         let Some(source_node) = self.nodes.get(&source) else {
-            return Err(GraphError::PathFind("source node not found".to_string()));
+            return Err(GraphError::PathFind(format!(
+                "source node not found: {:?}",
+                &source
+            )));
         };
         let Some(_target_node) = self.nodes.get(&target) else {
-            return Err(GraphError::PathFind("target node not found".to_string()));
+            return Err(GraphError::PathFind(format!(
+                "target node not found: {:?}",
+                &target
+            )));
         };
         // initialize the target node
         nodes_heap.push(NodeHeapElement {
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 8d769337..30662f4e 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1488,6 +1488,9 @@ where
             NetworkActorCommand::BroadcastLocalInfo(kind) => match kind {
                 LocalInfoKind::NodeAnnouncement => {
                     let message = state.get_or_create_new_node_announcement_message();
+                    // Need also to update our own graph with the new node announcement.
+                    let mut graph = self.network_graph.write().await;
+                    graph.process_node_announcement(message.clone());
                     myself
                         .send_message(NetworkActorMessage::new_command(
                             NetworkActorCommand::BroadcastMessage(
@@ -1610,6 +1613,13 @@ where
                 tx_index,
                 channel_announcement,
             ) => {
+                debug!(
+                    "Received channel announcement message for channel (confirmed at #{} block #{} tx) to peer {:?}: {:?}",
+                    &peer_id,
+                    &block_number,
+                    &tx_index,
+                    &channel_announcement
+                );
                 // Adding this owned channel to the network graph.
                 let channel_info = ChannelInfo {
                     funding_tx_block_number: block_number.into(),
@@ -1706,7 +1716,7 @@ where
         message: FiberBroadcastMessage,
     ) -> Result<(), Error> {
         match message {
-            FiberBroadcastMessage::NodeAnnouncement(ref node_announcement) => {
+            FiberBroadcastMessage::NodeAnnouncement(node_announcement) => {
                 let message = node_announcement.message_to_sign();
                 if !self
                     .network_graph
@@ -1728,20 +1738,18 @@ where
                             &node_announcement
                         );
 
-                        // Add the node to the network graph.
-                        let node_info = NodeInfo {
-                            node_id: node_announcement.node_id,
-                            timestamp: std::time::UNIX_EPOCH.elapsed().unwrap().as_millis() as u64,
-                            anouncement_msg: node_announcement.clone(),
-                        };
-                        self.network_graph.write().await.add_node(node_info);
-
-                        // TODO: bookkeeping how many nodes we have connected to. Stop connnecting once we surpass a threshold.
+                        // TODO: bookkeeping how many nodes we have connected to. Stop connecting once we surpass a threshold.
                         for addr in &node_announcement.addresses {
                             network.send_message(NetworkActorMessage::new_command(
                                 NetworkActorCommand::ConnectPeer(addr.clone()),
                             ))?;
                         }
+
+                        // Add the node to the network graph.
+                        self.network_graph
+                            .write()
+                            .await
+                            .process_node_announcement(node_announcement);
                         Ok(())
                     }
                     _ => {
@@ -3521,7 +3529,8 @@ where
             debug!("Tentacle service shutdown");
         });
 
-        let graph = self.network_graph.read().await;
+        let mut graph = self.network_graph.write().await;
+
         let peers_to_sync_network_graph = graph
             .get_peers_to_sync_network_graph()
             .into_iter()
@@ -3545,7 +3554,7 @@ where
             last_update,
             peers_to_sync_network_graph,
         );
-        let state = NetworkActorState {
+        let mut state = NetworkActorState {
             store: self.store.clone(),
             node_name: config.announced_node_name,
             peer_id: my_peer_id,
@@ -3581,6 +3590,10 @@ where
             broadcasted_message_queue: Default::default(),
         };
 
+        // Save our own NodeInfo to the network graph.
+        let node_announcement = state.get_or_create_new_node_announcement_message();
+        graph.process_node_announcement(node_announcement);
+
         // load the connected peers from the network graph
         let peers = graph.get_connected_peers();
         // TODO: we need to bootstrap the network if no peers are connected.

From 84aee3ecb7a02d8801e28214a5548b3579fe7055 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 00:06:34 +0800
Subject: [PATCH 40/66] fix fmt issue

---
 src/fiber/network.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 30662f4e..dabac46d 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -69,7 +69,7 @@ use crate::ckb::{CkbChainMessage, FundingRequest, FundingTx, TraceTxRequest, Tra
 use crate::fiber::channel::{
     AddTlcCommand, AddTlcResponse, TxCollaborationCommand, TxUpdateCommand,
 };
-use crate::fiber::graph::{ChannelInfo, NodeInfo, PaymentSession, PaymentSessionStatus};
+use crate::fiber::graph::{ChannelInfo, PaymentSession, PaymentSessionStatus};
 use crate::fiber::types::{secp256k1_instance, FiberChannelMessage, TxSignatures};
 use crate::fiber::types::{PaymentOnionPacket, PeeledPaymentOnionPacket};
 use crate::fiber::KeyPair;

From eb9117821ffa1b01a575f91e83fedb40a65ff448 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 09:55:52 +0800
Subject: [PATCH 41/66] fix errors for unkonwn peers

---
 src/fiber/network.rs | 145 +++++++++++++++++++++++++++----------------
 1 file changed, 92 insertions(+), 53 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index dabac46d..c942070c 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1975,60 +1975,82 @@ where
             packet,
             previous_tlc,
         } = command;
-        let res = if let Ok(peeled_packet) = PeeledPaymentOnionPacket::deserialize(&packet) {
-            let info = peeled_packet.current;
-            debug!("Processing onion packet info: {:?}", info);
 
-            let channel_outpoint = &info
-                .channel_outpoint
-                .expect("valid onion packet contains channel outpoint");
-            let channel_id = match state.outpoint_channel_map.get(channel_outpoint) {
-                Some(channel_id) => channel_id,
-                None => {
-                    error!(
-                            "Failed to process onion packet: channel id not found for channel outpoint {:?}. \
-                            Are we connected to the peer?",
-                            channel_outpoint
-                        );
-                    let error_detail = TlcFailDetail::new_channel_fail(
-                        TlcFailErrorCode::UnknownNextPeer,
-                        channel_outpoint.clone(),
-                        None,
-                    );
-                    reply
-                        .send(Err(RemoveTlcFail::new(error_detail)))
-                        .expect("send add tlc response");
-                    return;
-                }
-            };
-            let (send, recv) = oneshot::channel::<Result<AddTlcResponse, RemoveTlcFail>>();
-            let rpc_reply = RpcReplyPort::from(send);
-            let command = ChannelCommand::AddTlc(
-                AddTlcCommand {
-                    amount: info.amount,
-                    preimage: None,
-                    payment_hash: Some(info.payment_hash),
-                    expiry: info.expiry.into(),
-                    hash_algorithm: info.tlc_hash_algorithm,
-                    onion_packet: peeled_packet.next.map(|next| next.data).unwrap_or_default(),
-                    previous_tlc,
-                },
-                rpc_reply,
-            );
-
-            // we have already checked the channel_id is valid
-            state
-                .send_command_to_channel(*channel_id, command)
-                .await
-                .expect("send command failed");
+        let invalid_onion_error = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
+            let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
+            reply
+                .send(Err(RemoveTlcFail::new(error_detail)))
+                .expect("send error failed");
+        };
 
-            recv.await.expect("recv error").map(|res| res.tlc_id)
-        } else {
+        let Ok(peeled_packet) = PeeledPaymentOnionPacket::deserialize(&packet) else {
             info!("onion packet is empty, ignore it");
-            let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
-            Err(RemoveTlcFail::new(error_detail))
+            return invalid_onion_error(reply);
+        };
+
+        let info = peeled_packet.current;
+        debug!("Processing onion packet info: {:?}", info);
+
+        let Some(channel_outpoint) = &info.channel_outpoint else {
+            return invalid_onion_error(reply);
+        };
+
+        let unknown_next_peer = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
+            let error_detail = TlcFailDetail::new_channel_fail(
+                TlcFailErrorCode::UnknownNextPeer,
+                channel_outpoint.clone(),
+                None,
+            );
+            reply
+                .send(Err(RemoveTlcFail::new(error_detail)))
+                .expect("send add tlc response");
         };
-        reply.send(res).expect("send error");
+
+        let channel_id = match state.outpoint_channel_map.get(channel_outpoint) {
+            Some(channel_id) => channel_id,
+            None => {
+                error!(
+                        "Channel id not found in outpoint_channel_map with {:?}, are we connected to the peer?",
+                        channel_outpoint
+                    );
+                return unknown_next_peer(reply);
+            }
+        };
+        let (send, recv) = oneshot::channel::<Result<AddTlcResponse, RemoveTlcFail>>();
+        let rpc_reply = RpcReplyPort::from(send);
+        let command = ChannelCommand::AddTlc(
+            AddTlcCommand {
+                amount: info.amount,
+                preimage: None,
+                payment_hash: Some(info.payment_hash),
+                expiry: info.expiry.into(),
+                hash_algorithm: info.tlc_hash_algorithm,
+                onion_packet: peeled_packet.next.map(|next| next.data).unwrap_or_default(),
+                previous_tlc,
+            },
+            rpc_reply,
+        );
+
+        // we have already checked the channel_id is valid,
+        match state.send_command_to_channel(*channel_id, command).await {
+            Ok(()) => {}
+            Err(Error::ChannelNotFound(_)) => {
+                return unknown_next_peer(reply);
+            }
+            Err(err) => {
+                // must be some error fron tentacle, set it as temporary node failure
+                error!(
+                    "Failed to send onion packet to channel: {:?} with err: {:?}",
+                    channel_id, err
+                );
+                let error_detail = TlcFailDetail::new(TlcFailErrorCode::TemporaryNodeFailure);
+                return reply
+                    .send(Err(RemoveTlcFail::new(error_detail)))
+                    .expect("send add tlc response");
+            }
+        }
+        let add_tlc_res = recv.await.expect("recv error").map(|res| res.tlc_id);
+        reply.send(add_tlc_res).expect("send error");
     }
 
     async fn on_tlc_remove_received(
@@ -2377,7 +2399,8 @@ pub struct NetworkActorState<S> {
     peer_pubkey_map: HashMap<PeerId, Pubkey>,
     session_channels_map: HashMap<SessionId, HashSet<Hash256>>,
     channels: HashMap<Hash256, ActorRef<ChannelActorMessage>>,
-    // Outpoint to channel id mapping.
+    // Outpoint to channel id mapping, only contains channels with state of Ready.
+    // We need to remove the channel from this map when the channel is closed or peer disconnected.
     outpoint_channel_map: HashMap<OutPoint, Hash256>,
     // Channels in this hashmap are pending for acceptance. The user needs to
     // issue an AcceptChannelCommand with the amount of funding to accept the channel.
@@ -3028,11 +3051,27 @@ where
         self.maybe_sync_network_graph(remote_peer_id).await;
     }
 
+    fn remove_channel(&mut self, channel_id: &Hash256) -> Option<ActorRef<ChannelActorMessage>> {
+        self.channels.remove(channel_id).map(|channel| {
+            if let Some(outpoint) = self.outpoint_channel_map.iter().find_map(|(k, v)| {
+                if v == channel_id {
+                    Some(k.clone())
+                } else {
+                    None
+                }
+            }) {
+                self.outpoint_channel_map.remove(&outpoint);
+            }
+            channel
+        })
+    }
+
     fn on_peer_disconnected(&mut self, id: &PeerId) {
+        info!("Peer {:?} disconnected", id);
         if let Some(session) = self.peer_session_map.remove(id) {
             if let Some(channel_ids) = self.session_channels_map.remove(&session) {
                 for channel_id in channel_ids {
-                    if let Some(channel) = self.channels.remove(&channel_id) {
+                    if let Some(channel) = self.remove_channel(&channel_id) {
                         let _ = channel.send_message(ChannelActorMessage::Event(
                             ChannelEvent::PeerDisconnected,
                         ));
@@ -3169,7 +3208,7 @@ where
         channel_id: &Hash256,
         tx_hash: Byte32,
     ) {
-        self.channels.remove(channel_id);
+        self.remove_channel(channel_id);
         if let Some(session) = self.get_peer_session(peer_id) {
             if let Some(set) = self.session_channels_map.get_mut(&session) {
                 set.remove(channel_id);

From f231e896f3bd2a1ecd1c2a2eb2f4efe5cec6429c Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 10:07:35 +0800
Subject: [PATCH 42/66] fix the graph lock issue in update_with_tcl_fail_detail

---
 src/fiber/network.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index c942070c..fd300ca7 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -2103,10 +2103,13 @@ where
             }
         }
         match tcl_error_detail.error_code() {
-            TlcFailErrorCode::PermanentChannelFailure | TlcFailErrorCode::ChannelDisabled => {
+            TlcFailErrorCode::PermanentChannelFailure
+            | TlcFailErrorCode::ChannelDisabled
+            | TlcFailErrorCode::UnknownNextPeer => {
                 let channel_outpoint = tcl_error_detail
                     .error_channel_outpoint()
                     .expect("expect channel outpoint");
+                debug!("mark channel failed: {:?}", channel_outpoint);
                 let mut graph = self.network_graph.write().await;
                 graph.mark_channel_failed(&channel_outpoint);
             }
@@ -2134,11 +2137,11 @@ where
         state: &mut NetworkActorState<S>,
         mut payment_session: PaymentSession,
     ) -> Result<PaymentSession, Error> {
-        let graph = self.network_graph.read().await;
         let payment_data = payment_session.request.clone();
         let mut error = None;
         while payment_session.can_retry() {
             payment_session.retried_times += 1;
+            let graph = self.network_graph.read().await;
             let hops_infos = match graph.build_route(payment_data.clone()) {
                 Err(e) => {
                     error!("Failed to build route: {:?}", e);
@@ -2183,6 +2186,8 @@ where
                         error_detail.error_code_as_str()
                     );
                     error = Some(err);
+                    // drop the graph lock so that `update_with_tcl_fail_detail` can acquire it
+                    drop(graph);
                     self.update_with_tcl_fail_detail(&error_detail).await;
                     continue;
                 }

From c1dab4560e86680ebf8d3bcbf4db9527589d0932 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 10:44:07 +0800
Subject: [PATCH 43/66] tlc error struct and enum rename

---
 src/fiber/channel.rs     | 30 ++++++++++-----------
 src/fiber/network.rs     | 28 +++++++++----------
 src/fiber/tests/types.rs | 12 ++++-----
 src/fiber/types.rs       | 58 ++++++++++++++++++++--------------------
 4 files changed, 62 insertions(+), 66 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index b2df6574..e37baa49 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -7,7 +7,7 @@ use crate::{
     fiber::{
         fee::calculate_tlc_forward_fee,
         network::{get_chain_hash, SendOnionPacketCommand},
-        types::{ChannelUpdate, RemoveTlcFail, TlcFailDetail, TlcFailErrorCode},
+        types::{ChannelUpdate, RemoveTlcFail, TlcErr, TlcErrorCode},
     },
     invoice::InvoiceStore,
 };
@@ -679,40 +679,38 @@ where
         &self,
         state: &mut ChannelActorState,
         error: &ProcessingChannelError,
-    ) -> TlcFailDetail {
+    ) -> TlcErr {
         let error_code = match error {
-            ProcessingChannelError::PeelingOnionPacketError(_) => {
-                TlcFailErrorCode::InvalidOnionPayload
-            }
-            ProcessingChannelError::TlcForwardFeeIsTooLow => TlcFailErrorCode::FeeInsufficient,
+            ProcessingChannelError::PeelingOnionPacketError(_) => TlcErrorCode::InvalidOnionPayload,
+            ProcessingChannelError::TlcForwardFeeIsTooLow => TlcErrorCode::FeeInsufficient,
             ProcessingChannelError::FinalIncorrectPreimage
             | ProcessingChannelError::FinalIncorrectPaymentHash => {
-                TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
+                TlcErrorCode::IncorrectOrUnknownPaymentDetails
             }
             ProcessingChannelError::FinalIncorrectHTLCAmount => {
-                TlcFailErrorCode::FinalIncorrectHtlcAmount
+                TlcErrorCode::FinalIncorrectHtlcAmount
             }
-            ProcessingChannelError::TlcAmountIsTooLow => TlcFailErrorCode::AmountBelowMinimum,
+            ProcessingChannelError::TlcAmountIsTooLow => TlcErrorCode::AmountBelowMinimum,
             ProcessingChannelError::TlcNumberExceedLimit
             | ProcessingChannelError::TlcValueInflightExceedLimit => {
-                TlcFailErrorCode::TemporaryChannelFailure
+                TlcErrorCode::TemporaryChannelFailure
             }
             ProcessingChannelError::InvalidState(_) => match state.state {
-                ChannelState::Closed(_) => TlcFailErrorCode::PermanentChannelFailure,
+                ChannelState::Closed(_) => TlcErrorCode::PermanentChannelFailure,
                 // ShuttingDown is a temporary state
-                ChannelState::ShuttingDown(_) => TlcFailErrorCode::TemporaryChannelFailure,
+                ChannelState::ShuttingDown(_) => TlcErrorCode::TemporaryChannelFailure,
                 ChannelState::ChannelReady() => {
                     // we expect `ChannelReady` will be both OK for tlc forwarding,
                     // so here are the unreachable point in normal workflow,
                     // set `TemporaryNodeFailure` for general temporary failure of the processing node here
                     assert!(false, "unreachable point in normal workflow");
-                    TlcFailErrorCode::TemporaryNodeFailure
+                    TlcErrorCode::TemporaryNodeFailure
                 }
                 // otherwise, channel maybe not ready
-                _ => TlcFailErrorCode::TemporaryChannelFailure,
+                _ => TlcErrorCode::TemporaryChannelFailure,
             },
             // TODO: there maybe more error types here
-            _ => TlcFailErrorCode::IncorrectOrUnknownPaymentDetails,
+            _ => TlcErrorCode::IncorrectOrUnknownPaymentDetails,
         };
 
         let channel_update = if error_code.is_update() {
@@ -720,7 +718,7 @@ where
         } else {
             None
         };
-        TlcFailDetail::new_channel_fail(
+        TlcErr::new_channel_fail(
             error_code,
             state.get_funding_transaction_outpoint(),
             channel_update,
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index fd300ca7..db1fadc9 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -60,7 +60,7 @@ use super::types::{
     NodeAnnouncement, NodeAnnouncementQuery, OpenChannel, Privkey, Pubkey,
     QueryBroadcastMessagesWithinTimeRange, QueryBroadcastMessagesWithinTimeRangeResult,
     QueryChannelsWithinBlockRange, QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcFail,
-    RemoveTlcReason, TlcFailDetail, TlcFailDetailData, TlcFailErrorCode,
+    RemoveTlcReason, TlcErr, TlcErrData, TlcErrorCode,
 };
 use super::FiberConfig;
 
@@ -1977,7 +1977,7 @@ where
         } = command;
 
         let invalid_onion_error = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
-            let error_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionPayload);
+            let error_detail = TlcErr::new(TlcErrorCode::InvalidOnionPayload);
             reply
                 .send(Err(RemoveTlcFail::new(error_detail)))
                 .expect("send error failed");
@@ -1996,8 +1996,8 @@ where
         };
 
         let unknown_next_peer = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
-            let error_detail = TlcFailDetail::new_channel_fail(
-                TlcFailErrorCode::UnknownNextPeer,
+            let error_detail = TlcErr::new_channel_fail(
+                TlcErrorCode::UnknownNextPeer,
                 channel_outpoint.clone(),
                 None,
             );
@@ -2043,7 +2043,7 @@ where
                     "Failed to send onion packet to channel: {:?} with err: {:?}",
                     channel_id, err
                 );
-                let error_detail = TlcFailDetail::new(TlcFailErrorCode::TemporaryNodeFailure);
+                let error_detail = TlcErr::new(TlcErrorCode::TemporaryNodeFailure);
                 return reply
                     .send(Err(RemoveTlcFail::new(error_detail)))
                     .expect("send add tlc response");
@@ -2068,7 +2068,7 @@ where
                     }
                     RemoveTlcReason::RemoveTlcFail(reason) => {
                         let detail_error = reason.decode().expect("decoded error");
-                        self.update_with_tcl_fail_detail(&detail_error).await;
+                        self.update_with_tcl_fail(&detail_error).await;
                         if payment_session.can_retry() && !detail_error.error_code.payment_failed()
                         {
                             let res = self.try_payment_session(state, payment_session).await;
@@ -2085,14 +2085,14 @@ where
         }
     }
 
-    async fn update_with_tcl_fail_detail(&self, tcl_error_detail: &TlcFailDetail) {
+    async fn update_with_tcl_fail(&self, tcl_error_detail: &TlcErr) {
         let error_code = tcl_error_detail.error_code();
         // https://github.com/lightning/bolts/blob/master/04-onion-routing.md#rationale-6
         // we now still update the graph, maybe we need to remove it later?
         if error_code.is_update() {
             if let Some(extra_data) = &tcl_error_detail.extra_data {
                 match extra_data {
-                    TlcFailDetailData::ChannelFailed { channel_update, .. } => {
+                    TlcErrData::ChannelFailed { channel_update, .. } => {
                         if let Some(channel_update) = channel_update {
                             let mut graph = self.network_graph.write().await;
                             let _ = graph.process_channel_update(channel_update.clone());
@@ -2103,9 +2103,9 @@ where
             }
         }
         match tcl_error_detail.error_code() {
-            TlcFailErrorCode::PermanentChannelFailure
-            | TlcFailErrorCode::ChannelDisabled
-            | TlcFailErrorCode::UnknownNextPeer => {
+            TlcErrorCode::PermanentChannelFailure
+            | TlcErrorCode::ChannelDisabled
+            | TlcErrorCode::UnknownNextPeer => {
                 let channel_outpoint = tcl_error_detail
                     .error_channel_outpoint()
                     .expect("expect channel outpoint");
@@ -2113,7 +2113,7 @@ where
                 let mut graph = self.network_graph.write().await;
                 graph.mark_channel_failed(&channel_outpoint);
             }
-            TlcFailErrorCode::PermanentNodeFailure => {
+            TlcErrorCode::PermanentNodeFailure => {
                 let node_id = tcl_error_detail.error_node_id().expect("expect node id");
                 let mut graph = self.network_graph.write().await;
                 graph.mark_node_failed(node_id);
@@ -2186,9 +2186,9 @@ where
                         error_detail.error_code_as_str()
                     );
                     error = Some(err);
-                    // drop the graph lock so that `update_with_tcl_fail_detail` can acquire it
+                    // drop the graph lock so that `update_with_tcl_fail` can acquire it
                     drop(graph);
-                    self.update_with_tcl_fail_detail(&error_detail).await;
+                    self.update_with_tcl_fail(&error_detail).await;
                     continue;
                 }
                 Ok(tlc_id) => {
diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 0c6233c1..9d78ddd8 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -4,7 +4,7 @@ use crate::fiber::{
     tests::test_utils::generate_pubkey,
     types::{
         secp256k1_instance, AddTlc, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey,
-        RemoveTlcFail, TlcFailDetail, TlcFailErrorCode,
+        RemoveTlcFail, TlcErr, TlcErrorCode,
     },
 };
 use ckb_types::packed::OutPointBuilder;
@@ -100,19 +100,17 @@ fn test_peeled_onion_packet() {
 
 #[test]
 fn test_tlc_fail_error() {
-    let tlc_fail_detail = TlcFailDetail::new(TlcFailErrorCode::InvalidOnionVersion);
+    let tlc_fail_detail = TlcErr::new(TlcErrorCode::InvalidOnionVersion);
     assert!(!tlc_fail_detail.error_code.is_node());
     assert!(tlc_fail_detail.error_code.is_bad_onion());
     assert!(tlc_fail_detail.error_code.is_perm());
     let tlc_fail = RemoveTlcFail::new(tlc_fail_detail.clone());
 
-    let convert_back: TlcFailDetail = tlc_fail.decode().expect("decoded fail");
+    let convert_back: TlcErr = tlc_fail.decode().expect("decoded fail");
     assert_eq!(tlc_fail_detail, convert_back);
 
-    let node_fail = TlcFailDetail::new_node_fail(
-        TlcFailErrorCode::PermanentNodeFailure,
-        generate_pubkey().into(),
-    );
+    let node_fail =
+        TlcErr::new_node_fail(TlcErrorCode::PermanentNodeFailure, generate_pubkey().into());
     assert!(node_fail.error_code.is_node());
     let tlc_fail = RemoveTlcFail::new(node_fail.clone());
     let convert_back = tlc_fail.decode().expect("decoded fail");
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 755cdf3a..ff0a1327 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1257,7 +1257,7 @@ impl TryFrom<molecule_fiber::RemoveTlcFulfill> for RemoveTlcFulfill {
 
 #[serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
-pub enum TlcFailDetailData {
+pub enum TlcErrData {
     ChannelFailed {
         #[serde_as(as = "EntityHex")]
         channel_outpoint: OutPoint,
@@ -1270,34 +1270,34 @@ pub enum TlcFailDetailData {
 
 #[serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
-pub struct TlcFailDetail {
-    pub error_code: TlcFailErrorCode,
-    pub extra_data: Option<TlcFailDetailData>,
+pub struct TlcErr {
+    pub error_code: TlcErrorCode,
+    pub extra_data: Option<TlcErrData>,
 }
 
-impl TlcFailDetail {
-    pub fn new(error_code: TlcFailErrorCode) -> Self {
-        TlcFailDetail {
+impl TlcErr {
+    pub fn new(error_code: TlcErrorCode) -> Self {
+        TlcErr {
             error_code: error_code,
             extra_data: None,
         }
     }
 
-    pub fn new_node_fail(error_code: TlcFailErrorCode, node_id: Pubkey) -> Self {
-        TlcFailDetail {
+    pub fn new_node_fail(error_code: TlcErrorCode, node_id: Pubkey) -> Self {
+        TlcErr {
             error_code: error_code.into(),
-            extra_data: Some(TlcFailDetailData::NodeFailed { node_id }),
+            extra_data: Some(TlcErrData::NodeFailed { node_id }),
         }
     }
 
     pub fn new_channel_fail(
-        error_code: TlcFailErrorCode,
+        error_code: TlcErrorCode,
         channel_outpoint: OutPoint,
         channel_update: Option<ChannelUpdate>,
     ) -> Self {
-        TlcFailDetail {
+        TlcErr {
             error_code: error_code.into(),
-            extra_data: Some(TlcFailDetailData::ChannelFailed {
+            extra_data: Some(TlcErrData::ChannelFailed {
                 channel_outpoint,
                 channel_update,
             }),
@@ -1306,30 +1306,30 @@ impl TlcFailDetail {
 
     pub fn error_node_id(&self) -> Option<Pubkey> {
         match &self.extra_data {
-            Some(TlcFailDetailData::NodeFailed { node_id }) => Some(*node_id),
+            Some(TlcErrData::NodeFailed { node_id }) => Some(*node_id),
             _ => None,
         }
     }
 
     pub fn error_channel_outpoint(&self) -> Option<OutPoint> {
         match &self.extra_data {
-            Some(TlcFailDetailData::ChannelFailed {
+            Some(TlcErrData::ChannelFailed {
                 channel_outpoint, ..
             }) => Some(channel_outpoint.clone()),
             _ => None,
         }
     }
 
-    pub fn error_code(&self) -> TlcFailErrorCode {
+    pub fn error_code(&self) -> TlcErrorCode {
         self.error_code
     }
 
     pub fn error_code_as_str(&self) -> String {
-        let error_code: TlcFailErrorCode = self.error_code.into();
+        let error_code: TlcErrorCode = self.error_code.into();
         error_code.as_ref().to_string()
     }
 
-    pub fn set_extra_data(&mut self, extra_data: TlcFailDetailData) {
+    pub fn set_extra_data(&mut self, extra_data: TlcErrData) {
         self.extra_data = Some(extra_data);
     }
 
@@ -1343,8 +1343,8 @@ impl TlcFailDetail {
 }
 
 // This is the onion packet we need to encode and send back to the sender
-// currently it's the raw TlcFailDetail serialized data from the TlcFailDetail struct
-// sender should decode it and get the TlcFailDetail, then decide what to do
+// currently it's the raw TlcErr serialized data from the TlcErr struct
+// sender should decode it and get the TlcErr, then decide what to do
 // Note: this supposed to be only accessible by the sender, and it's not reliable since it
 // is not placed on-chain due to the possibility of hop failure.
 //
@@ -1357,14 +1357,14 @@ pub struct RemoveTlcFail {
 }
 
 impl RemoveTlcFail {
-    pub fn new(tlc_fail: TlcFailDetail) -> Self {
+    pub fn new(tlc_fail: TlcErr) -> Self {
         RemoveTlcFail {
             onion_packet: tlc_fail.serialize(),
         }
     }
 
-    pub fn decode(&self) -> Option<TlcFailDetail> {
-        TlcFailDetail::deserialize(&self.onion_packet)
+    pub fn decode(&self) -> Option<TlcErr> {
+        TlcErr::deserialize(&self.onion_packet)
     }
 }
 
@@ -1403,7 +1403,7 @@ const UPDATE: u16 = 0x1000;
 
 #[repr(u16)]
 #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, AsRefStr, IntoStaticStr)]
-pub enum TlcFailErrorCode {
+pub enum TlcErrorCode {
     TemporaryNodeFailure = NODE | 2,
     PermanentNodeFailure = PERM | NODE | 2,
     // unused right now
@@ -1432,7 +1432,7 @@ pub enum TlcFailErrorCode {
     InvalidOnionBlinding = BADONION | PERM | 24,
 }
 
-impl TlcFailErrorCode {
+impl TlcErrorCode {
     pub fn is_node(&self) -> bool {
         *self as u16 & NODE != 0
     }
@@ -1451,10 +1451,10 @@ impl TlcFailErrorCode {
 
     pub fn payment_failed(&self) -> bool {
         match self {
-            TlcFailErrorCode::IncorrectOrUnknownPaymentDetails
-            | TlcFailErrorCode::FinalIncorrectCltvExpiry
-            | TlcFailErrorCode::FinalIncorrectHtlcAmount
-            | TlcFailErrorCode::MppTimeout => true,
+            TlcErrorCode::IncorrectOrUnknownPaymentDetails
+            | TlcErrorCode::FinalIncorrectCltvExpiry
+            | TlcErrorCode::FinalIncorrectHtlcAmount
+            | TlcErrorCode::MppTimeout => true,
             _ => false,
         }
     }

From 53b4d38199c9dc6151892b650cb26e7625c991f7 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 12:26:26 +0800
Subject: [PATCH 44/66] fix cross-chain-hub spec for unstable issue

---
 tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru b/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru
index 1228fc99..974fa4f6 100644
--- a/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru
+++ b/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru
@@ -14,6 +14,11 @@ assert {
   res.body.channels: length 0
 }
 
+script:pre-request {
+  // Sleep for sometime to make sure current operation finishes before next request starts.
+  await new Promise(r => setTimeout(r, 1000));
+}
+
 script:post-response {
   console.log("list channel result: ", res.body);
 }
\ No newline at end of file

From 8cfb17ff0d45eac4649e5612995512fc79cd322b Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 17:03:02 +0800
Subject: [PATCH 45/66] code cleanup

---
 src/fiber/channel.rs |  6 ++----
 src/fiber/graph.rs   |  8 ++------
 src/fiber/network.rs | 43 ++++++++++++++++++++++---------------------
 src/fiber/types.rs   | 11 +++++------
 src/rpc/channel.rs   |  8 ++------
 5 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index e37baa49..2936d246 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -488,9 +488,9 @@ where
                         // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
                         // trying to add a duplicate tlc, so we use tlc count to make sure no new tlc was added
                         // and only send RemoveTlc message to peer if the TLC is not in our state
+                        assert!(tlc_count == state.tlcs.len());
                         error!("Error handling AddTlc message: {:?}", e);
                         let error_detail = self.get_tlc_detail_error(state, &e).await;
-                        assert!(tlc_count == state.tlcs.len());
                         if state.get_received_tlc(tlc_id).is_none() {
                             self.network
                                 .send_message(NetworkActorMessage::new_command(
@@ -893,9 +893,7 @@ where
                         command: ChannelCommand::RemoveTlc(
                             RemoveTlcCommand {
                                 id: added_tlc_id,
-                                reason: RemoveTlcReason::RemoveTlcFail(RemoveTlcFail {
-                                    onion_packet: vec![],
-                                }),
+                                reason: RemoveTlcReason::RemoveTlcFail(res),
                             },
                             port,
                         ),
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 9fca5eaf..0dfeb5c2 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -652,15 +652,11 @@ where
             next_hop: None,
             incoming_cltv_height: 0,
         });
-        loop {
-            nodes_visited += 1;
-            let Some(cur_hop) = nodes_heap.pop() else {
-                break;
-            };
-
+        while let Some(cur_hop) = nodes_heap.pop() {
             if cur_hop.node_id == source {
                 break;
             }
+            nodes_visited += 1;
 
             for (from, channel_info, channel_update) in self.get_node_inbounds(cur_hop.node_id) {
                 edges_expanded += 1;
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index db1fadc9..0f302263 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1270,7 +1270,7 @@ where
 
             // TODO: we should check the OnionPacket is valid or not, only the current node can decrypt it.
             NetworkActorCommand::SendPaymentOnionPacket(command, reply) => {
-                self.handle_onion_packet_send_command(state, command, reply)
+                self.handle_send_onion_packet_command(state, command, reply)
                     .await;
             }
             NetworkActorCommand::PeelPaymentOnionPacket(onion_packet, payment_hash, reply) => {
@@ -1965,7 +1965,7 @@ where
         }
     }
 
-    async fn handle_onion_packet_send_command(
+    async fn handle_send_onion_packet_command(
         &self,
         state: &mut NetworkActorState<S>,
         command: SendOnionPacketCommand,
@@ -2138,17 +2138,19 @@ where
         mut payment_session: PaymentSession,
     ) -> Result<PaymentSession, Error> {
         let payment_data = payment_session.request.clone();
+        let payment_hash = payment_data.payment_hash;
         let mut error = None;
         while payment_session.can_retry() {
             payment_session.retried_times += 1;
-            let graph = self.network_graph.read().await;
-            let hops_infos = match graph.build_route(payment_data.clone()) {
+            let hops_infos = match self
+                .network_graph
+                .read()
+                .await
+                .build_route(payment_data.clone())
+            {
                 Err(e) => {
                     error!("Failed to build route: {:?}", e);
-                    error = Some(format!(
-                        "Failed to build route: {:?}",
-                        payment_data.payment_hash
-                    ));
+                    error = Some(format!("Failed to build route: {:?}", payment_hash));
                     break;
                 }
                 Ok(onion_path) => onion_path,
@@ -2173,22 +2175,21 @@ where
                 packet: peeled_packet.serialize(),
                 previous_tlc: None,
             };
-            self.handle_onion_packet_send_command(state, command, rpc_reply)
+            self.handle_send_onion_packet_command(state, command, rpc_reply)
                 .await;
             match recv.await.expect("msg recv error") {
                 Err(e) => {
-                    let error_detail = e.decode().expect("decoded error");
-                    error!("Failed to send onion packet with error: {:?}", e);
-                    // This is the error implies we send payment request to the first hop failed
-                    let err = format!(
-                        "Failed to send onion packet: {:?} with error: {:?}",
-                        payment_data.payment_hash,
-                        error_detail.error_code_as_str()
-                    );
-                    error = Some(err);
-                    // drop the graph lock so that `update_with_tcl_fail` can acquire it
-                    drop(graph);
-                    self.update_with_tcl_fail(&error_detail).await;
+                    if let Some(error_detail) = e.decode() {
+                        error!("Failed to send onion packet with error: {:?}", e);
+                        // This is the error implies we send payment request to the first hop failed
+                        let err = format!(
+                            "Failed to send onion packet: {:?} with error {:?}",
+                            payment_hash,
+                            error_detail.error_code_as_str()
+                        );
+                        error = Some(err);
+                        self.update_with_tcl_fail(&error_detail).await;
+                    }
                     continue;
                 }
                 Ok(tlc_id) => {
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index ff0a1327..75f88b5c 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1342,14 +1342,13 @@ impl TlcErr {
     }
 }
 
-// This is the onion packet we need to encode and send back to the sender
-// currently it's the raw TlcErr serialized data from the TlcErr struct
-// sender should decode it and get the TlcErr, then decide what to do
+// This is the onion packet we need to encode and send back to the sender,
+// currently it's the raw TlcErr serialized data from the TlcErr struct,
+// sender should decode it and get the TlcErr, then decide what to do with the error
 // Note: this supposed to be only accessible by the sender, and it's not reliable since it
-// is not placed on-chain due to the possibility of hop failure.
+//       is not placed on-chain due to the possibility of hop failure.
 //
-// FIXME: a better name like `TlcFailOnionPacket`?, since it's not only caused by removing tlc now,
-//        maybe we will replace it later.
+// FIXME: a better name like `TlcFailOnionPacket`?, since it's not only caused by removing tlc now
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct RemoveTlcFail {
     // TODO: replace this with the real onion packet
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index 5472ce2a..64f7c4ce 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -7,7 +7,7 @@ use crate::fiber::{
     hash_algorithm::HashAlgorithm,
     network::{AcceptChannelCommand, OpenChannelCommand, SendPaymentCommand},
     serde_utils::{U128Hex, U16Hex, U64Hex},
-    types::{Hash256, LockTime, Pubkey, RemoveTlcFail, RemoveTlcFulfill},
+    types::{Hash256, LockTime, Pubkey, RemoveTlcFulfill},
     NetworkActorCommand, NetworkActorMessage,
 };
 use crate::{handle_actor_call, handle_actor_cast, log_and_error};
@@ -438,11 +438,7 @@ where
                                 }
                                 RemoveTlcReason::RemoveTlcFail { error_code: _ } => {
                                     // TODO: maybe we should remove this PRC or move add_tlc and remove_tlc to `test` module?
-                                    crate::fiber::types::RemoveTlcReason::RemoveTlcFail(
-                                        RemoveTlcFail {
-                                            onion_packet: vec![],
-                                        },
-                                    )
+                                    unimplemented!("RemoveTlcFail is only for internal use");
                                 }
                             },
                         },

From 0240ceaa209032d18324d767e7d2c6978a436bc3 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 17:52:33 +0800
Subject: [PATCH 46/66] rename RemoveTlcFail to TlcErrPacket

---
 src/fiber/channel.rs       |  10 ++--
 src/fiber/gen/fiber.rs     | 114 ++++++++++++++++++-------------------
 src/fiber/network.rs       |  22 +++----
 src/fiber/schema/fiber.mol |   4 +-
 src/fiber/tests/types.rs   |   8 +--
 src/fiber/types.rs         |  32 +++++------
 6 files changed, 94 insertions(+), 96 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 2936d246..4b416942 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -7,7 +7,7 @@ use crate::{
     fiber::{
         fee::calculate_tlc_forward_fee,
         network::{get_chain_hash, SendOnionPacketCommand},
-        types::{ChannelUpdate, RemoveTlcFail, TlcErr, TlcErrorCode},
+        types::{ChannelUpdate, TlcErr, TlcErrPacket, TlcErrorCode},
     },
     invoice::InvoiceStore,
 };
@@ -120,7 +120,7 @@ pub enum ChannelCommand {
     CommitmentSigned(),
     AddTlc(
         AddTlcCommand,
-        RpcReplyPort<Result<AddTlcResponse, RemoveTlcFail>>,
+        RpcReplyPort<Result<AddTlcResponse, TlcErrPacket>>,
     ),
     RemoveTlc(RemoveTlcCommand, RpcReplyPort<Result<(), String>>),
     Shutdown(ShutdownCommand, RpcReplyPort<Result<(), String>>),
@@ -501,7 +501,7 @@ where
                                                 channel_id: state.get_id(),
                                                 tlc_id,
                                                 reason: RemoveTlcReason::RemoveTlcFail(
-                                                    RemoveTlcFail::new(error_detail),
+                                                    TlcErrPacket::new(error_detail),
                                                 ),
                                             }),
                                         ),
@@ -864,7 +864,7 @@ where
         onion_packet: Vec<u8>,
         added_tlc_id: u64,
     ) -> Result<(), ProcessingChannelError> {
-        let (send, recv) = oneshot::channel::<Result<u64, RemoveTlcFail>>();
+        let (send, recv) = oneshot::channel::<Result<u64, TlcErrPacket>>();
         let rpc_reply = RpcReplyPort::from(send);
         self.network
             .send_message(NetworkActorMessage::Command(
@@ -1350,7 +1350,7 @@ where
                     }
                     Err(err) => {
                         let error_detail = self.get_tlc_detail_error(state, &err).await;
-                        let _ = reply.send(Err(RemoveTlcFail::new(error_detail)));
+                        let _ = reply.send(Err(TlcErrPacket::new(error_detail)));
                         Err(err)
                     }
                 }
diff --git a/src/fiber/gen/fiber.rs b/src/fiber/gen/fiber.rs
index 3949735d..42b0aa20 100644
--- a/src/fiber/gen/fiber.rs
+++ b/src/fiber/gen/fiber.rs
@@ -9496,8 +9496,8 @@ impl molecule::prelude::Builder for RemoveTlcFulfillBuilder {
     }
 }
 #[derive(Clone)]
-pub struct RemoveTlcFail(molecule::bytes::Bytes);
-impl ::core::fmt::LowerHex for RemoveTlcFail {
+pub struct TlcErrPacket(molecule::bytes::Bytes);
+impl ::core::fmt::LowerHex for TlcErrPacket {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         use molecule::hex_string;
         if f.alternate() {
@@ -9506,12 +9506,12 @@ impl ::core::fmt::LowerHex for RemoveTlcFail {
         write!(f, "{}", hex_string(self.as_slice()))
     }
 }
-impl ::core::fmt::Debug for RemoveTlcFail {
+impl ::core::fmt::Debug for TlcErrPacket {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{}({:#x})", Self::NAME, self)
     }
 }
-impl ::core::fmt::Display for RemoveTlcFail {
+impl ::core::fmt::Display for TlcErrPacket {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
         write!(f, "{}: {}", "onion_packet", self.onion_packet())?;
@@ -9522,13 +9522,13 @@ impl ::core::fmt::Display for RemoveTlcFail {
         write!(f, " }}")
     }
 }
-impl ::core::default::Default for RemoveTlcFail {
+impl ::core::default::Default for TlcErrPacket {
     fn default() -> Self {
         let v = molecule::bytes::Bytes::from_static(&Self::DEFAULT_VALUE);
-        RemoveTlcFail::new_unchecked(v)
+        TlcErrPacket::new_unchecked(v)
     }
 }
-impl RemoveTlcFail {
+impl TlcErrPacket {
     const DEFAULT_VALUE: [u8; 12] = [12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0];
     pub const FIELD_COUNT: usize = 1;
     pub fn total_size(&self) -> usize {
@@ -9557,15 +9557,15 @@ impl RemoveTlcFail {
             Bytes::new_unchecked(self.0.slice(start..))
         }
     }
-    pub fn as_reader<'r>(&'r self) -> RemoveTlcFailReader<'r> {
-        RemoveTlcFailReader::new_unchecked(self.as_slice())
+    pub fn as_reader<'r>(&'r self) -> TlcErrPacketReader<'r> {
+        TlcErrPacketReader::new_unchecked(self.as_slice())
     }
 }
-impl molecule::prelude::Entity for RemoveTlcFail {
-    type Builder = RemoveTlcFailBuilder;
-    const NAME: &'static str = "RemoveTlcFail";
+impl molecule::prelude::Entity for TlcErrPacket {
+    type Builder = TlcErrPacketBuilder;
+    const NAME: &'static str = "TlcErrPacket";
     fn new_unchecked(data: molecule::bytes::Bytes) -> Self {
-        RemoveTlcFail(data)
+        TlcErrPacket(data)
     }
     fn as_bytes(&self) -> molecule::bytes::Bytes {
         self.0.clone()
@@ -9574,10 +9574,10 @@ impl molecule::prelude::Entity for RemoveTlcFail {
         &self.0[..]
     }
     fn from_slice(slice: &[u8]) -> molecule::error::VerificationResult<Self> {
-        RemoveTlcFailReader::from_slice(slice).map(|reader| reader.to_entity())
+        TlcErrPacketReader::from_slice(slice).map(|reader| reader.to_entity())
     }
     fn from_compatible_slice(slice: &[u8]) -> molecule::error::VerificationResult<Self> {
-        RemoveTlcFailReader::from_compatible_slice(slice).map(|reader| reader.to_entity())
+        TlcErrPacketReader::from_compatible_slice(slice).map(|reader| reader.to_entity())
     }
     fn new_builder() -> Self::Builder {
         ::core::default::Default::default()
@@ -9587,8 +9587,8 @@ impl molecule::prelude::Entity for RemoveTlcFail {
     }
 }
 #[derive(Clone, Copy)]
-pub struct RemoveTlcFailReader<'r>(&'r [u8]);
-impl<'r> ::core::fmt::LowerHex for RemoveTlcFailReader<'r> {
+pub struct TlcErrPacketReader<'r>(&'r [u8]);
+impl<'r> ::core::fmt::LowerHex for TlcErrPacketReader<'r> {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         use molecule::hex_string;
         if f.alternate() {
@@ -9597,12 +9597,12 @@ impl<'r> ::core::fmt::LowerHex for RemoveTlcFailReader<'r> {
         write!(f, "{}", hex_string(self.as_slice()))
     }
 }
-impl<'r> ::core::fmt::Debug for RemoveTlcFailReader<'r> {
+impl<'r> ::core::fmt::Debug for TlcErrPacketReader<'r> {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{}({:#x})", Self::NAME, self)
     }
 }
-impl<'r> ::core::fmt::Display for RemoveTlcFailReader<'r> {
+impl<'r> ::core::fmt::Display for TlcErrPacketReader<'r> {
     fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         write!(f, "{} {{ ", Self::NAME)?;
         write!(f, "{}: {}", "onion_packet", self.onion_packet())?;
@@ -9613,7 +9613,7 @@ impl<'r> ::core::fmt::Display for RemoveTlcFailReader<'r> {
         write!(f, " }}")
     }
 }
-impl<'r> RemoveTlcFailReader<'r> {
+impl<'r> TlcErrPacketReader<'r> {
     pub const FIELD_COUNT: usize = 1;
     pub fn total_size(&self) -> usize {
         molecule::unpack_number(self.as_slice()) as usize
@@ -9642,14 +9642,14 @@ impl<'r> RemoveTlcFailReader<'r> {
         }
     }
 }
-impl<'r> molecule::prelude::Reader<'r> for RemoveTlcFailReader<'r> {
-    type Entity = RemoveTlcFail;
-    const NAME: &'static str = "RemoveTlcFailReader";
+impl<'r> molecule::prelude::Reader<'r> for TlcErrPacketReader<'r> {
+    type Entity = TlcErrPacket;
+    const NAME: &'static str = "TlcErrPacketReader";
     fn to_entity(&self) -> Self::Entity {
         Self::Entity::new_unchecked(self.as_slice().to_owned().into())
     }
     fn new_unchecked(slice: &'r [u8]) -> Self {
-        RemoveTlcFailReader(slice)
+        TlcErrPacketReader(slice)
     }
     fn as_slice(&self) -> &'r [u8] {
         self.0
@@ -9693,19 +9693,19 @@ impl<'r> molecule::prelude::Reader<'r> for RemoveTlcFailReader<'r> {
     }
 }
 #[derive(Clone, Debug, Default)]
-pub struct RemoveTlcFailBuilder {
+pub struct TlcErrPacketBuilder {
     pub(crate) onion_packet: Bytes,
 }
-impl RemoveTlcFailBuilder {
+impl TlcErrPacketBuilder {
     pub const FIELD_COUNT: usize = 1;
     pub fn onion_packet(mut self, v: Bytes) -> Self {
         self.onion_packet = v;
         self
     }
 }
-impl molecule::prelude::Builder for RemoveTlcFailBuilder {
-    type Entity = RemoveTlcFail;
-    const NAME: &'static str = "RemoveTlcFailBuilder";
+impl molecule::prelude::Builder for TlcErrPacketBuilder {
+    type Entity = TlcErrPacket;
+    const NAME: &'static str = "TlcErrPacketBuilder";
     fn expected_length(&self) -> usize {
         molecule::NUMBER_SIZE * (Self::FIELD_COUNT + 1) + self.onion_packet.as_slice().len()
     }
@@ -9725,7 +9725,7 @@ impl molecule::prelude::Builder for RemoveTlcFailBuilder {
         let mut inner = Vec::with_capacity(self.expected_length());
         self.write(&mut inner)
             .unwrap_or_else(|_| panic!("{} build should be ok", Self::NAME));
-        RemoveTlcFail::new_unchecked(inner.into())
+        TlcErrPacket::new_unchecked(inner.into())
     }
 }
 #[derive(Clone)]
@@ -9770,7 +9770,7 @@ impl RemoveTlcReason {
         let inner = self.0.slice(molecule::NUMBER_SIZE..);
         match self.item_id() {
             0 => RemoveTlcFulfill::new_unchecked(inner).into(),
-            1 => RemoveTlcFail::new_unchecked(inner).into(),
+            1 => TlcErrPacket::new_unchecked(inner).into(),
             _ => panic!("{}: invalid data", Self::NAME),
         }
     }
@@ -9835,7 +9835,7 @@ impl<'r> RemoveTlcReasonReader<'r> {
         let inner = &self.as_slice()[molecule::NUMBER_SIZE..];
         match self.item_id() {
             0 => RemoveTlcFulfillReader::new_unchecked(inner).into(),
-            1 => RemoveTlcFailReader::new_unchecked(inner).into(),
+            1 => TlcErrPacketReader::new_unchecked(inner).into(),
             _ => panic!("{}: invalid data", Self::NAME),
         }
     }
@@ -9862,7 +9862,7 @@ impl<'r> molecule::prelude::Reader<'r> for RemoveTlcReasonReader<'r> {
         let inner_slice = &slice[molecule::NUMBER_SIZE..];
         match item_id {
             0 => RemoveTlcFulfillReader::verify(inner_slice, compatible),
-            1 => RemoveTlcFailReader::verify(inner_slice, compatible),
+            1 => TlcErrPacketReader::verify(inner_slice, compatible),
             _ => ve!(Self, UnknownItem, Self::ITEMS_COUNT, item_id),
         }?;
         Ok(())
@@ -9900,12 +9900,12 @@ impl molecule::prelude::Builder for RemoveTlcReasonBuilder {
 #[derive(Debug, Clone)]
 pub enum RemoveTlcReasonUnion {
     RemoveTlcFulfill(RemoveTlcFulfill),
-    RemoveTlcFail(RemoveTlcFail),
+    TlcErrPacket(TlcErrPacket),
 }
 #[derive(Debug, Clone, Copy)]
 pub enum RemoveTlcReasonUnionReader<'r> {
     RemoveTlcFulfill(RemoveTlcFulfillReader<'r>),
-    RemoveTlcFail(RemoveTlcFailReader<'r>),
+    TlcErrPacket(TlcErrPacketReader<'r>),
 }
 impl ::core::default::Default for RemoveTlcReasonUnion {
     fn default() -> Self {
@@ -9918,8 +9918,8 @@ impl ::core::fmt::Display for RemoveTlcReasonUnion {
             RemoveTlcReasonUnion::RemoveTlcFulfill(ref item) => {
                 write!(f, "{}::{}({})", Self::NAME, RemoveTlcFulfill::NAME, item)
             }
-            RemoveTlcReasonUnion::RemoveTlcFail(ref item) => {
-                write!(f, "{}::{}({})", Self::NAME, RemoveTlcFail::NAME, item)
+            RemoveTlcReasonUnion::TlcErrPacket(ref item) => {
+                write!(f, "{}::{}({})", Self::NAME, TlcErrPacket::NAME, item)
             }
         }
     }
@@ -9930,8 +9930,8 @@ impl<'r> ::core::fmt::Display for RemoveTlcReasonUnionReader<'r> {
             RemoveTlcReasonUnionReader::RemoveTlcFulfill(ref item) => {
                 write!(f, "{}::{}({})", Self::NAME, RemoveTlcFulfill::NAME, item)
             }
-            RemoveTlcReasonUnionReader::RemoveTlcFail(ref item) => {
-                write!(f, "{}::{}({})", Self::NAME, RemoveTlcFail::NAME, item)
+            RemoveTlcReasonUnionReader::TlcErrPacket(ref item) => {
+                write!(f, "{}::{}({})", Self::NAME, TlcErrPacket::NAME, item)
             }
         }
     }
@@ -9940,7 +9940,7 @@ impl RemoveTlcReasonUnion {
     pub(crate) fn display_inner(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(ref item) => write!(f, "{}", item),
-            RemoveTlcReasonUnion::RemoveTlcFail(ref item) => write!(f, "{}", item),
+            RemoveTlcReasonUnion::TlcErrPacket(ref item) => write!(f, "{}", item),
         }
     }
 }
@@ -9948,7 +9948,7 @@ impl<'r> RemoveTlcReasonUnionReader<'r> {
     pub(crate) fn display_inner(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
         match self {
             RemoveTlcReasonUnionReader::RemoveTlcFulfill(ref item) => write!(f, "{}", item),
-            RemoveTlcReasonUnionReader::RemoveTlcFail(ref item) => write!(f, "{}", item),
+            RemoveTlcReasonUnionReader::TlcErrPacket(ref item) => write!(f, "{}", item),
         }
     }
 }
@@ -9957,9 +9957,9 @@ impl ::core::convert::From<RemoveTlcFulfill> for RemoveTlcReasonUnion {
         RemoveTlcReasonUnion::RemoveTlcFulfill(item)
     }
 }
-impl ::core::convert::From<RemoveTlcFail> for RemoveTlcReasonUnion {
-    fn from(item: RemoveTlcFail) -> Self {
-        RemoveTlcReasonUnion::RemoveTlcFail(item)
+impl ::core::convert::From<TlcErrPacket> for RemoveTlcReasonUnion {
+    fn from(item: TlcErrPacket) -> Self {
+        RemoveTlcReasonUnion::TlcErrPacket(item)
     }
 }
 impl<'r> ::core::convert::From<RemoveTlcFulfillReader<'r>> for RemoveTlcReasonUnionReader<'r> {
@@ -9967,9 +9967,9 @@ impl<'r> ::core::convert::From<RemoveTlcFulfillReader<'r>> for RemoveTlcReasonUn
         RemoveTlcReasonUnionReader::RemoveTlcFulfill(item)
     }
 }
-impl<'r> ::core::convert::From<RemoveTlcFailReader<'r>> for RemoveTlcReasonUnionReader<'r> {
-    fn from(item: RemoveTlcFailReader<'r>) -> Self {
-        RemoveTlcReasonUnionReader::RemoveTlcFail(item)
+impl<'r> ::core::convert::From<TlcErrPacketReader<'r>> for RemoveTlcReasonUnionReader<'r> {
+    fn from(item: TlcErrPacketReader<'r>) -> Self {
+        RemoveTlcReasonUnionReader::TlcErrPacket(item)
     }
 }
 impl RemoveTlcReasonUnion {
@@ -9977,31 +9977,31 @@ impl RemoveTlcReasonUnion {
     pub fn as_bytes(&self) -> molecule::bytes::Bytes {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(item) => item.as_bytes(),
-            RemoveTlcReasonUnion::RemoveTlcFail(item) => item.as_bytes(),
+            RemoveTlcReasonUnion::TlcErrPacket(item) => item.as_bytes(),
         }
     }
     pub fn as_slice(&self) -> &[u8] {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(item) => item.as_slice(),
-            RemoveTlcReasonUnion::RemoveTlcFail(item) => item.as_slice(),
+            RemoveTlcReasonUnion::TlcErrPacket(item) => item.as_slice(),
         }
     }
     pub fn item_id(&self) -> molecule::Number {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(_) => 0,
-            RemoveTlcReasonUnion::RemoveTlcFail(_) => 1,
+            RemoveTlcReasonUnion::TlcErrPacket(_) => 1,
         }
     }
     pub fn item_name(&self) -> &str {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(_) => "RemoveTlcFulfill",
-            RemoveTlcReasonUnion::RemoveTlcFail(_) => "RemoveTlcFail",
+            RemoveTlcReasonUnion::TlcErrPacket(_) => "TlcErrPacket",
         }
     }
     pub fn as_reader<'r>(&'r self) -> RemoveTlcReasonUnionReader<'r> {
         match self {
             RemoveTlcReasonUnion::RemoveTlcFulfill(item) => item.as_reader().into(),
-            RemoveTlcReasonUnion::RemoveTlcFail(item) => item.as_reader().into(),
+            RemoveTlcReasonUnion::TlcErrPacket(item) => item.as_reader().into(),
         }
     }
 }
@@ -10010,19 +10010,19 @@ impl<'r> RemoveTlcReasonUnionReader<'r> {
     pub fn as_slice(&self) -> &'r [u8] {
         match self {
             RemoveTlcReasonUnionReader::RemoveTlcFulfill(item) => item.as_slice(),
-            RemoveTlcReasonUnionReader::RemoveTlcFail(item) => item.as_slice(),
+            RemoveTlcReasonUnionReader::TlcErrPacket(item) => item.as_slice(),
         }
     }
     pub fn item_id(&self) -> molecule::Number {
         match self {
             RemoveTlcReasonUnionReader::RemoveTlcFulfill(_) => 0,
-            RemoveTlcReasonUnionReader::RemoveTlcFail(_) => 1,
+            RemoveTlcReasonUnionReader::TlcErrPacket(_) => 1,
         }
     }
     pub fn item_name(&self) -> &str {
         match self {
             RemoveTlcReasonUnionReader::RemoveTlcFulfill(_) => "RemoveTlcFulfill",
-            RemoveTlcReasonUnionReader::RemoveTlcFail(_) => "RemoveTlcFail",
+            RemoveTlcReasonUnionReader::TlcErrPacket(_) => "TlcErrPacket",
         }
     }
 }
@@ -10031,8 +10031,8 @@ impl From<RemoveTlcFulfill> for RemoveTlcReason {
         Self::new_builder().set(value).build()
     }
 }
-impl From<RemoveTlcFail> for RemoveTlcReason {
-    fn from(value: RemoveTlcFail) -> Self {
+impl From<TlcErrPacket> for RemoveTlcReason {
+    fn from(value: TlcErrPacket) -> Self {
         Self::new_builder().set(value).build()
     }
 }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 0f302263..bb56c403 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -59,8 +59,8 @@ use super::types::{
     FiberQueryInformation, GetBroadcastMessages, GetBroadcastMessagesResult, Hash256,
     NodeAnnouncement, NodeAnnouncementQuery, OpenChannel, Privkey, Pubkey,
     QueryBroadcastMessagesWithinTimeRange, QueryBroadcastMessagesWithinTimeRangeResult,
-    QueryChannelsWithinBlockRange, QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcFail,
-    RemoveTlcReason, TlcErr, TlcErrData, TlcErrorCode,
+    QueryChannelsWithinBlockRange, QueryChannelsWithinBlockRangeResult, RemoveTlc, RemoveTlcReason,
+    TlcErr, TlcErrData, TlcErrPacket, TlcErrorCode,
 };
 use super::FiberConfig;
 
@@ -153,7 +153,7 @@ pub enum NetworkActorCommand {
     // is for the current node.
     SendPaymentOnionPacket(
         SendOnionPacketCommand,
-        RpcReplyPort<Result<u64, RemoveTlcFail>>,
+        RpcReplyPort<Result<u64, TlcErrPacket>>,
     ),
     PeelPaymentOnionPacket(
         Vec<u8>, // onion_packet
@@ -1969,17 +1969,17 @@ where
         &self,
         state: &mut NetworkActorState<S>,
         command: SendOnionPacketCommand,
-        reply: RpcReplyPort<Result<u64, RemoveTlcFail>>,
+        reply: RpcReplyPort<Result<u64, TlcErrPacket>>,
     ) {
         let SendOnionPacketCommand {
             packet,
             previous_tlc,
         } = command;
 
-        let invalid_onion_error = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
+        let invalid_onion_error = |reply: RpcReplyPort<Result<u64, TlcErrPacket>>| {
             let error_detail = TlcErr::new(TlcErrorCode::InvalidOnionPayload);
             reply
-                .send(Err(RemoveTlcFail::new(error_detail)))
+                .send(Err(TlcErrPacket::new(error_detail)))
                 .expect("send error failed");
         };
 
@@ -1995,14 +1995,14 @@ where
             return invalid_onion_error(reply);
         };
 
-        let unknown_next_peer = |reply: RpcReplyPort<Result<u64, RemoveTlcFail>>| {
+        let unknown_next_peer = |reply: RpcReplyPort<Result<u64, TlcErrPacket>>| {
             let error_detail = TlcErr::new_channel_fail(
                 TlcErrorCode::UnknownNextPeer,
                 channel_outpoint.clone(),
                 None,
             );
             reply
-                .send(Err(RemoveTlcFail::new(error_detail)))
+                .send(Err(TlcErrPacket::new(error_detail)))
                 .expect("send add tlc response");
         };
 
@@ -2016,7 +2016,7 @@ where
                 return unknown_next_peer(reply);
             }
         };
-        let (send, recv) = oneshot::channel::<Result<AddTlcResponse, RemoveTlcFail>>();
+        let (send, recv) = oneshot::channel::<Result<AddTlcResponse, TlcErrPacket>>();
         let rpc_reply = RpcReplyPort::from(send);
         let command = ChannelCommand::AddTlc(
             AddTlcCommand {
@@ -2045,7 +2045,7 @@ where
                 );
                 let error_detail = TlcErr::new(TlcErrorCode::TemporaryNodeFailure);
                 return reply
-                    .send(Err(RemoveTlcFail::new(error_detail)))
+                    .send(Err(TlcErrPacket::new(error_detail)))
                     .expect("send add tlc response");
             }
         }
@@ -2169,7 +2169,7 @@ where
             )
             .map_err(|err| Error::InvalidOnionPacket(err))?;
 
-            let (send, recv) = oneshot::channel::<Result<u64, RemoveTlcFail>>();
+            let (send, recv) = oneshot::channel::<Result<u64, TlcErrPacket>>();
             let rpc_reply = RpcReplyPort::from(send);
             let command = SendOnionPacketCommand {
                 packet: peeled_packet.serialize(),
diff --git a/src/fiber/schema/fiber.mol b/src/fiber/schema/fiber.mol
index 4fc22c61..702bb2e7 100644
--- a/src/fiber/schema/fiber.mol
+++ b/src/fiber/schema/fiber.mol
@@ -128,13 +128,13 @@ struct RemoveTlcFulfill {
     payment_preimage:   Byte32,
 }
 
-table RemoveTlcFail {
+table TlcErrPacket {
     onion_packet:        Bytes,
 }
 
 union RemoveTlcReason {
     RemoveTlcFulfill,
-    RemoveTlcFail,
+    TlcErrPacket,
 }
 
 table RemoveTlc {
diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 9d78ddd8..8f386b97 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -3,8 +3,8 @@ use crate::fiber::{
     hash_algorithm::HashAlgorithm,
     tests::test_utils::generate_pubkey,
     types::{
-        secp256k1_instance, AddTlc, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey,
-        RemoveTlcFail, TlcErr, TlcErrorCode,
+        secp256k1_instance, AddTlc, PaymentHopData, PeeledOnionPacket, Privkey, Pubkey, TlcErr,
+        TlcErrPacket, TlcErrorCode,
     },
 };
 use ckb_types::packed::OutPointBuilder;
@@ -104,7 +104,7 @@ fn test_tlc_fail_error() {
     assert!(!tlc_fail_detail.error_code.is_node());
     assert!(tlc_fail_detail.error_code.is_bad_onion());
     assert!(tlc_fail_detail.error_code.is_perm());
-    let tlc_fail = RemoveTlcFail::new(tlc_fail_detail.clone());
+    let tlc_fail = TlcErrPacket::new(tlc_fail_detail.clone());
 
     let convert_back: TlcErr = tlc_fail.decode().expect("decoded fail");
     assert_eq!(tlc_fail_detail, convert_back);
@@ -112,7 +112,7 @@ fn test_tlc_fail_error() {
     let node_fail =
         TlcErr::new_node_fail(TlcErrorCode::PermanentNodeFailure, generate_pubkey().into());
     assert!(node_fail.error_code.is_node());
-    let tlc_fail = RemoveTlcFail::new(node_fail.clone());
+    let tlc_fail = TlcErrPacket::new(node_fail.clone());
     let convert_back = tlc_fail.decode().expect("decoded fail");
     assert_eq!(node_fail, convert_back);
 }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 75f88b5c..94083bf5 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1344,20 +1344,18 @@ impl TlcErr {
 
 // This is the onion packet we need to encode and send back to the sender,
 // currently it's the raw TlcErr serialized data from the TlcErr struct,
-// sender should decode it and get the TlcErr, then decide what to do with the error
+// sender should decode it and then decide what to do with the error.
 // Note: this supposed to be only accessible by the sender, and it's not reliable since it
 //       is not placed on-chain due to the possibility of hop failure.
-//
-// FIXME: a better name like `TlcFailOnionPacket`?, since it's not only caused by removing tlc now
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct RemoveTlcFail {
+pub struct TlcErrPacket {
     // TODO: replace this with the real onion packet
     pub onion_packet: Vec<u8>,
 }
 
-impl RemoveTlcFail {
+impl TlcErrPacket {
     pub fn new(tlc_fail: TlcErr) -> Self {
-        RemoveTlcFail {
+        TlcErrPacket {
             onion_packet: tlc_fail.serialize(),
         }
     }
@@ -1367,27 +1365,27 @@ impl RemoveTlcFail {
     }
 }
 
-impl From<RemoveTlcFail> for molecule_fiber::RemoveTlcFail {
-    fn from(remove_tlc_fail: RemoveTlcFail) -> Self {
-        molecule_fiber::RemoveTlcFail::new_builder()
+impl From<TlcErrPacket> for molecule_fiber::TlcErrPacket {
+    fn from(remove_tlc_fail: TlcErrPacket) -> Self {
+        molecule_fiber::TlcErrPacket::new_builder()
             .onion_packet(remove_tlc_fail.onion_packet.pack())
             .build()
     }
 }
 
-impl TryFrom<molecule_fiber::RemoveTlcFail> for RemoveTlcFail {
+impl TryFrom<molecule_fiber::TlcErrPacket> for TlcErrPacket {
     type Error = Error;
 
-    fn try_from(remove_tlc_fail: molecule_fiber::RemoveTlcFail) -> Result<Self, Self::Error> {
-        Ok(RemoveTlcFail {
+    fn try_from(remove_tlc_fail: molecule_fiber::TlcErrPacket) -> Result<Self, Self::Error> {
+        Ok(TlcErrPacket {
             onion_packet: remove_tlc_fail.onion_packet().unpack(),
         })
     }
 }
 
-impl std::fmt::Display for RemoveTlcFail {
+impl std::fmt::Display for TlcErrPacket {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        write!(f, "RemoveTlcFail")
+        write!(f, "TlcErrPacket")
     }
 }
 
@@ -1462,7 +1460,7 @@ impl TlcErrorCode {
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RemoveTlcReason {
     RemoveTlcFulfill(RemoveTlcFulfill),
-    RemoveTlcFail(RemoveTlcFail),
+    RemoveTlcFail(TlcErrPacket),
 }
 
 impl From<RemoveTlcReason> for molecule_fiber::RemoveTlcReasonUnion {
@@ -1472,7 +1470,7 @@ impl From<RemoveTlcReason> for molecule_fiber::RemoveTlcReasonUnion {
                 molecule_fiber::RemoveTlcReasonUnion::RemoveTlcFulfill(remove_tlc_fulfill.into())
             }
             RemoveTlcReason::RemoveTlcFail(remove_tlc_fail) => {
-                molecule_fiber::RemoveTlcReasonUnion::RemoveTlcFail(remove_tlc_fail.into())
+                molecule_fiber::RemoveTlcReasonUnion::TlcErrPacket(remove_tlc_fail.into())
             }
         }
     }
@@ -1494,7 +1492,7 @@ impl TryFrom<molecule_fiber::RemoveTlcReason> for RemoveTlcReason {
             molecule_fiber::RemoveTlcReasonUnion::RemoveTlcFulfill(remove_tlc_fulfill) => Ok(
                 RemoveTlcReason::RemoveTlcFulfill(remove_tlc_fulfill.try_into()?),
             ),
-            molecule_fiber::RemoveTlcReasonUnion::RemoveTlcFail(remove_tlc_fail) => {
+            molecule_fiber::RemoveTlcReasonUnion::TlcErrPacket(remove_tlc_fail) => {
                 Ok(RemoveTlcReason::RemoveTlcFail(remove_tlc_fail.try_into()?))
             }
         }

From 5d4e99846b5eda29b61d3770d5ef4265b00a900d Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 18:52:09 +0800
Subject: [PATCH 47/66] fix remove tlc e2e failure

---
 src/fiber/tests/types.rs                      |  8 +++-
 src/fiber/types.rs                            |  4 +-
 src/rpc/channel.rs                            | 41 ++++++++++++-------
 .../14-remove-tlc-from-NODE3.bru              |  2 +-
 .../17-remove-tlc-from-NODE1.bru              |  2 +-
 .../18-remove-tlc-from-NODE3.bru              |  2 +-
 .../20-remove-tlc-from-NODE1.bru              |  2 +-
 .../reestablish/12-remove-tlc-from-NODE1.bru  |  2 +-
 8 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/src/fiber/tests/types.rs b/src/fiber/tests/types.rs
index 8f386b97..3af7429f 100644
--- a/src/fiber/tests/types.rs
+++ b/src/fiber/tests/types.rs
@@ -1,3 +1,4 @@
+use super::test_utils::generate_seckey;
 use crate::fiber::{
     gen::fiber as molecule_fiber,
     hash_algorithm::HashAlgorithm,
@@ -10,8 +11,7 @@ use crate::fiber::{
 use ckb_types::packed::OutPointBuilder;
 use ckb_types::prelude::Builder;
 use secp256k1::{Secp256k1, SecretKey};
-
-use super::test_utils::generate_seckey;
+use std::str::FromStr;
 
 #[test]
 fn test_serde_public_key() {
@@ -115,4 +115,8 @@ fn test_tlc_fail_error() {
     let tlc_fail = TlcErrPacket::new(node_fail.clone());
     let convert_back = tlc_fail.decode().expect("decoded fail");
     assert_eq!(node_fail, convert_back);
+
+    let error_code = TlcErrorCode::PermanentNodeFailure;
+    let convert = TlcErrorCode::from_str("PermanentNodeFailure").expect("convert error");
+    assert_eq!(error_code, convert);
 }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 94083bf5..16ab543a 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -34,7 +34,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use std::marker::PhantomData;
 use std::str::FromStr;
-use strum::{AsRefStr, IntoStaticStr};
+use strum::{AsRefStr, EnumString};
 use tentacle::multiaddr::MultiAddr;
 use tentacle::secio::PeerId;
 use thiserror::Error;
@@ -1399,7 +1399,7 @@ const NODE: u16 = 0x2000;
 const UPDATE: u16 = 0x1000;
 
 #[repr(u16)]
-#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, AsRefStr, IntoStaticStr)]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, AsRefStr, EnumString)]
 pub enum TlcErrorCode {
     TemporaryNodeFailure = NODE | 2,
     PermanentNodeFailure = PERM | NODE | 2,
diff --git a/src/rpc/channel.rs b/src/rpc/channel.rs
index 64f7c4ce..6c72057d 100644
--- a/src/rpc/channel.rs
+++ b/src/rpc/channel.rs
@@ -6,8 +6,8 @@ use crate::fiber::{
     graph::PaymentSessionStatus,
     hash_algorithm::HashAlgorithm,
     network::{AcceptChannelCommand, OpenChannelCommand, SendPaymentCommand},
-    serde_utils::{U128Hex, U16Hex, U64Hex},
-    types::{Hash256, LockTime, Pubkey, RemoveTlcFulfill},
+    serde_utils::{U128Hex, U64Hex},
+    types::{Hash256, LockTime, Pubkey, RemoveTlcFulfill, TlcErr, TlcErrPacket, TlcErrorCode},
     NetworkActorCommand, NetworkActorMessage,
 };
 use crate::{handle_actor_call, handle_actor_cast, log_and_error};
@@ -22,6 +22,7 @@ use ractor::{call, ActorRef};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::cmp::Reverse;
+use std::str::FromStr;
 use tentacle::secio::PeerId;
 
 #[serde_as]
@@ -128,7 +129,7 @@ pub(crate) struct AddTlcResult {
 }
 
 #[serde_as]
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub(crate) struct RemoveTlcParams {
     channel_id: Hash256,
     #[serde_as(as = "U64Hex")]
@@ -137,16 +138,11 @@ pub(crate) struct RemoveTlcParams {
 }
 
 #[serde_as]
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(untagged)]
 enum RemoveTlcReason {
-    RemoveTlcFulfill {
-        payment_preimage: Hash256,
-    },
-    RemoveTlcFail {
-        #[serde_as(as = "U16Hex")]
-        error_code: u16,
-    },
+    RemoveTlcFulfill { payment_preimage: Hash256 },
+    RemoveTlcFail { error_code: String },
 }
 
 #[serde_as]
@@ -423,6 +419,15 @@ where
     }
 
     async fn remove_tlc(&self, params: RemoveTlcParams) -> Result<(), ErrorObjectOwned> {
+        let err_code = match &params.reason {
+            RemoveTlcReason::RemoveTlcFail { error_code } => {
+                let Ok(err) = TlcErrorCode::from_str(&error_code) else {
+                    return log_and_error!(params, format!("invalid error code: {}", error_code));
+                };
+                Some(err)
+            }
+            _ => None,
+        };
         let message = |rpc_reply| -> NetworkActorMessage {
             NetworkActorMessage::Command(NetworkActorCommand::ControlFiberChannel(
                 ChannelCommandWithId {
@@ -430,15 +435,21 @@ where
                     command: ChannelCommand::RemoveTlc(
                         RemoveTlcCommand {
                             id: params.tlc_id,
-                            reason: match params.reason {
+                            reason: match &params.reason {
                                 RemoveTlcReason::RemoveTlcFulfill { payment_preimage } => {
                                     crate::fiber::types::RemoveTlcReason::RemoveTlcFulfill(
-                                        RemoveTlcFulfill { payment_preimage },
+                                        RemoveTlcFulfill {
+                                            payment_preimage: *payment_preimage,
+                                        },
                                     )
                                 }
-                                RemoveTlcReason::RemoveTlcFail { error_code: _ } => {
+                                RemoveTlcReason::RemoveTlcFail { .. } => {
                                     // TODO: maybe we should remove this PRC or move add_tlc and remove_tlc to `test` module?
-                                    unimplemented!("RemoveTlcFail is only for internal use");
+                                    crate::fiber::types::RemoveTlcReason::RemoveTlcFail(
+                                        TlcErrPacket::new(TlcErr::new(
+                                            err_code.expect("expect error code"),
+                                        )),
+                                    )
                                 }
                             },
                         },
diff --git a/tests/bruno/e2e/open-use-close-a-channel/14-remove-tlc-from-NODE3.bru b/tests/bruno/e2e/open-use-close-a-channel/14-remove-tlc-from-NODE3.bru
index dd358ac0..2f9bfb1a 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/14-remove-tlc-from-NODE3.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/14-remove-tlc-from-NODE3.bru
@@ -25,7 +25,7 @@ body:json {
         "channel_id": "{{CHANNEL_ID}}",
         "tlc_id": "{{TLC_ID2}}",
         "reason": {
-          "error_code": "0x2a"
+          "error_code": "TemporaryNodeFailure"
         }
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/17-remove-tlc-from-NODE1.bru b/tests/bruno/e2e/open-use-close-a-channel/17-remove-tlc-from-NODE1.bru
index d11e68ac..2300af32 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/17-remove-tlc-from-NODE1.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/17-remove-tlc-from-NODE1.bru
@@ -25,7 +25,7 @@ body:json {
         "channel_id": "{{CHANNEL_ID}}",
         "tlc_id": "{{TLC_ID3}}",
         "reason": {
-          "error_code": "0x2a"
+          "error_code": "TemporaryNodeFailure"
         }
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/18-remove-tlc-from-NODE3.bru b/tests/bruno/e2e/open-use-close-a-channel/18-remove-tlc-from-NODE3.bru
index d51641c1..f3bead17 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/18-remove-tlc-from-NODE3.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/18-remove-tlc-from-NODE3.bru
@@ -25,7 +25,7 @@ body:json {
         "channel_id": "{{CHANNEL_ID}}",
         "tlc_id": "{{TLC_ID3}}",
         "reason": {
-          "error_code": "0x2a"
+          "error_code": "TemporaryNodeFailure"
         }
       }
     ]
diff --git a/tests/bruno/e2e/open-use-close-a-channel/20-remove-tlc-from-NODE1.bru b/tests/bruno/e2e/open-use-close-a-channel/20-remove-tlc-from-NODE1.bru
index d5b6b373..54ad022e 100644
--- a/tests/bruno/e2e/open-use-close-a-channel/20-remove-tlc-from-NODE1.bru
+++ b/tests/bruno/e2e/open-use-close-a-channel/20-remove-tlc-from-NODE1.bru
@@ -25,7 +25,7 @@ body:json {
         "channel_id": "{{CHANNEL_ID}}",
         "tlc_id": "{{TLC_ID6}}",
         "reason": {
-          "error_code": "0x2a"
+          "error_code": "TemporaryNodeFailure"
         }
       }
     ]
diff --git a/tests/bruno/e2e/reestablish/12-remove-tlc-from-NODE1.bru b/tests/bruno/e2e/reestablish/12-remove-tlc-from-NODE1.bru
index 878b0a0e..79ed9dfc 100644
--- a/tests/bruno/e2e/reestablish/12-remove-tlc-from-NODE1.bru
+++ b/tests/bruno/e2e/reestablish/12-remove-tlc-from-NODE1.bru
@@ -25,7 +25,7 @@ body:json {
         "channel_id": "{{CHANNEL_ID}}",
         "tlc_id": "{{TLC_ID1}}",
         "reason": {
-          "error_code": "0x2a"
+          "error_code": "TemporaryNodeFailure"
         }
       }
     ]

From 118d9bc9b7195d886bc16c1669e5ec8e20af1386 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Sun, 13 Oct 2024 23:47:17 +0800
Subject: [PATCH 48/66] fix PartialEq for NodeHeapElement

---
 src/fiber/network.rs | 3 +--
 src/fiber/path.rs    | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index bb56c403..af11bf88 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -2183,8 +2183,7 @@ where
                         error!("Failed to send onion packet with error: {:?}", e);
                         // This is the error implies we send payment request to the first hop failed
                         let err = format!(
-                            "Failed to send onion packet: {:?} with error {:?}",
-                            payment_hash,
+                            "Failed to send onion packet with error {:?}",
                             error_detail.error_code_as_str()
                         );
                         error = Some(err);
diff --git a/src/fiber/path.rs b/src/fiber/path.rs
index ac959019..1a6536ed 100644
--- a/src/fiber/path.rs
+++ b/src/fiber/path.rs
@@ -60,6 +60,7 @@ impl PartialEq for NodeHeapElement {
             && self.amount_received == other.amount_received
             && self.fee_charged == other.fee_charged
             && self.probability == other.probability
+            && self.incoming_cltv_height == other.incoming_cltv_height
             && self.next_hop == other.next_hop
     }
 }

From 856a2ada1705ab47cd8f5eb931238d64f2569839 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Tue, 15 Oct 2024 20:24:43 +0800
Subject: [PATCH 49/66] code cleanup and refactor

---
 src/fiber/channel.rs | 34 ++++++++++++++++------------------
 src/fiber/graph.rs   |  2 --
 2 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index b96cc6b1..e64a9341 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -485,11 +485,11 @@ where
                     Err(e) => {
                         // we assume that TLC was not inserted into our state,
                         // so we can safely send RemoveTlc message to the peer
-                        // note we can not use get_received_tlc_by_id here, because this new add_tlc may be
-                        // trying to add a duplicate tlc, so we use tlc count to make sure no new tlc was added
+                        // note this new add_tlc may be trying to add a duplicate tlc,
+                        // so we use tlc count to make sure no new tlc was added
                         // and only send RemoveTlc message to peer if the TLC is not in our state
-                        assert!(tlc_count == state.tlcs.len());
                         error!("Error handling AddTlc message: {:?}", e);
+                        assert!(tlc_count == state.tlcs.len());
                         let error_detail = self.get_tlc_detail_error(state, &e).await;
                         if state.get_received_tlc(tlc_id).is_none() {
                             self.network
@@ -766,7 +766,6 @@ where
         // try to fulfill the payment, find the corresponding payment preimage from payment hash.
         let mut preimage = None;
         let mut peeled_packet_bytes: Option<Vec<u8>> = None;
-        let mut forward_to_next_hop = false;
 
         if !add_tlc.onion_packet.is_empty() {
             // TODO: Here we call network actor to peel the onion packet. Indeed, this message is forwarded from
@@ -779,7 +778,7 @@ where
                     tx
                 )
             ))
-            .expect("call network")
+            .expect(ASSUME_NETWORK_ACTOR_ALIVE)
             .map_err(|err| ProcessingChannelError::PeelingOnionPacketError(err))?;
 
             // check the payment hash and amount
@@ -788,8 +787,20 @@ where
                     "Payment hash mismatch".to_string(),
                 ));
             }
+
+            let received_amount = add_tlc.amount;
+            let forward_amount = peeled_packet.current.amount;
+            debug!(
+                "received_amount: {} forward_amount: {}",
+                add_tlc.amount, forward_amount
+            );
+
             // TODO: check the expiry time, if it's expired, we should return an error.
             if peeled_packet.is_last() {
+                if forward_amount != add_tlc.amount {
+                    return Err(ProcessingChannelError::FinalIncorrectHTLCAmount);
+                }
+
                 // if this is the last hop, store the preimage.
                 // though we will RemoveTlcFulfill the TLC in try_to_settle_down_tlc function,
                 // here we can do error check early here for better error handling.
@@ -807,16 +818,6 @@ where
                 }
             } else {
                 peeled_packet_bytes = Some(peeled_packet.serialize());
-                forward_to_next_hop = true;
-            }
-
-            let received_amount = add_tlc.amount;
-            let forward_amount = peeled_packet.current.amount;
-            debug!(
-                "received_amount: {} forward_amount: {}",
-                add_tlc.amount, forward_amount
-            );
-            if forward_to_next_hop {
                 assert!(received_amount >= forward_amount);
                 let forward_fee = received_amount.saturating_sub(forward_amount);
                 let fee_rate: u128 = state
@@ -833,9 +834,6 @@ where
                     return Err(ProcessingChannelError::TlcForwardFeeIsTooLow);
                 }
             }
-            if !forward_to_next_hop && received_amount != add_tlc.amount {
-                return Err(ProcessingChannelError::FinalIncorrectHTLCAmount);
-            }
         }
 
         let tlc = state.create_inbounding_tlc(add_tlc.clone(), preimage)?;
diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 36d4aa53..dd51233c 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -732,8 +732,6 @@ where
                         continue;
                     }
                 }
-                // info!("weight: {:?} dist: {:?} fee: {:?}", weight, distance, fee);
-                // TODO: update weight and distance here
                 let node: NodeHeapElement = NodeHeapElement {
                     node_id: from,
                     weight,

From 5ccb247cb27f14cb3706b20cda99c97c262f0feb Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Wed, 16 Oct 2024 16:56:41 +0800
Subject: [PATCH 50/66] add invoice get rpc and check expire

---
 src/fiber/channel.rs                          | 13 ++++
 src/fiber/network.rs                          |  6 ++
 src/fiber/types.rs                            |  2 +
 src/invoice/invoice_impl.rs                   |  7 ++
 src/invoice/tests/invoice_impl.rs             | 15 ++++
 src/rpc/invoice.rs                            | 72 +++++++++++++++++--
 .../e2e/router-pay/12-node1-send-payment.bru  |  1 +
 .../22-node3-gen-expiring-invoice.bru         | 65 +++++++++++++++++
 .../23-node1-send-payment-will-fail.bru       | 48 +++++++++++++
 9 files changed, 224 insertions(+), 5 deletions(-)
 create mode 100644 tests/bruno/e2e/router-pay/22-node3-gen-expiring-invoice.bru
 create mode 100644 tests/bruno/e2e/router-pay/23-node1-send-payment-will-fail.bru

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index e64a9341..41c6771f 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -729,6 +729,19 @@ where
         let tlcs = state.get_tlcs_for_settle_down();
         for tlc_info in tlcs {
             let tlc = tlc_info.tlc.clone();
+            if let Some(invoice) = self.store.get_invoice(&tlc.payment_hash) {
+                if invoice.is_expired() {
+                    let command = RemoveTlcCommand {
+                        id: tlc.get_id(),
+                        reason: RemoveTlcReason::RemoveTlcFail(TlcErrPacket::new(TlcErr::new(
+                            TlcErrorCode::InvoiceExpired,
+                        ))),
+                    };
+                    let result = self.handle_remove_tlc_command(state, command);
+                    info!("try to settle down tlc: {:?} result: {:?}", &tlc, &result);
+                }
+            }
+
             let preimage = if let Some(preimage) = tlc.payment_preimage {
                 preimage
             } else if let Some(preimage) = self.store.get_invoice_preimage(&tlc.payment_hash) {
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index c51fe09d..6cb18370 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -280,6 +280,12 @@ impl SendPaymentData {
             .transpose()
             .map_err(|_| "invoice is invalid".to_string())?;
 
+        if let Some(invoice) = invoice.clone() {
+            if invoice.is_expired() {
+                return Err("invoice is expired".to_string());
+            }
+        }
+
         fn validate_field<T: PartialEq + Clone>(
             field: Option<T>,
             invoice_field: Option<T>,
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 8dd5ffad..096cec2a 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1420,6 +1420,7 @@ pub enum TlcErrorCode {
     IncorrectCltvExpiry = UPDATE | 13,
     ExpiryTooSoon = UPDATE | 14,
     IncorrectOrUnknownPaymentDetails = PERM | 15,
+    InvoiceExpired = PERM | 16,
     FinalIncorrectCltvExpiry = 18,
     FinalIncorrectHtlcAmount = 19,
     ChannelDisabled = UPDATE | 20,
@@ -1451,6 +1452,7 @@ impl TlcErrorCode {
             TlcErrorCode::IncorrectOrUnknownPaymentDetails
             | TlcErrorCode::FinalIncorrectCltvExpiry
             | TlcErrorCode::FinalIncorrectHtlcAmount
+            | TlcErrorCode::InvoiceExpired
             | TlcErrorCode::MppTimeout => true,
             _ => false,
         }
diff --git a/src/invoice/invoice_impl.rs b/src/invoice/invoice_impl.rs
index b0f0d359..135c7115 100644
--- a/src/invoice/invoice_impl.rs
+++ b/src/invoice/invoice_impl.rs
@@ -222,6 +222,13 @@ impl CkbInvoice {
         &self.data.payment_hash
     }
 
+    pub fn is_expired(&self) -> bool {
+        self.expiry_time().map_or(false, |expiry| {
+            self.data.timestamp + expiry.as_millis()
+                < std::time::UNIX_EPOCH.elapsed().unwrap().as_millis()
+        })
+    }
+
     /// Check that the invoice is signed correctly and that key recovery works
     pub fn check_signature(&self) -> Result<(), InvoiceError> {
         if self.signature.is_none() {
diff --git a/src/invoice/tests/invoice_impl.rs b/src/invoice/tests/invoice_impl.rs
index 300018d3..2e4fac1c 100644
--- a/src/invoice/tests/invoice_impl.rs
+++ b/src/invoice/tests/invoice_impl.rs
@@ -437,3 +437,18 @@ fn test_invoice_udt_script() {
     let decoded = serde_json::from_str::<CkbInvoice>(&res.unwrap()).unwrap();
     assert_eq!(decoded, invoice);
 }
+
+#[test]
+fn test_invoice_check_expired() {
+    let private_key = gen_rand_private_key();
+    let invoice = InvoiceBuilder::new(Currency::Fibb)
+        .amount(Some(1280))
+        .payment_hash(rand_sha256_hash())
+        .expiry_time(Duration::from_secs(1))
+        .build_with_sign(|hash| Secp256k1::new().sign_ecdsa_recoverable(hash, &private_key))
+        .unwrap();
+
+    assert_eq!(invoice.is_expired(), false);
+    std::thread::sleep(Duration::from_secs(2));
+    assert_eq!(invoice.is_expired(), true);
+}
diff --git a/src/rpc/invoice.rs b/src/rpc/invoice.rs
index 3ed4b34b..cdf64636 100644
--- a/src/rpc/invoice.rs
+++ b/src/rpc/invoice.rs
@@ -1,5 +1,6 @@
 use std::time::Duration;
 
+use crate::fiber::graph::{NetworkGraphStateStore, PaymentSessionStatus};
 use crate::fiber::hash_algorithm::HashAlgorithm;
 use crate::fiber::serde_utils::{U128Hex, U64Hex};
 use crate::fiber::types::Hash256;
@@ -32,7 +33,7 @@ pub(crate) struct NewInvoiceParams {
 }
 
 #[derive(Clone, Serialize, Deserialize)]
-pub(crate) struct NewInvoiceResult {
+pub(crate) struct InvoiceResult {
     invoice_address: String,
     invoice: CkbInvoice,
 }
@@ -47,19 +48,45 @@ pub(crate) struct ParseInvoiceResult {
     invoice: CkbInvoice,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct GetInvoiceParams {
+    payment_hash: Hash256,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+enum InvoiceStatus {
+    Unpaid,
+    Inflight,
+    Paid,
+    Expired,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub(crate) struct GetInvoiceResult {
+    invoice_address: String,
+    invoice: CkbInvoice,
+    status: InvoiceStatus,
+}
+
 #[rpc(server)]
 trait InvoiceRpc {
     #[method(name = "new_invoice")]
     async fn new_invoice(
         &self,
         params: NewInvoiceParams,
-    ) -> Result<NewInvoiceResult, ErrorObjectOwned>;
+    ) -> Result<InvoiceResult, ErrorObjectOwned>;
 
     #[method(name = "parse_invoice")]
     async fn parse_invoice(
         &self,
         params: ParseInvoiceParams,
     ) -> Result<ParseInvoiceResult, ErrorObjectOwned>;
+
+    #[method(name = "get_invoice")]
+    async fn get_invoice(
+        &self,
+        payment_hash: GetInvoiceParams,
+    ) -> Result<GetInvoiceResult, ErrorObjectOwned>;
 }
 
 pub(crate) struct InvoiceRpcServerImpl<S> {
@@ -76,12 +103,12 @@ impl<S> InvoiceRpcServerImpl<S> {
 #[async_trait]
 impl<S> InvoiceRpcServer for InvoiceRpcServerImpl<S>
 where
-    S: InvoiceStore + Send + Sync + 'static,
+    S: InvoiceStore + NetworkGraphStateStore + Send + Sync + 'static,
 {
     async fn new_invoice(
         &self,
         params: NewInvoiceParams,
-    ) -> Result<NewInvoiceResult, ErrorObjectOwned> {
+    ) -> Result<InvoiceResult, ErrorObjectOwned> {
         let mut invoice_builder = InvoiceBuilder::new(params.currency)
             .amount(Some(params.amount))
             .payment_preimage(params.payment_preimage);
@@ -116,7 +143,7 @@ where
                 .store
                 .insert_invoice(invoice.clone(), Some(params.payment_preimage))
             {
-                Ok(_) => Ok(NewInvoiceResult {
+                Ok(_) => Ok(InvoiceResult {
                     invoice_address: invoice.to_string(),
                     invoice,
                 }),
@@ -150,4 +177,39 @@ where
             )),
         }
     }
+
+    async fn get_invoice(
+        &self,
+        params: GetInvoiceParams,
+    ) -> Result<GetInvoiceResult, ErrorObjectOwned> {
+        let payment_hash = params.payment_hash;
+        match self.store.get_invoice(&payment_hash) {
+            Some(invoice) => {
+                let invoice_status = if invoice.is_expired() {
+                    InvoiceStatus::Expired
+                } else {
+                    InvoiceStatus::Unpaid
+                };
+                let payment_session = self.store.get_payment_session(payment_hash);
+                let status = match payment_session {
+                    Some(session) => match session.status {
+                        PaymentSessionStatus::Inflight => InvoiceStatus::Inflight,
+                        PaymentSessionStatus::Success => InvoiceStatus::Paid,
+                        _ => invoice_status,
+                    },
+                    None => invoice_status,
+                };
+                Ok(GetInvoiceResult {
+                    invoice_address: invoice.to_string(),
+                    invoice,
+                    status,
+                })
+            }
+            None => Err(ErrorObjectOwned::owned(
+                CALL_EXECUTION_FAILED_CODE,
+                "invoice not found".to_string(),
+                Some(payment_hash),
+            )),
+        }
+    }
 }
diff --git a/tests/bruno/e2e/router-pay/12-node1-send-payment.bru b/tests/bruno/e2e/router-pay/12-node1-send-payment.bru
index fd587bd9..4dc89dc3 100644
--- a/tests/bruno/e2e/router-pay/12-node1-send-payment.bru
+++ b/tests/bruno/e2e/router-pay/12-node1-send-payment.bru
@@ -37,4 +37,5 @@ assert {
 script:post-response {
   // Sleep for sometime to make sure current operation finishes before next request starts.
   await new Promise(r => setTimeout(r, 100));
+  console.log("12 step result: ", res.body);
 }
diff --git a/tests/bruno/e2e/router-pay/22-node3-gen-expiring-invoice.bru b/tests/bruno/e2e/router-pay/22-node3-gen-expiring-invoice.bru
new file mode 100644
index 00000000..c33c0e1e
--- /dev/null
+++ b/tests/bruno/e2e/router-pay/22-node3-gen-expiring-invoice.bru
@@ -0,0 +1,65 @@
+meta {
+  name: generate a invoice which will expiring in short time
+  type: http
+  seq: 22
+}
+
+post {
+  url: {{NODE3_RPC_URL}}
+  body: json
+  auth: none
+}
+
+headers {
+  Content-Type: application/json
+  Accept: application/json
+}
+
+body:json {
+  {
+    "id": "42",
+    "jsonrpc": "2.0",
+    "method": "new_invoice",
+    "params": [
+      {
+        "amount": "0x613",
+        "currency": "Fibb",
+        "description": "test invoice generated by node3",
+        "expiry": "0x2",
+        "final_cltv": "0x28",
+        "payment_preimage": "{{payment_preimage}}"
+      }
+    ]
+  }
+}
+
+assert {
+  res.body.error: isUndefined
+  res.body.result: isDefined
+}
+
+script:pre-request {
+  // generate random preimage
+  function generateRandomPreimage() {
+    let hash = '0x';
+    for (let i = 0; i < 64; i++) {
+        hash += Math.floor(Math.random() * 16).toString(16);
+    }
+    return hash;
+  }
+  const payment_preimage = generateRandomPreimage();
+  bru.setVar("payment_preimage", payment_preimage);
+  let hash_algorithm = bru.getEnvVar("HASH_ALGORITHM");
+  if (hash_algorithm !== null) {
+    let body = req.getBody();
+    body.params[0].hash_algorithm = hash_algorithm;
+    req.setBody(body);
+  }
+}
+
+script:post-response {
+  // Sleep for sometime to make sure current operation finishes before next request starts.
+  await new Promise(r => setTimeout(r, 3000));
+  console.log("generated result: ", res.body.result);
+  bru.setVar("encoded_invoice", res.body.result.invoice_address);
+}
diff --git a/tests/bruno/e2e/router-pay/23-node1-send-payment-will-fail.bru b/tests/bruno/e2e/router-pay/23-node1-send-payment-will-fail.bru
new file mode 100644
index 00000000..08969000
--- /dev/null
+++ b/tests/bruno/e2e/router-pay/23-node1-send-payment-will-fail.bru
@@ -0,0 +1,48 @@
+meta {
+  name: Node1 send payment with router
+  type: http
+  seq: 23
+}
+
+post {
+  url: {{NODE1_RPC_URL}}
+  body: json
+  auth: none
+}
+
+headers {
+  Content-Type: application/json
+  Accept: application/json
+}
+
+body:json {
+  {
+    "id": "42",
+    "jsonrpc": "2.0",
+    "method": "send_payment",
+    "params": [
+      {
+        "invoice": "{{encoded_invoice}}"
+      }
+    ]
+  }
+}
+
+assert {
+  res.body.error: isDefined
+}
+
+
+script:pre-request {
+  // sleep for a while
+  await new Promise(r => setTimeout(r, 1000));
+}
+
+
+script:post-response {
+  // Sleep for sometime to make sure current operation finishes before next request starts.
+  await new Promise(r => setTimeout(r, 100));
+  if (!(res.body.error.message.includes("invoice is expired"))) {
+    throw new Error("Assertion failed: error message is not right");
+  }
+}

From d737c67089839382c3db5ec74a07c5d3c323d896 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 18 Oct 2024 14:33:12 +0800
Subject: [PATCH 51/66] set ShuttingDown to PermanentChannelFailure

---
 src/fiber/channel.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 41c6771f..43bd2684 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -696,9 +696,10 @@ where
                 TlcErrorCode::TemporaryChannelFailure
             }
             ProcessingChannelError::InvalidState(_) => match state.state {
-                ChannelState::Closed(_) => TlcErrorCode::PermanentChannelFailure,
-                // ShuttingDown is a temporary state
-                ChannelState::ShuttingDown(_) => TlcErrorCode::TemporaryChannelFailure,
+                // we can not revert back up `ChannelReady` after `ShuttingDown`
+                ChannelState::Closed(_) | ChannelState::ShuttingDown(_) => {
+                    TlcErrorCode::PermanentChannelFailure
+                }
                 ChannelState::ChannelReady() => {
                     // we expect `ChannelReady` will be both OK for tlc forwarding,
                     // so here are the unreachable point in normal workflow,

From 9fc0a35bccf95427a82337cb793b4ad84732f9ea Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Fri, 18 Oct 2024 14:45:12 +0800
Subject: [PATCH 52/66] suppose public_channel_info must exists for TLC
 forwarding node

---
 src/fiber/channel.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/fiber/channel.rs b/src/fiber/channel.rs
index 43bd2684..ee072b97 100644
--- a/src/fiber/channel.rs
+++ b/src/fiber/channel.rs
@@ -837,7 +837,8 @@ where
                 let fee_rate: u128 = state
                     .public_channel_info
                     .as_ref()
-                    .map(|info| info.tlc_fee_proportional_millionths.unwrap_or_default())
+                    .expect("public channel exits")
+                    .tlc_fee_proportional_millionths
                     .unwrap_or_default();
                 let expected_fee = calculate_tlc_forward_fee(forward_amount, fee_rate);
                 if forward_fee < expected_fee {

From 0a442b1195d83e8a8496074ea576d62abd115853 Mon Sep 17 00:00:00 2001
From: ian <ian@cryptape.com>
Date: Tue, 22 Oct 2024 19:01:00 +0800
Subject: [PATCH 53/66] fix cross chain hub e2e test

Remove steps 16~18, which closes the ln channel.

Reasons to remove these steps:

1. The channel is opened in `setup-lnd.sh`. If we do need to test this,
   we should also open the lightning channel in bruno.
2. It is not our concern that lnd can successfully close a channel.
3. The implementation have bugs and fixing them is time consuming.
    1. Closing a channel is async in lnd, and we should block the test until
       the response stream closes, otherwise it is not to assert that we
       have closed the channel successfully.
    2. Closing a channel in lnd requires generating btc blocks so we
       need to call bitcoind rpc as well.
---
 .../16-node1-list-ln-channel.bru              | 20 -----------
 .../17-node1-close-ln-channel.bru             | 33 -------------------
 .../18-node1-list-ln-channel.bru              | 19 -----------
 3 files changed, 72 deletions(-)
 delete mode 100644 tests/bruno/e2e/cross-chain-hub/16-node1-list-ln-channel.bru
 delete mode 100644 tests/bruno/e2e/cross-chain-hub/17-node1-close-ln-channel.bru
 delete mode 100644 tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru

diff --git a/tests/bruno/e2e/cross-chain-hub/16-node1-list-ln-channel.bru b/tests/bruno/e2e/cross-chain-hub/16-node1-list-ln-channel.bru
deleted file mode 100644
index 43c75906..00000000
--- a/tests/bruno/e2e/cross-chain-hub/16-node1-list-ln-channel.bru
+++ /dev/null
@@ -1,20 +0,0 @@
-meta {
-  name: 16-node1-list-ln-channel
-  type: http
-  seq: 16
-}
-
-get {
-  url: {{LND_BOB_RPC_URL}}/v1/channels
-  body: json
-  auth: none
-}
-
-assert {
-  res.body.channels: length 1
-}
-
-script:post-response {
-  console.log("list ln channel result: ", res.body);
-  bru.setVar("LN_CHANNEL_ID", res.body.channels[0].channel_point.replace(":", "/"));
-}
diff --git a/tests/bruno/e2e/cross-chain-hub/17-node1-close-ln-channel.bru b/tests/bruno/e2e/cross-chain-hub/17-node1-close-ln-channel.bru
deleted file mode 100644
index 46b2201e..00000000
--- a/tests/bruno/e2e/cross-chain-hub/17-node1-close-ln-channel.bru
+++ /dev/null
@@ -1,33 +0,0 @@
-meta {
-  name: 17-node1-close-ln-channel
-  type: http
-  seq: 17
-}
-
-delete {
-  url: {{LND_BOB_RPC_URL}}/v1/channels/{{LN_CHANNEL_ID}}
-  body: json
-  auth: none
-}
-
-script:pre-request {
-  const axios = require('axios');
-
-  const url = bru.getEnvVar("LND_BOB_RPC_URL") + "/v1/channels/" + bru.getVar("LN_CHANNEL_ID");
-  console.log(url);
-
-  const resp = await axios({
-    method: 'DELETE',
-    url: url,
-    responseType: 'stream'
-  });
-  resp.data.destroy();
-}
-
-assert {
-  res.status: eq 500
-}
-
-docs {
-  This is a server-streaming RPC which will block Bruno. The workaround is sending the request in the pre-script so the Bruno request will return 500 because the channel is already closed.
-}
diff --git a/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru b/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru
deleted file mode 100644
index 1228fc99..00000000
--- a/tests/bruno/e2e/cross-chain-hub/18-node1-list-ln-channel.bru
+++ /dev/null
@@ -1,19 +0,0 @@
-meta {
-  name: 18-node1-list-ln-channel
-  type: http
-  seq: 18
-}
-
-get {
-  url: {{LND_BOB_RPC_URL}}/v1/channels
-  body: json
-  auth: none
-}
-
-assert {
-  res.body.channels: length 0
-}
-
-script:post-response {
-  console.log("list channel result: ", res.body);
-}
\ No newline at end of file

From 98beab48188cfaf4fc65cde780990783a3855d40 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 27 Sep 2024 06:23:59 +0000
Subject: [PATCH 54/66] Add a peer store to network actor

Save peer information to network actor state

Maybe connect to peer when on node annoouncement received

Persist state when peer addresses changed
---
 src/fiber/graph.rs            |  31 -----
 src/fiber/network.rs          | 215 +++++++++++++++++++++++++++-------
 src/fiber/tests/graph.rs      |  46 +-------
 src/fiber/tests/test_utils.rs |  70 ++++-------
 src/fiber/types.rs            |   4 +
 src/main.rs                   |  31 ++---
 src/store.rs                  | 114 ++++++++----------
 7 files changed, 260 insertions(+), 251 deletions(-)

diff --git a/src/fiber/graph.rs b/src/fiber/graph.rs
index 525c2f2c..a357cce5 100644
--- a/src/fiber/graph.rs
+++ b/src/fiber/graph.rs
@@ -11,8 +11,6 @@ use ckb_types::packed::{OutPoint, Script};
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use std::collections::HashMap;
-use tentacle::multiaddr::Multiaddr;
-use tentacle::secio::PeerId;
 use thiserror::Error;
 use tracing::log::error;
 use tracing::{debug, info, warn};
@@ -138,8 +136,6 @@ pub struct NetworkGraph<S> {
     // Similar to the best_height, this is the last update time of the network graph.
     // We assume that we have already synced the graph up to this time - ASSUME_MAX_MESSAGE_TIMESTAMP_GAP.
     last_update_timestamp: u64,
-    // when we restarting a node, we will reconnect to these peers
-    connected_peer_addresses: HashMap<PeerId, Multiaddr>,
     nodes: HashMap<Pubkey, NodeInfo>,
     store: S,
     chain_hash: Hash256,
@@ -172,7 +168,6 @@ where
             last_update_timestamp: 0,
             channels: HashMap::new(),
             nodes: HashMap::new(),
-            connected_peer_addresses: HashMap::new(),
             store,
             chain_hash: get_chain_hash(),
         };
@@ -212,9 +207,6 @@ where
             }
             self.nodes.insert(node.node_id, node.clone());
         }
-        for (peer, addr) in self.store.get_connected_peer(None) {
-            self.connected_peer_addresses.insert(peer, addr);
-        }
     }
 
     pub fn get_best_height(&self) -> u64 {
@@ -421,25 +413,6 @@ where
         self.chain_hash == chain_hash
     }
 
-    pub fn add_connected_peer(&mut self, peer_id: &PeerId, address: Multiaddr) {
-        self.connected_peer_addresses
-            .insert(peer_id.clone(), address.clone());
-        self.store.insert_connected_peer(peer_id.clone(), address);
-    }
-
-    pub fn get_connected_peers(&self) -> Vec<(&PeerId, &Multiaddr)> {
-        self.connected_peer_addresses.iter().collect()
-    }
-
-    pub fn get_peers_to_sync_network_graph(&self) -> Vec<(&PeerId, &Multiaddr)> {
-        self.connected_peer_addresses.iter().take(3).collect()
-    }
-
-    pub fn remove_connected_peer(&mut self, peer_id: &PeerId) {
-        self.connected_peer_addresses.remove(peer_id);
-        self.store.remove_connected_peer(peer_id);
-    }
-
     pub fn get_node_inbounds(
         &self,
         node_id: Pubkey,
@@ -479,7 +452,6 @@ where
     pub fn reset(&mut self) {
         self.channels.clear();
         self.nodes.clear();
-        self.connected_peer_addresses.clear();
     }
 
     pub fn init_payment_session(&self, payment_request: SendPaymentCommand) -> PaymentSession {
@@ -808,9 +780,6 @@ pub trait NetworkGraphStateStore {
     ) -> (Vec<ChannelInfo>, JsonBytes);
     fn insert_channel(&self, channel: ChannelInfo);
     fn insert_node(&self, node: NodeInfo);
-    fn insert_connected_peer(&self, peer_id: PeerId, multiaddr: Multiaddr);
-    fn get_connected_peer(&self, peer_id: Option<PeerId>) -> Vec<(PeerId, Multiaddr)>;
-    fn remove_connected_peer(&self, peer_id: &PeerId);
     fn get_payment_session(&self, payment_hash: Hash256) -> Option<PaymentSession>;
     fn insert_payment_session(&self, session: PaymentSession);
 }
diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index ded5b300..6d8fa3d6 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -1,5 +1,3 @@
-use crate::ckb::config::UdtCfgInfos;
-use crate::fiber::serde_utils::EntityHex;
 use ckb_hash::blake2b_256;
 use ckb_jsonrpc_types::{BlockNumber, Status, TxStatus};
 use ckb_types::core::TransactionView;
@@ -14,8 +12,9 @@ use ractor::{
 use rand::Rng;
 use secp256k1::{Message, Secp256k1};
 use serde::{Deserialize, Serialize};
-use serde_with::serde_as;
+use serde_with::{serde_as, DisplayFromStr};
 use std::borrow::Cow;
+use std::collections::hash_map::Entry;
 use tentacle::utils::extract_peer_id;
 
 use std::collections::{HashMap, HashSet};
@@ -67,12 +66,14 @@ use super::types::{
 };
 use super::{FiberConfig, ASSUME_NETWORK_ACTOR_ALIVE};
 
+use crate::ckb::config::UdtCfgInfos;
 use crate::ckb::contracts::{check_udt_script, get_udt_whitelist, is_udt_type_auto_accept};
 use crate::ckb::{CkbChainMessage, FundingRequest, FundingTx, TraceTxRequest, TraceTxResponse};
 use crate::fiber::channel::{
     AddTlcCommand, AddTlcResponse, TxCollaborationCommand, TxUpdateCommand,
 };
 use crate::fiber::graph::{ChannelInfo, PaymentSession};
+use crate::fiber::serde_utils::EntityHex;
 use crate::fiber::types::{
     secp256k1_instance, FiberChannelMessage, PaymentOnionPacket, PeeledPaymentOnionPacket,
     TxSignatures,
@@ -539,7 +540,8 @@ pub struct NetworkActor<S> {
 
 impl<S> NetworkActor<S>
 where
-    S: ChannelActorStateStore
+    S: NetworkActorStateStore
+        + ChannelActorStateStore
         + NetworkGraphStateStore
         + InvoiceStore
         + Clone
@@ -669,10 +671,7 @@ where
                         }
                     };
                     for message in messages {
-                        if let Err(e) = self
-                            .process_broadcasted_message(&state.network, message)
-                            .await
-                        {
+                        if let Err(e) = self.process_broadcasted_message(state, message).await {
                             let fail_message =
                                 format!("Failed to process broadcasted message: {:?}", &e);
                             error!("{}", &fail_message);
@@ -1007,10 +1006,6 @@ where
                 self.on_service_event(e).await;
             }
             NetworkActorEvent::PeerConnected(id, pubkey, session) => {
-                self.network_graph
-                    .write()
-                    .await
-                    .add_connected_peer(&id, session.address.clone());
                 state.on_peer_connected(&id, pubkey, &session).await;
                 // Notify outside observers.
                 myself
@@ -1023,7 +1018,6 @@ where
                     .expect(ASSUME_NETWORK_MYSELF_ALIVE);
             }
             NetworkActorEvent::PeerDisconnected(id, session) => {
-                self.network_graph.write().await.remove_connected_peer(&id);
                 state.on_peer_disconnected(&id);
                 // Notify outside observers.
                 myself
@@ -1535,10 +1529,7 @@ where
                 );
                 for message in broadcasted_message_queue {
                     let (_peer_id, message) = message;
-                    if let Err(e) = self
-                        .process_broadcasted_message(&state.network, message)
-                        .await
-                    {
+                    if let Err(e) = self.process_broadcasted_message(state, message).await {
                         error!("Failed to process broadcasted message: {:?}", e);
                     }
                 }
@@ -1753,13 +1744,12 @@ where
                 NetworkActorCommand::BroadcastMessage(message.clone()),
             ))
             .expect(ASSUME_NETWORK_MYSELF_ALIVE);
-        self.process_broadcasted_message(&state.network, message)
-            .await
+        self.process_broadcasted_message(state, message).await
     }
 
     async fn process_broadcasted_message(
         &self,
-        network: &ActorRef<NetworkActorMessage>,
+        state: &mut NetworkActorState<S>,
         message: FiberBroadcastMessage,
     ) -> Result<(), Error> {
         match message {
@@ -1787,16 +1777,21 @@ where
 
                         // TODO: bookkeeping how many nodes we have connected to. Stop connecting once we surpass a threshold.
                         for addr in &node_announcement.addresses {
-                            network.send_message(NetworkActorMessage::new_command(
-                                NetworkActorCommand::ConnectPeer(addr.clone()),
-                            ))?;
+                            state
+                                .network
+                                .send_message(NetworkActorMessage::new_command(
+                                    NetworkActorCommand::ConnectPeer(addr.clone()),
+                                ))?;
                         }
 
                         // Add the node to the network graph.
                         self.network_graph
                             .write()
                             .await
-                            .process_node_announcement(node_announcement);
+                            .process_node_announcement(node_announcement.clone());
+
+                        let peer_id = node_announcement.peer_id();
+                        state.on_node_announcement(peer_id, node_announcement.addresses.clone());
                         Ok(())
                     }
                     _ => {
@@ -2193,6 +2188,7 @@ enum RequestState {
 
 pub struct NetworkActorState<S> {
     store: S,
+    persistent_state: PersistentNetworkActorState,
     // The name of the node to be announced to the network, may be empty.
     node_name: Option<AnnouncedNodeName>,
     peer_id: PeerId,
@@ -2264,6 +2260,62 @@ pub struct NetworkActorState<S> {
     broadcasted_message_queue: Vec<(PeerId, FiberBroadcastMessage)>,
 }
 
+#[serde_as]
+#[derive(Default, Clone, Serialize, Deserialize)]
+pub struct PersistentNetworkActorState {
+    // when we restarting a node, we will reconnect to these peers
+    #[serde_as(as = "Vec<(DisplayFromStr, _)>")]
+    peer_store: HashMap<PeerId, Vec<Multiaddr>>,
+}
+
+impl PersistentNetworkActorState {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Save peer addresses to the peer store. If the peer addresses are updated,
+    /// return true, otherwise return false.
+    fn save_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
+        match self.peer_store.entry(peer_id) {
+            Entry::Occupied(mut entry) => {
+                if entry.get() == &addr {
+                    false
+                } else {
+                    entry.insert(addr);
+                    true
+                }
+            }
+            Entry::Vacant(entry) => {
+                entry.insert(addr);
+                true
+            }
+        }
+    }
+
+    fn get_peers_to_sync_graph(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
+        const NUM_PEERS_TO_SYNC_NETWORK_GRAPH: usize = 5;
+        self.peer_store
+            .iter()
+            .take(NUM_PEERS_TO_SYNC_NETWORK_GRAPH)
+            .map(|(id, addr)| (id.clone(), addr.clone()))
+            .collect()
+    }
+
+    fn get_peers_to_connect(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
+        const NUM_PEERS_TO_CONNECT: usize = 20;
+        self.peer_store
+            .iter()
+            .take(NUM_PEERS_TO_CONNECT)
+            .map(|(id, addr)| (id.clone(), addr.clone()))
+            .collect()
+    }
+}
+
+pub trait NetworkActorStateStore {
+    fn get_network_actor_state(&self, id: &PeerId) -> Option<PersistentNetworkActorState>;
+    fn insert_network_actor_state(&self, id: &PeerId, state: PersistentNetworkActorState);
+}
+
 static CHANNEL_ACTOR_NAME_PREFIX: AtomicU64 = AtomicU64::new(0u64);
 
 // ractor requires that the actor name is unique, so we add a prefix to the actor name.
@@ -2278,7 +2330,8 @@ fn generate_channel_actor_name(local_peer_id: &PeerId, remote_peer_id: &PeerId)
 
 impl<S> NetworkActorState<S>
 where
-    S: ChannelActorStateStore
+    S: NetworkActorStateStore
+        + ChannelActorStateStore
         + NetworkGraphStateStore
         + InvoiceStore
         + Clone
@@ -2943,6 +2996,52 @@ where
         self.maybe_tell_syncer_peer_disconnected(id);
     }
 
+    fn on_node_announcement(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
+        self.maybe_connect_to_peer(&peer_id, &addresses);
+        self.save_peer_addresses(peer_id, addresses);
+    }
+
+    fn save_peer_addresses(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
+        if self
+            .persistent_state
+            .save_peer_addresses(peer_id, addresses)
+        {
+            self.persist_state();
+        }
+    }
+
+    fn persist_state(&self) {
+        self.store
+            .insert_network_actor_state(&self.peer_id, self.persistent_state.clone());
+    }
+
+    fn maybe_connect_to_peer(&mut self, peer_id: &PeerId, addresses: &[Multiaddr]) {
+        // TODO: Make this configurable and add more complex connection logic,
+        // e.g. if a peer is banned, we should not connect to it.
+        const MAX_CONNECTED_PEERS: usize = 40;
+        if self.peer_session_map.len() >= MAX_CONNECTED_PEERS {
+            debug!(
+                "Already connected to {} peers, skipping connection to peer {:?}",
+                MAX_CONNECTED_PEERS, peer_id
+            );
+            return;
+        }
+        if let Some(session) = self.get_peer_session(peer_id) {
+            debug!(
+                "Peer {:?} already connected with session id {:?}, skipping connection",
+                peer_id, session
+            );
+            return;
+        }
+        for addr in addresses {
+            self.network
+                .send_message(NetworkActorMessage::new_command(
+                    NetworkActorCommand::ConnectPeer(addr.clone()),
+                ))
+                .expect(ASSUME_NETWORK_MYSELF_ALIVE);
+        }
+    }
+
     async fn maybe_sync_network_graph(&mut self, peer_id: &PeerId) {
         if let NetworkSyncStatus::Running(state) = &mut self.sync_status {
             if let Some(_) = state
@@ -3341,7 +3440,8 @@ pub struct NetworkActorStartArguments {
 #[rasync_trait]
 impl<S> Actor for NetworkActor<S>
 where
-    S: ChannelActorStateStore
+    S: NetworkActorStateStore
+        + ChannelActorStateStore
         + NetworkGraphStateStore
         + InvoiceStore
         + Clone
@@ -3431,16 +3531,33 @@ where
 
         let mut graph = self.network_graph.write().await;
 
-        let peers_to_sync_network_graph = graph
-            .get_peers_to_sync_network_graph()
-            .into_iter()
-            .map(|(a, b)| (a.clone(), b.clone()))
-            .collect();
+        let persistent_state = self
+            .store
+            .get_network_actor_state(&my_peer_id)
+            .unwrap_or_default();
+        let mut peers_to_connect = persistent_state.get_peers_to_connect();
+        let bootnodes = config.bootnode_addrs.clone();
+        for bootnode in &bootnodes {
+            let addr = Multiaddr::from_str(&bootnode).expect("valid bootnode");
+            let peer_id = extract_peer_id(&addr).expect("valid peer id");
+            // If we have already selected the boot node as a peer to connect, then
+            // add this address to the list of addresses to connect to.
+            // Otherwise, create a new list out of this address.
+            peers_to_connect
+                .entry(peer_id.clone())
+                .or_insert_with(Vec::new)
+                .push(addr);
+        }
+
+        // Don't sync graph with bootnodes, as that may overload the bootnodes.
+        // Instead select a few peers from the persistent state to sync graph.
+        let peers_to_sync_graph = persistent_state.get_peers_to_sync_graph();
+
         let height = graph.get_best_height();
         let last_update = graph.get_last_update_timestamp();
         debug!(
             "Trying to sync network graph with peers {:?} with height {} and last update {:?}",
-            &peers_to_sync_network_graph, &height, &last_update
+            &peers_to_sync_graph, &height, &last_update
         );
 
         let chain_actor = self.chain_actor.clone();
@@ -3452,10 +3569,15 @@ where
             height,
             current_block_number,
             last_update,
-            peers_to_sync_network_graph,
+            peers_to_sync_graph
+                .into_iter()
+                .map(|(peer_id, addrs)| (peer_id, addrs.into_iter().next().unwrap()))
+                .collect(),
         );
+
         let mut state = NetworkActorState {
             store: self.store.clone(),
+            persistent_state,
             node_name: config.announced_node_name,
             peer_id: my_peer_id,
             announced_addrs,
@@ -3494,16 +3616,12 @@ where
         let node_announcement = state.get_or_create_new_node_announcement_message();
         graph.process_node_announcement(node_announcement);
 
-        // load the connected peers from the network graph
-        let peers = graph.get_connected_peers();
-        // TODO: we need to bootstrap the network if no peers are connected.
-        if peers.is_empty() {
-            warn!("No connected peers found in the network graph");
-        }
-        for (_peer_id, addr) in peers {
-            myself.send_message(NetworkActorMessage::new_command(
-                NetworkActorCommand::ConnectPeer(addr.clone()),
-            ))?;
+        for (_peer_id, addrs) in peers_to_connect {
+            for addr in addrs {
+                myself.send_message(NetworkActorMessage::new_command(
+                    NetworkActorCommand::ConnectPeer(addr),
+                ))?;
+            }
         }
 
         let announce_node_interval_seconds = config.announce_node_interval_seconds();
@@ -3547,6 +3665,8 @@ where
         if let Err(err) = state.control.close().await {
             error!("Failed to close tentacle service: {}", err);
         }
+        debug!("Saving network actor state for {:?}", state.peer_id);
+        state.persist_state();
         debug!("Network service for {:?} shutdown", state.peer_id);
         // The event receiver may have been closed already.
         // We ignore the error here.
@@ -3695,7 +3815,14 @@ pub(crate) fn emit_service_event(
 }
 
 pub async fn start_network<
-    S: ChannelActorStateStore + NetworkGraphStateStore + InvoiceStore + Clone + Send + Sync + 'static,
+    S: NetworkActorStateStore
+        + ChannelActorStateStore
+        + NetworkGraphStateStore
+        + InvoiceStore
+        + Clone
+        + Send
+        + Sync
+        + 'static,
 >(
     config: FiberConfig,
     chain_actor: ActorRef<CkbChainMessage>,
diff --git a/src/fiber/tests/graph.rs b/src/fiber/tests/graph.rs
index ce9fd6d4..9d784dac 100644
--- a/src/fiber/tests/graph.rs
+++ b/src/fiber/tests/graph.rs
@@ -11,17 +11,8 @@ use ckb_types::{
     prelude::Entity,
 };
 use secp256k1::{PublicKey, SecretKey, XOnlyPublicKey};
-use tentacle::{multiaddr::Multiaddr, secio::PeerId};
 
-use super::test_utils::{generate_keypair, generate_pubkey};
-
-fn generate_keys(num: usize) -> Vec<PublicKey> {
-    let mut keys = vec![];
-    for _ in 0..num {
-        keys.push(generate_pubkey());
-    }
-    keys
-}
+use super::test_utils::generate_keypair;
 
 fn generate_key_pairs(num: usize) -> Vec<(SecretKey, PublicKey)> {
     let mut keys = vec![];
@@ -193,41 +184,6 @@ impl MockNetworkGraph {
     }
 }
 
-#[test]
-fn test_graph_connected_peers() {
-    let temp_path = tempfile::tempdir().unwrap();
-    let store = Store::new(temp_path.path());
-    let keys = generate_keys(1);
-    let public_key1 = keys[0];
-    let mut network_graph = NetworkGraph::new(store, public_key1.into());
-
-    let peer_id = PeerId::random();
-    let address: Multiaddr = "/ip4/127.0.0.1/tcp/10000".parse().unwrap();
-    network_graph.add_connected_peer(&peer_id, address.clone());
-
-    let connected_peers = network_graph.get_connected_peers();
-    assert_eq!(connected_peers.len(), 1);
-    assert_eq!(connected_peers[0], (&peer_id, &address));
-
-    network_graph.reset();
-    let connected_peers = network_graph.get_connected_peers();
-    assert_eq!(connected_peers.len(), 0);
-
-    // load from db
-    network_graph.load_from_store();
-    let connected_peers = network_graph.get_connected_peers();
-    assert_eq!(connected_peers.len(), 1);
-    assert_eq!(connected_peers[0], (&peer_id, &address));
-
-    network_graph.remove_connected_peer(&peer_id);
-    let connected_peers = network_graph.get_connected_peers();
-    assert_eq!(connected_peers.len(), 0);
-
-    network_graph.load_from_store();
-    let connected_peers = network_graph.get_connected_peers();
-    assert_eq!(connected_peers.len(), 0);
-}
-
 #[test]
 fn test_graph_channel_info() {
     let mut mock_network = MockNetworkGraph::new(1);
diff --git a/src/fiber/tests/test_utils.rs b/src/fiber/tests/test_utils.rs
index f123d1a4..63383fed 100644
--- a/src/fiber/tests/test_utils.rs
+++ b/src/fiber/tests/test_utils.rs
@@ -1,6 +1,3 @@
-use crate::fiber::graph::{ChannelInfo, NetworkGraph, NodeInfo};
-use crate::fiber::types::Pubkey;
-use crate::invoice::{CkbInvoice, InvoiceError, InvoiceStore};
 use ckb_jsonrpc_types::JsonBytes;
 use ckb_types::packed::OutPoint;
 use ckb_types::{core::TransactionView, packed::Byte32};
@@ -17,7 +14,6 @@ use std::{
     time::Duration,
 };
 use tempfile::TempDir as OldTempDir;
-use tentacle::multiaddr::Multiaddr;
 use tentacle::{multiaddr::MultiAddr, secio::PeerId};
 use tokio::sync::RwLock as TokioRwLock;
 use tokio::{
@@ -30,19 +26,20 @@ use crate::{
     actors::{RootActor, RootActorMessage},
     ckb::tests::test_utils::{submit_tx, trace_tx, trace_tx_hash, MockChainActor},
     ckb::CkbChainMessage,
+    fiber::channel::{ChannelActorState, ChannelActorStateStore, ChannelState},
+    fiber::graph::NetworkGraphStateStore,
+    fiber::graph::PaymentSession,
+    fiber::graph::{ChannelInfo, NetworkGraph, NodeInfo},
     fiber::network::NetworkActorStartArguments,
+    fiber::network::{NetworkActor, NetworkActorCommand, NetworkActorMessage},
+    fiber::network::{NetworkActorStateStore, PersistentNetworkActorState},
+    fiber::types::Hash256,
+    fiber::types::Pubkey,
+    invoice::{CkbInvoice, InvoiceError, InvoiceStore},
     tasks::{new_tokio_cancellation_token, new_tokio_task_tracker},
     FiberConfig, NetworkServiceEvent,
 };
 
-use crate::fiber::graph::NetworkGraphStateStore;
-use crate::fiber::graph::PaymentSession;
-use crate::fiber::{
-    channel::{ChannelActorState, ChannelActorStateStore, ChannelState},
-    types::Hash256,
-    NetworkActor, NetworkActorCommand, NetworkActorMessage,
-};
-
 static RETAIN_VAR: &str = "TEST_TEMP_RETAIN";
 
 #[derive(Debug)]
@@ -465,15 +462,28 @@ impl NetworkNode {
 
 #[derive(Clone, Default)]
 pub struct MemoryStore {
+    network_actor_sate_map: Arc<RwLock<HashMap<PeerId, PersistentNetworkActorState>>>,
     channel_actor_state_map: Arc<RwLock<HashMap<Hash256, ChannelActorState>>>,
     channels_map: Arc<RwLock<HashMap<OutPoint, ChannelInfo>>>,
-    pub nodes_map: Arc<RwLock<HashMap<Pubkey, NodeInfo>>>,
-    connected_peer_addresses: Arc<RwLock<HashMap<PeerId, Multiaddr>>>,
+    nodes_map: Arc<RwLock<HashMap<Pubkey, NodeInfo>>>,
     payment_sessions: Arc<RwLock<HashMap<Hash256, PaymentSession>>>,
     invoice_store: Arc<RwLock<HashMap<Hash256, CkbInvoice>>>,
     invoice_hash_to_preimage: Arc<RwLock<HashMap<Hash256, Hash256>>>,
 }
 
+impl NetworkActorStateStore for MemoryStore {
+    fn get_network_actor_state(&self, id: &PeerId) -> Option<PersistentNetworkActorState> {
+        self.network_actor_sate_map.read().unwrap().get(id).cloned()
+    }
+
+    fn insert_network_actor_state(&self, id: &PeerId, state: PersistentNetworkActorState) {
+        self.network_actor_sate_map
+            .write()
+            .unwrap()
+            .insert(id.clone(), state);
+    }
+}
+
 impl NetworkGraphStateStore for MemoryStore {
     fn get_channels(&self, outpoint: Option<OutPoint>) -> Vec<ChannelInfo> {
         if let Some(outpoint) = outpoint {
@@ -538,38 +548,6 @@ impl NetworkGraphStateStore for MemoryStore {
             .insert(node.node_id.clone(), node);
     }
 
-    fn insert_connected_peer(&self, peer_id: PeerId, multiaddr: Multiaddr) {
-        self.connected_peer_addresses
-            .write()
-            .unwrap()
-            .insert(peer_id, multiaddr);
-    }
-
-    fn get_connected_peer(&self, peer_id: Option<PeerId>) -> Vec<(PeerId, Multiaddr)> {
-        if let Some(peer_id) = peer_id {
-            let mut res = vec![];
-
-            if let Some(addr) = self.connected_peer_addresses.read().unwrap().get(&peer_id) {
-                res.push((peer_id, addr.clone()));
-            }
-            res
-        } else {
-            self.connected_peer_addresses
-                .read()
-                .unwrap()
-                .iter()
-                .map(|(peer_id, addr)| (peer_id.clone(), addr.clone()))
-                .collect()
-        }
-    }
-
-    fn remove_connected_peer(&self, peer_id: &PeerId) {
-        self.connected_peer_addresses
-            .write()
-            .unwrap()
-            .remove(peer_id);
-    }
-
     fn get_payment_session(&self, id: Hash256) -> Option<PaymentSession> {
         self.payment_sessions.read().unwrap().get(&id).cloned()
     }
diff --git a/src/fiber/types.rs b/src/fiber/types.rs
index 4009e37d..8b49f39a 100644
--- a/src/fiber/types.rs
+++ b/src/fiber/types.rs
@@ -1494,6 +1494,10 @@ impl NodeAnnouncement {
         };
         deterministically_hash(&unsigned_announcement)
     }
+
+    pub fn peer_id(&self) -> PeerId {
+        PeerId::from_public_key(&self.node_id.into())
+    }
 }
 
 impl From<UdtCellDep> for molecule_fiber::UdtCellDep {
diff --git a/src/main.rs b/src/main.rs
index 94d4c2e9..1d17c837 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,23 +1,24 @@
-use ckb_hash::blake2b_256;
-use core::default::Default;
 use fnn::actors::RootActor;
 use fnn::cch::CchMessage;
-use fnn::ckb::contracts::{get_script_by_contract, init_contracts_context, Contract};
-use fnn::ckb::CkbChainActor;
-use fnn::fiber::graph::NetworkGraph;
-use fnn::fiber::{channel::ChannelSubscribers, NetworkActorCommand, NetworkActorMessage};
+use fnn::ckb::{
+    contracts::{get_script_by_contract, init_contracts_context, Contract},
+    CkbChainActor,
+};
+use fnn::fiber::{channel::ChannelSubscribers, graph::NetworkGraph};
 use fnn::store::Store;
 use fnn::tasks::{
     cancel_tasks_and_wait_for_completion, new_tokio_cancellation_token, new_tokio_task_tracker,
 };
 use fnn::watchtower::{WatchtowerActor, WatchtowerMessage};
 use fnn::{start_cch, start_network, start_rpc, Config};
-use ractor::Actor;
-use secp256k1::Secp256k1;
-use std::str::FromStr;
+
+use core::default::Default;
 use std::sync::Arc;
 use std::time::Duration;
-use tentacle::multiaddr::Multiaddr;
+
+use ckb_hash::blake2b_256;
+use ractor::Actor;
+use secp256k1::Secp256k1;
 use tokio::sync::{mpsc, RwLock};
 use tokio::{select, signal};
 use tracing::{debug, error, info, info_span, trace};
@@ -85,8 +86,6 @@ pub async fn main() {
             const CHANNEL_SIZE: usize = 4000;
             let (event_sender, mut event_receiver) = mpsc::channel(CHANNEL_SIZE);
 
-            let bootnodes = fiber_config.bootnode_addrs.clone();
-
             let network_graph = Arc::new(RwLock::new(NetworkGraph::new(
                 store.clone(),
                 node_public_key.clone().into(),
@@ -112,14 +111,6 @@ pub async fn main() {
             )
             .await;
 
-            for bootnode in bootnodes {
-                let addr = Multiaddr::from_str(&bootnode).expect("valid bootnode");
-                let command = NetworkActorCommand::ConnectPeer(addr);
-                network_actor
-                    .send_message(NetworkActorMessage::new_command(command))
-                    .expect("ckb actor alive")
-            }
-
             let watchtower_actor = Actor::spawn_linked(
                 Some("watchtower".to_string()),
                 WatchtowerActor::new(store.clone()),
diff --git a/src/store.rs b/src/store.rs
index 4ef45fc6..c465f57d 100644
--- a/src/store.rs
+++ b/src/store.rs
@@ -2,6 +2,7 @@ use crate::{
     fiber::{
         channel::{ChannelActorState, ChannelActorStateStore, ChannelState},
         graph::{ChannelInfo, NetworkGraphStateStore, NodeInfo, PaymentSession},
+        network::{NetworkActorStateStore, PersistentNetworkActorState},
         types::{Hash256, Pubkey},
     },
     invoice::{CkbInvoice, InvoiceError, InvoiceStore},
@@ -13,7 +14,7 @@ use ckb_types::prelude::Entity;
 use rocksdb::{prelude::*, DBIterator, Direction, IteratorMode, WriteBatch, DB};
 use serde_json;
 use std::{path::Path, sync::Arc};
-use tentacle::{multiaddr::Multiaddr, secio::PeerId};
+use tentacle::secio::PeerId;
 
 #[derive(Clone)]
 pub struct Store {
@@ -157,18 +158,19 @@ impl Batch {
                     serde_json::to_vec(&node).expect("serialize NodeInfo should be OK"),
                 );
             }
-            KeyValue::PeerIdMultiAddr(peer_id, multiaddr) => {
-                let key = [&[PEER_ID_MULTIADDR_PREFIX], peer_id.as_bytes()].concat();
+            KeyValue::WatchtowerChannel(channel_id, channel_data) => {
+                let key = [&[WATCHTOWER_CHANNEL_PREFIX], channel_id.as_ref()].concat();
                 self.put(
                     key,
-                    serde_json::to_vec(&multiaddr).expect("serialize Multiaddr should be OK"),
+                    serde_json::to_vec(&channel_data).expect("serialize ChannelData should be OK"),
                 );
             }
-            KeyValue::WatchtowerChannel(channel_id, channel_data) => {
-                let key = [&[WATCHTOWER_CHANNEL_PREFIX], channel_id.as_ref()].concat();
+            KeyValue::NetworkActorState(peer_id, persistent_network_actor_state) => {
+                let key = [&[PEER_ID_NETWORK_ACTOR_STATE_PREFIX], peer_id.as_bytes()].concat();
                 self.put(
                     key,
-                    serde_json::to_vec(&channel_data).expect("serialize ChannelData should be OK"),
+                    serde_json::to_vec(&persistent_network_actor_state)
+                        .expect("serialize PersistentNetworkActorState should be OK"),
                 );
             }
         }
@@ -188,24 +190,26 @@ impl Batch {
 }
 
 ///
-/// +--------------+--------------------+--------------------------+
-/// | KeyPrefix::  | Key::              | Value::                  |
-/// +--------------+--------------------+--------------------------+
-/// | 0            | Hash256            | ChannelActorState        |
-/// | 32           | Hash256            | CkbInvoice               |
-/// | 64           | PeerId | Hash256   | ChannelState             |
-/// | 96           | ChannelId          | ChannelInfo              |
-/// | 97           | Block | Index      | ChannelId                |
-/// | 98           | Timestamp          | ChannelId                |
-/// | 128          | NodeId             | NodeInfo                 |
-/// | 129          | Timestamp          | NodeId                   |
-/// | 160          | PeerId             | MultiAddr                |
-/// | 192          | Hash256            | PaymentSession           |
-/// | 224          | Hash256            | ChannelData              |
-/// +--------------+--------------------+--------------------------+
+/// +--------------+--------------------+-----------------------------+
+/// | KeyPrefix::  | Key::              | Value::                     |
+/// +--------------+--------------------+-----------------------------+
+/// | 0            | Hash256            | ChannelActorState           |
+/// | 16           | PeerId             | PersistentNetworkActorState |
+/// | 32           | Hash256            | CkbInvoice                  |
+/// | 64           | PeerId | Hash256   | ChannelState                |
+/// | 96           | ChannelId          | ChannelInfo                 |
+/// | 97           | Block | Index      | ChannelId                   |
+/// | 98           | Timestamp          | ChannelId                   |
+/// | 128          | NodeId             | NodeInfo                    |
+/// | 129          | Timestamp          | NodeId                      |
+/// | 160          | PeerId             | MultiAddr                   |
+/// | 192          | Hash256            | PaymentSession              |
+/// | 224          | Hash256            | ChannelData                 |
+/// +--------------+--------------------+-----------------------------+
 ///
 
 const CHANNEL_ACTOR_STATE_PREFIX: u8 = 0;
+const PEER_ID_NETWORK_ACTOR_STATE_PREFIX: u8 = 16;
 const CKB_INVOICE_PREFIX: u8 = 32;
 const CKB_INVOICE_PREIMAGE_PREFIX: u8 = 33;
 const PEER_ID_CHANNEL_ID_PREFIX: u8 = 64;
@@ -214,7 +218,6 @@ const CHANNEL_ANNOUNCEMENT_INDEX_PREFIX: u8 = 97;
 const CHANNEL_UPDATE_INDEX_PREFIX: u8 = 98;
 pub(crate) const NODE_INFO_PREFIX: u8 = 128;
 const NODE_ANNOUNCEMENT_INDEX_PREFIX: u8 = 129;
-const PEER_ID_MULTIADDR_PREFIX: u8 = 160;
 const PAYMENT_SESSION_PREFIX: u8 = 192;
 const WATCHTOWER_CHANNEL_PREFIX: u8 = 224;
 
@@ -223,11 +226,33 @@ enum KeyValue {
     CkbInvoice(Hash256, CkbInvoice),
     CkbInvoicePreimage(Hash256, Hash256),
     PeerIdChannelId((PeerId, Hash256), ChannelState),
-    PeerIdMultiAddr(PeerId, Multiaddr),
     NodeInfo(Pubkey, NodeInfo),
     ChannelInfo(OutPoint, ChannelInfo),
     WatchtowerChannel(Hash256, ChannelData),
     PaymentSession(Hash256, PaymentSession),
+    NetworkActorState(PeerId, PersistentNetworkActorState),
+}
+
+impl NetworkActorStateStore for Store {
+    fn get_network_actor_state(&self, id: &PeerId) -> Option<PersistentNetworkActorState> {
+        let mut key = Vec::with_capacity(33);
+        key.push(PEER_ID_NETWORK_ACTOR_STATE_PREFIX);
+        key.extend_from_slice(id.as_bytes());
+        let iter = self
+            .db
+            .prefix_iterator(key.as_ref())
+            .find(|(col_key, _)| col_key.starts_with(&key));
+        iter.map(|(_key, value)| {
+            serde_json::from_slice(value.as_ref())
+                .expect("deserialize PersistentNetworkActorState should be OK")
+        })
+    }
+
+    fn insert_network_actor_state(&self, id: &PeerId, state: PersistentNetworkActorState) {
+        let mut batch = self.batch();
+        batch.put_kv(KeyValue::NetworkActorState(id.clone(), state));
+        batch.commit();
+    }
 }
 
 impl ChannelActorStateStore for Store {
@@ -446,30 +471,6 @@ impl NetworkGraphStateStore for Store {
         (nodes, JsonBytes::from_bytes(last_key.into()))
     }
 
-    fn get_connected_peer(&self, peer_id: Option<PeerId>) -> Vec<(PeerId, Multiaddr)> {
-        let key = match peer_id {
-            Some(peer_id) => {
-                let mut key = Vec::with_capacity(33);
-                key.push(PEER_ID_MULTIADDR_PREFIX);
-                key.extend_from_slice(peer_id.as_bytes());
-                key
-            }
-            None => vec![PEER_ID_MULTIADDR_PREFIX],
-        };
-        let iter = self
-            .db
-            .prefix_iterator(key.as_ref())
-            .take_while(|(col_key, _)| col_key.starts_with(&key));
-        iter.map(|(key, value)| {
-            let peer_id =
-                PeerId::from_bytes(key[1..].into()).expect("deserialize peer id should be OK");
-            let addr =
-                serde_json::from_slice(value.as_ref()).expect("deserialize Multiaddr should be OK");
-            (peer_id, addr)
-        })
-        .collect()
-    }
-
     fn insert_channel(&self, channel: ChannelInfo) {
         let mut batch = self.batch();
         batch.put_kv(KeyValue::ChannelInfo(channel.out_point(), channel.clone()));
@@ -482,23 +483,6 @@ impl NetworkGraphStateStore for Store {
         batch.commit();
     }
 
-    fn insert_connected_peer(&self, peer_id: PeerId, multiaddr: Multiaddr) {
-        let mut batch = self.batch();
-        batch.put_kv(KeyValue::PeerIdMultiAddr(peer_id, multiaddr));
-        batch.commit();
-    }
-
-    fn remove_connected_peer(&self, peer_id: &PeerId) {
-        let prefix = [&[PEER_ID_MULTIADDR_PREFIX], peer_id.as_bytes()].concat();
-        let iter = self
-            .db
-            .prefix_iterator(prefix.as_ref())
-            .take_while(|(key, _)| key.starts_with(&prefix));
-        for (key, _) in iter {
-            self.db.delete(key).expect("delete should be OK");
-        }
-    }
-
     fn get_payment_session(&self, payment_hash: Hash256) -> Option<PaymentSession> {
         let prefix = [&[PAYMENT_SESSION_PREFIX], payment_hash.as_ref()].concat();
         self.get(prefix).map(|v| {

From ddf07942b45342057b320f8e4b29e7fb6dfaf311 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Thu, 17 Oct 2024 19:27:20 +0800
Subject: [PATCH 55/66] Do scheduled maintenance for connected peers in network

---
 src/fiber/network.rs | 116 +++++++++++++++++++++++++------------------
 1 file changed, 69 insertions(+), 47 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 6d8fa3d6..92571842 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -99,6 +99,13 @@ const ASSUME_CHAIN_ACTOR_ALWAYS_ALIVE_FOR_NOW: &str =
 
 const ASSUME_NETWORK_MYSELF_ALIVE: &str = "network actor myself alive";
 
+// This is the approximate number of peers that we need to keep connection to to make the
+// network operating normally.
+const NUM_PEERS_IN_CONNECTION: usize = 40;
+
+// The duration for which we will try to maintain the number of peers in connection.
+const MAINTAINING_CONNECTED_PEERS_INTERVAL: Duration = Duration::from_secs(3600);
+
 pub(crate) fn get_chain_hash() -> Hash256 {
     Default::default()
 }
@@ -155,6 +162,7 @@ pub enum NetworkActorCommand {
     /// Network commands
     ConnectPeer(Multiaddr),
     DisconnectPeer(PeerId),
+    MaintainConnectedPeers(),
     // For internal use and debugging only. Most of the messages requires some
     // changes to local state. Even if we can send a message to a peer, some
     // part of the local state is not changed.
@@ -1229,6 +1237,41 @@ where
                 }
             }
 
+            NetworkActorCommand::MaintainConnectedPeers() => {
+                let num_connected_peers = state.peer_session_map.len();
+                if num_connected_peers >= NUM_PEERS_IN_CONNECTION {
+                    debug!(
+                        "Already connected to {} peers, skipping connecting to more peers",
+                        num_connected_peers,
+                    );
+                    return Ok(());
+                }
+                // Even though we have connected to some peers, we still try to select
+                // NUM_PEERS_IN_CONNECTION, because unless we're extremely lucky, we will connect
+                // to much less than 2 * NUM_PEERS_IN_CONNECTION peers. That should still be OK,
+                // because we will not be connecting to more peers in the next run.
+                let peers_to_connect = state
+                    .state_to_be_persisted
+                    .sample_n_peers_to_connect(NUM_PEERS_IN_CONNECTION);
+                for (peer_id, addresses) in peers_to_connect {
+                    if let Some(session) = state.get_peer_session(&peer_id) {
+                        debug!(
+                            "Randomly selected peer {:?} already connected with session id {:?}, skipping connection",
+                            peer_id, session
+                        );
+                        continue;
+                    }
+                    for addr in addresses {
+                        state
+                            .network
+                            .send_message(NetworkActorMessage::new_command(
+                                NetworkActorCommand::ConnectPeer(addr.clone()),
+                            ))
+                            .expect(ASSUME_NETWORK_MYSELF_ALIVE);
+                    }
+                }
+            }
+
             NetworkActorCommand::OpenChannel(open_channel, reply) => {
                 match state.create_outbound_channel(open_channel).await {
                     Ok((_, channel_id)) => {
@@ -2188,7 +2231,7 @@ enum RequestState {
 
 pub struct NetworkActorState<S> {
     store: S,
-    persistent_state: PersistentNetworkActorState,
+    state_to_be_persisted: PersistentNetworkActorState,
     // The name of the node to be announced to the network, may be empty.
     node_name: Option<AnnouncedNodeName>,
     peer_id: PeerId,
@@ -2301,11 +2344,10 @@ impl PersistentNetworkActorState {
             .collect()
     }
 
-    fn get_peers_to_connect(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
-        const NUM_PEERS_TO_CONNECT: usize = 20;
+    fn sample_n_peers_to_connect(&self, n: usize) -> HashMap<PeerId, Vec<Multiaddr>> {
         self.peer_store
             .iter()
-            .take(NUM_PEERS_TO_CONNECT)
+            .take(n)
             .map(|(id, addr)| (id.clone(), addr.clone()))
             .collect()
     }
@@ -2997,13 +3039,12 @@ where
     }
 
     fn on_node_announcement(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
-        self.maybe_connect_to_peer(&peer_id, &addresses);
         self.save_peer_addresses(peer_id, addresses);
     }
 
     fn save_peer_addresses(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
         if self
-            .persistent_state
+            .state_to_be_persisted
             .save_peer_addresses(peer_id, addresses)
         {
             self.persist_state();
@@ -3012,34 +3053,7 @@ where
 
     fn persist_state(&self) {
         self.store
-            .insert_network_actor_state(&self.peer_id, self.persistent_state.clone());
-    }
-
-    fn maybe_connect_to_peer(&mut self, peer_id: &PeerId, addresses: &[Multiaddr]) {
-        // TODO: Make this configurable and add more complex connection logic,
-        // e.g. if a peer is banned, we should not connect to it.
-        const MAX_CONNECTED_PEERS: usize = 40;
-        if self.peer_session_map.len() >= MAX_CONNECTED_PEERS {
-            debug!(
-                "Already connected to {} peers, skipping connection to peer {:?}",
-                MAX_CONNECTED_PEERS, peer_id
-            );
-            return;
-        }
-        if let Some(session) = self.get_peer_session(peer_id) {
-            debug!(
-                "Peer {:?} already connected with session id {:?}, skipping connection",
-                peer_id, session
-            );
-            return;
-        }
-        for addr in addresses {
-            self.network
-                .send_message(NetworkActorMessage::new_command(
-                    NetworkActorCommand::ConnectPeer(addr.clone()),
-                ))
-                .expect(ASSUME_NETWORK_MYSELF_ALIVE);
-        }
+            .insert_network_actor_state(&self.peer_id, self.state_to_be_persisted.clone());
     }
 
     async fn maybe_sync_network_graph(&mut self, peer_id: &PeerId) {
@@ -3531,11 +3545,12 @@ where
 
         let mut graph = self.network_graph.write().await;
 
-        let persistent_state = self
+        let mut state_to_be_persisted = self
             .store
             .get_network_actor_state(&my_peer_id)
             .unwrap_or_default();
-        let mut peers_to_connect = persistent_state.get_peers_to_connect();
+        let mut peers_to_connect =
+            state_to_be_persisted.sample_n_peers_to_connect(NUM_PEERS_IN_CONNECTION);
         let bootnodes = config.bootnode_addrs.clone();
         for bootnode in &bootnodes {
             let addr = Multiaddr::from_str(&bootnode).expect("valid bootnode");
@@ -3544,14 +3559,23 @@ where
             // add this address to the list of addresses to connect to.
             // Otherwise, create a new list out of this address.
             peers_to_connect
-                .entry(peer_id.clone())
+                .entry(peer_id)
                 .or_insert_with(Vec::new)
                 .push(addr);
         }
 
+        for (peer_id, addrs) in peers_to_connect {
+            state_to_be_persisted.save_peer_addresses(peer_id, addrs.clone());
+            for addr in addrs {
+                myself.send_message(NetworkActorMessage::new_command(
+                    NetworkActorCommand::ConnectPeer(addr),
+                ))?;
+            }
+        }
+
         // Don't sync graph with bootnodes, as that may overload the bootnodes.
         // Instead select a few peers from the persistent state to sync graph.
-        let peers_to_sync_graph = persistent_state.get_peers_to_sync_graph();
+        let peers_to_sync_graph = state_to_be_persisted.get_peers_to_sync_graph();
 
         let height = graph.get_best_height();
         let last_update = graph.get_last_update_timestamp();
@@ -3577,7 +3601,7 @@ where
 
         let mut state = NetworkActorState {
             store: self.store.clone(),
-            persistent_state,
+            state_to_be_persisted,
             node_name: config.announced_node_name,
             peer_id: my_peer_id,
             announced_addrs,
@@ -3616,14 +3640,6 @@ where
         let node_announcement = state.get_or_create_new_node_announcement_message();
         graph.process_node_announcement(node_announcement);
 
-        for (_peer_id, addrs) in peers_to_connect {
-            for addr in addrs {
-                myself.send_message(NetworkActorMessage::new_command(
-                    NetworkActorCommand::ConnectPeer(addr),
-                ))?;
-            }
-        }
-
         let announce_node_interval_seconds = config.announce_node_interval_seconds();
         if announce_node_interval_seconds > 0 {
             myself.send_interval(Duration::from_secs(announce_node_interval_seconds), || {
@@ -3633,6 +3649,12 @@ where
             });
         }
 
+        myself.send_interval(MAINTAINING_CONNECTED_PEERS_INTERVAL, || {
+            NetworkActorMessage::new_command(NetworkActorCommand::MaintainConnectedPeers())
+        });
+
+        // Save bootnodes to the network actor state.
+        state.persist_state();
         Ok(state)
     }
 

From 034eed469e88974dd0969f3a0183806e2572bb9f Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Thu, 17 Oct 2024 20:11:36 +0800
Subject: [PATCH 56/66] Add some unit tests for persistent network state

---
 src/fiber/network.rs       |  6 ++---
 src/fiber/tests/network.rs | 54 +++++++++++++++++++++++++++++++++++---
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 92571842..36cef3cf 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -2318,7 +2318,7 @@ impl PersistentNetworkActorState {
 
     /// Save peer addresses to the peer store. If the peer addresses are updated,
     /// return true, otherwise return false.
-    fn save_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
+    pub(crate) fn save_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
         match self.peer_store.entry(peer_id) {
             Entry::Occupied(mut entry) => {
                 if entry.get() == &addr {
@@ -2335,7 +2335,7 @@ impl PersistentNetworkActorState {
         }
     }
 
-    fn get_peers_to_sync_graph(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
+    pub(crate) fn get_peers_to_sync_graph(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
         const NUM_PEERS_TO_SYNC_NETWORK_GRAPH: usize = 5;
         self.peer_store
             .iter()
@@ -2344,7 +2344,7 @@ impl PersistentNetworkActorState {
             .collect()
     }
 
-    fn sample_n_peers_to_connect(&self, n: usize) -> HashMap<PeerId, Vec<Multiaddr>> {
+    pub(crate) fn sample_n_peers_to_connect(&self, n: usize) -> HashMap<PeerId, Vec<Multiaddr>> {
         self.peer_store
             .iter()
             .take(n)
diff --git a/src/fiber/tests/network.rs b/src/fiber/tests/network.rs
index 236a0c10..0bfcb835 100644
--- a/src/fiber/tests/network.rs
+++ b/src/fiber/tests/network.rs
@@ -2,7 +2,8 @@ use super::test_utils::{init_tracing, NetworkNode};
 use crate::{
     fiber::{
         graph::{ChannelInfo, NetworkGraphStateStore},
-        network::get_chain_hash,
+        network::{get_chain_hash, NetworkActorStateStore},
+        tests::test_utils::NetworkNodeConfigBuilder,
         types::{
             ChannelAnnouncement, ChannelUpdate, FiberBroadcastMessage, FiberMessage,
             NodeAnnouncement, Privkey, Pubkey,
@@ -21,8 +22,11 @@ use ckb_types::{
     packed::OutPoint,
     prelude::{Builder, Entity, Pack},
 };
-use std::str::FromStr;
-use tentacle::{multiaddr::MultiAddr, secio::PeerId};
+use std::{borrow::Cow, str::FromStr};
+use tentacle::{
+    multiaddr::{MultiAddr, Protocol},
+    secio::PeerId,
+};
 
 fn get_test_priv_key() -> Privkey {
     Privkey::from_slice(&[42u8; 32])
@@ -37,6 +41,21 @@ fn get_test_peer_id() -> PeerId {
     PeerId::from_public_key(&pub_key)
 }
 
+fn get_fake_peer_id_and_address() -> (PeerId, MultiAddr) {
+    let peer_id = PeerId::random();
+    let mut address = MultiAddr::from_str(&format!(
+        "/ip4/{}.{}.{}.{}/tcp/{}",
+        rand::random::<u8>(),
+        rand::random::<u8>(),
+        rand::random::<u8>(),
+        rand::random::<u8>(),
+        rand::random::<u16>()
+    ))
+    .expect("valid multiaddr");
+    address.push(Protocol::P2P(Cow::Owned(peer_id.clone().into_bytes())));
+    (peer_id, address)
+}
+
 fn create_fake_channel_announcement_mesage(
     priv_key: Privkey,
     capacity: u64,
@@ -520,3 +539,32 @@ async fn test_sync_node_announcement_after_restart() {
     let node = node2.store.get_nodes(Some(test_pub_key));
     assert!(!node.is_empty());
 }
+
+#[tokio::test]
+async fn test_persisting_network_state() {
+    let mut node = NetworkNode::new().await;
+    let state = node.store.clone();
+    let peer_id = node.peer_id.clone();
+    node.stop().await;
+    assert!(state.get_network_actor_state(&peer_id).is_some())
+}
+
+#[tokio::test]
+async fn test_persisting_bootnode() {
+    let (boot_peer_id, address) = get_fake_peer_id_and_address();
+    let address_string = format!("{}", &address);
+
+    let mut node = NetworkNode::new_with_config(
+        NetworkNodeConfigBuilder::new()
+            .fiber_config_updater(move |config| config.bootnode_addrs = vec![address_string])
+            .build(),
+    )
+    .await;
+    let state = node.store.clone();
+    let peer_id = node.peer_id.clone();
+    node.stop().await;
+
+    let state = state.get_network_actor_state(&peer_id).unwrap();
+    let peers = state.sample_n_peers_to_connect(1);
+    assert_eq!(peers.get(&boot_peer_id), Some(&vec![address]));
+}

From 41e099a9dd78e1a5dd9d60d77789f6eb4e1796e2 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 10:53:36 +0800
Subject: [PATCH 57/66] Only connect to peers on maintaining connections

---
 src/fiber/network.rs | 76 +++++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 36cef3cf..184dc90c 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -99,12 +99,12 @@ const ASSUME_CHAIN_ACTOR_ALWAYS_ALIVE_FOR_NOW: &str =
 
 const ASSUME_NETWORK_MYSELF_ALIVE: &str = "network actor myself alive";
 
-// This is the approximate number of peers that we need to keep connection to to make the
+// This is the default approximate number of peers that we need to keep connection to to make the
 // network operating normally.
-const NUM_PEERS_IN_CONNECTION: usize = 40;
+const NUM_PEER_CONNECTIONS: usize = 40;
 
 // The duration for which we will try to maintain the number of peers in connection.
-const MAINTAINING_CONNECTED_PEERS_INTERVAL: Duration = Duration::from_secs(3600);
+const MAINTAINING_CONNECTIONS_INTERVAL: Duration = Duration::from_secs(3600);
 
 pub(crate) fn get_chain_hash() -> Hash256 {
     Default::default()
@@ -162,7 +162,8 @@ pub enum NetworkActorCommand {
     /// Network commands
     ConnectPeer(Multiaddr),
     DisconnectPeer(PeerId),
-    MaintainConnectedPeers(),
+    // We need to maintain a certain number of peers connections to keep the network running.
+    MaintainConnections(usize),
     // For internal use and debugging only. Most of the messages requires some
     // changes to local state. Even if we can send a message to a peer, some
     // part of the local state is not changed.
@@ -1237,22 +1238,18 @@ where
                 }
             }
 
-            NetworkActorCommand::MaintainConnectedPeers() => {
+            NetworkActorCommand::MaintainConnections(num_peers) => {
                 let num_connected_peers = state.peer_session_map.len();
-                if num_connected_peers >= NUM_PEERS_IN_CONNECTION {
+                if num_connected_peers >= num_peers {
                     debug!(
                         "Already connected to {} peers, skipping connecting to more peers",
                         num_connected_peers,
                     );
                     return Ok(());
                 }
-                // Even though we have connected to some peers, we still try to select
-                // NUM_PEERS_IN_CONNECTION, because unless we're extremely lucky, we will connect
-                // to much less than 2 * NUM_PEERS_IN_CONNECTION peers. That should still be OK,
-                // because we will not be connecting to more peers in the next run.
                 let peers_to_connect = state
                     .state_to_be_persisted
-                    .sample_n_peers_to_connect(NUM_PEERS_IN_CONNECTION);
+                    .sample_n_peers_to_connect(num_peers - num_connected_peers);
                 for (peer_id, addresses) in peers_to_connect {
                     if let Some(session) = state.get_peer_session(&peer_id) {
                         debug!(
@@ -2316,8 +2313,27 @@ impl PersistentNetworkActorState {
         Default::default()
     }
 
+    /// Save a single peer address to the peer store. If this address for the peer does not exist,
+    /// then return false, otherwise return true.
+    pub(crate) fn add_peer_address(&mut self, peer_id: PeerId, addr: Multiaddr) -> bool {
+        match self.peer_store.entry(peer_id) {
+            Entry::Occupied(mut entry) => {
+                if entry.get().contains(&addr) {
+                    false
+                } else {
+                    entry.get_mut().push(addr);
+                    true
+                }
+            }
+            Entry::Vacant(entry) => {
+                entry.insert(vec![addr]);
+                true
+            }
+        }
+    }
+
     /// Save peer addresses to the peer store. If the peer addresses are updated,
-    /// return true, otherwise return false.
+    /// return true, otherwise return false. This method will NOT keep the old addresses.
     pub(crate) fn save_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
         match self.peer_store.entry(peer_id) {
             Entry::Occupied(mut entry) => {
@@ -3549,28 +3565,11 @@ where
             .store
             .get_network_actor_state(&my_peer_id)
             .unwrap_or_default();
-        let mut peers_to_connect =
-            state_to_be_persisted.sample_n_peers_to_connect(NUM_PEERS_IN_CONNECTION);
-        let bootnodes = config.bootnode_addrs.clone();
-        for bootnode in &bootnodes {
-            let addr = Multiaddr::from_str(&bootnode).expect("valid bootnode");
-            let peer_id = extract_peer_id(&addr).expect("valid peer id");
-            // If we have already selected the boot node as a peer to connect, then
-            // add this address to the list of addresses to connect to.
-            // Otherwise, create a new list out of this address.
-            peers_to_connect
-                .entry(peer_id)
-                .or_insert_with(Vec::new)
-                .push(addr);
-        }
 
-        for (peer_id, addrs) in peers_to_connect {
-            state_to_be_persisted.save_peer_addresses(peer_id, addrs.clone());
-            for addr in addrs {
-                myself.send_message(NetworkActorMessage::new_command(
-                    NetworkActorCommand::ConnectPeer(addr),
-                ))?;
-            }
+        for bootnode in &config.bootnode_addrs {
+            let addr = Multiaddr::from_str(bootnode.as_str()).expect("valid bootnode");
+            let peer_id = extract_peer_id(&addr).expect("valid peer id");
+            state_to_be_persisted.add_peer_address(peer_id, addr);
         }
 
         // Don't sync graph with bootnodes, as that may overload the bootnodes.
@@ -3649,12 +3648,15 @@ where
             });
         }
 
-        myself.send_interval(MAINTAINING_CONNECTED_PEERS_INTERVAL, || {
-            NetworkActorMessage::new_command(NetworkActorCommand::MaintainConnectedPeers())
-        });
-
         // Save bootnodes to the network actor state.
         state.persist_state();
+
+        myself.send_interval(MAINTAINING_CONNECTIONS_INTERVAL, || {
+            NetworkActorMessage::new_command(NetworkActorCommand::MaintainConnections(
+                NUM_PEER_CONNECTIONS,
+            ))
+        });
+
         Ok(state)
     }
 

From adeea7ee0f7fc57efc8e3766f30b4aa43db8619f Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 11:17:40 +0800
Subject: [PATCH 58/66] Update the comment of getting peers to sync graph

---
 src/fiber/network.rs | 31 ++++++++-----------------------
 1 file changed, 8 insertions(+), 23 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 184dc90c..7356ee9f 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -2085,6 +2085,11 @@ struct NetworkSyncState {
     // The timestamp we started syncing.
     starting_time: u64,
     // All the pinned peers that we are going to sync with.
+    // TODO: the intention of passing a few peer addresses to the sync status was to let the user
+    // select a few peers to sync network graph (these peers may have faster connection to the node).
+    // After some refactoring, the code below is a little bit clouded. We are currently only connecting
+    // to random peers. If this functionality is desired, we should make a config option for it.
+    // Otherwise, remove this completely.
     pinned_syncing_peers: Vec<(PeerId, Multiaddr)>,
     active_syncers: HashMap<PeerId, ActorRef<GraphSyncerMessage>>,
     // Number of peers with whom we succeeded to sync.
@@ -2185,13 +2190,13 @@ impl NetworkSyncStatus {
         starting_height: u64,
         ending_height: u64,
         starting_time: u64,
-        syncing_peers: Vec<(PeerId, Multiaddr)>,
+        pinned_syncing_peers: Vec<(PeerId, Multiaddr)>,
     ) -> Self {
         let state = NetworkSyncState {
             starting_height,
             ending_height,
             starting_time,
-            pinned_syncing_peers: syncing_peers,
+            pinned_syncing_peers,
             active_syncers: Default::default(),
             succeeded: 0,
             failed: 0,
@@ -2351,15 +2356,6 @@ impl PersistentNetworkActorState {
         }
     }
 
-    pub(crate) fn get_peers_to_sync_graph(&self) -> HashMap<PeerId, Vec<Multiaddr>> {
-        const NUM_PEERS_TO_SYNC_NETWORK_GRAPH: usize = 5;
-        self.peer_store
-            .iter()
-            .take(NUM_PEERS_TO_SYNC_NETWORK_GRAPH)
-            .map(|(id, addr)| (id.clone(), addr.clone()))
-            .collect()
-    }
-
     pub(crate) fn sample_n_peers_to_connect(&self, n: usize) -> HashMap<PeerId, Vec<Multiaddr>> {
         self.peer_store
             .iter()
@@ -3572,16 +3568,8 @@ where
             state_to_be_persisted.add_peer_address(peer_id, addr);
         }
 
-        // Don't sync graph with bootnodes, as that may overload the bootnodes.
-        // Instead select a few peers from the persistent state to sync graph.
-        let peers_to_sync_graph = state_to_be_persisted.get_peers_to_sync_graph();
-
         let height = graph.get_best_height();
         let last_update = graph.get_last_update_timestamp();
-        debug!(
-            "Trying to sync network graph with peers {:?} with height {} and last update {:?}",
-            &peers_to_sync_graph, &height, &last_update
-        );
 
         let chain_actor = self.chain_actor.clone();
         let current_block_number = call!(chain_actor, CkbChainMessage::GetCurrentBlockNumber, ())
@@ -3592,10 +3580,7 @@ where
             height,
             current_block_number,
             last_update,
-            peers_to_sync_graph
-                .into_iter()
-                .map(|(peer_id, addrs)| (peer_id, addrs.into_iter().next().unwrap()))
-                .collect(),
+            vec![],
         );
 
         let mut state = NetworkActorState {

From d7ca6f36810cd93a1731aa3b9b570b985742384b Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 11:37:16 +0800
Subject: [PATCH 59/66] Add test for bootnode connection

---
 src/fiber/tests/network.rs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/fiber/tests/network.rs b/src/fiber/tests/network.rs
index 0bfcb835..52040d01 100644
--- a/src/fiber/tests/network.rs
+++ b/src/fiber/tests/network.rs
@@ -568,3 +568,22 @@ async fn test_persisting_bootnode() {
     let peers = state.sample_n_peers_to_connect(1);
     assert_eq!(peers.get(&boot_peer_id), Some(&vec![address]));
 }
+
+#[tokio::test]
+async fn test_connecting_to_bootnode() {
+    let boot_node = NetworkNode::new().await;
+    let boot_node_address = format!("{}", boot_node.get_node_address());
+    let boot_node_id = &boot_node.peer_id;
+
+    let mut node = NetworkNode::new_with_config(
+        NetworkNodeConfigBuilder::new()
+            .fiber_config_updater(move |config| config.bootnode_addrs = vec![boot_node_address])
+            .build(),
+    )
+    .await;
+
+    node.expect_event(
+        |event| matches!(event, NetworkServiceEvent::PeerConnected(id, _addr) if id == boot_node_id),
+    )
+    .await;
+}

From 7d84f35d975a535ec9e7afbdfe4bc5e5f10dac30 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 11:37:54 +0800
Subject: [PATCH 60/66] Fire MaintainConnection immediately

---
 src/fiber/network.rs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 7356ee9f..30ce7991 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -3636,13 +3636,25 @@ where
         // Save bootnodes to the network actor state.
         state.persist_state();
 
+        Ok(state)
+    }
+
+    async fn post_start(
+        &self,
+        myself: ActorRef<Self::Msg>,
+        _state: &mut Self::State,
+    ) -> Result<(), ActorProcessingErr> {
+        myself
+            .send_message(NetworkActorMessage::new_command(
+                NetworkActorCommand::MaintainConnections(NUM_PEER_CONNECTIONS),
+            ))
+            .expect(ASSUME_NETWORK_MYSELF_ALIVE);
         myself.send_interval(MAINTAINING_CONNECTIONS_INTERVAL, || {
             NetworkActorMessage::new_command(NetworkActorCommand::MaintainConnections(
                 NUM_PEER_CONNECTIONS,
             ))
         });
-
-        Ok(state)
+        Ok(())
     }
 
     async fn handle(

From 9725d761b6d4a28225339c9ea11b7f48489607eb Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 19:51:53 +0800
Subject: [PATCH 61/66] Add command to save peer address to peer store

---
 src/fiber/network.rs       | 136 +++++++++++++++++++++++++++++--------
 src/fiber/tests/network.rs |  28 ++++++++
 src/rpc/peer.rs            |  12 +++-
 3 files changed, 147 insertions(+), 29 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index 30ce7991..e31ee8b9 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::borrow::Cow;
 use std::collections::hash_map::Entry;
+use std::hash::RandomState;
 use tentacle::utils::extract_peer_id;
 
 use std::collections::{HashMap, HashSet};
@@ -160,8 +161,12 @@ pub struct NodeInfoResponse {
 #[derive(Debug)]
 pub enum NetworkActorCommand {
     /// Network commands
+    // Connect to a peer, and optionally also save the peer to the peer store.
     ConnectPeer(Multiaddr),
     DisconnectPeer(PeerId),
+    // Save the address of a peer to the peer store, the address here must be a valid
+    // multiaddr with the peer id.
+    SavePeerAddress(Multiaddr),
     // We need to maintain a certain number of peers connections to keep the network running.
     MaintainConnections(usize),
     // For internal use and debugging only. Most of the messages requires some
@@ -1199,7 +1204,6 @@ where
         state: &mut NetworkActorState<S>,
         command: NetworkActorCommand,
     ) -> crate::Result<()> {
-        debug!("Handling command: {:?}", command);
         match command {
             NetworkActorCommand::SendFiberMessage(FiberMessageWithPeerId { peer_id, message }) => {
                 state.send_message_to_peer(&peer_id, message).await?;
@@ -1238,7 +1242,19 @@ where
                 }
             }
 
+            NetworkActorCommand::SavePeerAddress(addr) => match extract_peer_id(&addr) {
+                Some(peer) => {
+                    debug!("Saved peer id {:?} with address {:?}", &peer, &addr);
+                    state.save_peer_address(peer, addr);
+                }
+                None => {
+                    error!("Failed to save address to peer store: unable to extract peer id from address {:?}", &addr);
+                }
+            },
+
             NetworkActorCommand::MaintainConnections(num_peers) => {
+                debug!("Maintaining connections to {} peers", num_peers);
+
                 let num_connected_peers = state.peer_session_map.len();
                 if num_connected_peers >= num_peers {
                     debug!(
@@ -1250,6 +1266,10 @@ where
                 let peers_to_connect = state
                     .state_to_be_persisted
                     .sample_n_peers_to_connect(num_peers - num_connected_peers);
+                debug!(
+                    "Randomly selected peers to connect: {:?}",
+                    &peers_to_connect
+                );
                 for (peer_id, addresses) in peers_to_connect {
                     if let Some(session) = state.get_peer_session(&peer_id) {
                         debug!(
@@ -1815,15 +1835,6 @@ where
                             &node_announcement
                         );
 
-                        // TODO: bookkeeping how many nodes we have connected to. Stop connecting once we surpass a threshold.
-                        for addr in &node_announcement.addresses {
-                            state
-                                .network
-                                .send_message(NetworkActorMessage::new_command(
-                                    NetworkActorCommand::ConnectPeer(addr.clone()),
-                                ))?;
-                        }
-
                         // Add the node to the network graph.
                         self.network_graph
                             .write()
@@ -1831,7 +1842,10 @@ where
                             .process_node_announcement(node_announcement.clone());
 
                         let peer_id = node_announcement.peer_id();
-                        state.on_node_announcement(peer_id, node_announcement.addresses.clone());
+                        state.save_announced_peer_addresses(
+                            peer_id,
+                            node_announcement.addresses.clone(),
+                        );
                         Ok(())
                     }
                     _ => {
@@ -2308,9 +2322,14 @@ pub struct NetworkActorState<S> {
 #[serde_as]
 #[derive(Default, Clone, Serialize, Deserialize)]
 pub struct PersistentNetworkActorState {
-    // when we restarting a node, we will reconnect to these peers
+    // These addresses are announced by the peer itself to the network.
+    // When a new NodeAnnouncement message is received, we will overwrite the old addresses.
+    #[serde_as(as = "Vec<(DisplayFromStr, _)>")]
+    announced_peer_addresses: HashMap<PeerId, Vec<Multiaddr>>,
+    // These addresses are saved by the user (e.g. the user sends a ConnectPeer rpc to the node),
+    // we will then save these addresses to the peer store.
     #[serde_as(as = "Vec<(DisplayFromStr, _)>")]
-    peer_store: HashMap<PeerId, Vec<Multiaddr>>,
+    saved_peer_addresses: HashMap<PeerId, Vec<Multiaddr>>,
 }
 
 impl PersistentNetworkActorState {
@@ -2318,10 +2337,26 @@ impl PersistentNetworkActorState {
         Default::default()
     }
 
+    fn get_peer_addresses(&self, peer_id: &PeerId) -> HashSet<Multiaddr> {
+        let empty = vec![];
+        self.announced_peer_addresses
+            .get(peer_id)
+            .unwrap_or(&empty)
+            .iter()
+            .chain(
+                self.saved_peer_addresses
+                    .get(peer_id)
+                    .unwrap_or(&empty)
+                    .iter(),
+            )
+            .map(|addr| addr.clone())
+            .collect::<HashSet<_, RandomState>>()
+    }
+
     /// Save a single peer address to the peer store. If this address for the peer does not exist,
     /// then return false, otherwise return true.
-    pub(crate) fn add_peer_address(&mut self, peer_id: PeerId, addr: Multiaddr) -> bool {
-        match self.peer_store.entry(peer_id) {
+    fn save_peer_address(&mut self, peer_id: PeerId, addr: Multiaddr) -> bool {
+        match self.saved_peer_addresses.entry(peer_id) {
             Entry::Occupied(mut entry) => {
                 if entry.get().contains(&addr) {
                     false
@@ -2337,10 +2372,10 @@ impl PersistentNetworkActorState {
         }
     }
 
-    /// Save peer addresses to the peer store. If the peer addresses are updated,
-    /// return true, otherwise return false. This method will NOT keep the old addresses.
-    pub(crate) fn save_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
-        match self.peer_store.entry(peer_id) {
+    /// Save announced peer addresses to the peer store. If the peer addresses are updated,
+    /// return true, otherwise return false. This method will NOT keep the old announced addresses.
+    fn save_announced_peer_addresses(&mut self, peer_id: PeerId, addr: Vec<Multiaddr>) -> bool {
+        match self.announced_peer_addresses.entry(peer_id) {
             Entry::Occupied(mut entry) => {
                 if entry.get() == &addr {
                     false
@@ -2357,10 +2392,22 @@ impl PersistentNetworkActorState {
     }
 
     pub(crate) fn sample_n_peers_to_connect(&self, n: usize) -> HashMap<PeerId, Vec<Multiaddr>> {
-        self.peer_store
-            .iter()
+        let nodes = self
+            .saved_peer_addresses
+            .keys()
+            .into_iter()
+            .chain(self.saved_peer_addresses.keys().into_iter())
+            .collect::<HashSet<_, RandomState>>();
+
+        nodes
+            .into_iter()
             .take(n)
-            .map(|(id, addr)| (id.clone(), addr.clone()))
+            .map(|peer_id| {
+                (
+                    peer_id.clone(),
+                    self.get_peer_addresses(peer_id).into_iter().collect(),
+                )
+            })
             .collect()
     }
 }
@@ -3050,14 +3097,30 @@ where
         self.maybe_tell_syncer_peer_disconnected(id);
     }
 
-    fn on_node_announcement(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
-        self.save_peer_addresses(peer_id, addresses);
+    pub(crate) fn get_peer_addresses(&self, peer_id: &PeerId) -> HashSet<Multiaddr> {
+        self.state_to_be_persisted.get_peer_addresses(peer_id)
+    }
+
+    pub(crate) fn save_peer_address(&mut self, peer_id: PeerId, address: Multiaddr) -> bool {
+        if self
+            .state_to_be_persisted
+            .save_peer_address(peer_id, address)
+        {
+            self.persist_state();
+            true
+        } else {
+            false
+        }
     }
 
-    fn save_peer_addresses(&mut self, peer_id: PeerId, addresses: Vec<Multiaddr>) {
+    pub(crate) fn save_announced_peer_addresses(
+        &mut self,
+        peer_id: PeerId,
+        addresses: Vec<Multiaddr>,
+    ) {
         if self
             .state_to_be_persisted
-            .save_peer_addresses(peer_id, addresses)
+            .save_announced_peer_addresses(peer_id, addresses)
         {
             self.persist_state();
         }
@@ -3565,7 +3628,7 @@ where
         for bootnode in &config.bootnode_addrs {
             let addr = Multiaddr::from_str(bootnode.as_str()).expect("valid bootnode");
             let peer_id = extract_peer_id(&addr).expect("valid peer id");
-            state_to_be_persisted.add_peer_address(peer_id, addr);
+            state_to_be_persisted.save_peer_address(peer_id, addr);
         }
 
         let height = graph.get_best_height();
@@ -3642,8 +3705,25 @@ where
     async fn post_start(
         &self,
         myself: ActorRef<Self::Msg>,
-        _state: &mut Self::State,
+        state: &mut Self::State,
     ) -> Result<(), ActorProcessingErr> {
+        debug!("Trying to connect to peers with mutual channels");
+        for (peer_id, channel_id, channel_state) in self.store.get_channel_states(None) {
+            let addresses = state.get_peer_addresses(&peer_id);
+
+            debug!(
+                "Reconnecting channel {:x} peers {:?} in state {:?}",
+                &channel_id, &peer_id, &channel_state
+            );
+            for addr in addresses {
+                myself
+                    .send_message(NetworkActorMessage::new_command(
+                        NetworkActorCommand::ConnectPeer(addr),
+                    ))
+                    .expect(ASSUME_NETWORK_MYSELF_ALIVE);
+            }
+        }
+
         myself
             .send_message(NetworkActorMessage::new_command(
                 NetworkActorCommand::MaintainConnections(NUM_PEER_CONNECTIONS),
diff --git a/src/fiber/tests/network.rs b/src/fiber/tests/network.rs
index 52040d01..e7952a85 100644
--- a/src/fiber/tests/network.rs
+++ b/src/fiber/tests/network.rs
@@ -587,3 +587,31 @@ async fn test_connecting_to_bootnode() {
     )
     .await;
 }
+
+#[tokio::test]
+async fn test_saving_and_connecting_to_node() {
+    init_tracing();
+
+    let node1 = NetworkNode::new().await;
+    let node1_address = node1.get_node_address().clone();
+    let node1_id = &node1.peer_id;
+
+    let mut node2 = NetworkNode::new().await;
+
+    node2
+        .network_actor
+        .send_message(NetworkActorMessage::new_command(
+            NetworkActorCommand::SavePeerAddress(node1_address),
+        ))
+        .expect("send message to network actor");
+
+    // Wait for the above message to be processed.
+    tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+
+    node2.restart().await;
+
+    node2.expect_event(
+        |event| matches!(event, NetworkServiceEvent::PeerConnected(id, _addr) if id == node1_id),
+    )
+    .await;
+}
diff --git a/src/rpc/peer.rs b/src/rpc/peer.rs
index f1ffa4be..b5a7b480 100644
--- a/src/rpc/peer.rs
+++ b/src/rpc/peer.rs
@@ -9,9 +9,10 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use tentacle::{multiaddr::MultiAddr, secio::PeerId};
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub(crate) struct ConnectPeerParams {
     address: MultiAddr,
+    save: bool,
 }
 
 #[serde_as]
@@ -45,6 +46,15 @@ impl PeerRpcServer for PeerRpcServerImpl {
     async fn connect_peer(&self, params: ConnectPeerParams) -> Result<(), ErrorObjectOwned> {
         let message =
             NetworkActorMessage::Command(NetworkActorCommand::ConnectPeer(params.address.clone()));
+        if params.save {
+            crate::handle_actor_cast!(
+                self.actor,
+                NetworkActorMessage::Command(NetworkActorCommand::ConnectPeer(
+                    params.address.clone()
+                )),
+                params.clone()
+            )?;
+        }
         crate::handle_actor_cast!(self.actor, message, params)
     }
 

From ad5675d191cc4d2e024b5b9094d527ee5e6ccffd Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Fri, 18 Oct 2024 20:57:36 +0800
Subject: [PATCH 62/66] Add test for reconnect peer on restart

---
 src/fiber/tests/channel.rs | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/fiber/tests/channel.rs b/src/fiber/tests/channel.rs
index 68f7be5f..ee495709 100644
--- a/src/fiber/tests/channel.rs
+++ b/src/fiber/tests/channel.rs
@@ -1233,3 +1233,20 @@ async fn test_commitment_tx_capacity() {
         output_capacity as u128
     );
 }
+
+#[tokio::test]
+async fn test_connect_to_peers_with_mutual_channel_on_restart() {
+    let node_a_funding_amount = 100000000000;
+    let node_b_funding_amount = 6200000000;
+
+    let (mut node_a, node_b, _new_channel_id) =
+        create_nodes_with_established_channel(node_a_funding_amount, node_b_funding_amount, true)
+            .await;
+
+    node_a.restart().await;
+
+    node_a.expect_event(
+        |event| matches!(event, NetworkServiceEvent::PeerConnected(id, _addr) if id == &node_b.peer_id),
+    )
+    .await;
+}

From 21b4ed545936768cc6fe35385f5ce8b292bdf1b0 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Mon, 21 Oct 2024 18:35:50 +0800
Subject: [PATCH 63/66] Make the parameter save to connect_peer optional

---
 src/rpc/README.md | 1 +
 src/rpc/peer.rs   | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/rpc/README.md b/src/rpc/README.md
index b939ecef..2d154b8b 100644
--- a/src/rpc/README.md
+++ b/src/rpc/README.md
@@ -241,6 +241,7 @@ Attempts to connect to a peer.
 ###### Params
 
 * `address` - The address of the peer to connect to
+* `save` - Whether to save the peer address, an optional parameter (default value true)
 
 ###### Returns
 
diff --git a/src/rpc/peer.rs b/src/rpc/peer.rs
index b5a7b480..7022ccd8 100644
--- a/src/rpc/peer.rs
+++ b/src/rpc/peer.rs
@@ -12,7 +12,7 @@ use tentacle::{multiaddr::MultiAddr, secio::PeerId};
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub(crate) struct ConnectPeerParams {
     address: MultiAddr,
-    save: bool,
+    save: Option<bool>,
 }
 
 #[serde_as]
@@ -46,10 +46,10 @@ impl PeerRpcServer for PeerRpcServerImpl {
     async fn connect_peer(&self, params: ConnectPeerParams) -> Result<(), ErrorObjectOwned> {
         let message =
             NetworkActorMessage::Command(NetworkActorCommand::ConnectPeer(params.address.clone()));
-        if params.save {
+        if params.save.unwrap_or(true) {
             crate::handle_actor_cast!(
                 self.actor,
-                NetworkActorMessage::Command(NetworkActorCommand::ConnectPeer(
+                NetworkActorMessage::Command(NetworkActorCommand::SavePeerAddress(
                     params.address.clone()
                 )),
                 params.clone()

From a7aac1adac718d0eb404eacfd45ca6e0023e8aa2 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Mon, 21 Oct 2024 20:19:24 +0800
Subject: [PATCH 64/66] Add test for persisting announced node

---
 src/fiber/network.rs       |  2 +-
 src/fiber/tests/network.rs | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index e31ee8b9..3467266e 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -2396,7 +2396,7 @@ impl PersistentNetworkActorState {
             .saved_peer_addresses
             .keys()
             .into_iter()
-            .chain(self.saved_peer_addresses.keys().into_iter())
+            .chain(self.announced_peer_addresses.keys().into_iter())
             .collect::<HashSet<_, RandomState>>();
 
         nodes
diff --git a/src/fiber/tests/network.rs b/src/fiber/tests/network.rs
index e7952a85..6b7cfe46 100644
--- a/src/fiber/tests/network.rs
+++ b/src/fiber/tests/network.rs
@@ -569,6 +569,33 @@ async fn test_persisting_bootnode() {
     assert_eq!(peers.get(&boot_peer_id), Some(&vec![address]));
 }
 
+#[tokio::test]
+async fn test_persisting_announced_nodes() {
+    let mut node = new_synced_node("test").await;
+
+    let announcement = create_fake_node_announcement_mesage_version1();
+    let node_pk = announcement.node_id;
+    let peer_id = node_pk.tentacle_peer_id();
+
+    node.network_actor
+        .send_message(NetworkActorMessage::Event(NetworkActorEvent::PeerMessage(
+            peer_id.clone(),
+            FiberMessage::BroadcastMessage(FiberBroadcastMessage::NodeAnnouncement(
+                create_fake_node_announcement_mesage_version1(),
+            )),
+        )))
+        .expect("send message to network actor");
+
+    // Wait for the above message to be processed.
+    tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+
+    node.stop().await;
+    let state = node.store.clone();
+    let state = state.get_network_actor_state(&node.peer_id).unwrap();
+    let peers = state.sample_n_peers_to_connect(1);
+    assert!(peers.get(&peer_id).is_some());
+}
+
 #[tokio::test]
 async fn test_connecting_to_bootnode() {
     let boot_node = NetworkNode::new().await;

From ecc717c919dfc684c28fe99b87b70a114afefcdd Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Wed, 23 Oct 2024 10:54:41 +0800
Subject: [PATCH 65/66] Fix ctrl-c not working

See https://github.com/slawlor/ractor/issues/226#issuecomment-2428871642 for additional context
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 765272a4..1c9b989a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2977,9 +2977,9 @@ dependencies = [
 
 [[package]]
 name = "ractor"
-version = "0.9.8"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d551077e2f2f354bd9cbe5520fe493cf57618bc70c1de4609f022ee9cc5d33"
+checksum = "7dd876f0d609ba2ddc8a36136e9b81299312bd9fc9b71131381d16c9ce8e495a"
 dependencies = [
  "async-trait",
  "dashmap",
diff --git a/Cargo.toml b/Cargo.toml
index 6b386269..3c3c1d16 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,7 +37,7 @@ bitflags = { version = "2.5.0", features = ["serde"] }
 ckb-hash = "0.115.0"
 secp256k1 = { version = "0.28.0", features = ["serde", "recovery", "rand-std"] }
 musig2 = { version = "0.0.11", features = ["secp256k1", "serde"] }
-ractor = "0.9.7"
+ractor = "=0.9.7"
 arcode = "0.2.4"
 nom = "7.1.3"
 regex = "1.10.5"

From 1252ff5c211c0f1144b8aa2407889e36449aac8c Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Wed, 23 Oct 2024 21:53:17 +0800
Subject: [PATCH 66/66] code clean up

---
 src/fiber/network.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/fiber/network.rs b/src/fiber/network.rs
index b2edc3fa..cd946b9f 100644
--- a/src/fiber/network.rs
+++ b/src/fiber/network.rs
@@ -3349,18 +3349,9 @@ where
     }
 
     fn remove_channel(&mut self, channel_id: &Hash256) -> Option<ActorRef<ChannelActorMessage>> {
-        self.channels.remove(channel_id).map(|channel| {
-            if let Some(outpoint) = self.outpoint_channel_map.iter().find_map(|(k, v)| {
-                if v == channel_id {
-                    Some(k.clone())
-                } else {
-                    None
-                }
-            }) {
-                self.outpoint_channel_map.remove(&outpoint);
-            }
-            channel
-        })
+        self.channels
+            .remove(channel_id)
+            .inspect(|_| self.outpoint_channel_map.retain(|_, v| v != channel_id))
     }
 
     fn on_peer_disconnected(&mut self, id: &PeerId) {