chore: firewall counter exporter (#343)

The replica node and API boundary node firewall has connection- and
rate-limits. At the moment, each packet that is dropped based on these
rules is logged.

In this PR, we switch to counters and add a service to export the
counters as a metric in order to not spam logs and to easily monitor
whether the rules are actually active.

---------

Co-authored-by: IDX GitLab Automation <infra+gitlab-automation@dfinity.org>

Cargo.toml

@@ -132,6 +132,7 @@ members = [
     "rs/ic_os/inject_files",
     "rs/ic_os/launch-single-vm",
     "rs/ic_os/network",
+    "rs/ic_os/nft_exporter",
     "rs/ic_os/nss_icos",
     "rs/ic_os/setupos-inject-configuration",
     "rs/ic_os/partition_tools",
@@ -539,6 +540,7 @@ mockall = "0.12.1"
 mockito = "1.2.0"
 minicbor = { version = "0.19.1", features = ["alloc", "derive"] }
 minicbor-derive = "0.13.0"
+nftables = "0.4"
 nix = "0.24.3"
 num-traits = { version = "0.2.12", features = ["libm"] }
 num-bigint = "0.4.6"

ic-os/components/guestos.bzl

@@ -82,6 +82,8 @@ component_files = {
     Label("monitoring/metrics-proxy/guestos/metrics-proxy.yaml"): "/etc/metrics-proxy.yaml",
     Label("monitoring/metrics-proxy/metrics-proxy.service"): "/etc/systemd/system/metrics-proxy.service",
     Label("monitoring/journald.conf"): "/etc/systemd/journald.conf",
+    Label("monitoring/nft-exporter/nft-exporter.service"): "/etc/systemd/system/nft-exporter.service",
+    Label("monitoring/nft-exporter/nft-exporter.timer"): "/etc/systemd/system/nft-exporter.timer",
 
     # networking
     Label("networking/generate-network-config/guestos/generate-network-config.service"): "/etc/systemd/system/generate-network-config.service",

ic-os/components/ic/ic.json5.template

@@ -187,15 +187,18 @@
     size 65535\n\
     flags dynamic\n\
   }\n\
+\n\
+  counter rate_limit_v4_counter {}\n\
+  counter connection_limit_v4_counter {}\n\
 \n\
   chain INPUT {\n\
     type filter hook input priority 0; policy drop;\n\
     iif lo accept\n\
-    ct state new add @rate_limit { ip saddr limit rate over 1000/minute burst 500 packets } log prefix \"Drop - rate limit: \" drop\n\
+    ct state new add @rate_limit { ip saddr limit rate over 1000/minute burst 500 packets } counter name rate_limit_v4_counter drop\n\
     #  Notes about the rule below:\n\
     #   - The rule allows a maximum of <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> persistent connections to any ip address.\n\
     #   - The rule drops all new connections that goes over the configured limit.\n\
-    ct state new add @connection_limit { ip saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } log prefix \"Drop - connection limit: \" drop\n\
+    ct state new add @connection_limit { ip saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } counter name connection_limit_v4_counter drop\n\
     icmp type parameter-problem accept\n\
     icmp type echo-request accept\n\
     icmp type echo-reply accept\n\
@@ -230,17 +233,20 @@ table ip6 filter {\n\
     size 65535\n\
     flags dynamic\n\
   }\n\
+\n\
+  counter rate_limit_v6_counter {}\n\
+  counter connection_limit_v6_counter {}\n\
 \n\
   chain INPUT {\n\
     type filter hook input priority 0; policy drop;\n\
     iif lo accept\n\
     ct state { invalid } drop\n\
     ct state { established, related } accept\n\
-    ct state new add @rate_limit { ip6 saddr limit rate over 1000/minute burst 500 packets } log prefix \"Drop - rate limit: \" drop\n\
+    ct state new add @rate_limit { ip6 saddr limit rate over 1000/minute burst 500 packets } counter name rate_limit_v6_counter drop\n\
     #  Notes about the rule below:\n\
     #   - The rule allows a maximum of <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> persistent connections to any ip6 address.\n\
     #   - The rule drops all new connections that goes over the configured limit.\n\
-    ct state new add @connection_limit { ip6 saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } log prefix \"Drop - connection limit: \" drop\n\
+    ct state new add @connection_limit { ip6 saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } counter name connection_limit_v6_counter drop\n\
     icmpv6 type destination-unreachable accept\n\
     icmpv6 type packet-too-big accept\n\
     icmpv6 type time-exceeded accept\n\
@@ -381,16 +387,19 @@ table ip6 filter {\n\
     type ipv4_addr\n\
     size 65535\n\
   }\n\
+\n\
+  counter rate_limit_v4_counter {}\n\
+  counter connection_limit_v4_counter {}\n\
 \n\
   chain INPUT {\n\
     type filter hook input priority 0; policy drop;\n\
     iif lo accept\n\
     ip saddr @blackhole drop\n\
-    ct state new add @rate_limit { ip saddr limit rate over 2000/minute burst 1000 packets } log prefix \"Drop - rate limit: \" drop\n\
+    ct state new add @rate_limit { ip saddr limit rate over 2000/minute burst 1000 packets } counter name rate_limit_v4_counter drop\n\
     #  Notes about the rule below:\n\
     #   - The rule allows a maximum of <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> persistent connections to any ip address.\n\
     #   - The rule drops all new connections that goes over the configured limit.\n\
-    ct state new add @connection_limit { ip saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } log prefix \"Drop - connection limit: \" drop\n\
+    ct state new add @connection_limit { ip saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } counter name connection_limit_v4_counter drop\n\
     icmp type { echo-reply, destination-unreachable, source-quench, echo-request, time-exceeded } accept\n\
     ct state invalid drop\n\
     ct state { established, related } accept\n\
@@ -432,16 +441,19 @@ table ip6 filter {\n\
     type ipv6_addr\n\
     size 65535\n\
   }\n\
+\n\
+  counter rate_limit_v6_counter {}\n\
+  counter connection_limit_v6_counter {}\n\
 \n\
   chain INPUT {\n\
     type filter hook input priority 0; policy drop;\n\
     iif lo accept\n\
     ip6 saddr @blackhole6 drop\n\
-    ct state new add @rate_limit { ip6 saddr limit rate over 2000/minute burst 1000 packets } log prefix \"Drop - rate limit: \" drop\n\
+    ct state new add @rate_limit { ip6 saddr limit rate over 2000/minute burst 1000 packets } counter name rate_limit_v6_counter drop\n\
     #  Notes about the rule below:\n\
     #   - The rule allows a maximum of <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> persistent connections to any ip6 address.\n\
     #   - The rule drops all new connections that goes over the configured limit.\n\
-    ct state new add @connection_limit { ip6 saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } log prefix \"Drop - connection limit: \" drop\n\
+    ct state new add @connection_limit { ip6 saddr ct count over <<MAX_SIMULTANEOUS_CONNECTIONS_PER_IP_ADDRESS>> } counter name connection_limit_v6_counter drop\n\
     icmpv6 type { destination-unreachable, packet-too-big, time-exceeded, echo-request, echo-reply, nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } accept\n\
     ct state { invalid } drop\n\
     ct state { established, related } accept\n\

ic-os/components/monitoring/nft-exporter/nft-exporter.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Collect nft counters and write them to a file for node exporter
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/ic/bin/nft-exporter

ic-os/components/monitoring/nft-exporter/nft-exporter.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=nft counter export timer
+
+[Timer]
+OnBootSec=60s
+OnUnitActiveSec=60s
+Unit=nft-exporter.service
+
+[Install]
+WantedBy=timers.target

ic-os/guestos/defs.bzl

@@ -42,6 +42,7 @@ def image_deps(mode, malicious = False):
             "//publish/binaries:ic-boundary-tls": "/opt/ic/bin/ic-boundary:0755",
             ("//publish/malicious:replica" if malicious else "//publish/binaries:replica"): "/opt/ic/bin/replica:0755",  # Install the malicious replica if set
             "//publish/binaries:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755",
+            "//publish/binaries:nft-exporter": "/opt/ic/bin/nft-exporter:0755",
             "//publish/binaries:sandbox_launcher": "/opt/ic/bin/sandbox_launcher:0755",
             "//publish/binaries:state-tool": "/opt/ic/bin/state-tool:0755",
             "//publish/binaries:vsock_guest": "/opt/ic/bin/vsock_guest:0755",

publish/binaries/BUILD.bazel

@@ -58,6 +58,7 @@ BINARIES = {
     "icx-proxy": "//rs/boundary_node/icx_proxy:icx-proxy",
     "icx-proxy-dev": "//rs/boundary_node/icx_proxy:icx-proxy-dev",
     "metrics-proxy": "@crate_index//:metrics-proxy__metrics-proxy",
+    "nft-exporter": "//rs/ic_os/nft_exporter:nft-exporter",
     "nfplot": "//rs/nervous_system/neurons_fund/nfplot",
     "orchestrator": "//rs/orchestrator",
     "sandbox_launcher": "//rs/canister_sandbox:sandbox_launcher",

rs/boundary_node/ic_boundary/Cargo.toml

@@ -63,7 +63,7 @@ little-loadshedder = "0.1.0"
 maxminddb = "0.24"
 mockall = { workspace = true }
 moka = { version = "0.12", features = ["sync", "future"] }
-nftables = "0.4"
+nftables = { workspace = true }
 nix = { workspace = true }
 prometheus = { workspace = true }
 rand = { workspace = true }

rs/ic_os/nft_exporter/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@rules_rust//rust:defs.bzl", "rust_binary")
+
+package(default_visibility = ["//visibility:public"])
+
+DEPENDENCIES = [
+    "@crate_index//:anyhow",
+    "@crate_index//:clap",
+    "@crate_index//:serde_json",
+    "@crate_index//:serde",
+]
+
+rust_binary(
+    name = "nft-exporter",
+    srcs = glob(
+        ["src/**/*.rs"],
+    ),
+    aliases = {},
+    proc_macro_deps = [],
+    version = "0.1.0",
+    deps = DEPENDENCIES,
+)

rs/ic_os/nft_exporter/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "nft_exporter"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"