From 1f686b97ae468fbc89ef50f6f831a58d68b9e958 Mon Sep 17 00:00:00 2001 From: Nikhil Dixit Limaye Date: Fri, 15 Sep 2023 14:11:44 -0700 Subject: [PATCH] TPR Misrouting check after decap Summary: In xdpdecap, check inner packet's tpr server id and see if it matches hosts' server-id (from bpf map). This diff only adds the check and unit tests. Next diffs will add changes in proxygen to set the server-id Reviewed By: frankfeir Differential Revision: D49208122 fbshipit-source-id: b55b475fb1a697b558c610c78f66ed750611f24d --- katran/decap/XdpDecap.cpp | 14 +++++++ katran/decap/XdpDecap.h | 4 ++ katran/decap/XdpDecapStructs.h | 2 + katran/decap/bpf/decap.bpf.c | 41 +++++++++++++++++++ katran/decap/bpf/decap_maps.h | 11 +++++ .../decap/testing/XdpDecapGueTestFixtures.h | 16 ++++++++ katran/decap/testing/xdpdecap_tester.cpp | 18 ++++++-- katran/lib/bpf/balancer_consts.h | 2 +- katran/lib/bpf/pckt_parsing.h | 28 +++++++++---- 9 files changed, 124 insertions(+), 12 deletions(-) diff --git a/katran/decap/XdpDecap.cpp b/katran/decap/XdpDecap.cpp index 571e194ee..3be3d2fa6 100644 --- a/katran/decap/XdpDecap.cpp +++ b/katran/decap/XdpDecap.cpp @@ -15,6 +15,7 @@ */ #include +#include #include "katran/decap/XdpDecap.h" @@ -137,6 +138,8 @@ decap_stats XdpDecap::getXdpDecapStats() { stats.decap_v4 += stat.decap_v4; stats.decap_v6 += stat.decap_v6; stats.total += stat.total; + stats.tpr_misrouted += stat.tpr_misrouted; + stats.tpr_total += stat.tpr_total; } } else { LOG(ERROR) << "Error while trying to get decap stats"; @@ -144,4 +147,15 @@ decap_stats XdpDecap::getXdpDecapStats() { return stats; } +void XdpDecap::setSeverId(int id) { + auto map_fd = bpfAdapter_.getMapFdByName("tpr_server_id"); + uint32_t key = 0; + uint32_t value = id; + if (bpfAdapter_.bpfUpdateMap(map_fd, &key, &value)) { + LOG(FATAL) << "Was not able to update pinned bpf map " << config_.mapPath + << " with elem on position " << config_.progPos; + return; + } +} + } // namespace katran diff --git a/katran/decap/XdpDecap.h b/katran/decap/XdpDecap.h index 4753c4190..e53c42626 100644 --- a/katran/decap/XdpDecap.h +++ b/katran/decap/XdpDecap.h @@ -59,6 +59,10 @@ class XdpDecap { return bpfAdapter_.getProgFdByName("xdpdecap"); } + // Used only for unit testing + // Updates server-id in bpf sever-id-map + void setSeverId(int id); + private: /** * main configuration diff --git a/katran/decap/XdpDecapStructs.h b/katran/decap/XdpDecapStructs.h index 236587e0b..1148faf2d 100644 --- a/katran/decap/XdpDecapStructs.h +++ b/katran/decap/XdpDecapStructs.h @@ -36,6 +36,8 @@ struct decap_stats { uint64_t decap_v4; uint64_t decap_v6; uint64_t total; + uint64_t tpr_misrouted; + uint64_t tpr_total; }; /** diff --git a/katran/decap/bpf/decap.bpf.c b/katran/decap/bpf/decap.bpf.c index ce505616b..eefa4e688 100644 --- a/katran/decap/bpf/decap.bpf.c +++ b/katran/decap/bpf/decap.bpf.c @@ -208,6 +208,45 @@ __attribute__((__always_inline__)) static inline int process_encaped_gue_pckt( } #endif // INLINE_DECAP_GUE +__attribute__((__always_inline__)) static inline void validate_tpr_server_id( + void* data, + __u64 off, + void* data_end, + bool is_ipv6, + struct xdp_md* xdp, + struct decap_stats* data_stats) { + __u16 inner_pkt_bytes; + struct packet_description inner_pckt = {}; + __u8 inner_protocol; + if (process_l3_headers( + &inner_pckt, + &inner_protocol, + off, + &inner_pkt_bytes, + data, + data_end, + is_ipv6) >= 0) { + return; + } + // only check for TCP non SYN packets + if (inner_protocol == IPPROTO_TCP && !(inner_pckt.flags & F_SYN_SET)) { + // lookup server id from tpr header option and compare against server_id on + // this host (if available) + __u32 s_key = 0; + __u32* server_id_host = bpf_map_lookup_elem(&tpr_server_id, &s_key); + if (server_id_host && *server_id_host > 0) { + __u32 server_id = 0; + tcp_hdr_opt_lookup_server_id(xdp, is_ipv6, &server_id); + if (server_id > 0) { + data_stats->tpr_total += 1; + if (*server_id_host != server_id) { + data_stats->tpr_misrouted += 1; + } + } + } + } +} + __attribute__((__always_inline__)) static inline int process_packet( void* data, __u64 off, @@ -278,6 +317,8 @@ __attribute__((__always_inline__)) static inline int process_packet( if (action >= 0) { return action; } + // For inner packet - check TPR server id and capture stats + validate_tpr_server_id(data, off, data_end, is_ipv6, xdp, data_stats); } } #endif // INLINE_DECAP_GUE diff --git a/katran/decap/bpf/decap_maps.h b/katran/decap/bpf/decap_maps.h index 11cb5b4d6..5b929ab4e 100644 --- a/katran/decap/bpf/decap_maps.h +++ b/katran/decap/bpf/decap_maps.h @@ -34,6 +34,8 @@ struct decap_stats { __u64 decap_v4; __u64 decap_v6; __u64 total; + __u64 tpr_misrouted; + __u64 tpr_total; }; // map w/ per vip statistics @@ -45,4 +47,13 @@ struct { __uint(map_flags, NO_FLAGS); } decap_counters SEC(".maps"); +// map, which contains server_id info +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 1); + __uint(map_flags, NO_FLAGS); +} tpr_server_id SEC(".maps"); + #endif // of _DECAP_MAPS diff --git a/katran/decap/testing/XdpDecapGueTestFixtures.h b/katran/decap/testing/XdpDecapGueTestFixtures.h index dd8f9afe7..8690a8b3f 100644 --- a/katran/decap/testing/XdpDecapGueTestFixtures.h +++ b/katran/decap/testing/XdpDecapGueTestFixtures.h @@ -128,6 +128,22 @@ const std::vector gueTestFixtures = { .expectedReturnValue = "XDP_PASS", .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAAg6QPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABgACHtgAAAAA=" }, + // 12 + { // data_value = int(100).to_bytes(4, byteorder='little') + // Ether(src="0x1", dst="0x2")/IPv6(src="100::1", dst="100::2")/UDP(sport=1337, dport=9886)/IPv6(src="fc00:2::1", dst="fc00:1::1")/TCP(sport=31337, dport=80,flags="A", options=[(0xB7, data_value)])/"katran test pkt" + .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAFsRQAEAAAAAAAAAAAAAAAAAAAEBAAAAAAAAAAAAAAAAAAACBTkmngBbayRgAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAMJAAAC3BmQAAAAAAGthdHJhbiB0ZXN0IHBrdA==", + .description = "ipv6 gue ipv6 innner with TPR option set with server id 100", + .expectedReturnValue = "XDP_PASS", + .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAMJAAAC3BmQAAAAAAGthdHJhbiB0ZXN0IHBrdA==" + }, + // 13 + { // data_value = int(200).to_bytes(4, byteorder='little') + // Ether(src="0x1", dst="0x2")/IPv6(src="100::1", dst="100::2")/UDP(sport=1337, dport=9886)/IPv6(src="fc00:2::1", dst="fc00:1::1")/TCP(sport=31337, dport=80,flags="A", options=[(0xB7, data_value)])/"katran test pkt" + .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAFsRQAEAAAAAAAAAAAAAAAAAAAEBAAAAAAAAAAAAAAAAAAACBTkmngBbayRgAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAF5AAAC3BsgAAAAAAGthdHJhbiB0ZXN0IHBrdA==", + .description = "ipv6 gue ipv6 innner with TPR option set with server id 200", + .expectedReturnValue = "XDP_PASS", + .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAF5AAAC3BsgAAAAAAGthdHJhbiB0ZXN0IHBrdA==" + }, }; } // namespace testing diff --git a/katran/decap/testing/xdpdecap_tester.cpp b/katran/decap/testing/xdpdecap_tester.cpp index 609f6fd41..5e94102d7 100644 --- a/katran/decap/testing/xdpdecap_tester.cpp +++ b/katran/decap/testing/xdpdecap_tester.cpp @@ -40,17 +40,25 @@ void testXdpDecapCounters(katran::XdpDecap& decap) { LOG(INFO) << "Testing counter's sanity"; auto stats = decap.getXdpDecapStats(); int expectedV4DecapPkts = 1; - int expectedV6DecapPkts = FLAGS_gue ? 3 : 2; - int expectedTotalPkts = FLAGS_gue ? 9 : 7; + int expectedV6DecapPkts = FLAGS_gue ? 5 : 2; + int expectedTotalPkts = FLAGS_gue ? 6 : 7; + int expectedTotalTPRPkts = 2; + int expectedMisroutedTPRPkts = 1; if (stats.decap_v4 != expectedV4DecapPkts || stats.decap_v6 != expectedV6DecapPkts || - stats.total != expectedTotalPkts) { + stats.total != expectedTotalPkts || + stats.tpr_total != expectedTotalTPRPkts || + stats.tpr_misrouted != expectedMisroutedTPRPkts) { LOG(ERROR) << "decap_v4 pkts: " << stats.decap_v4 << ", expected decap_v4 pkts: " << expectedV4DecapPkts << ", decap_v6: " << stats.decap_v6 << ", expected decap_v6 pkts: " << expectedV6DecapPkts << " total: " << stats.total - << ", expected total_pkts: " << expectedTotalPkts; + << ", expected total_pkts: " << expectedTotalPkts + << " tpr total: " << stats.tpr_total + << ", expected tpr total pkts: " << expectedTotalTPRPkts + << " tpr misrouted: " << stats.tpr_misrouted + << ", expected tpr misrouted: " << expectedMisroutedTPRPkts; LOG(ERROR) << "[FAIL] Incorrect decap counters"; return; } @@ -79,6 +87,8 @@ int main(int argc, char** argv) { decap.loadXdpDecap(); auto decap_prog_fd = decap.getXdpDecapFd(); tester.setBpfProgFd(decap_prog_fd); + decap.setSeverId(100); + if (!FLAGS_pcap_input.empty()) { tester.testPcktsFromPcap(); return 0; diff --git a/katran/lib/bpf/balancer_consts.h b/katran/lib/bpf/balancer_consts.h index 9e21a8cf6..5d727c75b 100644 --- a/katran/lib/bpf/balancer_consts.h +++ b/katran/lib/bpf/balancer_consts.h @@ -185,7 +185,7 @@ // Constants related to the feature for routing of TCP packets // using server_id (also referred as TPR: TCP Packet Routing). -#ifdef TCP_SERVER_ID_ROUTING +#if defined(TCP_SERVER_ID_ROUTING) || defined(DECAP_TPR_STATS) // the structure of the header-option used to embed server_id is: // __u8 kind | __u8 len | __u32 server_id // Arbitrarily picked unused value from IANA TCP Option Kind Numbers diff --git a/katran/lib/bpf/pckt_parsing.h b/katran/lib/bpf/pckt_parsing.h index 948a51d8f..a1bbb7e33 100644 --- a/katran/lib/bpf/pckt_parsing.h +++ b/katran/lib/bpf/pckt_parsing.h @@ -142,7 +142,7 @@ struct hdr_opt_state { __u8 hdr_bytes_remaining; }; -#ifdef TCP_SERVER_ID_ROUTING +#if defined(TCP_SERVER_ID_ROUTING) || defined(DECAP_TPR_STATS) #ifdef TCP_HDR_OPT_SKIP_UNROLL_LOOP __attribute__ ((noinline)) #else @@ -203,14 +203,13 @@ int parse_hdr_opt(const struct xdp_md *xdp, struct hdr_opt_state *state) return 0; } -__attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup( +__attribute__((__always_inline__)) static inline int +tcp_hdr_opt_lookup_server_id( const struct xdp_md* xdp, bool is_ipv6, - struct real_definition** real, - struct packet_description* pckt) { + __u32* server_id) { const void* data = (void*)(long)xdp->data; const void* data_end = (void*)(long)xdp->data_end; - struct real_pos_lru* dst_lru; struct tcphdr* tcp_hdr; __u8 tcp_hdr_opt_len = 0; __u64 tcp_offset = 0; @@ -241,12 +240,27 @@ __attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup( break; } } - if (!opt_state.server_id) { return FURTHER_PROCESSING; } + *server_id = opt_state.server_id; + return 0; +} +#endif // TCP_SERVER_ID_ROUTING) || DECAP_TPR_STATS - __u32 key = opt_state.server_id; +#ifdef TCP_SERVER_ID_ROUTING +__attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup( + const struct xdp_md* xdp, + bool is_ipv6, + struct real_definition** real, + struct packet_description* pckt) { + __u32 server_id = 0; + int err = 0; + if (tcp_hdr_opt_lookup_server_id(xdp, is_ipv6, &server_id) == + FURTHER_PROCESSING) { + return FURTHER_PROCESSING; + } + __u32 key = server_id; __u32* real_pos = bpf_map_lookup_elem(&server_id_map, &key); if (!real_pos) { return FURTHER_PROCESSING;