TPR Misrouting check after decap

Summary:
In xdpdecap, check inner packet's tpr server id and see if it matches hosts' server-id (from bpf map).
This diff only adds the check and unit tests.
Next diffs will add changes in proxygen to set the server-id

Reviewed By: frankfeir

Differential Revision: D49208122

fbshipit-source-id: b55b475fb1a697b558c610c78f66ed750611f24d

katran/decap/XdpDecap.cpp

@@ -15,6 +15,7 @@
  */
 
 #include <glog/logging.h>
+#include <cstdint>
 
 #include "katran/decap/XdpDecap.h"
 
@@ -137,11 +138,24 @@ decap_stats XdpDecap::getXdpDecapStats() {
       stats.decap_v4 += stat.decap_v4;
       stats.decap_v6 += stat.decap_v6;
       stats.total += stat.total;
+      stats.tpr_misrouted += stat.tpr_misrouted;
+      stats.tpr_total += stat.tpr_total;
     }
   } else {
     LOG(ERROR) << "Error while trying to get decap stats";
   }
   return stats;
 }
 
+void XdpDecap::setSeverId(int id) {
+  auto map_fd = bpfAdapter_.getMapFdByName("tpr_server_id");
+  uint32_t key = 0;
+  uint32_t value = id;
+  if (bpfAdapter_.bpfUpdateMap(map_fd, &key, &value)) {
+    LOG(FATAL) << "Was not able to update pinned bpf map " << config_.mapPath
+               << " with elem on position " << config_.progPos;
+    return;
+  }
+}
+
 } // namespace katran

katran/decap/XdpDecap.h

@@ -59,6 +59,10 @@ class XdpDecap {
     return bpfAdapter_.getProgFdByName("xdpdecap");
   }
 
+  // Used only for unit testing
+  // Updates server-id in bpf sever-id-map
+  void setSeverId(int id);
+
  private:
   /**
    * main configuration

katran/decap/XdpDecapStructs.h

@@ -36,6 +36,8 @@ struct decap_stats {
   uint64_t decap_v4;
   uint64_t decap_v6;
   uint64_t total;
+  uint64_t tpr_misrouted;
+  uint64_t tpr_total;
 };
 
 /**

katran/decap/bpf/decap.bpf.c

@@ -208,6 +208,45 @@ __attribute__((__always_inline__)) static inline int process_encaped_gue_pckt(
 }
 #endif // INLINE_DECAP_GUE
 
+__attribute__((__always_inline__)) static inline void validate_tpr_server_id(
+    void* data,
+    __u64 off,
+    void* data_end,
+    bool is_ipv6,
+    struct xdp_md* xdp,
+    struct decap_stats* data_stats) {
+  __u16 inner_pkt_bytes;
+  struct packet_description inner_pckt = {};
+  __u8 inner_protocol;
+  if (process_l3_headers(
+          &inner_pckt,
+          &inner_protocol,
+          off,
+          &inner_pkt_bytes,
+          data,
+          data_end,
+          is_ipv6) >= 0) {
+    return;
+  }
+  // only check for TCP non SYN packets
+  if (inner_protocol == IPPROTO_TCP && !(inner_pckt.flags & F_SYN_SET)) {
+    // lookup server id from tpr header option and compare against server_id on
+    // this host (if available)
+    __u32 s_key = 0;
+    __u32* server_id_host = bpf_map_lookup_elem(&tpr_server_id, &s_key);
+    if (server_id_host && *server_id_host > 0) {
+      __u32 server_id = 0;
+      tcp_hdr_opt_lookup_server_id(xdp, is_ipv6, &server_id);
+      if (server_id > 0) {
+        data_stats->tpr_total += 1;
+        if (*server_id_host != server_id) {
+          data_stats->tpr_misrouted += 1;
+        }
+      }
+    }
+  }
+}
+
 __attribute__((__always_inline__)) static inline int process_packet(
     void* data,
     __u64 off,
@@ -278,6 +317,8 @@ __attribute__((__always_inline__)) static inline int process_packet(
       if (action >= 0) {
         return action;
       }
+      // For inner packet - check TPR server id and capture stats
+      validate_tpr_server_id(data, off, data_end, is_ipv6, xdp, data_stats);
     }
   }
 #endif // INLINE_DECAP_GUE

katran/decap/bpf/decap_maps.h

@@ -34,6 +34,8 @@ struct decap_stats {
   __u64 decap_v4;
   __u64 decap_v6;
   __u64 total;
+  __u64 tpr_misrouted;
+  __u64 tpr_total;
 };
 
 // map w/ per vip statistics
@@ -45,4 +47,13 @@ struct {
   __uint(map_flags, NO_FLAGS);
 } decap_counters SEC(".maps");
 
+// map, which contains server_id info
+struct {
+  __uint(type, BPF_MAP_TYPE_ARRAY);
+  __type(key, __u32);
+  __type(value, __u32);
+  __uint(max_entries, 1);
+  __uint(map_flags, NO_FLAGS);
+} tpr_server_id SEC(".maps");
+
 #endif // of _DECAP_MAPS

katran/decap/testing/XdpDecapGueTestFixtures.h

@@ -128,6 +128,22 @@ const std::vector<PacketAttributes> gueTestFixtures = {
     .expectedReturnValue = "XDP_PASS",
     .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAAg6QPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABgACHtgAAAAA="
   },
+  // 12
+  { // data_value = int(100).to_bytes(4, byteorder='little')
+    // Ether(src="0x1", dst="0x2")/IPv6(src="100::1", dst="100::2")/UDP(sport=1337, dport=9886)/IPv6(src="fc00:2::1", dst="fc00:1::1")/TCP(sport=31337, dport=80,flags="A", options=[(0xB7, data_value)])/"katran test pkt"
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAFsRQAEAAAAAAAAAAAAAAAAAAAEBAAAAAAAAAAAAAAAAAAACBTkmngBbayRgAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAMJAAAC3BmQAAAAAAGthdHJhbiB0ZXN0IHBrdA==",
+    .description = "ipv6 gue ipv6 innner with TPR option set with server id 100",
+    .expectedReturnValue = "XDP_PASS",
+    .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAMJAAAC3BmQAAAAAAGthdHJhbiB0ZXN0IHBrdA=="
+  },
+  // 13
+  { // data_value = int(200).to_bytes(4, byteorder='little')
+    // Ether(src="0x1", dst="0x2")/IPv6(src="100::1", dst="100::2")/UDP(sport=1337, dport=9886)/IPv6(src="fc00:2::1", dst="fc00:1::1")/TCP(sport=31337, dport=80,flags="A", options=[(0xB7, data_value)])/"katran test pkt"
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAFsRQAEAAAAAAAAAAAAAAAAAAAEBAAAAAAAAAAAAAAAAAAACBTkmngBbayRgAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAF5AAAC3BsgAAAAAAGthdHJhbiB0ZXN0IHBrdA==",
+    .description = "ipv6 gue ipv6 innner with TPR option set with server id 200",
+    .expectedReturnValue = "XDP_PASS",
+    .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAACsGQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAABemkAUAAAAAAAAAAAcBAgAF5AAAC3BsgAAAAAAGthdHJhbiB0ZXN0IHBrdA=="
+  },
 };
 
 } // namespace testing

katran/decap/testing/xdpdecap_tester.cpp

@@ -40,17 +40,25 @@ void testXdpDecapCounters(katran::XdpDecap& decap) {
   LOG(INFO) << "Testing counter's sanity";
   auto stats = decap.getXdpDecapStats();
   int expectedV4DecapPkts = 1;
-  int expectedV6DecapPkts = FLAGS_gue ? 3 : 2;
-  int expectedTotalPkts = FLAGS_gue ? 9 : 7;
+  int expectedV6DecapPkts = FLAGS_gue ? 5 : 2;
+  int expectedTotalPkts = FLAGS_gue ? 6 : 7;
+  int expectedTotalTPRPkts = 2;
+  int expectedMisroutedTPRPkts = 1;
   if (stats.decap_v4 != expectedV4DecapPkts ||
       stats.decap_v6 != expectedV6DecapPkts ||
-      stats.total != expectedTotalPkts) {
+      stats.total != expectedTotalPkts ||
+      stats.tpr_total != expectedTotalTPRPkts ||
+      stats.tpr_misrouted != expectedMisroutedTPRPkts) {
     LOG(ERROR) << "decap_v4 pkts: " << stats.decap_v4
                << ", expected decap_v4 pkts: " << expectedV4DecapPkts
                << ", decap_v6: " << stats.decap_v6
                << ", expected decap_v6 pkts: " << expectedV6DecapPkts
                << " total: " << stats.total
-               << ", expected total_pkts: " << expectedTotalPkts;
+               << ", expected total_pkts: " << expectedTotalPkts
+               << " tpr total: " << stats.tpr_total
+               << ", expected tpr total pkts: " << expectedTotalTPRPkts
+               << " tpr misrouted: " << stats.tpr_misrouted
+               << ", expected tpr misrouted: " << expectedMisroutedTPRPkts;
     LOG(ERROR) << "[FAIL] Incorrect decap counters";
     return;
   }
@@ -79,6 +87,8 @@ int main(int argc, char** argv) {
   decap.loadXdpDecap();
   auto decap_prog_fd = decap.getXdpDecapFd();
   tester.setBpfProgFd(decap_prog_fd);
+  decap.setSeverId(100);
+
   if (!FLAGS_pcap_input.empty()) {
     tester.testPcktsFromPcap();
     return 0;

katran/lib/bpf/balancer_consts.h

@@ -185,7 +185,7 @@
 
 // Constants related to the feature for routing of TCP packets
 // using server_id (also referred as TPR: TCP Packet Routing).
-#ifdef TCP_SERVER_ID_ROUTING
+#if defined(TCP_SERVER_ID_ROUTING) || defined(DECAP_TPR_STATS)
 // the structure of the header-option used to embed server_id is:
 //  __u8 kind | __u8 len | __u32 server_id
 // Arbitrarily picked unused value from IANA TCP Option Kind Numbers

katran/lib/bpf/pckt_parsing.h

@@ -142,7 +142,7 @@ struct hdr_opt_state {
   __u8 hdr_bytes_remaining;
 };
 
-#ifdef TCP_SERVER_ID_ROUTING
+#if defined(TCP_SERVER_ID_ROUTING) || defined(DECAP_TPR_STATS)
 #ifdef TCP_HDR_OPT_SKIP_UNROLL_LOOP
 __attribute__ ((noinline))
 #else
@@ -203,14 +203,13 @@ int parse_hdr_opt(const struct xdp_md *xdp, struct hdr_opt_state *state)
   return 0;
 }
 
-__attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup(
+__attribute__((__always_inline__)) static inline int
+tcp_hdr_opt_lookup_server_id(
     const struct xdp_md* xdp,
     bool is_ipv6,
-    struct real_definition** real,
-    struct packet_description* pckt) {
+    __u32* server_id) {
   const void* data = (void*)(long)xdp->data;
   const void* data_end = (void*)(long)xdp->data_end;
-  struct real_pos_lru* dst_lru;
   struct tcphdr* tcp_hdr;
   __u8 tcp_hdr_opt_len = 0;
   __u64 tcp_offset = 0;
@@ -241,12 +240,27 @@ __attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup(
       break;
     }
   }
-
   if (!opt_state.server_id) {
     return FURTHER_PROCESSING;
   }
+  *server_id = opt_state.server_id;
+  return 0;
+}
+#endif // TCP_SERVER_ID_ROUTING) || DECAP_TPR_STATS
 
-  __u32 key = opt_state.server_id;
+#ifdef TCP_SERVER_ID_ROUTING
+__attribute__((__always_inline__)) static inline int tcp_hdr_opt_lookup(
+    const struct xdp_md* xdp,
+    bool is_ipv6,
+    struct real_definition** real,
+    struct packet_description* pckt) {
+  __u32 server_id = 0;
+  int err = 0;
+  if (tcp_hdr_opt_lookup_server_id(xdp, is_ipv6, &server_id) ==
+      FURTHER_PROCESSING) {
+    return FURTHER_PROCESSING;
+  }
+  __u32 key = server_id;
   __u32* real_pos = bpf_map_lookup_elem(&server_id_map, &key);
   if (!real_pos) {
     return FURTHER_PROCESSING;