Add stable routing stats

Summary: Adding service handler API to support fetching stable routing stats

Reviewed By: frankfeir

Differential Revision: D64874281

fbshipit-source-id: caa6183f88ebae0cc6045db43212e8d3f55ad633

katran/lib/KatranLb.cpp

@@ -2013,6 +2013,33 @@ lb_tpr_packets_stats KatranLb::getTcpServerIdRoutingStats() {
   return sum_stat;
 }
 
+lb_stable_rt_packets_stats KatranLb::getUdpStableRoutingStats() {
+  unsigned int nr_cpus = BpfAdapter::getPossibleCpus();
+  if (nr_cpus < 0) {
+    LOG(ERROR) << "Error while getting number of possible cpus";
+    return lb_stable_rt_packets_stats{};
+  }
+  lb_stable_rt_packets_stats stats[nr_cpus];
+  lb_stable_rt_packets_stats sum_stat = {};
+
+  if (!config_.testing) {
+    int position = 0;
+    auto res = bpfAdapter_->bpfMapLookupElement(
+        bpfAdapter_->getMapFdByName("stable_rt_stats"), &position, stats);
+    if (!res) {
+      for (auto& stat : stats) {
+        sum_stat.ch_routed += stat.ch_routed;
+        sum_stat.cid_routed += stat.cid_routed;
+        sum_stat.cid_invalid_server_id += stat.cid_invalid_server_id;
+        sum_stat.cid_unknown_real_dropped += stat.cid_unknown_real_dropped;
+      }
+    } else {
+      lbStats_.bpfFailedCalls++;
+    }
+  }
+  return sum_stat;
+}
+
 lb_stats KatranLb::getChDropStats() {
   return getLbStats(config_.maxVips + kChDropOffset);
 }

katran/lib/KatranLb.h

@@ -545,6 +545,15 @@ class KatranLb {
    */
   lb_tpr_packets_stats getTcpServerIdRoutingStats();
 
+  /**
+   * @return struct lb_stable_rt_packets_stats w/ statistic of UDP stable
+   * routing
+   *
+   * helper function which returns how many UDP packets were routed
+   * using the default 5-tuple hash vs using the connection-id
+   */
+  lb_stable_rt_packets_stats getUdpStableRoutingStats();
+
   /**
    * @return struct lb_stats w/ src routing related statistics
    *

katran/lib/testing/KatranGueTestFixtures.h

@@ -353,7 +353,6 @@ const std::vector<::katran::PacketAttributes> gueTestFixtures = {
      .expectedReturnValue = "XDP_DROP",
      .expectedOutputPacket = "AgAAAAAAAQAAAAAAht1gAAAAABMRQPwAAAIAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAARemkAUAATUc5rYXRyYW4gdGVzdA=="
     },
-
 };
 
 } // namespace testing

katran/lib/testing/KatranTestProvision.cpp

@@ -98,6 +98,15 @@ void addQuicMappings(katran::KatranLb& lb) {
     constexpr uint32_t twJobMask = 0x030000; // tw job set to 3
     qreal.id = twJobMask | ids[i];
     qreals.push_back(qreal);
+    // log server address and id in hex
+    LOG(INFO) << "Adding mapping for " << qreal.address << " with id "
+              << qreal.id;
+    printf(
+        "%02X%02X%02X%02X\n",
+        (qreal.id >> 24) & 0xFF,
+        (qreal.id >> 16) & 0xFF,
+        (qreal.id >> 8) & 0xFF,
+        qreal.id & 0xFF);
   }
   lb.modifyQuicRealsMapping(action, qreals);
 }
@@ -192,6 +201,68 @@ void prepareOptionalLbData(katran::KatranLb& lb) {
   lb.modifyReal("10.0.0.6", kLocalReal);
 }
 
+void prepareLbDataStableRt(katran::KatranLb& lb) {
+  lb.restartKatranMonitor(kMonitorLimit);
+  katran::VipKey vip;
+  // adding few reals to test
+  std::vector<std::string> reals = {"10.0.0.1", "10.0.0.2", "10.0.0.3"};
+  std::vector<std::string> reals6 = {"fc00::1", "fc00::2", "fc00::3"};
+  vip.address = "fc00:1::9";
+  vip.proto = kUdp;
+  vip.port = kVipPort;
+  lb.addVip(vip);
+  addReals(lb, vip, reals6);
+  lb.modifyVip(vip, kUdpStableRouting);
+  // add v4 vip to test
+  vip.address = "10.200.1.90";
+  lb.addVip(vip);
+  // adding few reals to test
+  addReals(lb, vip, reals);
+  lb.modifyVip(vip, kUdpStableRouting);
+  // vip which ignores dst_port (testing for TURN-like services)
+  vip.address = "10.200.1.2";
+  vip.port = 0;
+  lb.addVip(vip);
+  // adding few reals to test
+  addReals(lb, vip, reals);
+  // vip which is using only dst port to pick up real
+  vip.address = "10.200.1.4";
+  lb.addVip(vip);
+  // adding few reals to test
+  addReals(lb, vip, reals);
+  lb.modifyVip(vip, kDportHash);
+  // v4inv6 vip. tcp
+  vip.address = "10.200.1.3";
+  vip.port = kVipPort;
+  lb.addVip(vip);
+  // adding few reals to test
+  addReals(lb, vip, reals6);
+  // v6inv6 vip. tcp
+  vip.address = "fc00:1::1";
+  lb.addVip(vip);
+  // adding few reals to test
+  addReals(lb, vip, reals6);
+  // adding mappings for quic.
+  addQuicMappings(lb);
+  // adding quic v4 vip.
+  vip.proto = kUdp;
+  vip.port = 443;
+  vip.address = "10.200.1.5";
+  lb.addVip(vip);
+  lb.modifyVip(vip, kQuicVip);
+  addReals(lb, vip, reals);
+  // adding quic v6 vip.
+  vip.address = "fc00:1::2";
+  lb.addVip(vip);
+  lb.modifyVip(vip, kQuicVip);
+  addReals(lb, vip, reals6);
+
+  // adding healthchecking dst
+  lb.addHealthcheckerDst(1, "10.0.0.1");
+  lb.addHealthcheckerDst(2, "10.0.0.2");
+  lb.addHealthcheckerDst(3, "fc00::1");
+}
+
 void prepareVipUninitializedLbData(katran::KatranLb& lb) {
   katran::VipKey vip;
   vip.address = "10.200.1.99";
@@ -270,6 +341,22 @@ uint64_t KatranTestParam::expectedTcpServerIdRoutingCounts() noexcept {
 uint64_t KatranTestParam::expectedTcpServerIdRoutingFallbackCounts() noexcept {
   return _lookup_counter(KatranTestCounters::TCP_SERVER_ID_ROUTING_FALLBACK_CH);
 }
+uint64_t KatranTestParam::expectedUdpStableRoutingWithCh() noexcept {
+  return _lookup_counter(KatranTestCounters::STABLE_RT_CH_ROUTING);
+}
+uint64_t KatranTestParam::expectedUdpStableRoutingWithCid() noexcept {
+  return _lookup_counter(KatranTestCounters::STABLE_RT_CID_ROUTING);
+}
+uint64_t KatranTestParam::expectedUdpStableRoutingInvalidSid() noexcept {
+  return _lookup_counter(KatranTestCounters::STABLE_RT_CID_INVALID_SERVER_ID);
+}
+uint64_t KatranTestParam::expectedUdpStableRoutingUnknownReals() noexcept {
+  return _lookup_counter(
+      KatranTestCounters::STABLE_RT_CID_UNKNOWN_REAL_DROPPED);
+}
+uint64_t KatranTestParam::expectedUdpStableRoutingInvalidPacketType() noexcept {
+  return _lookup_counter(KatranTestCounters::STABLE_RT_INVALID_PACKET_TYPE);
+}
 uint64_t KatranTestParam::expectedTotalFailedBpfCalls() noexcept {
   return _lookup_counter(KatranTestCounters::TOTAL_FAILED_BPF_CALLS);
 }

katran/lib/testing/KatranTestProvision.h

@@ -40,11 +40,20 @@ constexpr uint16_t kVipPort = 80;
 constexpr uint8_t kUdp = 17;
 constexpr uint8_t kTcp = 6;
 constexpr uint32_t kDefaultWeight = 1;
-constexpr uint32_t kDportHash = 8;
+
+// Flags from katran/lib/bpf/balancer_consts.h
+// real is specified as local (1 << 1)
+constexpr uint8_t kLocalReal = 2;
+// use quic's connection id for the hash calculation (1 << 2)
 constexpr uint32_t kQuicVip = 4;
+// use only dst port for the hash calculation (1 << 3)
+constexpr uint32_t kDportHash = 8;
+// check if src based routing should be used (1 << 4)
 constexpr uint32_t kSrcRouting = 16;
+// vip flag to optimize local delivery (1 << 5)
 constexpr uint32_t kLocalVip = 32;
-constexpr uint8_t kLocalReal = 2;
+// parse udp stable routing header to get server-id (1 << 8)
+constexpr uint32_t kUdpStableRouting = 256;
 
 // Each of the TestMode correspond to the TestFixtures
 enum class TestMode : uint8_t {
@@ -81,6 +90,12 @@ enum class KatranTestCounters : uint8_t {
   SRC_ROUTING_PKTS_LOCAL = 18,
   SRC_ROUTING_PKTS_REMOTE = 19,
   INLINE_DECAP_PKTS = 20,
+  // udp stable routing counters
+  STABLE_RT_CH_ROUTING = 21,
+  STABLE_RT_CID_ROUTING = 22,
+  STABLE_RT_CID_INVALID_SERVER_ID = 23,
+  STABLE_RT_CID_UNKNOWN_REAL_DROPPED = 24,
+  STABLE_RT_INVALID_PACKET_TYPE = 25,
 };
 
 struct KatranTestParam {
@@ -103,6 +118,11 @@ struct KatranTestParam {
   uint64_t expectedQuicCidDropsNoRealCounts() noexcept;
   uint64_t expectedTcpServerIdRoutingCounts() noexcept;
   uint64_t expectedTcpServerIdRoutingFallbackCounts() noexcept;
+  uint64_t expectedUdpStableRoutingWithCh() noexcept;
+  uint64_t expectedUdpStableRoutingWithCid() noexcept;
+  uint64_t expectedUdpStableRoutingInvalidSid() noexcept;
+  uint64_t expectedUdpStableRoutingUnknownReals() noexcept;
+  uint64_t expectedUdpStableRoutingInvalidPacketType() noexcept;
   uint64_t expectedTotalFailedBpfCalls() noexcept;
   uint64_t expectedTotalAddressValidations() noexcept;
   // optional counters
@@ -138,6 +158,8 @@ void prepareLbData(katran::KatranLb& lb);
 
 void prepareOptionalLbData(katran::KatranLb& lb);
 
+void prepareLbDataStableRt(katran::KatranLb& lb);
+
 void prepareVipUninitializedLbData(katran::KatranLb& lb);
 
 void preparePerfTestingLbData(katran::KatranLb& lb);

katran/lib/testing/KatranTestUtil.cpp

@@ -18,6 +18,7 @@
 #include "katran/lib/testing/KatranGueTestFixtures.h"
 #include "katran/lib/testing/KatranTPRTestFixtures.h"
 #include "katran/lib/testing/KatranTestFixtures.h"
+#include "katran/lib/testing/KatranUdpStableRtTestFixtures.h"
 
 namespace katran {
 namespace testing {
@@ -180,5 +181,27 @@ KatranTestParam createTPRTestParam() {
   return testParam;
 }
 
+KatranTestParam createUdpStableRtTestParam() {
+  katran::VipKey vip;
+  vip.address = "fc00:1::9";
+  vip.port = kVipPort;
+  vip.proto = kUdp;
+  KatranTestParam testParam = {
+      .mode = TestMode::GUE,
+      .testData = katran::testing::udpStableRtFixtures,
+      .expectedCounters =
+          {
+              {KatranTestCounters::TOTAL_PKTS, 5},
+              {KatranTestCounters::STABLE_RT_CH_ROUTING, 2},
+              {KatranTestCounters::STABLE_RT_CID_ROUTING, 3},
+              {KatranTestCounters::STABLE_RT_CID_INVALID_SERVER_ID, 0},
+              {KatranTestCounters::STABLE_RT_CID_UNKNOWN_REAL_DROPPED, 0},
+              {KatranTestCounters::STABLE_RT_INVALID_PACKET_TYPE, 0},
+          },
+      .perVipCounters = {{vip, std::pair<uint64_t, uint64_t>(4, 244)}}};
+  return testParam;
+
+} // namespace testing
+
 } // namespace testing
 } // namespace katran

katran/lib/testing/KatranTestUtil.h

@@ -29,5 +29,7 @@ KatranTestParam createDefaultTestParam(TestMode testMode);
 
 KatranTestParam createTPRTestParam();
 
+KatranTestParam createUdpStableRtTestParam();
+
 } // namespace testing
 } // namespace katran

katran/lib/testing/KatranUdpStableRtTestFixtures.h

@@ -0,0 +1,86 @@
+// clang-format off
+
+/* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#pragma once
+#include <string>
+#include <utility>
+#include <vector>
+#include "katran/lib/testing/PacketAttributes.h"
+
+namespace katran {
+namespace testing {
+/**
+ * input packets has been generated with scapy. above each of em you can find
+ * a command which has been used to do so.
+ *
+ * format of the input data: <string, string>; 1st string is a base64 encoded
+ * packet. 2nd string is test's description
+ *
+ * format of the output data: <string, string>; 1st string is a base64 encoded
+ * packet which we are expecting to see after bpf program's run.
+ * 2nd string = bpf's program return code.
+ *
+ * to create pcap w/ scapy:
+ * 1) create packets
+ * 2) pckts = [ <created packets from above> ]
+ * 3) wrpcap(<path_to_file>, pckts)
+ */
+const std::vector<::katran::PacketAttributes> udpStableRtFixtures = {
+    // 1
+    {// Ether(src="0x1", dst="0x2")/IPv6(src="fc00:1::1",
+    // dst="fc00:1::9")/UDP(sport=31337, dport=80)/Raw(load=b'\x80\x00\x00\x00\x00\x00\x00\x00local test pkt')
+     .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeXTuAAAAAAAAAAGxvY2FsIHRlc3QgcGt0",
+     .description = "Stable Rt packet with conn-id 0",
+     .expectedReturnValue = "XDP_TX",
+     .expectedOutputPacket = "AADerb6vAgAAAAAAht1gAAAAAE4RQPwAIwcAAAAAAAAAAAAAEzf8AAAAAAAAAAAAAAAAAAADemkmngBOvthgAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeXTuAAAAAAAAAAGxvY2FsIHRlc3QgcGt0"
+    },
+    // 2
+    {// Ether(src="0x1", dst="0x2")/IPv6(src="fc00:1::1",
+     // dst="fc00:1::9")/UDP(sport=31337, dport=80)/Raw(load=b'\x80\x03\x04\x03\x00\x00\x00\x00local test pkt')
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeWTWAAwQDAAAAAGxvY2FsIHRlc3QgcGt0",
+    .description = "Stable Rt packet from same src, with conn-id for fc00::3 real",
+    .expectedReturnValue = "XDP_TX",
+    .expectedOutputPacket = "AADerb6vAgAAAAAAht1gAAAAAE4RQPwAIwcAAAAAAAAAAAAAEzf8AAAAAAAAAAAAAAAAAAADemkmngBOvthgAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeWTWAAwQDAAAAAGxvY2FsIHRlc3QgcGt0"
+    },
+    // 3
+    {// Ether(src="0x1", dst="0x2")/IPv6(src="fc00:1::1",
+    // dst="fc00:1::9")/UDP(sport=31339, dport=80)/Raw(load=b'\x80\x03\x04\x03\x00\x00\x00\x00local test pkt')
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemsAUAAeWTOAAwQDAAAAAGxvY2FsIHRlc3QgcGt0",
+    .description = "Stable Rt packet from different src port, with conn-id for fc00::3 real",
+    .expectedReturnValue = "XDP_TX",
+    .expectedOutputPacket = "AADerb6vAgAAAAAAht1gAAAAAE4RQPwAIwcAAAAAAAAAAAAAEzf8AAAAAAAAAAAAAAAAAAADemsmngBOvtZgAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemsAUAAeWTOAAwQDAAAAAGxvY2FsIHRlc3QgcGt0"
+    },
+    // 4
+    {// Ether(src="0x1", dst="0x2")/IPv6(src="fc00:1::5",
+    // dst="fc00:1::9")/UDP(sport=31339, dport=80)/Raw(load=b'\x80\x03\x04\x03\x00\x00\x00\x00local test pkt')
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAX8AAABAAAAAAAAAAAAAAAJemsAUAAeWS+AAwQDAAAAAGxvY2FsIHRlc3QgcGt0",
+    .description = "Stable Rt packet from different src ip, with same conn-id for fc00::3 real",
+    .expectedReturnValue = "XDP_TX",
+    .expectedOutputPacket = "AADerb6vAgAAAAAAht1gAAAAAE4RQPwAIwcAAAAAAAAAAAAAEzf8AAAAAAAAAAAAAAAAAAADemsmngBOvtZgAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAX8AAABAAAAAAAAAAAAAAAJemsAUAAeWS+AAwQDAAAAAGxvY2FsIHRlc3QgcGt0"
+    },
+    // 5
+    {// Ether(src="0x1", dst="0x2")/IPv6(src="fc00:1::1",
+    // dst="fc00:1::9")/UDP(sport=31337, dport=80)/Raw(load=b'\x80\x00\x00\x00\x00\x00\x00\x00local test pkt')
+    .inputPacket = "AgAAAAAAAQAAAAAAht1gAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeXTuAAAAAAAAAAGxvY2FsIHRlc3QgcGt0",
+    .description = "Stable Rt packet with conn-id 0, from same original src ip/port, so LRU hit",
+    .expectedReturnValue = "XDP_TX",
+    .expectedOutputPacket = "AADerb6vAgAAAAAAht1gAAAAAE4RQPwAIwcAAAAAAAAAAAAAEzf8AAAAAAAAAAAAAAAAAAADemkmngBOvthgAAAAAB4RQPwAAAEAAAAAAAAAAAAAAAH8AAABAAAAAAAAAAAAAAAJemkAUAAeXTuAAAAAAAAAAGxvY2FsIHRlc3QgcGt0"
+    }
+};
+}
+}