Skip to content

Commit

Permalink
DPDK: Support DPDK up to 18.05
Browse files Browse the repository at this point in the history
  • Loading branch information
Tom Barbette committed Aug 3, 2018
1 parent a460414 commit 8247b32
Show file tree
Hide file tree
Showing 3 changed files with 205 additions and 35 deletions.
54 changes: 40 additions & 14 deletions include/click/dpdkdevice.hh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
#ifndef CLICK_DPDKDEVICE_HH
#define CLICK_DPDKDEVICE_HH

//Prevent bug under some configurations (like travis-ci's one) where these macros get undefined
/**
* Prevent bug under some configurations
* (like travis-ci's one) where these
* macros get undefined.
*/
#ifndef UINT8_MAX
#define UINT8_MAX 255
#endif
Expand All @@ -24,6 +28,17 @@
#include <click/vector.hh>
#include <click/args.hh>

/**
* Unified type for DPDK port IDs.
* Until DPDK v17.05 was uint8_t
* After DPDK v17.05 has been uint16_t
*/
#if RTE_VERSION >= RTE_VERSION_NUM(17,05,0,0)
typedef uint16_t portid_t;
#else
typedef uint8_t portid_t;
#endif

CLICK_DECLS
class DPDKDeviceArg;

Expand All @@ -32,13 +47,10 @@ extern bool dpdk_enabled;
class DPDKDevice {
public:

unsigned port_id;

DPDKDevice() : port_id(-1), info() {
}
portid_t port_id;

DPDKDevice(unsigned port_id) : port_id(port_id) {
} CLICK_COLD;
DPDKDevice() CLICK_COLD;
DPDKDevice(portid_t port_id) CLICK_COLD;

int add_rx_queue(int &queue_id, bool promisc,
unsigned n_desc, ErrorHandler *errh) CLICK_COLD;
Expand All @@ -48,9 +60,19 @@ public:

unsigned int get_nb_txdesc();

const char *get_device_driver();

static unsigned int dev_count() {
#if RTE_VERSION >= RTE_VERSION_NUM(18,05,0,0)
return rte_eth_dev_count_avail();
#else
return rte_eth_dev_count();
#endif
}

static struct rte_mempool *get_mpool(unsigned int);

static int get_port_numa_node(unsigned port_id);
static int get_port_numa_node(portid_t port_id);

static int initialize(ErrorHandler *errh);

Expand Down Expand Up @@ -95,7 +117,7 @@ private:
DevInfo info;

static bool _is_initialized;
static HashTable<unsigned, DPDKDevice> _devs;
static HashTable<portid_t, DPDKDevice> _devs;
static struct rte_mempool** _pktmbuf_pools;
static bool no_more_buffer_msg_printed;

Expand All @@ -105,13 +127,17 @@ private:

static bool alloc_pktmbufs() CLICK_COLD;

static DPDKDevice* get_device(unsigned port_id) {
return &(_devs.find_insert(port_id, DPDKDevice(port_id)).value());
static DPDKDevice* get_device(const portid_t &port_id) {
return &(_devs.find_insert(port_id, DPDKDevice(port_id)).value());
}

static int get_port_from_pci(uint16_t domain, uint8_t bus, uint8_t dev_id, uint8_t function) {

#if RTE_VERSION < RTE_VERSION_NUM(18,05,0,0)
static int get_port_from_pci(uint32_t domain, uint8_t bus, uint8_t dev_id, uint8_t function) {
struct rte_eth_dev_info dev_info;
for (uint8_t port_id = 0 ; port_id < rte_eth_dev_count(); ++port_id) {

uint16_t count = rte_eth_dev_count();
for (portid_t port_id = 0 ; port_id < count; ++port_id) {
rte_eth_dev_info_get(port_id, &dev_info);
struct rte_pci_addr addr = dev_info.pci_dev->addr;
if (addr.domain == domain &&
Expand All @@ -122,6 +148,7 @@ private:
}
return -1;
}
#endif

friend class DPDKDeviceArg;
};
Expand Down Expand Up @@ -153,7 +180,6 @@ class DPDKDeviceArg { public:

template<> struct DefaultArg<DPDKDevice*> : public DPDKDeviceArg {};


CLICK_ENDDECLS

#endif
63 changes: 43 additions & 20 deletions lib/dpdkdevice.cc
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
/*
* dpdkdevice.{cc,hh} -- library for interfacing with Intel's DPDK
* Cyril Soldani, Tom Barbette
* dpdkdevice.{cc,hh} -- library for interfacing with DPDK
* Cyril Soldani, Tom Barbette, Georgios Katsikas
*
* Copyright (c) 2014-2016 University of Liege
* Copyright (c) 2014-2018 University of Liege
* Copyright (c) 2016 Cisco Meraki
* Copyright (c) 2017-2018 RISE SICS
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
Expand All @@ -21,11 +22,18 @@

CLICK_DECLS

DPDKDevice::DPDKDevice() : port_id(-1), info() {
}

DPDKDevice::DPDKDevice(portid_t port_id) : port_id(port_id) {
};


/* Wraps rte_eth_dev_socket_id(), which may return -1 for valid ports when NUMA
* is not well supported. This function will return 0 instead in that case. */
int DPDKDevice::get_port_numa_node(unsigned port_id)
int DPDKDevice::get_port_numa_node(portid_t port_id)
{
if (port_id >= rte_eth_dev_count())
if (port_id >= dev_count())
return -1;
int numa_node = rte_eth_dev_socket_id(port_id);
return (numa_node == -1) ? 0 : numa_node;
Expand All @@ -40,7 +48,7 @@ bool DPDKDevice::alloc_pktmbufs()
{
// Count NUMA sockets
int max_socket = -1;
for (HashTable<unsigned, DPDKDevice>::const_iterator it = _devs.begin();
for (HashTable<portid_t, DPDKDevice>::const_iterator it = _devs.begin();
it != _devs.end(); ++it) {
int numa_node = DPDKDevice::get_port_numa_node(it.key());
if (numa_node > max_socket)
Expand All @@ -57,7 +65,7 @@ bool DPDKDevice::alloc_pktmbufs()
memset(_pktmbuf_pools, 0, (max_socket + 1) * sizeof(rte_mempool_p));

// Create a pktmbuf pool for each active socket
for (HashTable<unsigned, DPDKDevice>::const_iterator it = _devs.begin();
for (HashTable<portid_t, DPDKDevice>::const_iterator it = _devs.begin();
it != _devs.end(); ++it) {
int numa_node = DPDKDevice::get_port_numa_node(it.key());
if (!_pktmbuf_pools[numa_node]) {
Expand Down Expand Up @@ -109,7 +117,7 @@ int DPDKDevice::initialize_device(ErrorHandler *errh)
"Cannot initialize DPDK port %u with %u RX and %u TX queues",
port_id, info.rx_queues.size(), info.tx_queues.size());
struct rte_eth_rxconf rx_conf;
#if RTE_VERSION >= (RTE_VERSION_NUM(2,1,0,0))
#if RTE_VERSION >= (RTE_VERSION_NUM(2,0,0,0))
memcpy(&rx_conf, &dev_info.default_rxconf, sizeof rx_conf);
#else
bzero(&rx_conf,sizeof rx_conf);
Expand All @@ -119,7 +127,7 @@ int DPDKDevice::initialize_device(ErrorHandler *errh)
rx_conf.rx_thresh.wthresh = RX_WTHRESH;

struct rte_eth_txconf tx_conf;
#if RTE_VERSION >= (RTE_VERSION_NUM(2,1,0,0))
#if RTE_VERSION >= RTE_VERSION_NUM(2,0,0,0)
memcpy(&tx_conf, &dev_info.default_txconf, sizeof tx_conf);
#else
bzero(&tx_conf,sizeof tx_conf);
Expand Down Expand Up @@ -251,19 +259,19 @@ int DPDKDevice::initialize(ErrorHandler *errh)
return errh->error("Cannot probe the PCI bus");
#endif

const unsigned n_ports = rte_eth_dev_count();
const unsigned n_ports = dev_count();
if (n_ports == 0)
return errh->error("No DPDK-enabled ethernet port found");

for (HashTable<unsigned, DPDKDevice>::const_iterator it = _devs.begin();
for (HashTable<portid_t, DPDKDevice>::const_iterator it = _devs.begin();
it != _devs.end(); ++it)
if (it.key() >= n_ports)
return errh->error("Cannot find DPDK port %u", it.key());

if (!alloc_pktmbufs())
return errh->error("Could not allocate packet MBuf pools");

for (HashTable<unsigned, DPDKDevice>::iterator it = _devs.begin();
for (HashTable<portid_t, DPDKDevice>::iterator it = _devs.begin();
it != _devs.end(); ++it) {
int ret = it.value().initialize_device(errh);
if (ret < 0)
Expand All @@ -281,11 +289,19 @@ void DPDKDevice::free_pkt(unsigned char *, size_t, void *pktmbuf)


bool
DPDKDeviceArg::parse(const String &str, DPDKDevice* &result, const ArgContext &ctx)
DPDKDeviceArg::parse(
const String &str, DPDKDevice* &result, const ArgContext &ctx)
{
int port_id;
portid_t port_id;

if (!IntArg().parse(str, port_id)) {
#if RTE_VERSION >= RTE_VERSION_NUM(18,05,0,0)
uint16_t id;
if (rte_eth_dev_get_port_by_name(str.c_str(), &id) != 0)
return false;
else
port_id = id;
#else
//Try parsing a ffff:ff:ff.f format. Code adapted from EtherAddressArg::parse
unsigned data[4];
int d = 0, p = 0;
Expand All @@ -300,15 +316,19 @@ DPDKDeviceArg::parse(const String &str, DPDKDevice* &result, const ArgContext &c
else if (*s >= 'A' && *s <= 'F')
digit = *s - 'A' + 10;
else {
if (((*s == ':' && d < 2) || (*s == '.' && d == 2)) && (p == 1 || (d < 3 && p == 2) || (d == 0 && (p == 3 || p == 4))) && d < 3) {
if (((*s == ':' && d < 2) ||
(*s == '.' && d == 2)) &&
(p == 1 || (d < 3 && p == 2) || (d == 0 && (p == 3 || p == 4)))
&& d < 3) {
p = 0;
++d;
continue;
} else
break;
}

if ((d == 0 && p == 4) || (d > 0 && p == 2) || (d == 3 && p == 1) || d == 4)
if ((d == 0 && p == 4) || (d > 0 && p == 2)||
(d == 3 && p == 1) || d == 4)
break;

data[d] = (p ? data[d] << 4 : 0) + digit;
Expand All @@ -320,12 +340,15 @@ DPDKDeviceArg::parse(const String &str, DPDKDevice* &result, const ArgContext &c
return false;
}

port_id = DPDKDevice::get_port_from_pci(data[0],data[1],data[2],data[3]);
port_id = DPDKDevice::get_port_from_pci(
data[0], data[1], data[2], data[3]
);
#endif
}

if (port_id >= 0 && port_id < rte_eth_dev_count())
if (port_id >= 0 && port_id < DPDKDevice::dev_count()) {
result = DPDKDevice::get_device(port_id);
else {
} else {
ctx.error("Cannot resolve PCI address to DPDK device");
return false;
}
Expand All @@ -346,7 +369,7 @@ int DPDKDevice::TX_HTHRESH = 0;
int DPDKDevice::TX_WTHRESH = 0;

bool DPDKDevice::_is_initialized = false;
HashTable<unsigned, DPDKDevice> DPDKDevice::_devs;
HashTable<portid_t, DPDKDevice> DPDKDevice::_devs;
struct rte_mempool** DPDKDevice::_pktmbuf_pools;
bool DPDKDevice::no_more_buffer_msg_printed = false;

Expand Down
Loading

0 comments on commit 8247b32

Please sign in to comment.