Tuesday, January 19, 2016

Syn flood network virtual server in line rate by mTCP and DPDK

there is a need at my work to come up a solution to do tcp syn flood subnet network destination addresses (network virtual server) like 10.1.1.0/24 or even wildcard virtual server 0.0.0.0/0 in line rate speed to validate the FPGA hardware acceleration for network virtual servers syn cookie handling.

hping3 is well known open source software that can be used tcp syn flooding using kernel raw socket underneath, but only  one destination ip address can be given to hping3 to flood at a time, and kernel raw socket still use Linux kernel driver which is not as fast as DPDK poll mode driver.

and here mTCP and DPDK come to resecue, I could syn flood network virtual server in line rate high speed. here is the mTCP patch idea:

1, add libcidr and link example synflood code with libcidr to parse IP CIDR format at command line
2, modify mTCP mtcp_init_rss to add number of destination addresses as one of parameter
3, comment out the receive side scaling (RSS) for CPU affinity check, otherwise perf top shows GetRSSHash high cpu and low packet rate, as mtcp_connect->FetchAddress->GetRSSCPUCore->GetRSSHash
4, modify mtcp/src/ip_out.c to generate random source ip, be aware not to use rand() as it is not thread-safe function which could cause spin lock contention(_raw_spin_lock high cpu cycles from perf top and low packet rate also see make your program slower with threads  )


detail patch below:



diff --git a/apps/example/synflood.c b/apps/example/synflood.c
index be5129b..9394a0b 100644
--- a/apps/example/synflood.c
+++ b/apps/example/synflood.c
@@ -18,6 +18,7 @@
 #include
 #include
+#include
 #include "cpu.h"
 #include "rss.h"
 #include "http_parsing.h"
@@ -31,6 +32,7 @@

 #define IP_RANGE 16
 #define MAX_IP_STR_LEN 16
+#define MAX_DADDR 254

 #define BUF_SIZE (8*1024)

@@ -71,6 +73,16 @@ static char url[MAX_URL_LEN + 1];
 static in_addr_t daddr;
 static in_port_t dport;
 static in_addr_t saddr;
+
+static struct in_addr minip, ip_index, dest[MAX_DADDR];
+static int a, b, c, d;
+static int ret;
+static int num_daddr;
+static CIDR *addr, *addr2;
+static char *astr;
+static char *szIP;
+static const char *cstr;
+
 /*----------------------------------------------------------------------------*/
 static int total_flows;
 static int flows[MAX_CPUS];
@@ -165,6 +177,8 @@ DestroyContext(thread_context_t ctx)
        mtcp_destroy_context(ctx->mctx);
        free(ctx);
 }
+
+
 /*----------------------------------------------------------------------------*/
 inline int
 CreateConnection(thread_context_t ctx)
@@ -174,6 +188,7 @@ CreateConnection(thread_context_t ctx)
        struct sockaddr_in addr;
        int sockid;
        int ret;
+       int off = 0;
        sockid = mtcp_socket(mctx, AF_INET, SOCK_STREAM, 0);
        if (sockid < 0) {
@@ -187,6 +202,9 @@ CreateConnection(thread_context_t ctx)
                exit(-1);
        }

+       off = ctx->started++ % num_daddr;
+       daddr = dest[off].s_addr;
+
        addr.sin_family = AF_INET;
        addr.sin_addr.s_addr = daddr;
        addr.sin_port = dport;
@@ -201,7 +219,6 @@ CreateConnection(thread_context_t ctx)
                }
        }

-       ctx->started++;
        ctx->pending++;
        ctx->stat.connects++;

@@ -514,7 +531,7 @@ RunWgetMain(void *arg)
        g_stat[core] = &ctx->stat;
        srand(time(NULL));

-       mtcp_init_rss(mctx, saddr, num_ip, daddr, dport);
+       mtcp_init_rss(mctx, saddr, num_ip, daddr, num_daddr, dport);

        n = flows[core];
        if (n == 0) {
@@ -674,6 +691,38 @@ main(int argc, char **argv)
                return FALSE;
        }

+       astr = NULL;
+       addr = cidr_from_str(argv[1]);
+       if(addr ==NULL) {
+               TRACE_CONFIG("Error: Couldn't parse address %s\n", argv[1]);
+               return FALSE;
+       }
+
+        addr2 = cidr_addr_hostmin(addr);
+        astr = cidr_to_str(addr2, CIDR_ONLYADDR);
+
+        ret = sscanf(astr, "%i.%i.%i.%i", &a, &b, &c, &d);
+        if (ret != 4 ) {
+               fprintf(stderr, "Error: Invalid syntax.\n");
+        }
+        minip.s_addr = a << 24 | b << 16 | c << 8 | d;
+
+        /* Num of hosts */
+        cstr = cidr_numhost(addr);
+        num_daddr = atoi(cstr);
+       TRACE_CONFIG("%s: %d\n", "NumOfDestinationHosts", num_daddr);
+
+        for( i=0; i+        {
+               ip_index = minip;
+               ip_index.s_addr += i;
+               ip_index.s_addr = htonl(ip_index.s_addr);
+               dest[i] = ip_index;
+               //szIP = inet_ntoa(ip_index);
+               //szIP = inet_ntoa(dest[i]);
+               //TRACE_CONFIG("IP: %s\n", szIP);
+        }
+
        char* slash_p = strchr(argv[1], '/');
        if (slash_p) {
                strncpy(host, argv[1], slash_p - argv[1]);
@@ -683,8 +732,13 @@ main(int argc, char **argv)
                strncpy(url, "/", 1);
        }

-       daddr = inet_addr(host);
+       daddr = inet_addr(astr);
        dport = htons(80);
+
+       free(astr);
+       cidr_free(addr);
+       cidr_free(addr2);
+
        saddr = INADDR_ANY;

        total_flows = atoi(argv[2]);
diff --git a/mtcp/src/addr_pool.c b/mtcp/src/addr_pool.c
index 2bc4944..5482cb0 100644
--- a/mtcp/src/addr_pool.c
+++ b/mtcp/src/addr_pool.c
@@ -7,6 +7,7 @@

 #define MIN_PORT (1025)
 #define MAX_PORT (65535 + 1)
+#define MAX_NUM_DADDR  254

 /*----------------------------------------------------------------------------*/
 struct addr_entry
@@ -23,10 +24,12 @@ struct addr_map
 struct addr_pool
 {
        struct addr_entry *pool;                /* address pool */
-       struct addr_map *mapper;                /* address map  */
+       struct addr_map *mapper[MAX_NUM_DADDR];         /* address map  */

        uint32_t addr_base;                             /* in host order */
+       uint32_t daddr_base;                            /* in host order */
        int num_addr;                                   /* number of addresses in use */
+       int num_daddr;                                  /* number of addresses in use */

        int num_entry;
        int num_free;
@@ -38,11 +41,11 @@ struct addr_pool
 };
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr)
+CreateAddressPool(in_addr_t addr_base, int num_addr,  in_addr_t daddr_base, int num_daddr)
 {
        struct addr_pool *ap;
        int num_entry;
-       int i, j, cnt;
+       int i, j, k, cnt;
        in_addr_t addr;
        uint32_t addr_h;

@@ -51,7 +54,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
                return NULL;

        /* initialize address pool */
-       num_entry = num_addr * (MAX_PORT - MIN_PORT);
+       num_entry = num_addr * num_daddr * (MAX_PORT - MIN_PORT);
        ap->pool = (struct addr_entry *)calloc(num_entry, sizeof(struct addr_entry));
        if (!ap->pool) {
                free(ap);
@@ -59,11 +62,13 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
        }

        /* initialize address map */
-       ap->mapper = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
-       if (!ap->mapper) {
-               free(ap->pool);
-               free(ap);
-               return NULL;
+       for ( i = 0; i < num_daddr; i++) {
+               ap->mapper[i] = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
+               if (!ap->mapper[i]) {
+                       free(ap->pool);
+                       free(ap);
+                       return NULL;
+               }
        }

        TAILQ_INIT(&ap->free_list);
@@ -78,21 +83,25 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
        pthread_mutex_lock(&ap->lock);

        ap->addr_base = ntohl(addr_base);
+       ap->daddr_base = ntohl(daddr_base);
        ap->num_addr = num_addr;
+       ap->num_daddr = num_daddr;

        cnt = 0;
-       for (i = 0; i < num_addr; i++) {
-               addr_h = ap->addr_base + i;
-               addr = htonl(addr_h);
-               for (j = MIN_PORT; j < MAX_PORT; j++) {
-                       ap->pool[cnt].addr.sin_addr.s_addr = addr;
-                       ap->pool[cnt].addr.sin_port = htons(j);
-                       ap->mapper[i].addrmap[j] = &ap->pool[cnt];
+       for (k = 0; k < num_daddr; k++) {
+               for (i = 0; i < num_addr; i++) {
+                       addr_h = ap->addr_base + i;
+                       addr = htonl(addr_h);
+                       for (j = MIN_PORT; j < MAX_PORT; j++) {
+                               ap->pool[cnt].addr.sin_addr.s_addr = addr;
+                               ap->pool[cnt].addr.sin_port = htons(j);
+                               ap->mapper[k][i].addrmap[j] = &ap->pool[cnt];

-                       TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
+                               TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);

-                       if ((++cnt) >= num_entry)
-                               break;
+                               if ((++cnt) >= num_entry)
+                                       break;
+                       }
                }
        }
        ap->num_entry = cnt;
@@ -106,11 +115,11 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
 /*----------------------------------------------------------------------------*/
 addr_pool_t
 CreateAddressPoolPerCore(int core, int num_queues,
-               in_addr_t saddr_base, int num_addr, in_addr_t daddr, in_port_t dport)
+               in_addr_t saddr_base, int num_addr, in_addr_t daddr_base, int num_daddr, in_port_t dport)
 {
        struct addr_pool *ap;
        int num_entry;
-       int i, j, cnt;
+       int i, j, k, cnt;
        in_addr_t saddr;
        uint32_t saddr_h, daddr_h;
        uint16_t sport_h, dport_h;
@@ -123,7 +132,7 @@ CreateAddressPoolPerCore(int core, int num_queues,
                return NULL;

        /* initialize address pool */
-       num_entry = (num_addr * (MAX_PORT - MIN_PORT)) / num_queues;
+       num_entry = (num_addr * num_daddr * (MAX_PORT - MIN_PORT)) / num_queues;
        ap->pool = (struct addr_entry *)calloc(num_entry, sizeof(struct addr_entry));
        if (!ap->pool) {
                free(ap);
@@ -131,11 +140,13 @@ CreateAddressPoolPerCore(int core, int num_queues,
        }

        /* initialize address map */
-       ap->mapper = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
-       if (!ap->mapper) {
-               free(ap->pool);
-               free(ap);
-               return NULL;
+       for ( i = 0; i < num_daddr; i++) {
+               ap->mapper[i] = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
+               if (!ap->mapper[i]) {
+                       free(ap->pool);
+                       free(ap);
+                       return NULL;
+               }
        }

        TAILQ_INIT(&ap->free_list);
@@ -150,29 +161,36 @@ CreateAddressPoolPerCore(int core, int num_queues,
        pthread_mutex_lock(&ap->lock);

        ap->addr_base = ntohl(saddr_base);
+       ap->daddr_base = ntohl(daddr_base);
        ap->num_addr = num_addr;
-       daddr_h = ntohl(daddr);
+       ap->num_daddr = num_daddr;
+       daddr_h = ntohl(daddr_base);
        dport_h = ntohs(dport);

        /* search address space to get RSS-friendly addresses */
        cnt = 0;
-       for (i = 0; i < num_addr; i++) {
-               saddr_h = ap->addr_base + i;
-               saddr = htonl(saddr_h);
-               for (j = MIN_PORT; j < MAX_PORT; j++) {
-                       if (cnt >= num_entry)
-                               break;
-
-                       sport_h = j;
-                       rss_core = GetRSSCPUCore(daddr_h, saddr_h, dport_h, sport_h, num_queues, endian_check);
-                       if (rss_core != core)
-                               continue;
-
-                       ap->pool[cnt].addr.sin_addr.s_addr = saddr;
-                       ap->pool[cnt].addr.sin_port = htons(sport_h);
-                       ap->mapper[i].addrmap[j] = &ap->pool[cnt];
-                       TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
-                       cnt++;
+       for (k = 0; k < num_daddr; k++) {
+               daddr_h = ap->daddr_base + k;
+               for (i = 0; i < num_addr; i++) {
+                       saddr_h = ap->addr_base + i;
+                       saddr = htonl(saddr_h);
+                       for (j = MIN_PORT; j < MAX_PORT; j++) {
+                               if (cnt >= num_entry)
+                                       break;
+
+                               sport_h = j;
+#if 0
+                               rss_core = GetRSSCPUCore(daddr_h, saddr_h, dport_h, sport_h, num_queues, endian_check);
+                               if (rss_core != core)
+                                       continue;
+
+#endif
+                               ap->pool[cnt].addr.sin_addr.s_addr = saddr;
+                               ap->pool[cnt].addr.sin_port = htons(sport_h);
+                               ap->mapper[k][i].addrmap[j] = &ap->pool[cnt];
+                               TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
+                               cnt++;
+                       }
                }
        }

@@ -194,6 +212,8 @@ CreateAddressPoolPerCore(int core, int num_queues,
 void
 DestroyAddressPool(addr_pool_t ap)
 {
+       int i;
+
        if (!ap)
                return;

@@ -201,10 +221,13 @@ DestroyAddressPool(addr_pool_t ap)
                free(ap->pool);
                ap->pool = NULL;
        }
+
+       for ( i = 0; i < ap->num_daddr; i++) {

-       if (ap->mapper) {
-               free(ap->mapper);
-               ap->mapper = NULL;
+               if (ap->mapper[i]) {
+                       free(ap->mapper[i]);
+                       ap->mapper[i] = NULL;
+               }
        }

        pthread_mutex_destroy(&ap->lock);
@@ -228,6 +251,7 @@ FetchAddress(addr_pool_t ap, int core, int num_queues,
        pthread_mutex_lock(&ap->lock);

        walk = TAILQ_FIRST(&ap->free_list);
+#if 0
        while (walk) {
                next = TAILQ_NEXT(walk, addr_link);

@@ -240,6 +264,7 @@ FetchAddress(addr_pool_t ap, int core, int num_queues,

                walk = next;
        }
+#endif

        if (walk) {
                *saddr = walk->addr;
@@ -260,35 +285,38 @@ FreeAddress(addr_pool_t ap, const struct sockaddr_in *addr)
 {
        struct addr_entry *walk, *next;
        int ret = -1;
+       int i;

        if (!ap || !addr)
                return -1;

        pthread_mutex_lock(&ap->lock);

-       if (ap->mapper) {
-               uint32_t addr_h = ntohl(addr->sin_addr.s_addr);
-               uint16_t port_h = ntohs(addr->sin_port);
-               int index = addr_h - ap->addr_base;
+       for (i = 0; i < ap->num_daddr; i++) {
+               if (ap->mapper[i]) {
+                       uint32_t addr_h = ntohl(addr->sin_addr.s_addr);
+                       uint16_t port_h = ntohs(addr->sin_port);
+                       int index = addr_h - ap->addr_base;

-               if (index >= 0 || index < ap->num_addr) {
-                       walk = ap->mapper[addr_h - ap->addr_base].addrmap[port_h];
-               } else {
-                       walk = NULL;
-               }
+                       if (index >= 0 || index < ap->num_addr) {
+                               walk = ap->mapper[i][addr_h - ap->addr_base].addrmap[port_h];
+                       } else {
+                               walk = NULL;
+                       }

-       } else {
-               walk = TAILQ_FIRST(&ap->used_list);
-               while (walk) {
-                       next = TAILQ_NEXT(walk, addr_link);
-                       if (addr->sin_port == walk->addr.sin_port &&
-                                       addr->sin_addr.s_addr == walk->addr.sin_addr.s_addr) {
-                               break;
+               } else {
+                       walk = TAILQ_FIRST(&ap->used_list);
+                       while (walk) {
+                               next = TAILQ_NEXT(walk, addr_link);
+                               if (addr->sin_port == walk->addr.sin_port &&
+                                               addr->sin_addr.s_addr == walk->addr.sin_addr.s_addr) {
+                                       break;
+                               }
+
+                               walk = next;
                        }

-                       walk = next;
                }
-
        }

        if (walk) {
diff --git a/mtcp/src/api.c b/mtcp/src/api.c
index 8e25b32..d8aee5b 100644
--- a/mtcp/src/api.c
+++ b/mtcp/src/api.c
@@ -496,7 +496,7 @@ mtcp_accept(mctx_t mctx, int sockid, struct sockaddr *addr, socklen_t *addrlen)
 /*----------------------------------------------------------------------------*/
 int
 mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
-               in_addr_t daddr, in_addr_t dport)
+               in_addr_t daddr_base, int num_daddr, in_addr_t dport)
 {
        mtcp_manager_t mtcp;
        addr_pool_t ap;
@@ -511,12 +511,12 @@ mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
                /* for the INADDR_ANY, find the output interface for the destination
                   and set the saddr_base as the ip address of the output interface */
-               nif_out = GetOutputInterface(daddr);
+               nif_out = GetOutputInterface(daddr_base);
                saddr_base = CONFIG.eths[nif_out].ip_addr;
        }

        ap = CreateAddressPoolPerCore(mctx->cpu, num_cpus,
-                       saddr_base, num_addr, daddr, dport);
+                       saddr_base, num_addr, daddr_base, num_daddr, dport);
        if (!ap) {
                errno = ENOMEM;
                return -1;
diff --git a/mtcp/src/core.c b/mtcp/src/core.c
index 82f6fc6..ece4ae0 100644
--- a/mtcp/src/core.c
+++ b/mtcp/src/core.c
@@ -1396,11 +1396,13 @@ mtcp_init(char *config_file)
        PrintConfiguration();

        /* TODO: this should be fixed */
+#if 0
        ap = CreateAddressPool(CONFIG.eths[0].ip_addr, 1);
        if (!ap) {
                TRACE_CONFIG("Error occured while creating address pool.\n");
                return -1;
        }
+#endif

        PrintInterfaceInfo();

diff --git a/mtcp/src/include/addr_pool.h b/mtcp/src/include/addr_pool.h
index 7447513..452f934 100644
--- a/mtcp/src/include/addr_pool.h
+++ b/mtcp/src/include/addr_pool.h
@@ -13,7 +13,7 @@ typedef struct addr_pool *addr_pool_t;
 /* num_addr: number of addresses to use as source IP                          */
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr);
+CreateAddressPool(in_addr_t addr_base, int num_addr, in_addr_t daddr_base, int num_daddr);
 /*----------------------------------------------------------------------------*/
 /* CreateAddressPoolPerCore()                                                 */
 /* Create address pool only for the given core number.                        */
@@ -21,7 +21,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr);
+#if 0
        ap = CreateAddressPool(CONFIG.eths[0].ip_addr, 1);
        if (!ap) {
                TRACE_CONFIG("Error occured while creating address pool.\n");
                return -1;
        }
+#endif

        PrintInterfaceInfo();

diff --git a/mtcp/src/include/addr_pool.h b/mtcp/src/include/addr_pool.h
index 7447513..452f934 100644
--- a/mtcp/src/include/addr_pool.h
+++ b/mtcp/src/include/addr_pool.h
@@ -13,7 +13,7 @@ typedef struct addr_pool *addr_pool_t;
 /* num_addr: number of addresses to use as source IP                          */
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr);
+CreateAddressPool(in_addr_t addr_base, int num_addr, in_addr_t daddr_base, int num_daddr);
 /*----------------------------------------------------------------------------*/
 /* CreateAddressPoolPerCore()                                                 */
 /* Create address pool only for the given core number.                        */
@@ -21,7 +21,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr);
 /*----------------------------------------------------------------------------*/
 addr_pool_t
 CreateAddressPoolPerCore(int core, int num_queues,
-               in_addr_t saddr_base, int num_addr, in_addr_t daddr, in_port_t dport);
+               in_addr_t saddr_base, int num_addr, in_addr_t daddr_base, int num_daddr, in_port_t dport);
 /*----------------------------------------------------------------------------*/
 void
 DestroyAddressPool(addr_pool_t ap);
diff --git a/mtcp/src/include/mtcp_api.h b/mtcp/src/include/mtcp_api.h
index 84cbfc5..719fff4 100644
--- a/mtcp/src/include/mtcp_api.h
+++ b/mtcp/src/include/mtcp_api.h
@@ -97,7 +97,7 @@ mtcp_accept(mctx_t mctx, int sockid, struct sockaddr *addr, socklen_t *addrlen);

 int
 mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
-               in_addr_t daddr, in_addr_t dport);
+               in_addr_t daddr_base, int num_daddr, in_addr_t dport);

 int
 mtcp_connect(mctx_t mctx, int sockid,

  diff --git a/mtcp/src/ip_out.c b/mtcp/src/ip_out.c
index 2473112..6873df6 100644
--- a/mtcp/src/ip_out.c
+++ b/mtcp/src/ip_out.c
@@ -3,6 +3,38 @@
 #include "eth_out.h"
 #include "arp.h"
 #include "debug.h"
+#include
+#include

Followers