Vincent Tech Blog

Wednesday, March 23, 2016

Patch draft to make mTCP+DPDK work in vlan tagged network

Here is a patch idea draft to make mTCP + DPDK work in vlan tagged environment,  the next thing is to figure out how to run mTCP + DPDK in vlan tagged VMware ESXi environment, which would be great to run mTCP + DPDK in VMware ESXi VM and easy to clone the VM for everybody need it

 diff --git a/mtcp/src/dpdk_module.c b/mtcp/src/dpdk_module.c  

 index 33d349e..3c08e25 100644
 --- a/mtcp/src/dpdk_module.c
 +++ b/mtcp/src/dpdk_module.c
 @@ -66,7 +66,7 @@ static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
  /* packet memory pools for storing packet bufs */
  static struct rte_mempool *pktmbuf_pool[MAX_CPUS] = {NULL};
 -//#define DEBUG                1
 +#define DEBUG             1
  #ifdef DEBUG
  /* ethernet addresses of ports */
  static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
 @@ -79,7 +79,8 @@ static struct rte_eth_conf port_conf = {
         .split_hdr_size =    0,
         .header_split  =    0, /**< Header Split disabled */
         .hw_ip_checksum =    1, /**< IP checksum offload enabled */
 -        .hw_vlan_filter =    0, /**< VLAN filtering disabled */
 +        .hw_vlan_filter =    1, /**< VLAN filtering disabled */
 +        .hw_vlan_strip =    1, /**< VLAN strip enabled */
         .jumbo_frame  =    0, /**< Jumbo Frame Support disabled */
         .hw_strip_crc  =    1, /**< CRC stripped by hardware */
     },
 @@ -127,6 +128,7 @@ static const struct rte_eth_txconf tx_conf = {
     .txq_flags =          0x0,
  };
 +
  struct mbuf_table {
     unsigned len; /* length of queued packets */
     struct rte_mbuf *m_table[MAX_PKT_BURST];
 @@ -266,6 +268,8 @@ dpdk_send_pkts(struct mtcp_thread_context *ctxt, int nif)
                       ctxt->cpu, i, nif);
                 exit(EXIT_FAILURE);
             }
 +            dpc->wmbufs[nif].m_table[i]->ol_flags = PKT_TX_VLAN_PKT;
 +            dpc->wmbufs[nif].m_table[i]->vlan_tci = 4094;
         }
         /* reset the len of mbufs var after flushing of packets */
         dpc->wmbufs[nif].len = 0;
 @@ -534,6 +538,12 @@ dpdk_load_module(void)
             if (ret < 0)
                 rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
                      ret, (unsigned) portid);
 +
 +            ret = rte_eth_dev_vlan_filter(portid, 4094, 1);
 +
 +            if (ret < 0)
 +                rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
 +                    ret, (unsigned) portid);
             /* init one RX queue per CPU */
             fflush(stdout);

Friday, March 18, 2016

Patch to make lighttpd run in multiple core properly with mtcp with the configuration

diff --git a/apps/lighttpd-1.4.32/src/server.c b/apps/lighttpd-1.4.32/src/server.c
index 7c76fd7..f0dde58 100644
--- a/apps/lighttpd-1.4.32/src/server.c
+++ b/apps/lighttpd-1.4.32/src/server.c
@@ -1213,7 +1213,8 @@ int
 main(int argc, char **argv) {
 #ifdef MULTI_THREADED
        server **srv_states = NULL;
-       char *conf_file = NULL;
+       //char *conf_file = NULL;
+       char *conf_file = "/etc/mtcp/config/m-lighttpd.conf";
 #ifdef USE_MTCP
        struct mtcp_conf mcfg;
 #endif
@@ -1594,7 +1595,7 @@ main(int argc, char **argv) {
        mcfg.num_cores = cpus;
        mtcp_setconf(&mcfg);
        /* initialize the mtcp context */
-       if (mtcp_init("mtcp.conf")) {
+       if (mtcp_init("/etc/mtcp/config/lighttpd-mtcp.conf")) {
                fprintf(stderr, "Failed to initialize mtcp\n");
                goto clean_up;
        }

diff --git a/mtcp/src/config.c b/mtcp/src/config.c
index c4faea5..b4e24d0 100644
--- a/mtcp/src/config.c
+++ b/mtcp/src/config.c
@@ -23,8 +23,8 @@
 #define MAX_OPTLINE_LEN 1024
 #define ALL_STRING "all"

-static const char *route_file = "config/route.conf";
-static const char *arp_file = "config/arp.conf";
+static const char *route_file = "/etc/mtcp/config/route.conf";
+static const char *arp_file = "/etc/mtcp/config/arp.conf";


the configuration directory looks like:

root@pktgen:/home/pktgen/mtcp# ls -l /etc/mtcp/config/
total 48
-rw-r--r-- 1 root root   530 Mar  4 14:18 arp.conf
-rw-r--r-- 1 root root  1360 Nov 13 10:34 brute-shake.conf
drwxr-xr-x 2 root root  4096 Mar  4 14:43 conf.d
-rw-r--r-- 1 root root  1370 Nov 13 10:32 epwget.conf
-rw-r--r-- 1 root root  1237 Mar  4 14:15 lighttpd-mtcp.conf
-rw-r--r-- 1 root root 11857 Mar  4 14:40 m-lighttpd.conf
-rw-r--r-- 1 root root  3235 Mar  4 14:42 modules.conf
-rw-r--r-- 1 root root   646 Nov 12 20:18 mtcp.conf
-rw-r--r-- 1 root root   352 Mar  4 14:19 route.conf
-rw-r--r-- 1 root root  1366 Nov 13 10:38 synflood.conf


top output:


top - 14:14:15 up 18 days, 35 min,  4 users,  load average: 7.98, 5.51, 2.53
Threads: 304 total,   9 running, 295 sleeping,   0 stopped,   0 zombie

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND                                                                                                                                                                  P
15707 root      20   0 14.071g 0.010t   9680 R 99.9 14.9   5:44.92 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        1
15730 root      20   0 14.071g 0.010t   9680 R 99.9 14.9   5:44.93 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        0
15708 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:44.95 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        2
15709 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:45.08 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        3
15710 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:44.99 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        4
15711 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:44.94 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        5
15712 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:44.89 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                                                                                                                        6
15713 root      20   0 14.071g 0.010t   9680 R 99.7 14.9   5:44.96 lighttpd -n 8 -f /etc/mtcp/config/m-lighttpd.conf                

How to re-injecting TCP segment after TCP 4-way close with scapy

At work, I do lot of in-house reproduction of customers issue with lots of different tools, scapy is one I used often to simulate some odd TCP behavior. for example, a customer has a PoS terminal establish tcp connections to our load balancer,  the terminal would FIN+ACK the TCP connection after sending a transaction request (PSH+ACK), our load balancer would FIN+ACK , the PoS terminal send final ACK to close the TCP connection. this is all sound and good.

here is an interesting problem though,  intermittently, some customers PoS terminal would re-send the transaction request (PSH+ACK) about ~1 second later after the TCP 4-way close. This of course will be dropped by load balancer with RST since the TCP connections has been closed and TCP connection flows in memory has been cleared. I think this is a good example to use scapy to simulate the PoS terminal client behavior and show how flexible with scapy to simulate some odd TCP behavior, see the script from my githup:




https://github.com/vincentmli/bash_scripts/blob/master/scapy-http-responder/pshack-after-tcp4wayclose.py

Tuesday, January 19, 2016

Syn flood network virtual server in line rate by mTCP and DPDK

there is a need at my work to come up a solution to do tcp syn flood subnet network destination addresses (network virtual server) like 10.1.1.0/24 or even wildcard virtual server 0.0.0.0/0 in line rate speed to validate the FPGA hardware acceleration for network virtual servers syn cookie handling.

hping3 is well known open source software that can be used tcp syn flooding using kernel raw socket underneath, but only  one destination ip address can be given to hping3 to flood at a time, and kernel raw socket still use Linux kernel driver which is not as fast as DPDK poll mode driver.

and here mTCP and DPDK come to resecue, I could syn flood network virtual server in line rate high speed. here is the mTCP patch idea:

1, add libcidr and link example synflood code with libcidr to parse IP CIDR format at command line
2, modify mTCP mtcp_init_rss to add number of destination addresses as one of parameter
3, comment out the receive side scaling (RSS) for CPU affinity check, otherwise perf top shows GetRSSHash high cpu and low packet rate, as mtcp_connect->FetchAddress->GetRSSCPUCore->GetRSSHash
4, modify mtcp/src/ip_out.c to generate random source ip, be aware not to use rand() as it is not thread-safe function which could cause spin lock contention(_raw_spin_lock high cpu cycles from perf top and low packet rate also see make your program slower with threads  )


detail patch below:



diff --git a/apps/example/synflood.c b/apps/example/synflood.c
index be5129b..9394a0b 100644
--- a/apps/example/synflood.c
+++ b/apps/example/synflood.c
@@ -18,6 +18,7 @@
 #include
 #include
+#include
 #include "cpu.h"
 #include "rss.h"
 #include "http_parsing.h"
@@ -31,6 +32,7 @@

 #define IP_RANGE 16
 #define MAX_IP_STR_LEN 16
+#define MAX_DADDR 254

 #define BUF_SIZE (8*1024)

@@ -71,6 +73,16 @@ static char url[MAX_URL_LEN + 1];
 static in_addr_t daddr;
 static in_port_t dport;
 static in_addr_t saddr;
+
+static struct in_addr minip, ip_index, dest[MAX_DADDR];
+static int a, b, c, d;
+static int ret;
+static int num_daddr;
+static CIDR *addr, *addr2;
+static char *astr;
+static char *szIP;
+static const char *cstr;
+
 /*----------------------------------------------------------------------------*/
 static int total_flows;
 static int flows[MAX_CPUS];
@@ -165,6 +177,8 @@ DestroyContext(thread_context_t ctx)
        mtcp_destroy_context(ctx->mctx);
        free(ctx);
 }
+
+
 /*----------------------------------------------------------------------------*/
 inline int
 CreateConnection(thread_context_t ctx)
@@ -174,6 +188,7 @@ CreateConnection(thread_context_t ctx)
        struct sockaddr_in addr;
        int sockid;
        int ret;
+       int off = 0;
        sockid = mtcp_socket(mctx, AF_INET, SOCK_STREAM, 0);
        if (sockid < 0) {
@@ -187,6 +202,9 @@ CreateConnection(thread_context_t ctx)
                exit(-1);
        }

+       off = ctx->started++ % num_daddr;
+       daddr = dest[off].s_addr;
+
        addr.sin_family = AF_INET;
        addr.sin_addr.s_addr = daddr;
        addr.sin_port = dport;
@@ -201,7 +219,6 @@ CreateConnection(thread_context_t ctx)
                }
        }

-       ctx->started++;
        ctx->pending++;
        ctx->stat.connects++;

@@ -514,7 +531,7 @@ RunWgetMain(void *arg)
        g_stat[core] = &ctx->stat;
        srand(time(NULL));

-       mtcp_init_rss(mctx, saddr, num_ip, daddr, dport);
+       mtcp_init_rss(mctx, saddr, num_ip, daddr, num_daddr, dport);

        n = flows[core];
        if (n == 0) {
@@ -674,6 +691,38 @@ main(int argc, char **argv)
                return FALSE;
        }

+       astr = NULL;
+       addr = cidr_from_str(argv[1]);
+       if(addr ==NULL) {
+               TRACE_CONFIG("Error: Couldn't parse address %s\n", argv[1]);
+               return FALSE;
+       }
+
+        addr2 = cidr_addr_hostmin(addr);
+        astr = cidr_to_str(addr2, CIDR_ONLYADDR);
+
+        ret = sscanf(astr, "%i.%i.%i.%i", &a, &b, &c, &d);
+        if (ret != 4 ) {
+               fprintf(stderr, "Error: Invalid syntax.\n");
+        }
+        minip.s_addr = a << 24 | b << 16 | c << 8 | d;
+
+        /* Num of hosts */
+        cstr = cidr_numhost(addr);
+        num_daddr = atoi(cstr);
+       TRACE_CONFIG("%s: %d\n", "NumOfDestinationHosts", num_daddr);
+
+        for( i=0; i+        {
+               ip_index = minip;
+               ip_index.s_addr += i;
+               ip_index.s_addr = htonl(ip_index.s_addr);
+               dest[i] = ip_index;
+               //szIP = inet_ntoa(ip_index);
+               //szIP = inet_ntoa(dest[i]);
+               //TRACE_CONFIG("IP: %s\n", szIP);
+        }
+
        char* slash_p = strchr(argv[1], '/');
        if (slash_p) {
                strncpy(host, argv[1], slash_p - argv[1]);
@@ -683,8 +732,13 @@ main(int argc, char **argv)
                strncpy(url, "/", 1);
        }

-       daddr = inet_addr(host);
+       daddr = inet_addr(astr);
        dport = htons(80);
+
+       free(astr);
+       cidr_free(addr);
+       cidr_free(addr2);
+
        saddr = INADDR_ANY;

        total_flows = atoi(argv[2]);
diff --git a/mtcp/src/addr_pool.c b/mtcp/src/addr_pool.c
index 2bc4944..5482cb0 100644
--- a/mtcp/src/addr_pool.c
+++ b/mtcp/src/addr_pool.c
@@ -7,6 +7,7 @@

 #define MIN_PORT (1025)
 #define MAX_PORT (65535 + 1)
+#define MAX_NUM_DADDR  254

 /*----------------------------------------------------------------------------*/
 struct addr_entry
@@ -23,10 +24,12 @@ struct addr_map
 struct addr_pool
 {
        struct addr_entry *pool;                /* address pool */
-       struct addr_map *mapper;                /* address map  */
+       struct addr_map *mapper[MAX_NUM_DADDR];         /* address map  */

        uint32_t addr_base;                             /* in host order */
+       uint32_t daddr_base;                            /* in host order */
        int num_addr;                                   /* number of addresses in use */
+       int num_daddr;                                  /* number of addresses in use */

        int num_entry;
        int num_free;
@@ -38,11 +41,11 @@ struct addr_pool
 };
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr)
+CreateAddressPool(in_addr_t addr_base, int num_addr,  in_addr_t daddr_base, int num_daddr)
 {
        struct addr_pool *ap;
        int num_entry;
-       int i, j, cnt;
+       int i, j, k, cnt;
        in_addr_t addr;
        uint32_t addr_h;

@@ -51,7 +54,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
                return NULL;

        /* initialize address pool */
-       num_entry = num_addr * (MAX_PORT - MIN_PORT);
+       num_entry = num_addr * num_daddr * (MAX_PORT - MIN_PORT);
        ap->pool = (struct addr_entry *)calloc(num_entry, sizeof(struct addr_entry));
        if (!ap->pool) {
                free(ap);
@@ -59,11 +62,13 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
        }

        /* initialize address map */
-       ap->mapper = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
-       if (!ap->mapper) {
-               free(ap->pool);
-               free(ap);
-               return NULL;
+       for ( i = 0; i < num_daddr; i++) {
+               ap->mapper[i] = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
+               if (!ap->mapper[i]) {
+                       free(ap->pool);
+                       free(ap);
+                       return NULL;
+               }
        }

        TAILQ_INIT(&ap->free_list);
@@ -78,21 +83,25 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
        pthread_mutex_lock(&ap->lock);

        ap->addr_base = ntohl(addr_base);
+       ap->daddr_base = ntohl(daddr_base);
        ap->num_addr = num_addr;
+       ap->num_daddr = num_daddr;

        cnt = 0;
-       for (i = 0; i < num_addr; i++) {
-               addr_h = ap->addr_base + i;
-               addr = htonl(addr_h);
-               for (j = MIN_PORT; j < MAX_PORT; j++) {
-                       ap->pool[cnt].addr.sin_addr.s_addr = addr;
-                       ap->pool[cnt].addr.sin_port = htons(j);
-                       ap->mapper[i].addrmap[j] = &ap->pool[cnt];
+       for (k = 0; k < num_daddr; k++) {
+               for (i = 0; i < num_addr; i++) {
+                       addr_h = ap->addr_base + i;
+                       addr = htonl(addr_h);
+                       for (j = MIN_PORT; j < MAX_PORT; j++) {
+                               ap->pool[cnt].addr.sin_addr.s_addr = addr;
+                               ap->pool[cnt].addr.sin_port = htons(j);
+                               ap->mapper[k][i].addrmap[j] = &ap->pool[cnt];

-                       TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
+                               TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);

-                       if ((++cnt) >= num_entry)
-                               break;
+                               if ((++cnt) >= num_entry)
+                                       break;
+                       }
                }
        }
        ap->num_entry = cnt;
@@ -106,11 +115,11 @@ CreateAddressPool(in_addr_t addr_base, int num_addr)
 /*----------------------------------------------------------------------------*/
 addr_pool_t
 CreateAddressPoolPerCore(int core, int num_queues,
-               in_addr_t saddr_base, int num_addr, in_addr_t daddr, in_port_t dport)
+               in_addr_t saddr_base, int num_addr, in_addr_t daddr_base, int num_daddr, in_port_t dport)
 {
        struct addr_pool *ap;
        int num_entry;
-       int i, j, cnt;
+       int i, j, k, cnt;
        in_addr_t saddr;
        uint32_t saddr_h, daddr_h;
        uint16_t sport_h, dport_h;
@@ -123,7 +132,7 @@ CreateAddressPoolPerCore(int core, int num_queues,
                return NULL;

        /* initialize address pool */
-       num_entry = (num_addr * (MAX_PORT - MIN_PORT)) / num_queues;
+       num_entry = (num_addr * num_daddr * (MAX_PORT - MIN_PORT)) / num_queues;
        ap->pool = (struct addr_entry *)calloc(num_entry, sizeof(struct addr_entry));
        if (!ap->pool) {
                free(ap);
@@ -131,11 +140,13 @@ CreateAddressPoolPerCore(int core, int num_queues,
        }

        /* initialize address map */
-       ap->mapper = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
-       if (!ap->mapper) {
-               free(ap->pool);
-               free(ap);
-               return NULL;
+       for ( i = 0; i < num_daddr; i++) {
+               ap->mapper[i] = (struct addr_map *)calloc(num_addr, sizeof(struct addr_map));
+               if (!ap->mapper[i]) {
+                       free(ap->pool);
+                       free(ap);
+                       return NULL;
+               }
        }

        TAILQ_INIT(&ap->free_list);
@@ -150,29 +161,36 @@ CreateAddressPoolPerCore(int core, int num_queues,
        pthread_mutex_lock(&ap->lock);

        ap->addr_base = ntohl(saddr_base);
+       ap->daddr_base = ntohl(daddr_base);
        ap->num_addr = num_addr;
-       daddr_h = ntohl(daddr);
+       ap->num_daddr = num_daddr;
+       daddr_h = ntohl(daddr_base);
        dport_h = ntohs(dport);

        /* search address space to get RSS-friendly addresses */
        cnt = 0;
-       for (i = 0; i < num_addr; i++) {
-               saddr_h = ap->addr_base + i;
-               saddr = htonl(saddr_h);
-               for (j = MIN_PORT; j < MAX_PORT; j++) {
-                       if (cnt >= num_entry)
-                               break;
-
-                       sport_h = j;
-                       rss_core = GetRSSCPUCore(daddr_h, saddr_h, dport_h, sport_h, num_queues, endian_check);
-                       if (rss_core != core)
-                               continue;
-
-                       ap->pool[cnt].addr.sin_addr.s_addr = saddr;
-                       ap->pool[cnt].addr.sin_port = htons(sport_h);
-                       ap->mapper[i].addrmap[j] = &ap->pool[cnt];
-                       TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
-                       cnt++;
+       for (k = 0; k < num_daddr; k++) {
+               daddr_h = ap->daddr_base + k;
+               for (i = 0; i < num_addr; i++) {
+                       saddr_h = ap->addr_base + i;
+                       saddr = htonl(saddr_h);
+                       for (j = MIN_PORT; j < MAX_PORT; j++) {
+                               if (cnt >= num_entry)
+                                       break;
+
+                               sport_h = j;
+#if 0
+                               rss_core = GetRSSCPUCore(daddr_h, saddr_h, dport_h, sport_h, num_queues, endian_check);
+                               if (rss_core != core)
+                                       continue;
+
+#endif
+                               ap->pool[cnt].addr.sin_addr.s_addr = saddr;
+                               ap->pool[cnt].addr.sin_port = htons(sport_h);
+                               ap->mapper[k][i].addrmap[j] = &ap->pool[cnt];
+                               TAILQ_INSERT_TAIL(&ap->free_list, &ap->pool[cnt], addr_link);
+                               cnt++;
+                       }
                }
        }

@@ -194,6 +212,8 @@ CreateAddressPoolPerCore(int core, int num_queues,
 void
 DestroyAddressPool(addr_pool_t ap)
 {
+       int i;
+
        if (!ap)
                return;

@@ -201,10 +221,13 @@ DestroyAddressPool(addr_pool_t ap)
                free(ap->pool);
                ap->pool = NULL;
        }
+
+       for ( i = 0; i < ap->num_daddr; i++) {

-       if (ap->mapper) {
-               free(ap->mapper);
-               ap->mapper = NULL;
+               if (ap->mapper[i]) {
+                       free(ap->mapper[i]);
+                       ap->mapper[i] = NULL;
+               }
        }

        pthread_mutex_destroy(&ap->lock);
@@ -228,6 +251,7 @@ FetchAddress(addr_pool_t ap, int core, int num_queues,
        pthread_mutex_lock(&ap->lock);

        walk = TAILQ_FIRST(&ap->free_list);
+#if 0
        while (walk) {
                next = TAILQ_NEXT(walk, addr_link);

@@ -240,6 +264,7 @@ FetchAddress(addr_pool_t ap, int core, int num_queues,

                walk = next;
        }
+#endif

        if (walk) {
                *saddr = walk->addr;
@@ -260,35 +285,38 @@ FreeAddress(addr_pool_t ap, const struct sockaddr_in *addr)
 {
        struct addr_entry *walk, *next;
        int ret = -1;
+       int i;

        if (!ap || !addr)
                return -1;

        pthread_mutex_lock(&ap->lock);

-       if (ap->mapper) {
-               uint32_t addr_h = ntohl(addr->sin_addr.s_addr);
-               uint16_t port_h = ntohs(addr->sin_port);
-               int index = addr_h - ap->addr_base;
+       for (i = 0; i < ap->num_daddr; i++) {
+               if (ap->mapper[i]) {
+                       uint32_t addr_h = ntohl(addr->sin_addr.s_addr);
+                       uint16_t port_h = ntohs(addr->sin_port);
+                       int index = addr_h - ap->addr_base;

-               if (index >= 0 || index < ap->num_addr) {
-                       walk = ap->mapper[addr_h - ap->addr_base].addrmap[port_h];
-               } else {
-                       walk = NULL;
-               }
+                       if (index >= 0 || index < ap->num_addr) {
+                               walk = ap->mapper[i][addr_h - ap->addr_base].addrmap[port_h];
+                       } else {
+                               walk = NULL;
+                       }

-       } else {
-               walk = TAILQ_FIRST(&ap->used_list);
-               while (walk) {
-                       next = TAILQ_NEXT(walk, addr_link);
-                       if (addr->sin_port == walk->addr.sin_port &&
-                                       addr->sin_addr.s_addr == walk->addr.sin_addr.s_addr) {
-                               break;
+               } else {
+                       walk = TAILQ_FIRST(&ap->used_list);
+                       while (walk) {
+                               next = TAILQ_NEXT(walk, addr_link);
+                               if (addr->sin_port == walk->addr.sin_port &&
+                                               addr->sin_addr.s_addr == walk->addr.sin_addr.s_addr) {
+                                       break;
+                               }
+
+                               walk = next;
                        }

-                       walk = next;
                }
-
        }

        if (walk) {
diff --git a/mtcp/src/api.c b/mtcp/src/api.c
index 8e25b32..d8aee5b 100644
--- a/mtcp/src/api.c
+++ b/mtcp/src/api.c
@@ -496,7 +496,7 @@ mtcp_accept(mctx_t mctx, int sockid, struct sockaddr *addr, socklen_t *addrlen)
 /*----------------------------------------------------------------------------*/
 int
 mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
-               in_addr_t daddr, in_addr_t dport)
+               in_addr_t daddr_base, int num_daddr, in_addr_t dport)
 {
        mtcp_manager_t mtcp;
        addr_pool_t ap;
@@ -511,12 +511,12 @@ mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
                /* for the INADDR_ANY, find the output interface for the destination
                   and set the saddr_base as the ip address of the output interface */
-               nif_out = GetOutputInterface(daddr);
+               nif_out = GetOutputInterface(daddr_base);
                saddr_base = CONFIG.eths[nif_out].ip_addr;
        }

        ap = CreateAddressPoolPerCore(mctx->cpu, num_cpus,
-                       saddr_base, num_addr, daddr, dport);
+                       saddr_base, num_addr, daddr_base, num_daddr, dport);
        if (!ap) {
                errno = ENOMEM;
                return -1;
diff --git a/mtcp/src/core.c b/mtcp/src/core.c
index 82f6fc6..ece4ae0 100644
--- a/mtcp/src/core.c
+++ b/mtcp/src/core.c
@@ -1396,11 +1396,13 @@ mtcp_init(char *config_file)
        PrintConfiguration();

        /* TODO: this should be fixed */
+#if 0
        ap = CreateAddressPool(CONFIG.eths[0].ip_addr, 1);
        if (!ap) {
                TRACE_CONFIG("Error occured while creating address pool.\n");
                return -1;
        }
+#endif

        PrintInterfaceInfo();

diff --git a/mtcp/src/include/addr_pool.h b/mtcp/src/include/addr_pool.h
index 7447513..452f934 100644
--- a/mtcp/src/include/addr_pool.h
+++ b/mtcp/src/include/addr_pool.h
@@ -13,7 +13,7 @@ typedef struct addr_pool *addr_pool_t;
 /* num_addr: number of addresses to use as source IP                          */
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr);
+CreateAddressPool(in_addr_t addr_base, int num_addr, in_addr_t daddr_base, int num_daddr);
 /*----------------------------------------------------------------------------*/
 /* CreateAddressPoolPerCore()                                                 */
 /* Create address pool only for the given core number.                        */
@@ -21,7 +21,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr);
+#if 0
        ap = CreateAddressPool(CONFIG.eths[0].ip_addr, 1);
        if (!ap) {
                TRACE_CONFIG("Error occured while creating address pool.\n");
                return -1;
        }
+#endif

        PrintInterfaceInfo();

diff --git a/mtcp/src/include/addr_pool.h b/mtcp/src/include/addr_pool.h
index 7447513..452f934 100644
--- a/mtcp/src/include/addr_pool.h
+++ b/mtcp/src/include/addr_pool.h
@@ -13,7 +13,7 @@ typedef struct addr_pool *addr_pool_t;
 /* num_addr: number of addresses to use as source IP                          */
 /*----------------------------------------------------------------------------*/
 addr_pool_t
-CreateAddressPool(in_addr_t addr_base, int num_addr);
+CreateAddressPool(in_addr_t addr_base, int num_addr, in_addr_t daddr_base, int num_daddr);
 /*----------------------------------------------------------------------------*/
 /* CreateAddressPoolPerCore()                                                 */
 /* Create address pool only for the given core number.                        */
@@ -21,7 +21,7 @@ CreateAddressPool(in_addr_t addr_base, int num_addr);
 /*----------------------------------------------------------------------------*/
 addr_pool_t
 CreateAddressPoolPerCore(int core, int num_queues,
-               in_addr_t saddr_base, int num_addr, in_addr_t daddr, in_port_t dport);
+               in_addr_t saddr_base, int num_addr, in_addr_t daddr_base, int num_daddr, in_port_t dport);
 /*----------------------------------------------------------------------------*/
 void
 DestroyAddressPool(addr_pool_t ap);
diff --git a/mtcp/src/include/mtcp_api.h b/mtcp/src/include/mtcp_api.h
index 84cbfc5..719fff4 100644
--- a/mtcp/src/include/mtcp_api.h
+++ b/mtcp/src/include/mtcp_api.h
@@ -97,7 +97,7 @@ mtcp_accept(mctx_t mctx, int sockid, struct sockaddr *addr, socklen_t *addrlen);

 int
 mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr,
-               in_addr_t daddr, in_addr_t dport);
+               in_addr_t daddr_base, int num_daddr, in_addr_t dport);

 int
 mtcp_connect(mctx_t mctx, int sockid,

  diff --git a/mtcp/src/ip_out.c b/mtcp/src/ip_out.c
index 2473112..6873df6 100644
--- a/mtcp/src/ip_out.c
+++ b/mtcp/src/ip_out.c
@@ -3,6 +3,38 @@
 #include "eth_out.h"
 #include "arp.h"
 #include "debug.h"
+#include
+#include

Wednesday, December 9, 2015

Concurrent 10 million connection and Performance Test


Develop an opensource based packet generator for BIGIP load/performance evaluation/trouble-shooting using technology below

DPDK  is an user space drivers and libraries for fast packet
processing, it can generates 10Mpps, 10Mcps

mTCP A Highly Scalable User-level TCP Stack for Multicore Systems.

MoonGen  to generate raw packet like SYN/RST/ACK/UDP/ICMP flooding


Background:


The Secret to 10 Million Concurrent Connections -The Kernel is the Problem, Not the Solution



Problem:


Simplified packet processing in Linux:




Real packet processing in Linux:Linux network data flow




  System calls
  Context switching on blocking I/O
  Data Copying from kernel to user space
  Interrupt handing in kernel

Expense of sendto :

  sendto -  system call:  96ns
  sosend_dgram - lock sock_buff, alloc mbuf, copy in: 137ns
  udp_output - UDP header setup: 57ns
  ip_output - route lookup, ip header setup: 198ns
  ether_output - MAC lookup, MAC header setup: 162ns
  ixgbe_xmit - device programing: 220ns
Total: 950ns

Solution:


Packet processing with DPDK




  Processor affinity (separate cores)
  Huge pages( no swap, TLB)
  UIO (no copying from kernel)
  Polling (no interrupts overhead)
  Lockless synchronization(avoid waiting)
  Batch packets handling
  SSE, NUMA awareness UIO for example:

Kernel space (UIO framework) <------>/dev/uioX<------>userspace epoll/mmap<-------->App 

Problem:

http://www.ndsl.kaist.edu/~kyoungsoo/papers/mtcp.pdf
Limitaions of the Kernel's TCP stack
  Lack of  connection locality
  Shared file descriptor space
  Inefficient per-packet processing
  System call overhead

Solution:

  Batching in packet I/O, TCP processing, user applications ( reduce system call overhead)
  Connection locality on multicore systems - handling same connection on same core, avoid cache
pollution (solve connection locality)
  No descriptor sharing between mTCP thread


mTCP: A Highly Scalable User-level TCP Stack for Multicore
Systems  https://github.com/eunyoung14/mtcp



clone of mTCP in ES codeshare http://git.es.f5net.com/index.cgi/codeshare/tree/vli/mtcp
clone addition:
  Change apachebench configure script to compile with dpdk support
  Ported SSL BIO onto mTCP to enable apachebench to perform SSL test
  Add SSL clienthello stress test based on epwget and ssl-dos
  Add command line option in epwget and apachebenach to enable source address pool to congest
servers
  Increase mTCP SYN BACKLOG to increase concurrent connection
  Changed DPDK .config to compile DPDK as combined shared library
  Tuned send/receive buffer size in epwget.conf to achieve ~7 million concurrent connection
on Dell Poweredge R710 II 72G MEM, 16 core, Intel NIC 82599ES

mTCP installation

https://github.com/eunyoung14/mtcp has detail installation
- DPDK VERSION -
----------------
1. Set up Intel's DPDK driver. Please use our version of DPDK.    We have only changed the lib/igb_uio/ submodule. The best
   method to compile DPDK package is to use DPDK's tools/setup.sh
   script. Please compile your package based on your own hardware
   configuration. We tested the mTCP stack on Intel Xeon E5-2690
   (x86_64) machine with Intel 82599 Ethernet adapters (10G). We
   used the following steps in the setup.sh script for our setup:
 
     - Press [10] to compile the package
           - Press [13] to install the driver
           - Press [17] to setup 1024 2MB hugepages
           - Press [19] to register the Ethernet ports
           - Press [31] to quit the tool

  - check that DPDK package creates a new directory of compiled
  libraries. For x86_64 machines, the new subdirectory should be
  *dpdk-2.1.0/x86_64-native-linuxapp-gcc*

  - only those devices will work with DPDK drivers that are listed
  on this page: http://dpdk.org/doc/nics. Please make sure that your
  NIC is compatible before moving on to the next step.

2. Next bring the dpdk-registered interfaces up. Please use the
   setup_iface_single_process.sh script file present in dpdk-2.1.0/tools/
   directory for this purpose. Please change lines 49-51 to change the IP
   address. Under default settings, run the script as:
        # ./setup_iface_single_process.sh 3

   This sets the IP address of your interfaces as 10.0.x.3.

3. Create soft links for include/ and lib/ directories inside
   empty dpdk/ directory:
        # cd dpdk/
     # ln -s /x86_64-native-linuxapp-
gcc/lib lib
     # ln -s /x86_64-native-linuxapp-
gcc/include include

4. Setup mtcp library:
     # ./configure --with-dpdk-lib=$/dpdk 
       ## And not dpdk-2.1.0!
       ## e.g. ./configure --with-dpdk-lib=`echo $PWD`/dpdk
     # cd mtcp/src
        # make
  - check libmtcp.a in mtcp/lib
  - check header files in mtcp/include

5. make in util/:
        # make

6. make in apps/example:
        # make
  - check example binary files

7. Check the configurations
  - epserver.conf for server-side configuration
  - epwget.conf for client-side configuration
  - you may write your own configuration file for your application   - please see README.config for more details
    -- for the latest version, dyanmic ARP learning is *DISABLED*

8. Run the applications!

mTCP App configuration

############### mtcp configuration file ###############

# The underlying I/O module you want to use. Please
# enable only one out of the two.
io = dpdk
num_cores = 8
num_ip = 64
# Number of memory channels per processor socket (dpdk-only)
num_mem_ch = 4
#------ DPDK ports -------#
#port = dpdk0 dpdk1
port = dpdk0
#port = dpdk0:0
#port = dpdk0:1

# Enable multi-process support (under development)
#multiprocess = 0 master
#multiprocess = 1

# Receive buffer size of sockets
rcvbuf = 512
# Send buffer size of sockets sndbuf = 512
# Maximum concurrency per core
max_concurrency = 1000000
# Maximum number of socket buffers per core
# Set this to small value if there are many idle connections
max_num_buffers = 1000000
# TCO timeout seconds
# (tcp_timeout = -1 can disable the timeout check)
tcp_timeout = 30

mTCP APP call path(epwget):


epwget.c
main()
773         ret = mtcp_init("/etc/mtcp/config/epwget.conf");  //initialize mTCP and i/o modules

799                 if (pthread_create(&app_thread[i],
800                                         NULL, RunWgetMain, (void *)&cores[i])) { //app main thread

}
523 void *
524 RunWgetMain(void *arg)
525 {
526         thread_context_t ctx;
  540         mtcp_core_affinitize(core);
541
542         ctx = CreateContext(core); //create app context with mTCP
675 }

mtcp/src/core.c

1099 mctx_t
1100 mtcp_create_context(int cpu)
1101 {
1141         /* Wake up mTCP threads (wake up I/O threads) */
1142         if (current_iomodule_func == &dpdk_module_func) {
1143 #ifndef DISABLE_DPDK
1144                 int master;
1145                 master = rte_get_master_lcore();
1146                 if (master == cpu) {
1147                         lcore_config[master].ret = 0;
1148                         lcore_config[master].state = FINISHED;
1149                         if (pthread_create(&g_thread[cpu],
1150                                            NULL, MTCPRunThread, (void *)mctx) != 0) {
1151                                 TRACE_ERROR("pthread_create of mtcp thread failed!\n");
1152                                 return NULL;
1153                         }
1154                 } else
1155                         rte_eal_remote_launch(MTCPDPDKRunThread, mctx, cpu); 1156 #endif /* !DISABLE_DPDK */

1179         return mctx;
1180 }

1363 int
1364 mtcp_init(char *config_file)
1365 {
1391         ret = LoadConfiguration(config_file); //parse config and initilize DPDK environment
1399         ap = CreateAddressPool(CONFIG.eths[0].ip_addr, 1);
1428         current_iomodule_func->load_module(); //load dpdk i/o module
1429
1430         return 0;
1431 }

 mtcp/src/config.c
555 int
556 LoadConfiguration(char *fname)
557 {
604                 if (ParseConfiguration(p) < 0)
611 }

453 static int
454 ParseConfiguration(char *line)
455 {
534         } else if (strcmp(p, "port") == 0) {
535                 if(strncmp(q, ALL_STRING, sizeof(ALL_STRING)) == 0) {
536                         SetInterfaceInfo(q); //DPDK rte_eal_init
537                 } else {
538                         SetInterfaceInfo(line + strlen(p) + 1);
539                 }
540         } else if (strcmp(p, "io") == 0) {
541                 AssignIOModule(q); //Assign IO modules like psio/dpdk/netmap

553 }

mtcp/src/io_module.c
 66 int
 67 SetInterfaceInfo(char* dev_name_list)
 68 {

151         } else if (current_iomodule_func == &dpdk_module_func) {
152 #ifndef DISABLE_DPDK

173                 sprintf(external_driver, "%s", "/usr/src/mtcp/dpdk/lib/librte_pmd_e1000.so");
//load the specific NIC PMD driver if DPDK compiled as combined shared lib
1
175                 /* initialize the rte env first, what a waste of implementation effort!  */
176                 char *argv[] = {"",
177                                 "-c", 178                                 cpumaskbuf,
179                                 "-n",
180                                 mem_channels,
181                                 "-d",
182                                 external_driver,
183                                 "--proc-type=auto",
184                                 ""
185                 };
186                 const int argc = 8;

188                 /*
189                  * re-set getopt extern variable optind.
190                  * this issue was a bitch to debug
191                  * rte_eal_init() internally uses getopt() syscall
192                  * mtcp applications that also use an `external' getopt
193                  * will cause a violent crash if optind is not reset to zero
194                  * prior to calling the func below...
195                  * see man getopt(3) for more details
196                  */
197                 optind = 0;
198
199                 /* initialize the dpdk eal env */
200                 ret = rte_eal_init(argc, argv); //initialize DPDK environment

286 #endif /* !DISABLE_DPDK */ 287         }
288         return 0;
289 }

mtcp/src/core.c
1012 static void *
1013 MTCPRunThread(void *arg)
1014 {
1022         mtcp_core_affinitize(cpu);
1034         mtcp = ctx->mtcp_manager = InitializeMTCPManager(ctx);

1040         /* assign mtcp context's underlying I/O module */
1041         mtcp->iom = current_iomodule_func;

1043         /* I/O initializing */
1044         mtcp->iom->init_handle(ctx);

1085         /* start the main loop */
1086         RunMainLoop(ctx); //main packet receiving/sending loop

1090         return 0;
1091 }

 720 static void
 721 RunMainLoop(struct mtcp_thread_context *ctx)  722 {

735         while ((!ctx->done || mtcp->flow_cnt) && !ctx->exit) {

 744                 for (rx_inf = 0; rx_inf < CONFIG.eths_num; rx_inf++) {
 745
 746                         recv_cnt = mtcp->iom->recv_pkts(ctx, rx_inf); //receive packets
 747                         STAT_COUNT(mtcp->runstat.rounds_rx_try);
 748
 749                         for (i = 0; i < recv_cnt; i++) {
 750                                 uint16_t len;
 751                                 uint8_t *pktbuf;
 752                                 pktbuf = mtcp->iom->get_rptr(mtcp->ctx, rx_inf, i, &len); //
 753                                 ProcessPacket(mtcp, rx_inf, ts, pktbuf, len); //process receiving packet
 754                         }
 755                 }

 791                 if (mtcp->flow_cnt > 0) {
 792                         /* hadnle stream queues  */
 793                         HandleApplicationCalls(mtcp, ts);
 794                 }
 795
 796                 WritePacketsToChunks(mtcp, ts);
 797
 798                 /* send packets from write buffer */  799                 /* send until tx is available */
 800                 for (tx_inf = 0; tx_inf < CONFIG.eths_num; tx_inf++) {
 801                         mtcp->iom->send_pkts(ctx, tx_inf); //send packets
 802                 }

830 }

MoonGen: fully scriptable high-speed packet generator built on DPDK and LuaJIT.

https://github.com/emmericp/MoonGen

Design


 



 

 

 

 

 

 

 


Userscript Master



Userscript slave



 


clone of MoonGen in ES codeshare http://git.es.f5net.com/index.cgi/codeshare/tree/vli/MoonGen

  improved tcp syn flooding with random src ip and src port
  added DNS flooding script to test Victoria2 DNS DDOS Hardware protection
  added icmp echo flooding

MoonGen Installation:

1.  Install the dependencies (see below)
2.  git submodule update --init
3.  ./build.sh
4.  ./setup-hugetlbfs.sh
5.  Run MoonGen from the build directory
How to Run MoonGen script:
command syntax: build/MoonGen examples/ <# of src ip>  
#build/MoonGen examples/dns-flood-victoria.lua 0 10.0.0.1 16000000 10000

Hardware SPEC:   

Dell Poweredge R710  72G MEM, 16 core, Intel NIC 82599                                  
DUT: Victoria B2250 Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz 20 cores 64G MEM

Dell PowerEdge R210 II (used $300) 8 core, 32G MEM Intel 1G NIC I350            
DUT: BIGIP KVM VE CPU: QEMU Virtual CPU version (cpu64-rhel6) 4 cores 16G MEM

Load Test Example


1, DNS flooding without HW acceleration

#build/MoonGen examples/dns-flood-victoria.lua 0 10.0.0.1 16000000 10000
Device: id=0] Sent 13710082176 packets, current rate 4.51 Mpps, 3246.01 MBit/s, 3967.35 MBit/s
wire rate.
[Device: id=0] Sent 13714591360 packets, current rate 4.51 Mpps, 3246.53 MBit/s, 3967.98 MBit/s
wire rate.
[Device: id=0] Sent 13719099520 packets, current rate 4.51 Mpps, 3245.79 MBit/s, 3967.08 MBit/s
wire rate.
 
top - 12:07:02 up 1 day, 20:38,  1 user,  load average: 5.22, 7.46, 9.27
 Tasks: 777 total,  19 running, 758 sleeping,   0 stopped,   0 zombie
 Cpu(s): 50.6%us, 40.2%sy,  0.0%ni,  9.2%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
 Mem:  66080376k total, 65722732k used,   357644k free,   108700k buffers
 Swap:  5242872k total,        0k used,  5242872k free,  4048612k cached
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 17859 root       1 -19 57.9g 145m 123m R 100.8  0.2  56:44.33 tmm.0 -T 10 --tmid
 17741 root       1 -19 57.9g 145m 123m R 100.5  0.2  58:08.51 tmm.0 -T 10 --tmid
 17853 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:46.73 tmm.0 -T 10 --tmid
 17854 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:46.97 tmm.0 -T 10 --tmid
 17855 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:46.06 tmm.0 -T 10 --tmid
 17856 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:37.67 tmm.0 -T 10 --tmid  17857 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:45.54 tmm.0 -T 10 --tmid
 17858 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:45.70 tmm.0 -T 10 --tmid
 17860 root       1 -19 57.9g 145m 123m R 100.5  0.2  56:45.65 tmm.0 -T 10 --tmid
 17852 root       1 -19 57.9g 145m 123m R 100.2  0.2  56:50.91 tmm.0 -T 10 --tmid
 20110 root      RT   0     0    0    0 S 80.6  0.0   0:27.55 [enforcer/11]
 20111 root      RT   0     0    0    0 R 80.6  0.0   0:27.56 [enforcer/15]
 20116 root      RT   0     0    0    0 R 80.6  0.0   0:27.50 [enforcer/13]
 20108 root      RT   0     0    0    0 R 80.2  0.0   0:27.55 [enforcer/19]
 20109 root      RT   0     0    0    0 R 80.2  0.0   0:27.57 [enforcer/17]
 20112 root      RT   0     0    0    0 S 80.2  0.0   0:27.55 [enforcer/5]
 20113 root      RT   0     0    0    0 R 80.2  0.0   0:27.52 [enforcer/1]

------------------------------------------------------------------
Ltm::Virtual Server: vs_dns_10g
------------------------------------------------------------------
Status
  Availability     : unknown
  State            : enabled
  Reason           : The children pool member(s) either don't have service check
  CMP              : enabled
  CMP Mode         : all-cpus
  Destination      : 10.3.3.249:53
  PVA Acceleration : none
Traffic                             ClientSide  Ephemeral  General
  Bits In                                11.5G          0        -   Bits Out                               16.7G          0        -
  Packets In                             20.0M          0        -
  Packets Out                            20.0M          0        -
  Current Connections                    27.1M          0        -
  Maximum Connections                    27.1M          0        -
  Total Connections                      28.8M          0        -

2, DNS flooding with HW acceleration

#build/MoonGen examples/dns-flood-victoria.lua 0 10.0.0.1 16000000 10000
 Device: id=0] Sent 13710082176 packets, current rate 4.51 Mpps, 3246.01 MBit/s, 3967.35 MBit/s
wire rate.
 [Device: id=0] Sent 13714591360 packets, current rate 4.51 Mpps, 3246.53 MBit/s, 3967.98 MBit/s
wire rate.
 [Device: id=0] Sent 13719099520 packets, current rate 4.51 Mpps, 3245.79 MBit/s, 3967.08 MBit/s
wire rate.
https://docs.f5net.com/display/PDDESIGN/DNS+DDoS+HW+Acceleration+-+Validation
sys fpga firmware-config {
    type l7-intelligent-fpga
}
ltm profile dns /Common/dns_fpga {
    app-service none
    enable-hardware-query-validation yes
    enable-hardware-response-cache yes
}
ltm virtual /Common/vs_dns_10g {
    destination /Common/10.3.3.249:53
    ip-protocol udp
    mask 255.255.255.255     profiles {
        /Common/dns_fpga { }
        /Common/udp_immediate { }
    }
    rules {
        /Common/dns_responder
    }
    source 0.0.0.0/0
    translate-address enabled
    translate-port enabled
}

top - 14:51:05 up  3:30,  1 user,  load average: 0.12, 0.05, 0.01
Tasks: 771 total,   1 running, 770 sleeping,   0 stopped,   0 zombie
Cpu(s):  4.2%us,  0.5%sy,  0.0%ni, 95.2%id,  0.0%wa,  0.1%hi,  0.0%si,  0.0%st
Mem:  66080272k total, 63094488k used,  2985784k free,    61152k buffers
Swap:  5242876k total,        0k used,  5242876k free,  1352852k cached
  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 6428 root       1 -19 58.4g 151m 122m S 12.6  0.2   3:19.62 tmm.0
 6435 root       1 -19 58.4g 151m 122m S 11.3  0.2   2:42.67 tmm.4
 6432 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:44.57 tmm.1
 6433 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:42.78 tmm.2
 6434 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:40.69 tmm.3
 6436 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:41.53 tmm.5
 6437 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:42.68 tmm.6  6438 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:40.92 tmm.7
 6439 root       1 -19 58.4g 151m 122m S 10.9  0.2   2:41.87 tmm.8
 6440 root       1 -19 58.4g 151m 122m S 10.6  0.2   2:41.49 tmm.9
28351 root     -91   0 97592  81m  31m S  2.0  0.1   7:00.29 bcmINTR
28589 root      20   0 97592  81m  31m S  2.0  0.1   5:43.36 bcmCNTR.0

profile_dns_stat
name               vs_name                    queries          drops         hw_malformed        hw_inspected            hw_cache_lookups         hw_cache_responses
---------------- ------------------ ------- ----- ------------ ------------ ---------------- --------------------------------------------------------------------------------------------------
/Common/dns_fpga   /Common/vs_dns_10g      6981      0         0                         29727032                 297          26590                         29720435

3, SYN flooding without hardware acceleration


#build/MoonGen examples/l3-tcp-syn-flood.lua 0 10.0.0.1 16000000 10000

 [Device: id=0] Sent 7061632 packets, current rate 7.06 Mpps, 3615.47 MBit/s, 4745.31 MBit/s wire
rate.
 ltm profile fastl4 /Common/fl4_fpga {
     app-service none
     defaults-from /Common/fastL4
     hardware-syn-cookie disabled
     pva-offload-dynamic disabled
     software-syn-cookie enabled
 }
top - 10:53:51 up 42 min,  1 user,  load average: 0.24, 0.23, 0.65
Tasks: 769 total,  10 running, 759 sleeping,   0 stopped,   0 zombie Cpu(s): 35.4%us,  1.7%sy,  0.1%ni, 62.9%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:  66080376k total, 62740700k used,  3339676k free,    45784k buffers
Swap:  5242872k total,        0k used,  5242872k free,  1199508k cached
  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
19290 root       1 -19 57.9g 145m 123m R 71.5  0.2   0:14.38 tmm.0 -T 10 --tmid
CPU
 19291 root       1 -19 57.9g 145m 123m R 70.2  0.2   0:14.53 tmm.0 -T 10 --tmid
 19293 root       1 -19 57.9g 145m 123m S 70.2  0.2   0:14.37 tmm.0 -T 10 --tmid
 19267 root       1 -19 57.9g 145m 123m R 69.8  0.2   0:30.32 tmm.0 -T 10 --tmid
 19292 root       1 -19 57.9g 145m 123m R 69.8  0.2   0:14.38 tmm.0 -T 10 --tmid
 19298 root       1 -19 57.9g 145m 123m R 69.8  0.2   0:14.72 tmm.0 -T 10 --tmid
 19295 root       1 -19 57.9g 145m 123m R 69.5  0.2   0:14.73 tmm.0 -T 10 --tmid
 19296 root       1 -19 57.9g 145m 123m R 69.5  0.2   0:14.03 tmm.0 -T 10 --tmid
 19297 root       1 -19 57.9g 145m 123m R 69.2  0.2   0:14.14 tmm.0 -T 10 --tmid
 19294 root       1 -19 57.9g 145m 123m R 65.2  0.2   0:13.31 tmm.0 -T 10 --tmid

4 SYN flooding with HW acceleration

 #build/MoonGen examples/l3-tcp-syn-flood.lua 0 10.0.0.1 16000000 10000

  [Device: id=0] Sent 7061632 packets, current rate 7.06 Mpps, 3615.47 MBit/s, 4745.31 MBit/s wire rate.

ltm profile fastl4 /Common/fl4_fpga {
    app-service none
    defaults-from /Common/fastL4
    hardware-syn-cookie enabled
    pva-offload-dynamic enabled
    software-syn-cookie enabled
}

 top - 10:50:08 up 38 min,  1 user,  load average: 0.06, 0.36, 0.81
 Tasks: 769 total,   1 running, 768 sleeping,   0 stopped,   0 zombie
 Cpu(s):  0.8%us,  0.2%sy,  0.0%ni, 98.5%id,  0.5%wa,  0.0%hi,  0.0%si,  0.0%st
 Mem:  66080376k total, 62740552k used,  3339824k free,    45324k buffers
 Swap:  5242872k total,        0k used,  5242872k free,  1199492k cached
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 19267 root       1 -19 57.9g 145m 123m S  3.6  0.2   0:11.87 tmm
 19293 root       1 -19 57.9g 145m 123m S  1.3  0.2   0:01.72 tmm
 19296 root       1 -19 57.9g 145m 123m S  1.3  0.2   0:01.36 tmm
 19297 root       1 -19 57.9g 145m 123m S  1.3  0.2   0:01.37 tmm
 19290 root       1 -19 57.9g 145m 123m S  1.0  0.2   0:01.38 tmm
 19292 root       1 -19 57.9g 145m 123m S  1.0  0.2   0:01.73 tmm
 19294 root       1 -19 57.9g 145m 123m S  1.0  0.2   0:01.35 tmm
 19295 root       1 -19 57.9g 145m 123m S  1.0  0.2   0:02.12 tmm
 19298 root       1 -19 57.9g 145m 123m S  1.0  0.2   0:02.11 tmm
 19291 root       1 -19 57.9g 145m 123m S  0.7  0.2   0:01.73 tmm


5, 10M concurrent HTTP connection


#epwget 10.3.3.249/ 160000000 -N 16 –c 10000000

 [CPU 0] dpdk0 flows: 625000, RX:   96382(pps) (err:     0),  0.10(Gbps), TX:  413888(pps),  0.64(Gbps)
 [CPU 1] dpdk0 flows: 625000, RX:  101025(pps) (err:     0),  0.10(Gbps), TX:  398592(pps),  0.61(Gbps)
 [CPU 2] dpdk0 flows: 625000, RX:  106882(pps) (err:     0),  0.11(Gbps), TX:  418432(pps),  0.64(Gbps)
 [CPU 3] dpdk0 flows: 625000, RX:  101497(pps) (err:     0),  0.10(Gbps), TX:  405952(pps),  0.62(Gbps)
 [CPU 4] dpdk0 flows: 625000, RX:  107375(pps) (err:     0),  0.11(Gbps), TX:  427008(pps),  0.66(Gbps)  [CPU 5] dpdk0 flows: 625000, RX:   96012(pps) (err:     0),  0.10(Gbps), TX:  404352(pps),  0.62(Gbps)
 [CPU 6] dpdk0 flows: 625000, RX:  100834(pps) (err:     0),  0.10(Gbps), TX:  405504(pps),  0.62(Gbps)
 [CPU 7] dpdk0 flows: 625000, RX:  102572(pps) (err:     0),  0.11(Gbps), TX:  401024(pps),  0.62(Gbps)
 [CPU 8] dpdk0 flows: 635366, RX:  111319(pps) (err:     0),  0.12(Gbps), TX:  410880(pps),  0.63(Gbps)
 [CPU 9] dpdk0 flows: 625000, RX:  102179(pps) (err:     0),  0.11(Gbps), TX:  391104(pps),  0.60(Gbps)
 [CPU10] dpdk0 flows: 625000, RX:   98014(pps) (err:     0),  0.10(Gbps), TX:  408320(pps),  0.63(Gbps)
 [CPU11] dpdk0 flows: 625000, RX:  102712(pps) (err:     0),  0.11(Gbps), TX:  398976(pps),  0.61(Gbps)
 [CPU12] dpdk0 flows: 625000, RX:  105891(pps) (err:     0),  0.11(Gbps), TX:  415616(pps),  0.64(Gbps)
 [CPU13] dpdk0 flows: 625000, RX:   97728(pps) (err:     0),  0.10(Gbps), TX:  390592(pps),  0.60(Gbps)
 [CPU14] dpdk0 flows: 625001, RX:  100570(pps) (err:     0),  0.10(Gbps), TX:  407872(pps),  0.63(Gbps)
 [CPU15] dpdk0 flows: 625000, RX:  103412(pps) (err:     0),  0.11(Gbps), TX:  391296(pps),  0.60(Gbps)
 [ ALL ] dpdk0 flows: 10010366, RX: 1634404(pps) (err:     0),  1.69(Gbps), TX: 6489408(pps),  9.96(Gbps)

top - 15:25:26 up 23:57,  1 user,  load average: 0.16, 0.33, 0.43
 Tasks: 778 total,  17 running, 761 sleeping,   0 stopped,   0 zombie
 Cpu(s): 45.1%us, 30.6%sy,  0.0%ni, 24.3%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
 Mem:  66080376k total, 62855960k used,  3224416k free,   136316k buffers
 Swap:  5242872k total,        0k used,  5242872k free,  1182216k cached
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 17283 root       1 -19 57.9g 145m 123m R 94.1  0.2   1322:36 tmm.0 -T 10 --tmid
 17286 root       1 -19 57.9g 145m 123m R 94.1  0.2   1322:37 tmm.0 -T 10 --tmid
 17281 root       1 -19 57.9g 145m 123m R 93.8  0.2   1322:39 tmm.0 -T 10 --tmid
 17284 root       1 -19 57.9g 145m 123m R 93.8  0.2   1322:37 tmm.0 -T 10 --tmid
 17282 root       1 -19 57.9g 145m 123m R 93.4  0.2   1322:37 tmm.0 -T 10 --tmid
 17287 root       1 -19 57.9g 145m 123m R 93.4  0.2   1322:37 tmm.0 -T 10 --tmid
 17289 root       1 -19 57.9g 145m 123m R 93.4  0.2   1322:36 tmm.0 -T 10 --tmid
 17043 root       1 -19 57.9g 145m 123m R 92.8  0.2   1325:48 tmm.0 -T 10 --tmid
 17285 root       1 -19 57.9g 145m 123m R 92.1  0.2   1322:37 tmm.0 -T 10 --tmid
 31507 root      RT   0     0    0    0 R 32.0  0.0   0:00.97 [enforcer/19]
 31508 root      RT   0     0    0    0 R 32.0  0.0   0:00.97 [enforcer/13]
 31509 root      RT   0     0    0    0 R 32.0  0.0   0:00.97 [enforcer/15]
 31510 root      RT   0     0    0    0 S 31.7  0.0   0:00.96 [enforcer/9]
 31511 root      RT   0     0    0    0 S 31.7  0.0   0:00.96 [enforcer/7]
 31512 root      RT   0     0    0    0 R 31.4  0.0   0:00.95 [enforcer/3]
 31515 root      RT   0     0    0    0 S 16.8  0.0   0:00.51 [enforcer/1]

[root@localhost:/S1-green-P:Active:Standalone] config # tail -f /var/log/ltm
 Nov  4 15:25:29 slot1/bigip1 warning tmm7[17043]: 011e0003:4: Aggressive mode sweeper:
/Common/default-eviction-policy (70000000002d6) (global memory) 9864 Connections killed
 Nov  4 15:25:29 slot1/bigip1 warning tmm7[17043]: 011e0002:4:
sweeper_policy_bind_deactivation_update: Aggressive mode /Common/default-eviction-policy
deactivated (70000000002d6) (global memory). (12793204/15051776 pages)
 Nov  4 15:25:29 slot1/bigip1 warning tmm6[17043]: 011e0003:4: Aggressive mode sweeper:
/Common/default-eviction-policy (60000000002d2) (global memory) 10122 Connections killed
 Nov  4 15:25:29 slot1/bigip1 warning tmm6[17043]: 011e0002:4:
sweeper_policy_bind_deactivation_update: Aggressive mode /Common/default-eviction-policy
deactivated (60000000002d2) (global memory). (12792703/15051776 pages)
 Nov  4 15:25:29 slot1/bigip1 warning tmm3[17043]: 011e0003:4: Aggressive mode sweeper:
/Common/default-eviction-policy (30000000002de) (global memory) 10877 Connections killed
 Nov  4 15:25:29 slot1/bigip1 warning tmm3[17043]: 011e0002:4:
sweeper_policy_bind_deactivation_update: Aggressive mode /Common/default-eviction-policy
deactivated (30000000002de) (global memory). (12787088/15051776 pages)
 Nov  4 15:25:29 slot1/bigip1 warning tmm4[17043]: 011e0003:4: Aggressive mode sweeper:
/Common/default-eviction-policy (40000000002c2) (global memory) 10306 Connections killed  Nov  4 15:25:29 slot1/bigip1 warning tmm4[17043]: 011e0002:4:
sweeper_policy_bind_deactivation_update: Aggressive mode /Common/default-eviction-policy
deactivated (40000000002c2) (global memory). (12787088/15051776 pages)


Every 1.0s: tmsh show ltm virtual vs_http_10g           Wed Nov  4 15:27:15 2015
Availability     : unknown
  State            : enabled
  Reason           : The children pool member(s) either don't have service check
ing enabled, or service check results are not available yet
  CMP              : enabled
  CMP Mode         : all-cpus
  Destination      : 10.3.3.249:80
  PVA Acceleration : none
Traffic                             ClientSide  Ephemeral  General
  Bits In                               329.8G          0        -
  Bits Out                             90.4G          0        -
  Packets In                          287.6M          0        -
   Packets Out                       150.2M          0        -
   Current Connections         6.1M          0        -
   Maximum Connections     6.7M          0        -
   Total Connections               39.8M          0        -

mTCP perf top output ~70% cycles in Userspace

 Samples: 1M of event 'cycles', Event count (approx.): 441906428558
   8.25%  epwget              [.] SendTCPPacket
   7.93%  [kernel]            [k] _raw_spin_lock    7.16%  epwget              [.] GetRSSCPUCore
   7.15%  epwget              [.] IPOutput
   4.26%  libc-2.19.so        [.] memset
   4.10%  epwget              [.] ixgbe_xmit_pkts
   3.62%  [kernel]            [k] clear_page_c
   3.26%  epwget              [.] WriteTCPControlList
   3.24%  [vdso]              [.] 0x0000000000000cf9
   2.95%  epwget              [.] AddtoControlList
   2.70%  epwget              [.] MTCPRunThread
   2.66%  epwget              [.] HandleRTO
   2.51%  epwget              [.] CheckRtmTimeout
   2.10%  libpthread-2.19.so  [.] pthread_mutex_unlock
   1.83%  epwget              [.] dpdk_send_pkts
   1.68%  epwget              [.] HTInsert
   1.65%  epwget              [.] CreateTCPStream
   1.42%  epwget              [.] MPAllocateChunk
   1.29%  epwget              [.] TCPCalcChecksum
   1.24%  epwget              [.] dpdk_recv_pkts
   1.20%  epwget              [.] mtcp_getsockopt
   1.12%  epwget              [.] rx_recv_pkts

6, SSL DDOS test using mTCP



 top - 09:10:21 up 22:58,  1 user,  load average: 10.45, 4.43, 1.67
 Tasks: 782 total,  19 running, 763 sleeping,   0 stopped,   0 zombie
 Cpu(s): 50.6%us, 40.1%sy,  0.1%ni,  9.1%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
 Mem:  66080376k total, 62923192k used,  3157184k free,   138624k buffers
 Swap:  5242872k total,        0k used,  5242872k free,  1259132k cached
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 21480 root       1 -19 57.9g 145m 123m R 100.0  0.2  81:24.41 tmm
 21503 root       1 -19 57.9g 145m 123m R 100.0  0.2  48:05.30 tmm
 21504 root       1 -19 57.9g 145m 123m R 100.0  0.2  47:23.12 tmm
 21505 root       1 -19 57.9g 145m 123m R 100.0  0.2  47:06.70 tmm
 21506 root       1 -19 57.9g 145m 123m R 100.0  0.2  46:55.21 tmm
 21507 root       1 -19 57.9g 145m 123m R 100.0  0.2  46:12.27 tmm
 21508 root       1 -19 57.9g 145m 123m R 100.0  0.2  46:56.27 tmm
 21509 root       1 -19 57.9g 145m 123m R 100.0  0.2  47:01.32 tmm
 21510 root       1 -19 57.9g 145m 123m R 100.0  0.2  46:48.54 tmm
 21511 root       1 -19 57.9g 145m 123m R 100.0  0.2  47:06.64 tmm
   1670 root      RT   0     0    0    0 R 80.2  0.0   2:07.03 enforcer/15
  1673 root      RT   0     0    0    0 R 80.2  0.0   2:07.03 enforcer/3
  1677 root      RT   0     0    0    0 R 80.2  0.0   2:07.02 enforcer/13
  1671 root      RT   0     0    0    0 S 79.9  0.0   2:07.04 enforcer/19
  1672 root      RT   0     0    0    0 R 79.9  0.0   2:07.02 enforcer/5


7, ApacheBench(ab) mTCP port https test to Victoria blade
 #ab -n 16000 -N 16 -c 8000  -L 64 https://10.3.3.249/

 ---------------------------------------------------------------------------------
 Loading mtcp configuration from : /etc/mtcp/config/mtcp.conf
 Loading interface setting
 EAL: Detected lcore 0 as core 0 on socket 0
 .................................................  Checking link statusdone
 Port 0 Link Up - speed 10000 Mbps - full-duplex
 Benchmarking 10.3.3.249 (be patient)
 CPU6 connecting to port 443
 CPU7 connecting to port 443
 CPU8 connecting to port 443
 CPU9 connecting to port 443
 CPU10 connecting to port 443
 CPU5 connecting to port 443
 CPU11 connecting to port 443
 CPU12 connecting to port 443
 CPU13 connecting to port 443
 CPU14 connecting to port 443
 CPU15 connecting to port 443
 CPU4 connecting to port 443
 CPU2 connecting to port 443
 CPU3 connecting to port 443
 CPU1 connecting to port 443
 CPU0 connecting to port 443
 .......................................
 [ ALL ] dpdk0 flows:   5016, RX:    9651(pps) (err:     0),  0.04(Gbps), TX:   14784(pps),  0.02(Gbps)
 ------------------------------------------------------------------
 Ltm::Virtual Server: vs_https
 ------------------------------------------------------------------
 CMP              : enabled    CMP Mode         : all-cpus
   Destination      : 10.3.3.249:443
   PVA Acceleration : none
 Traffic                             ClientSide  Ephemeral  General
   Bits In                                49.2G          0        -
   Bits Out                               71.0G          0        -
   Packets In                             47.1M          0        -
   Packets Out                            30.4M          0        -
   Current Connections                     6.3K          0        -
   Maximum Connections                   146.0K          0        -
   Total Connections                       4.3M          0        -
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 12864 root       1 -19 57.9g 145m 123m S  5.0  0.2  53:09.44 tmm
 13087 root       1 -19 57.9g 145m 123m S  3.0  0.2  14:01.00 tmm
 13088 root       1 -19 57.9g 145m 123m S  3.0  0.2  13:32.00 tmm
 13091 root       1 -19 57.9g 145m 123m S  3.0  0.2  13:25.59 tmm
 13093 root       1 -19 57.9g 145m 123m S  3.0  0.2  13:34.57 tmm
 13094 root       1 -19 57.9g 145m 123m S  3.0  0.2  13:46.66 tmm
 13086 root       1 -19 57.9g 145m 123m S  2.6  0.2  14:09.38 tmm
 13089 root       1 -19 57.9g 145m 123m S  2.6  0.2  13:42.05 tmm
 13090 root       1 -19 57.9g 145m 123m S  2.6  0.2  13:47.88 tmm
 13092 root       1 -19 57.9g 145m 123m S  2.3  0.2  13:40.11 tmm

8, ApacheBench(ab) mTCP port https test to BIGIP VE(KVM)
 #ab -n 1000000 -c 8000 -N 8 -L 64  https://10.9.3.6/

Checking link status......................................done
Port 0 Link Up - speed 1000 Mbps - full-duplex
Benchmarking 10.9.3.6 (be patient)
CPU6 connecting to port 443
CPU7 connecting to port 443
CPU5 connecting to port 443
CPU4 connecting to port 443
CPU3 connecting to port 443
CPU2 connecting to port 443
CPU1 connecting to port 443
CPU0 connecting to port 443
[ ALL ] dpdk0 flows:   8000, RX:   13443(pps) (err:     0),  0.01(Gbps), TX:   13953(pps),  0.01(Gbps)

top - 13:12:22 up 4 min,  1 user,  load average: 3.34, 2.01, 0.82
Tasks: 395 total,   4 running, 391 sleeping,   0 stopped,   0 zombie
Cpu(s): 13.2%us,  6.5%sy,  0.0%ni, 64.5%id, 15.6%wa,  0.0%hi,  0.1%si,  0.0%st
Mem:  14403128k total, 14060912k used,   342216k free,    22400k buffers
Swap:  1048568k total,        0k used,  1048568k free,   863780k cached
  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  P COMMAND
13954 root      RT   0 12.0g 124m 104m R 92.4  0.9   0:27.17 0 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
 14125 root      RT   0 12.0g 124m 104m R 92.0  0.9   0:13.28 1 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
 14126 root      RT   0 12.0g 124m 104m S 92.0  0.9   0:12.36 2 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
 14127 root      RT   0 12.0g 124m 104m S 92.0  0.9   0:13.15 3 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088  ------------------------------------------------------------------
 Ltm::Virtual Server: vs_https
 ------------------------------------------------------------------
 Status
 Traffic                             ClientSide  Ephemeral  General
   Bits In                               428.8M          0        -
   Bits Out                              786.2M          0        -
   Packets In                            505.9K          0        -
   Packets Out                           423.1K          0        -
   Current Connections                     9.4K          0        -
   Maximum Connections                    12.9K          0        -


9 Generate  TCP/HTTP connection with random src MAC
http://sourceforge.net/p/curl-loader/mailman/message/33614941/







10 ICMP ping flooding to BIGIP VE
build/MoonGen examples/icmp-flood.lua 0 10.0.0.1 16000000 10000
 top - 12:10:54 up  1:55,  1 user,  load average: 0.24, 0.06, 0.02
 Tasks: 381 total,   2 running, 379 sleeping,   0 stopped,   0 zombie
 Cpu0  : 16.2%us, 11.3%sy,  0.0%ni, 13.1%id,  0.0%wa,  0.3%hi, 59.1%si,  0.0%st
 Cpu1  :  2.1%us,  2.4%sy,  0.0%ni, 94.6%id,  0.0%wa,  0.0%hi,  0.3%si,  0.6%st
 Cpu2  :  3.5%us,  3.2%sy,  0.0%ni, 90.6%id,  0.0%wa,  0.0%hi,  1.8%si,  0.9%st
 Cpu3  :  1.2%us,  1.8%sy,  0.3%ni, 96.0%id,  0.0%wa,  0.0%hi,  0.3%si,  0.3%st
 Mem:  14403128k total, 14267112k used,   136016k free,    22252k buffers
 Swap:  1048568k total,     1224k used,  1047344k free,   571908k cached
   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  P COMMAND
  2889 root      RT   0 12.0g 124m 104m S 154.7  0.9   3:58.44 0 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
  3054 root      RT   0 12.0g 124m 104m S  8.8  0.9   2:48.53 3 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
  3053 root      RT   0 12.0g 124m 104m S  8.5  0.9   2:12.29 2 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088
  3052 root      RT   0 12.0g 124m 104m R  7.6  0.9   2:06.63 1 tmm.0 -T 4 --tmid 0 --npus 4 --platform
Z100 -m -s 12088

Technical tips for load generation 1 mTCP thread pre-allocate memory pools for TCP send and receive buffer from configured maximum  number of buffers.
   when the load generator has limited memory, it is recommended to reduce the size of TCP send and  receive buffer and  number of buffers in application configuration file.
  also configure the BIGIP DUT to respond small packet size ( < 64 bytes) because large response
payload size would trigger mTCP payload merge length error
for example: setting TCP receive and send buffer in epwget.conf
# Receive buffer size of sockets
rcvbuf = 1024
# Send buffer size of sockets
sndbuf = 1024

 mTCP pre-allocate memory (mtcp/src/core.c)
static mtcp_manager_t
InitializeMTCPManager(struct mtcp_thread_context* ctx)
{

        mtcp->flow_pool = MPCreate(sizeof(tcp_stream),
                                                                sizeof(tcp_stream) * CONFIG.max_concurrency,
IS_HUGEPAGE);
        if (!mtcp->flow_pool) {
                CTRACE_ERROR("Failed to allocate tcp flow pool.\n");
                return NULL;
        }
        mtcp->rv_pool = MPCreate(sizeof(struct tcp_recv_vars),
                        sizeof(struct tcp_recv_vars) * CONFIG.max_concurrency, IS_HUGEPAGE);
        if (!mtcp->rv_pool) {
                CTRACE_ERROR("Failed to allocate tcp recv variable pool.\n");                 return NULL;
        }
        mtcp->sv_pool = MPCreate(sizeof(struct tcp_send_vars),
                        sizeof(struct tcp_send_vars) * CONFIG.max_concurrency, IS_HUGEPAGE);
        if (!mtcp->sv_pool) {
                CTRACE_ERROR("Failed to allocate tcp send variable pool.\n");
                return NULL;
        }

        mtcp->rbm_snd = SBManagerCreate(CONFIG.sndbuf_size, CONFIG.max_num_buffers);
        if (!mtcp->rbm_snd) {
                CTRACE_ERROR("Failed to create send ring buffer.\n");
                return NULL;
        }
        mtcp->rbm_rcv = RBManagerCreate(CONFIG.rcvbuf_size, CONFIG.max_num_buffers);
        if (!mtcp->rbm_rcv) {
                CTRACE_ERROR("Failed to create recv ring buffer.\n");
                return NULL;
        }


References
http://highscalability.com/blog/2013/5/13/the-secret-to-10-million-concurrent-connections-the-kernel-i.html
http://www.dpdk.org/
http://www.ndsl.kaist.edu/~kyoungsoo/papers/mtcp.pdf
https://github.com/eunyoung14/mtcp
https://github.com/emmericp/MoonGen
http://git.es.f5net.com/index.cgi/codeshare/tree/vli/mtcp  (git log --author="Vincent Li“)
http://git.es.f5net.com/index.cgi/codeshare/tree/vli/MoonGen

Followers