28 #include <sys/epoll.h> 29 #include <sys/eventfd.h> 30 #include <rte_config.h> 31 #include <rte_malloc.h> 32 #include <rte_cycles.h> 34 #include <tas_memif.h> 39 #define DATAPLANE_TSCS 41 #ifdef DATAPLANE_STATS 42 # ifdef DATAPLANE_TSCS 43 # define STATS_TS(n) uint64_t n = rte_get_tsc_cycles() 44 # define STATS_TSADD(c, f, n) __sync_fetch_and_add(&c->stat_##f, n) 46 # define STATS_TS(n) do { } while (0) 47 # define STATS_TSADD(c, f, n) do { } while (0) 49 # define STATS_ADD(c, f, n) __sync_fetch_and_add(&c->stat_##f, n) 51 # define STATS_TS(n) do { } while (0) 52 # define STATS_TSADD(c, f, n) do { } while (0) 53 # define STATS_ADD(c, f, n) do { } while (0) 59 uint64_t tsc) __attribute__((noinline));
60 static unsigned poll_queues(
struct dataplane_context *ctx, uint32_t
ts) __attribute__((noinline));
61 static unsigned poll_kernel(
struct dataplane_context *ctx, uint32_t
ts) __attribute__((noinline));
62 static unsigned poll_qman(
struct dataplane_context *ctx, uint32_t
ts) __attribute__((noinline));
63 static unsigned poll_qman_fwd(
struct dataplane_context *ctx, uint32_t
ts) __attribute__((noinline));
66 static inline uint8_t bufcache_prealloc(
struct dataplane_context *ctx, uint16_t num,
67 struct network_buf_handle ***handles);
70 struct network_buf_handle *handle);
74 struct network_buf_handle *nbh, uint16_t off, uint16_t len);
76 static void arx_cache_flush(
struct dataplane_context *ctx, uint64_t tsc) __attribute__((noinline));
78 int dataplane_init(
void)
81 fprintf(stderr,
"dataplane_init: internal flexnic memory size not " 82 "sufficient (got %x, need %zx)\n", FLEXNIC_INTERNAL_MEM_SIZE,
87 if (fp_cores_max > FLEXNIC_PL_APPST_CTX_MCS) {
88 fprintf(stderr,
"dataplane_init: more cores than FLEXNIC_PL_APPST_CTX_MCS " 89 "(%u)\n", FLEXNIC_PL_APPST_CTX_MCS);
92 if (FLEXNIC_PL_FLOWST_NUM > FLEXNIC_NUM_QMQUEUES) {
93 fprintf(stderr,
"dataplane_init: more flow states than queue manager queues" 94 "(%u > %u)\n", FLEXNIC_PL_FLOWST_NUM, FLEXNIC_NUM_QMQUEUES);
106 sprintf(name,
"qman_fwd_ring_%u", ctx->id);
107 if ((ctx->qman_fwd_ring = rte_ring_create(name, 32 * 1024, rte_socket_id(),
108 RING_F_SC_DEQ)) == NULL)
110 fprintf(stderr,
"initializing rte_ring_create");
115 if (qman_thread_init(ctx) != 0) {
116 fprintf(stderr,
"initializing qman thread failed\n");
121 if (network_thread_init(ctx) != 0) {
122 fprintf(stderr,
"initializing rx thread failed\n");
126 ctx->poll_next_ctx = ctx->id;
128 ctx->evfd = eventfd(0, EFD_NONBLOCK);
129 assert(ctx->evfd != -1);
130 ctx->ev.epdata.event = EPOLLIN;
131 int r = rte_epoll_ctl(RTE_EPOLL_PER_THREAD, EPOLL_CTL_ADD, ctx->evfd, &ctx->ev);
133 fp_state->kctx[ctx->id].evfd = ctx->evfd;
146 uint64_t cyc, prev_cyc;
149 notify_canblock_reset(&nbs);
155 cyc = rte_get_tsc_cycles();
157 ctx->loadmon_cyc_busy += cyc - prev_cyc;
160 ts = qman_timestamp(cyc);
163 n += poll_rx(ctx, ts, cyc);
167 n += poll_qman_fwd(ctx, ts);
169 STATS_TSADD(ctx, cyc_rx, rx - start);
170 n += poll_qman(ctx, ts);
172 STATS_TSADD(ctx, cyc_qm, qm - rx);
173 n += poll_queues(ctx, ts);
175 STATS_TSADD(ctx, cyc_qs, qs - qm);
176 n += poll_kernel(ctx, ts);
185 if (config.
fp_interrupts && notify_canblock(&nbs, !was_idle, cyc)) {
186 dataplane_block(ctx, ts);
187 notify_canblock_reset(&nbs);
194 uint32_t max_timeout;
197 struct rte_epoll_event event[2];
199 if (network_rx_interrupt_ctl(&ctx->net, 1) != 0) {
203 max_timeout = qman_next_ts(&ctx->qman, ts);
205 ret = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, 2,
206 max_timeout == (uint32_t) -1 ? -1 : max_timeout / 1000);
208 perror(
"dataplane_block: rte_epoll_wait failed");
212 for(i = 0; i < ret; i++) {
213 if(event[i].fd == ctx->evfd) {
214 ret = read(ctx->evfd, &val,
sizeof(uint64_t));
215 if ((ret > 0 && ret !=
sizeof(uint64_t)) ||
216 (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK))
218 perror(
"dataplane_block: read failed");
223 network_rx_interrupt_ctl(&ctx->net, 0);
226 #ifdef DATAPLANE_STATS 227 static inline uint64_t read_stat(uint64_t *p)
229 return __sync_lock_test_and_set(p, 0);
232 void dataplane_dump_stats(
void)
237 for (i = 0; i < fp_cores_max; i++) {
239 fprintf(stderr,
"dp stats %u: " 240 "qm=(%"PRIu64
",%"PRIu64
",%"PRIu64
") " 241 "rx=(%"PRIu64
",%"PRIu64
",%"PRIu64
") " 242 "qs=(%"PRIu64
",%"PRIu64
",%"PRIu64
") " 243 "cyc=(%"PRIu64
",%"PRIu64
",%"PRIu64
",%"PRIu64
")\n", i,
244 read_stat(&ctx->stat_qm_poll), read_stat(&ctx->stat_qm_empty),
245 read_stat(&ctx->stat_qm_total),
246 read_stat(&ctx->stat_rx_poll), read_stat(&ctx->stat_rx_empty),
247 read_stat(&ctx->stat_rx_total),
248 read_stat(&ctx->stat_qs_poll), read_stat(&ctx->stat_qs_empty),
249 read_stat(&ctx->stat_qs_total),
250 read_stat(&ctx->stat_cyc_db), read_stat(&ctx->stat_cyc_qm),
251 read_stat(&ctx->stat_cyc_rx), read_stat(&ctx->stat_cyc_qs));
261 uint8_t freebuf[BATCH_SIZE] = { 0 };
262 void *fss[BATCH_SIZE];
263 struct tcp_opts tcpopts[BATCH_SIZE];
264 struct network_buf_handle *bhs[BATCH_SIZE];
267 if (TXBUF_SIZE - ctx->tx_num < n)
268 n = TXBUF_SIZE - ctx->tx_num;
270 STATS_ADD(ctx, rx_poll, 1);
273 ret = network_poll(&ctx->net, n, bhs);
275 STATS_ADD(ctx, rx_empty, 1);
278 STATS_ADD(ctx, rx_total, n);
282 for (i = 0; i < n; i++) {
283 rte_prefetch0(network_buf_bufoff(bhs[i]));
287 fast_flows_packet_fss(ctx, bhs, fss, n);
290 for (i = 0; i < n; i++) {
291 rte_prefetch0(network_buf_bufoff(bhs[i]) + 64);
295 fast_flows_packet_parse(ctx, bhs, fss, tcpopts, n);
297 for (i = 0; i < n; i++) {
299 if (fss[i] != NULL) {
300 ret = fast_flows_packet(ctx, bhs[i], fss[i], &tcpopts[i], ts);
307 }
else if (ret < 0) {
308 fast_kernel_packet(ctx, bhs[i]);
312 arx_cache_flush(ctx, tsc);
315 for (i = 0; i < n; i++) {
317 bufcache_free(ctx, bhs[i]);
325 struct network_buf_handle **handles;
326 void *aqes[BATCH_SIZE];
327 unsigned n, i, total = 0;
328 uint16_t max, k = 0, num_bufs = 0, j;
331 STATS_ADD(ctx, qs_poll, 1);
334 if (TXBUF_SIZE - ctx->tx_num < max)
335 max = TXBUF_SIZE - ctx->tx_num;
338 max = bufcache_prealloc(ctx, max, &handles);
340 for (n = 0; n < FLEXNIC_PL_APPCTX_NUM; n++) {
341 fast_appctx_poll_pf(ctx, (ctx->poll_next_ctx + n) % FLEXNIC_PL_APPCTX_NUM);
344 for (n = 0; n < FLEXNIC_PL_APPCTX_NUM && k < max; n++) {
345 for (i = 0; i < BATCH_SIZE && k < max; i++) {
346 ret = fast_appctx_poll_fetch(ctx, ctx->poll_next_ctx, &aqes[k]);
355 ctx->poll_next_ctx = (ctx->poll_next_ctx + 1) %
356 FLEXNIC_PL_APPCTX_NUM;
359 for (j = 0; j < k; j++) {
360 ret = fast_appctx_poll_bump(ctx, aqes[j], handles[num_bufs], ts);
366 bufcache_alloc(ctx, num_bufs);
368 for (n = 0; n < FLEXNIC_PL_APPCTX_NUM; n++)
369 fast_actx_rxq_probe(ctx, n);
371 STATS_ADD(ctx, qs_total, total);
373 STATS_ADD(ctx, qs_empty, total);
380 struct network_buf_handle **handles;
386 if (TXBUF_SIZE - ctx->tx_num < max)
387 max = TXBUF_SIZE - ctx->tx_num;
389 max = (max > 8 ? 8 : max);
391 max = bufcache_prealloc(ctx, max, &handles);
393 for (k = 0; k < max;) {
394 ret = fast_kernel_poll(ctx, handles[k], ts);
405 bufcache_alloc(ctx, k);
412 unsigned q_ids[BATCH_SIZE];
413 uint16_t q_bytes[BATCH_SIZE];
414 struct network_buf_handle **handles;
415 uint16_t off = 0, max;
419 if (TXBUF_SIZE - ctx->tx_num < max)
420 max = TXBUF_SIZE - ctx->tx_num;
422 STATS_ADD(ctx, qm_poll, 1);
425 max = bufcache_prealloc(ctx, max, &handles);
428 ret = qman_poll(&ctx->qman, max, q_ids, q_bytes);
430 STATS_ADD(ctx, qm_empty, 1);
434 STATS_ADD(ctx, qm_total, ret);
436 for (i = 0; i < ret; i++) {
437 rte_prefetch0(handles[i]);
440 for (i = 0; i < ret; i++) {
441 rte_prefetch0((uint8_t *) handles[i] + 64);
445 for (i = 0; i < ret; i++) {
446 rte_prefetch0(network_buf_buf(handles[i]));
449 fast_flows_qman_pf(ctx, q_ids, ret);
451 fast_flows_qman_pfbufs(ctx, q_ids, ret);
453 for (i = 0; i < ret; i++) {
454 use = fast_flows_qman(ctx, q_ids[i], handles[off], ts);
461 bufcache_alloc(ctx, off);
468 void *flow_states[4 * BATCH_SIZE];
472 ret = rte_ring_dequeue_burst(ctx->qman_fwd_ring, flow_states, 4 * BATCH_SIZE, NULL);
473 for (i = 0; i < ret; i++) {
474 fast_flows_qman_fwd(ctx, flow_states[i]);
480 static inline uint8_t bufcache_prealloc(
struct dataplane_context *ctx, uint16_t num,
481 struct network_buf_handle ***handles)
483 uint16_t grow, res, head, g, i;
484 struct network_buf_handle *nbh;
487 if (ctx->bufcache_num < num) {
488 grow = BUFCACHE_SIZE - ctx->bufcache_num;
489 head = (ctx->bufcache_head + ctx->bufcache_num) & (BUFCACHE_SIZE - 1);
491 if (head + grow <= BUFCACHE_SIZE) {
492 res = network_buf_alloc(&ctx->net, grow, ctx->bufcache_handles + head);
494 g = BUFCACHE_SIZE - head;
495 res = network_buf_alloc(&ctx->net, g, ctx->bufcache_handles + head);
497 res += network_buf_alloc(&ctx->net, grow - g, ctx->bufcache_handles);
501 for (i = 0; i < res; i++) {
502 g = (head + i) & (BUFCACHE_SIZE - 1);
503 nbh = ctx->bufcache_handles[g];
504 ctx->bufcache_handles[g] = (
struct network_buf_handle *)
508 ctx->bufcache_num += res;
510 num = MIN(num, (ctx->bufcache_head + ctx->bufcache_num <= BUFCACHE_SIZE ?
511 ctx->bufcache_num : BUFCACHE_SIZE - ctx->bufcache_head));
513 *handles = ctx->bufcache_handles + ctx->bufcache_head;
520 assert(num <= ctx->bufcache_num);
522 ctx->bufcache_head = (ctx->bufcache_head + num) & (BUFCACHE_SIZE - 1);
523 ctx->bufcache_num -= num;
527 struct network_buf_handle *handle)
531 num = ctx->bufcache_num;
532 if (num < BUFCACHE_SIZE) {
534 head = (ctx->bufcache_head + num) & (BUFCACHE_SIZE - 1);
535 ctx->bufcache_handles[head] = handle;
536 ctx->bufcache_num = num + 1;
537 network_buf_reset(handle);
540 network_free(1, &handle);
549 if (ctx->tx_num == 0) {
554 ret = network_send(&ctx->net, ctx->tx_num, ctx->tx_handles);
556 if (ret == ctx->tx_num) {
559 }
else if (ret > 0) {
561 for (i = ret; i < ctx->tx_num; i++) {
562 ctx->tx_handles[i - ret] = ctx->tx_handles[i];
570 unsigned st = fp_scale_to;
575 fprintf(stderr,
"Scaling fast path from %u to %u\n", fp_cores_cur, st);
576 if (st < fp_cores_cur) {
577 if (network_scale_down(fp_cores_cur, st) != 0) {
578 fprintf(stderr,
"network_scale_down failed\n");
581 }
else if (st > fp_cores_cur) {
582 if (network_scale_up(fp_cores_cur, st) != 0) {
583 fprintf(stderr,
"network_scale_up failed\n");
587 fprintf(stderr,
"poll_scale: warning core number didn't change\n");
600 for (i = 0; i < ctx->arx_num; i++) {
601 actx = &fp_state->appctx[ctx->id][ctx->arx_ctx[i]];
602 if (fast_actx_rxq_alloc(ctx, actx, &parx[i]) != 0) {
604 fprintf(stderr,
"arx_cache_flush: no space in app rx queue\n");
609 for (i = 0; i < ctx->arx_num; i++) {
610 rte_prefetch0(parx[i]);
613 for (i = 0; i < ctx->arx_num; i++) {
614 *parx[i] = ctx->arx_cache[i];
617 for (i = 0; i < ctx->arx_num; i++) {
618 actx = &fp_state->appctx[ctx->id][ctx->arx_ctx[i]];
619 notify_appctx(actx, tsc);
struct tcp_timestamp_opt * ts