TAS
TCP Acceleration as an OS Service
fast_flows.c
1 /*
2  * Copyright 2019 University of Washington, Max Planck Institute for
3  * Software Systems, and The University of Texas at Austin
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be
14  * included in all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23  */
24 
25 #include <assert.h>
26 #include <rte_config.h>
27 #include <rte_ip.h>
28 #include <rte_hash_crc.h>
29 
30 #include <tas_memif.h>
31 #include <utils_sync.h>
32 
33 #include "internal.h"
34 #include "fastemu.h"
35 #include "tcp_common.h"
36 
37 #define TCP_MSS 1448
38 #define TCP_MAX_RTT 100000
39 
40 //#define SKIP_ACK 1
41 
42 struct flow_key {
43  ip_addr_t local_ip;
44  ip_addr_t remote_ip;
45  beui16_t local_port;
46  beui16_t remote_port;
47 } __attribute__((packed));
48 
49 #if 1
50 #define fs_lock(fs) util_spin_lock(&fs->lock)
51 #define fs_unlock(fs) util_spin_unlock(&fs->lock)
52 #else
53 #define fs_lock(fs) do {} while (0)
54 #define fs_unlock(fs) do {} while (0)
55 #endif
56 
57 
58 static void flow_tx_read(struct flextcp_pl_flowst *fs, uint32_t pos,
59  uint16_t len, void *dst);
60 static void flow_rx_write(struct flextcp_pl_flowst *fs, uint32_t pos,
61  uint16_t len, const void *src);
62 #ifdef FLEXNIC_PL_OOO_RECV
63 static void flow_rx_seq_write(struct flextcp_pl_flowst *fs, uint32_t seq,
64  uint16_t len, const void *src);
65 #endif
66 static void flow_tx_segment(struct dataplane_context *ctx,
67  struct network_buf_handle *nbh, struct flextcp_pl_flowst *fs,
68  uint32_t seq, uint32_t ack, uint32_t rxwnd, uint16_t payload,
69  uint32_t payload_pos, uint32_t ts_echo, uint32_t ts_my, uint8_t fin);
70 static void flow_tx_ack(struct dataplane_context *ctx, uint32_t seq,
71  uint32_t ack, uint32_t rxwnd, uint32_t echo_ts, uint32_t my_ts,
72  struct network_buf_handle *nbh, struct tcp_timestamp_opt *ts_opt);
73 static void flow_reset_retransmit(struct flextcp_pl_flowst *fs);
74 
75 static inline void tcp_checksums(struct network_buf_handle *nbh,
76  struct pkt_tcp *p, beui32_t ip_s, beui32_t ip_d, uint16_t l3_paylen);
77 
78 void fast_flows_qman_pf(struct dataplane_context *ctx, uint32_t *queues,
79  uint16_t n)
80 {
81  uint16_t i;
82 
83  for (i = 0; i < n; i++) {
84  rte_prefetch0(&fp_state->flowst[queues[i]]);
85  }
86 }
87 
88 void fast_flows_qman_pfbufs(struct dataplane_context *ctx, uint32_t *queues,
89  uint16_t n)
90 {
91  struct flextcp_pl_flowst *fs;
92  uint16_t i;
93  void *p;
94 
95  for (i = 0; i < n; i++) {
96  fs = &fp_state->flowst[queues[i]];
97  p = dma_pointer(fs->tx_base + fs->tx_next_pos, 1);
98  rte_prefetch0(p);
99  rte_prefetch0(p + 64);
100  }
101 }
102 
103 
104 int fast_flows_qman(struct dataplane_context *ctx, uint32_t queue,
105  struct network_buf_handle *nbh, uint32_t ts)
106 {
107  uint32_t flow_id = queue;
108  struct flextcp_pl_flowst *fs = &fp_state->flowst[flow_id];
109  uint32_t avail, len, tx_pos, tx_seq, ack, rx_wnd;
110  uint16_t new_core;
111  uint8_t fin;
112  int ret = 0;
113 
114  fs_lock(fs);
115 
116  /* if connection has been moved, add to forwarding queue and stop */
117  new_core = fp_state->flow_group_steering[fs->flow_group];
118  if (new_core != ctx->id) {
119  /*fprintf(stderr, "fast_flows_qman: arrived on wrong core, forwarding "
120  "%u -> %u (fs=%p, fg=%u)\n", ctx->id, new_core, fs, fs->flow_group);*/
121 
122  /* enqueue flo state on forwarding queue */
123  if (rte_ring_enqueue(ctxs[new_core]->qman_fwd_ring, fs) != 0) {
124  fprintf(stderr, "fast_flows_qman: rte_ring_enqueue failed\n");
125  abort();
126  }
127 
128  /* clear queue manager queue */
129  if (qman_set(&ctx->qman, flow_id, 0, 0, 0,
130  QMAN_SET_RATE | QMAN_SET_MAXCHUNK | QMAN_SET_AVAIL) != 0)
131  {
132  fprintf(stderr, "flast_flows_qman: qman_set clear failed, UNEXPECTED\n");
133  abort();
134  }
135 
136  notify_fastpath_core(new_core);
137 
138  ret = -1;
139  goto unlock;
140  }
141 
142  /* calculate how much is available to be sent */
143  avail = tcp_txavail(fs, NULL);
144 
145 #if PL_DEBUG_ATX
146  fprintf(stderr, "ATX try_sendseg local=%08x:%05u remote=%08x:%05u "
147  "tx_avail=%x tx_next_pos=%x avail=%u\n",
148  f_beui32(fs->local_ip), f_beui16(fs->local_port),
149  f_beui32(fs->remote_ip), f_beui16(fs->remote_port),
150  fs->tx_avail, fs->tx_next_pos, avail);
151 #endif
152 #ifdef FLEXNIC_TRACING
153  struct flextcp_pl_trev_afloqman te_afloqman = {
154  .flow_id = flow_id,
155  .tx_base = fs->tx_base,
156  .tx_avail = fs->tx_avail,
157  .tx_next_pos = fs->tx_next_pos,
158  .tx_len = fs->tx_len,
159  .rx_remote_avail = fs->rx_remote_avail,
160  .tx_sent = fs->tx_sent,
161  };
162  trace_event(FLEXNIC_PL_TREV_AFLOQMAN, sizeof(te_afloqman), &te_afloqman);
163 #endif
164 
165  /* if there is no data available, stop */
166  if (avail == 0) {
167  ret = -1;
168  goto unlock;
169  }
170  len = MIN(avail, TCP_MSS);
171 
172  /* state snapshot for creating segment */
173  tx_seq = fs->tx_next_seq;
174  tx_pos = fs->tx_next_pos;
175  rx_wnd = fs->rx_avail;
176  ack = fs->rx_next_seq;
177 
178  /* update tx flow state */
179  fs->tx_next_seq += len;
180  fs->tx_next_pos += len;
181  if (fs->tx_next_pos >= fs->tx_len) {
182  fs->tx_next_pos -= fs->tx_len;
183  }
184  fs->tx_sent += len;
185  fs->tx_avail -= len;
186 
187  fin = (fs->rx_base_sp & FLEXNIC_PL_FLOWST_TXFIN) == FLEXNIC_PL_FLOWST_TXFIN &&
188  !fs->tx_avail;
189 
190  /* make sure we don't send out dummy byte for FIN */
191  if (fin) {
192  assert(len > 0);
193  len--;
194  }
195 
196  /* send out segment */
197  flow_tx_segment(ctx, nbh, fs, tx_seq, ack, rx_wnd, len, tx_pos,
198  fs->tx_next_ts, ts, fin);
199 unlock:
200  fs_unlock(fs);
201  return ret;
202 }
203 
204 int fast_flows_qman_fwd(struct dataplane_context *ctx,
205  struct flextcp_pl_flowst *fs)
206 {
207  unsigned avail;
208  uint16_t flow_id = fs - fp_state->flowst;
209 
210  /*fprintf(stderr, "fast_flows_qman_fwd: fs=%p\n", fs);*/
211 
212  fs_lock(fs);
213 
214  avail = tcp_txavail(fs, NULL);
215 
216  /* re-arm queue manager */
217  if (qman_set(&ctx->qman, flow_id, fs->tx_rate, avail, TCP_MSS,
218  QMAN_SET_RATE | QMAN_SET_MAXCHUNK | QMAN_SET_AVAIL) != 0)
219  {
220  fprintf(stderr, "fast_flows_qman_fwd: qman_set failed, UNEXPECTED\n");
221  abort();
222  }
223 
224  fs_unlock(fs);
225  return 0;
226 }
227 
228 void fast_flows_packet_parse(struct dataplane_context *ctx,
229  struct network_buf_handle **nbhs, void **fss, struct tcp_opts *tos,
230  uint16_t n)
231 {
232  struct pkt_tcp *p;
233  uint16_t i, len;
234 
235  for (i = 0; i < n; i++) {
236  if (fss[i] == NULL)
237  continue;
238 
239  p = network_buf_bufoff(nbhs[i]);
240  len = network_buf_len(nbhs[i]);
241 
242  int cond =
243  (len < sizeof(*p)) |
244  (f_beui16(p->eth.type) != ETH_TYPE_IP) |
245  (p->ip.proto != IP_PROTO_TCP) |
246  (IPH_V(&p->ip) != 4) |
247  (IPH_HL(&p->ip) != 5) |
248  (TCPH_HDRLEN(&p->tcp) < 5) |
249  (len < f_beui16(p->ip.len) + sizeof(p->eth)) |
250  (tcp_parse_options(p, len, &tos[i]) != 0) |
251  (tos[i].ts == NULL);
252 
253  if (cond)
254  fss[i] = NULL;
255  }
256 }
257 
258 void fast_flows_packet_pfbufs(struct dataplane_context *ctx,
259  void **fss, uint16_t n)
260 {
261  uint16_t i;
262  uint64_t rx_base;
263  void *p;
264  struct flextcp_pl_flowst *fs;
265 
266  for (i = 0; i < n; i++) {
267  if (fss[i] == NULL)
268  continue;
269 
270  fs = fss[i];
271  rx_base = fs->rx_base_sp & FLEXNIC_PL_FLOWST_RX_MASK;
272  p = dma_pointer(rx_base + fs->rx_next_pos, 1);
273  rte_prefetch0(p);
274  }
275 }
276 
277 /* Received packet */
278 int fast_flows_packet(struct dataplane_context *ctx,
279  struct network_buf_handle *nbh, void *fsp, struct tcp_opts *opts,
280  uint32_t ts)
281 {
282  struct pkt_tcp *p = network_buf_bufoff(nbh);
283  struct flextcp_pl_flowst *fs = fsp;
284  uint32_t payload_bytes, payload_off, seq, ack, old_avail, new_avail,
285  orig_payload;
286  uint8_t *payload;
287  uint32_t rx_bump = 0, tx_bump = 0, rx_pos, rtt;
288  int no_permanent_sp = 0;
289  uint16_t tcp_extra_hlen, trim_start, trim_end;
290  uint16_t flow_id = fs - fp_state->flowst;
291  int trigger_ack = 0, fin_bump = 0;
292 
293  tcp_extra_hlen = (TCPH_HDRLEN(&p->tcp) - 5) * 4;
294  payload_off = sizeof(*p) + tcp_extra_hlen;
295  payload_bytes =
296  f_beui16(p->ip.len) - (sizeof(p->ip) + sizeof(p->tcp) + tcp_extra_hlen);
297  orig_payload = payload_bytes;
298 
299 #if PL_DEBUG_ARX
300  fprintf(stderr, "FLOW local=%08x:%05u remote=%08x:%05u RX: seq=%u ack=%u "
301  "flags=%x payload=%u\n",
302  f_beui32(p->ip.dest), f_beui16(p->tcp.dest),
303  f_beui32(p->ip.src), f_beui16(p->tcp.src), f_beui32(p->tcp.seqno),
304  f_beui32(p->tcp.ackno), TCPH_FLAGS(&p->tcp), payload_bytes);
305 #endif
306 
307  fs_lock(fs);
308 
309 #ifdef FLEXNIC_TRACING
310  struct flextcp_pl_trev_rxfs te_rxfs = {
311  .local_ip = f_beui32(p->ip.dest),
312  .remote_ip = f_beui32(p->ip.src),
313  .local_port = f_beui16(p->tcp.dest),
314  .remote_port = f_beui16(p->tcp.src),
315 
316  .flow_id = flow_id,
317  .flow_seq = f_beui32(p->tcp.seqno),
318  .flow_ack = f_beui32(p->tcp.ackno),
319  .flow_flags = TCPH_FLAGS(&p->tcp),
320  .flow_len = payload_bytes,
321 
322  .fs_rx_nextpos = fs->rx_next_pos,
323  .fs_rx_nextseq = fs->rx_next_seq,
324  .fs_rx_avail = fs->rx_avail,
325  .fs_tx_nextpos = fs->tx_next_pos,
326  .fs_tx_nextseq = fs->tx_next_seq,
327  .fs_tx_sent = fs->tx_sent,
328  .fs_tx_avail = fs->tx_avail,
329  };
330  trace_event(FLEXNIC_PL_TREV_RXFS, sizeof(te_rxfs), &te_rxfs);
331 #endif
332 
333 #if PL_DEBUG_ARX
334  fprintf(stderr, "FLOW local=%08x:%05u remote=%08x:%05u ST: op=%"PRIx64
335  " rx_pos=%x rx_next_seq=%u rx_avail=%x tx_pos=%x tx_next_seq=%u"
336  " tx_sent=%u sp=%u\n",
337  f_beui32(p->ip.dest), f_beui16(p->tcp.dest),
338  f_beui32(p->ip.src), f_beui16(p->tcp.src), fs->opaque, fs->rx_next_pos,
339  fs->rx_next_seq, fs->rx_avail, fs->tx_next_pos, fs->tx_next_seq,
340  fs->tx_sent, fs->slowpath);
341 #endif
342 
343  /* state indicates slow path */
344  if (UNLIKELY((fs->rx_base_sp & FLEXNIC_PL_FLOWST_SLOWPATH) != 0)) {
345  fprintf(stderr, "dma_krx_pkt_fastpath: slowpath because of state\n");
346  goto slowpath;
347  }
348 
349  /* if we get weird flags -> kernel */
350  if (UNLIKELY((TCPH_FLAGS(&p->tcp) & ~(TCP_ACK | TCP_PSH | TCP_ECE | TCP_CWR |
351  TCP_FIN)) != 0))
352  {
353  if ((TCPH_FLAGS(&p->tcp) & TCP_SYN) != 0) {
354  /* for SYN/SYN-ACK we'll let the kernel handle them out of band */
355  no_permanent_sp = 1;
356  } else {
357  fprintf(stderr, "dma_krx_pkt_fastpath: slow path because of flags (%x)\n",
358  TCPH_FLAGS(&p->tcp));
359  }
360  goto slowpath;
361  }
362 
363  /* calculate how much data is available to be sent before processing this
364  * packet, to detect whether more data can be sent afterwards */
365  old_avail = tcp_txavail(fs, NULL);
366 
367  seq = f_beui32(p->tcp.seqno);
368  ack = f_beui32(p->tcp.ackno);
369  rx_pos = fs->rx_next_pos;
370 
371  /* trigger an ACK if there is payload (even if we discard it) */
372 #ifndef SKIP_ACK
373  if (payload_bytes > 0)
374  trigger_ack = 1;
375 #endif
376 
377  /* Stats for CC */
378  if ((TCPH_FLAGS(&p->tcp) & TCP_ACK) == TCP_ACK) {
379  fs->cnt_rx_acks++;
380  }
381 
382  /* if there is a valid ack, process it */
383  if (LIKELY((TCPH_FLAGS(&p->tcp) & TCP_ACK) == TCP_ACK &&
384  tcp_valid_rxack(fs, ack, &tx_bump) == 0))
385  {
386  fs->cnt_rx_ack_bytes += tx_bump;
387  if ((TCPH_FLAGS(&p->tcp) & TCP_ECE) == TCP_ECE) {
388  fs->cnt_rx_ecn_bytes += tx_bump;
389  }
390 
391  if (LIKELY(tx_bump <= fs->tx_sent)) {
392  fs->tx_sent -= tx_bump;
393  } else {
394 #ifdef ALLOW_FUTURE_ACKS
395  fs->tx_next_seq += tx_bump - fs->tx_sent;
396  fs->tx_next_pos += tx_bump - fs->tx_sent;
397  if (fs->tx_next_pos >= fs->tx_len)
398  fs->tx_next_pos -= fs->tx_len;
399  fs->tx_avail -= tx_bump - fs->tx_sent;
400  fs->tx_sent = 0;
401 #else
402  /* this should not happen */
403  fprintf(stderr, "dma_krx_pkt_fastpath: acked more bytes than sent\n");
404  abort();
405 #endif
406  }
407 
408  /* duplicate ack */
409  if (UNLIKELY(tx_bump != 0)) {
410  fs->rx_dupack_cnt = 0;
411  } else if (UNLIKELY(orig_payload == 0 && ++fs->rx_dupack_cnt >= 3)) {
412  /* reset to last acknowledged position */
413  flow_reset_retransmit(fs);
414  goto unlock;
415  }
416  }
417 
418 #ifdef FLEXNIC_PL_OOO_RECV
419  /* check if we should drop this segment */
420  if (UNLIKELY(tcp_trim_rxbuf(fs, seq, payload_bytes, &trim_start, &trim_end) != 0)) {
421  /* packet is completely outside of unused receive buffer */
422  trigger_ack = 1;
423  goto unlock;
424  }
425 
426  /* trim payload to what we can actually use */
427  payload_bytes -= trim_start + trim_end;
428  payload_off += trim_start;
429  payload = (uint8_t *) p + payload_off;
430  seq += trim_start;
431 
432  /* handle out of order segment */
433  if (UNLIKELY(seq != fs->rx_next_seq)) {
434  trigger_ack = 1;
435 
436  /* if there is no payload abort immediately */
437  if (payload_bytes == 0) {
438  goto unlock;
439  }
440 
441  /* otherwise check if we can add it to the out of order interval */
442  if (fs->rx_ooo_len == 0) {
443  fs->rx_ooo_start = seq;
444  fs->rx_ooo_len = payload_bytes;
445  flow_rx_seq_write(fs, seq, payload_bytes, payload);
446  /*fprintf(stderr, "created OOO interval (%p start=%u len=%u)\n",
447  fs, fs->rx_ooo_start, fs->rx_ooo_len);*/
448  } else if (seq + payload_bytes == fs->rx_ooo_start) {
449  /* TODO: those two overlap checks should be more sophisticated */
450  fs->rx_ooo_start = seq;
451  fs->rx_ooo_len += payload_bytes;
452  flow_rx_seq_write(fs, seq, payload_bytes, payload);
453  /*fprintf(stderr, "extended OOO interval (%p start=%u len=%u)\n",
454  fs, fs->rx_ooo_start, fs->rx_ooo_len);*/
455  } else if (fs->rx_ooo_start + fs->rx_ooo_len == seq) {
456  /* TODO: those two overlap checks should be more sophisticated */
457  fs->rx_ooo_len += payload_bytes;
458  flow_rx_seq_write(fs, seq, payload_bytes, payload);
459  /*fprintf(stderr, "extended OOO interval (%p start=%u len=%u)\n",
460  fs, fs->rx_ooo_start, fs->rx_ooo_len);*/
461  } else {
462  /*fprintf(stderr, "Sad, no luck with OOO interval (%p ooo.start=%u "
463  "ooo.len=%u seq=%u bytes=%u)\n", fs, fs->rx_ooo_start,
464  fs->rx_ooo_len, seq, payload_bytes);*/
465  }
466  goto unlock;
467  }
468 
469 #else
470  /* check if we should drop this segment */
471  if (tcp_valid_rxseq(fs, seq, payload_bytes, &trim_start, &trim_end) != 0) {
472  trigger_ack = 1;
473 #if 0
474  fprintf(stderr, "dma_krx_pkt_fastpath: packet with bad seq "
475  "(got %u, expect %u, avail %u, payload %u)\n", seq, fs->rx_next_seq,
476  fs->rx_avail, payload_bytes);
477 #endif
478  goto unlock;
479  }
480 
481  /* trim payload to what we can actually use */
482  payload_bytes -= trim_start + trim_end;
483  payload_off += trim_start;
484  payload = (uint8_t *) p + payload_off;
485 #endif
486 
487  /* update rtt estimate */
488  fs->tx_next_ts = f_beui32(opts->ts->ts_val);
489  if (LIKELY((TCPH_FLAGS(&p->tcp) & TCP_ACK) == TCP_ACK &&
490  f_beui32(opts->ts->ts_ecr) != 0))
491  {
492  rtt = ts - f_beui32(opts->ts->ts_ecr);
493  if (rtt < TCP_MAX_RTT) {
494  if (LIKELY(fs->rtt_est != 0)) {
495  fs->rtt_est = (fs->rtt_est * 7 + rtt) / 8;
496  } else {
497  fs->rtt_est = rtt;
498  }
499  }
500  }
501 
502  fs->rx_remote_avail = f_beui16(p->tcp.wnd);
503 
504  /* make sure we don't receive anymore payload after FIN */
505  if ((fs->rx_base_sp & FLEXNIC_PL_FLOWST_RXFIN) == FLEXNIC_PL_FLOWST_RXFIN &&
506  payload_bytes > 0)
507  {
508  fprintf(stderr, "fast_flows_packet: data after FIN dropped\n");
509  goto unlock;
510  }
511 
512  /* if there is payload, dma it to the receive buffer */
513  if (payload_bytes > 0) {
514  flow_rx_write(fs, fs->rx_next_pos, payload_bytes, payload);
515 
516  rx_bump = payload_bytes;
517  fs->rx_avail -= payload_bytes;
518  fs->rx_next_pos += payload_bytes;
519  if (fs->rx_next_pos >= fs->rx_len) {
520  fs->rx_next_pos -= fs->rx_len;
521  }
522  assert(fs->rx_next_pos < fs->rx_len);
523  fs->rx_next_seq += payload_bytes;
524 #ifndef SKIP_ACK
525  trigger_ack = 1;
526 #endif
527 
528 #ifdef FLEXNIC_PL_OOO_RECV
529  /* if we have out of order segments, check whether buffer is continuous
530  * or superfluous */
531  if (UNLIKELY(fs->rx_ooo_len != 0)) {
532  if (tcp_trim_rxbuf(fs, fs->rx_ooo_start, fs->rx_ooo_len, &trim_start,
533  &trim_end) != 0) {
534  /*fprintf(stderr, "dropping ooo (%p ooo.start=%u ooo.len=%u seq=%u "
535  "len=%u next_seq=%u)\n", fs, fs->rx_ooo_start, fs->rx_ooo_len, seq,
536  payload_bytes, fs->rx_next_seq);*/
537  /* completely superfluous: drop out of order interval */
538  fs->rx_ooo_len = 0;
539  } else {
540  /* adjust based on overlap */
541  fs->rx_ooo_start += trim_start;
542  fs->rx_ooo_len -= trim_start + trim_end;
543  /*fprintf(stderr, "adjusting ooo (%p ooo.start=%u ooo.len=%u seq=%u "
544  "len=%u next_seq=%u)\n", fs, fs->rx_ooo_start, fs->rx_ooo_len, seq,
545  payload_bytes, fs->rx_next_seq);*/
546  if (fs->rx_ooo_len > 0 && fs->rx_ooo_start == fs->rx_next_seq) {
547  /* yay, we caught up, make continuous and drop OOO interval */
548  /*fprintf(stderr, "caught up with ooo buffer (%p start=%u len=%u)\n",
549  fs, fs->rx_ooo_start, fs->rx_ooo_len);*/
550 
551  rx_bump += fs->rx_ooo_len;
552  fs->rx_avail -= fs->rx_ooo_len;
553  fs->rx_next_pos += fs->rx_ooo_len;
554  if (fs->rx_next_pos >= fs->rx_len) {
555  fs->rx_next_pos -= fs->rx_len;
556  }
557  assert(fs->rx_next_pos < fs->rx_len);
558  fs->rx_next_seq += fs->rx_ooo_len;
559 
560  fs->rx_ooo_len = 0;
561  }
562  }
563  }
564 #endif
565  }
566 
567  if ((TCPH_FLAGS(&p->tcp) & TCP_FIN) == TCP_FIN &&
568  !(fs->rx_base_sp & FLEXNIC_PL_FLOWST_RXFIN))
569  {
570  if (fs->rx_next_seq == f_beui32(p->tcp.seqno) + orig_payload && !fs->rx_ooo_len) {
571  fin_bump = 1;
572  fs->rx_base_sp |= FLEXNIC_PL_FLOWST_RXFIN;
573  /* FIN takes up sequence number space */
574  fs->rx_next_seq++;
575  trigger_ack = 1;
576  } else {
577  fprintf(stderr, "fast_flows_packet: ignored fin because out of order\n");
578  }
579  }
580 
581 unlock:
582  /* if we bumped at least one, then we need to add a notification to the
583  * queue */
584  if (LIKELY(rx_bump != 0 || tx_bump != 0 || fin_bump)) {
585 #if PL_DEBUG_ARX
586  fprintf(stderr, "dma_krx_pkt_fastpath: updating application state\n");
587 #endif
588 
589  uint16_t type;
590  type = FLEXTCP_PL_ARX_CONNUPDATE;
591 
592  if (fin_bump) {
593  type |= FLEXTCP_PL_ARX_FLRXDONE << 8;
594  }
595 
596 #ifdef FLEXNIC_TRACING
597  struct flextcp_pl_trev_arx te_arx = {
598  .opaque = fs->opaque,
599  .rx_bump = rx_bump,
600  .tx_bump = tx_bump,
601  .rx_pos = rx_pos,
602  .flags = type,
603 
604  .flow_id = flow_id,
605  .db_id = fs->db_id,
606 
607  .local_ip = f_beui32(p->ip.dest),
608  .remote_ip = f_beui32(p->ip.src),
609  .local_port = f_beui16(p->tcp.dest),
610  .remote_port = f_beui16(p->tcp.src),
611  };
612  trace_event(FLEXNIC_PL_TREV_ARX, sizeof(te_arx), &te_arx);
613 #endif
614 
615  arx_cache_add(ctx, fs->db_id, fs->opaque, rx_bump, rx_pos, tx_bump, type);
616  }
617 
618  /* Flow control: More receiver space? -> might need to start sending */
619  new_avail = tcp_txavail(fs, NULL);
620  if (new_avail > old_avail) {
621  /* update qman queue */
622  if (qman_set(&ctx->qman, flow_id, fs->tx_rate, new_avail -
623  old_avail, TCP_MSS, QMAN_SET_RATE | QMAN_SET_MAXCHUNK
624  | QMAN_ADD_AVAIL) != 0)
625  {
626  fprintf(stderr, "fast_flows_packet: qman_set 1 failed, UNEXPECTED\n");
627  abort();
628  }
629  }
630 
631  /* if we need to send an ack, also send packet to TX pipeline to do so */
632  if (trigger_ack) {
633  flow_tx_ack(ctx, fs->tx_next_seq, fs->rx_next_seq, fs->rx_avail,
634  fs->tx_next_ts, ts, nbh, opts->ts);
635  }
636 
637  fs_unlock(fs);
638  return trigger_ack;
639 
640 slowpath:
641  if (!no_permanent_sp) {
642  fs->rx_base_sp |= FLEXNIC_PL_FLOWST_SLOWPATH;
643  }
644 
645  fs_unlock(fs);
646  /* TODO: should pass current flow state to kernel as well */
647  return -1;
648 }
649 
650 /* Update receive and transmit queue pointers from application */
651 int fast_flows_bump(struct dataplane_context *ctx, uint32_t flow_id,
652  uint16_t bump_seq, uint32_t rx_bump, uint32_t tx_bump, uint8_t flags,
653  struct network_buf_handle *nbh, uint32_t ts)
654 {
655  struct flextcp_pl_flowst *fs = &fp_state->flowst[flow_id];
656  uint32_t rx_avail_prev, old_avail, new_avail, tx_avail;
657  int ret = -1;
658 
659  fs_lock(fs);
660 #ifdef FLEXNIC_TRACING
661  struct flextcp_pl_trev_atx te_atx = {
662  .rx_bump = rx_bump,
663  .tx_bump = tx_bump,
664  .bump_seq_ent = bump_seq,
665  .bump_seq_flow = fs->bump_seq,
666  .flags = flags,
667 
668  .local_ip = f_beui32(fs->local_ip),
669  .remote_ip = f_beui32(fs->remote_ip),
670  .local_port = f_beui16(fs->local_port),
671  .remote_port = f_beui16(fs->remote_port),
672 
673  .flow_id = flow_id,
674  .db_id = fs->db_id,
675 
676  .tx_next_pos = fs->tx_next_pos,
677  .tx_next_seq = fs->tx_next_seq,
678  .tx_avail_prev = fs->tx_avail,
679  .rx_next_pos = fs->rx_next_pos,
680  .rx_avail = fs->rx_avail,
681  .tx_len = fs->tx_len,
682  .rx_len = fs->rx_len,
683  .rx_remote_avail = fs->rx_remote_avail,
684  .tx_sent = fs->tx_sent,
685  };
686  trace_event(FLEXNIC_PL_TREV_ATX, sizeof(te_atx), &te_atx);
687 #endif
688 
689  /* TODO: is this still necessary? */
690  /* catch out of order bumps */
691  if ((bump_seq >= fs->bump_seq &&
692  bump_seq - fs->bump_seq > (UINT16_MAX / 2)) ||
693  (bump_seq < fs->bump_seq &&
694  (fs->bump_seq < ((UINT16_MAX / 4) * 3) ||
695  bump_seq > (UINT16_MAX / 4))))
696  {
697  goto unlock;
698  }
699  fs->bump_seq = bump_seq;
700 
701  if ((fs->rx_base_sp & FLEXNIC_PL_FLOWST_TXFIN) == FLEXNIC_PL_FLOWST_TXFIN &&
702  tx_bump != 0)
703  {
704  /* TX already closed, don't accept anything for transmission */
705  fprintf(stderr, "fast_flows_bump: tx bump while TX is already closed\n");
706  tx_bump = 0;
707  } else if ((flags & FLEXTCP_PL_ATX_FLTXDONE) == FLEXTCP_PL_ATX_FLTXDONE &&
708  !(fs->rx_base_sp & FLEXNIC_PL_FLOWST_TXFIN) &&
709  !tx_bump)
710  {
711  /* Closing TX requires at least one byte (dummy) */
712  fprintf(stderr, "fast_flows_bump: tx eos without dummy byte\n");
713  goto unlock;
714  }
715 
716  tx_avail = fs->tx_avail + tx_bump;
717 
718  /* validate tx bump */
719  if (tx_bump > fs->tx_len || tx_avail > fs->tx_len ||
720  tx_avail + fs->tx_sent > fs->tx_len)
721  {
722  fprintf(stderr, "fast_flows_bump: tx bump too large\n");
723  goto unlock;
724  }
725  /* validate rx bump */
726  if (rx_bump > fs->rx_len || rx_bump + fs->rx_avail > fs->tx_len) {
727  fprintf(stderr, "fast_flows_bump: rx bump too large\n");
728  goto unlock;
729  }
730  /* calculate how many bytes can be sent before and after this bump */
731  old_avail = tcp_txavail(fs, NULL);
732  new_avail = tcp_txavail(fs, &tx_avail);
733 
734  /* mark connection as closed if requested */
735  if ((flags & FLEXTCP_PL_ATX_FLTXDONE) == FLEXTCP_PL_ATX_FLTXDONE &&
736  !(fs->rx_base_sp & FLEXNIC_PL_FLOWST_TXFIN))
737  {
738  fs->rx_base_sp |= FLEXNIC_PL_FLOWST_TXFIN;
739  }
740 
741  /* update queue manager queue */
742  if (old_avail < new_avail) {
743  if (qman_set(&ctx->qman, flow_id, fs->tx_rate, new_avail -
744  old_avail, TCP_MSS, QMAN_SET_RATE | QMAN_SET_MAXCHUNK
745  | QMAN_ADD_AVAIL) != 0)
746  {
747  fprintf(stderr, "flast_flows_bump: qman_set 1 failed, UNEXPECTED\n");
748  abort();
749  }
750  }
751 
752  /* update flow state */
753  fs->tx_avail = tx_avail;
754  rx_avail_prev = fs->rx_avail;
755  fs->rx_avail += rx_bump;
756 
757  /* receive buffer freed up from empty, need to send out a window update, if
758  * we're not sending anyways. */
759  if (new_avail == 0 && rx_avail_prev == 0 && fs->rx_avail != 0) {
760  flow_tx_segment(ctx, nbh, fs, fs->tx_next_seq, fs->rx_next_seq,
761  fs->rx_avail, 0, 0, fs->tx_next_ts, ts, 0);
762  ret = 0;
763  }
764 
765 unlock:
766  fs_unlock(fs);
767  return ret;
768 }
769 
770 /* start retransmitting */
771 void fast_flows_retransmit(struct dataplane_context *ctx, uint32_t flow_id)
772 {
773  struct flextcp_pl_flowst *fs = &fp_state->flowst[flow_id];
774  uint32_t old_avail, new_avail = -1;
775 
776  fs_lock(fs);
777 
778 #ifdef FLEXNIC_TRACING
779  struct flextcp_pl_trev_rexmit te_rexmit = {
780  .flow_id = flow_id,
781  .tx_avail = fs->tx_avail,
782  .tx_sent = fs->tx_sent,
783  .tx_next_pos = fs->tx_next_pos,
784  .tx_next_seq = fs->tx_next_seq,
785  .rx_remote_avail = fs->rx_remote_avail,
786  };
787  trace_event(FLEXNIC_PL_TREV_REXMIT, sizeof(te_rexmit), &te_rexmit);
788 #endif
789 
790 
791  /* uint32_t old_head = fs->tx_head;
792  uint32_t old_sent = fs->tx_sent;
793  uint32_t old_pos = fs->tx_next_pos;*/
794 
795  old_avail = tcp_txavail(fs, NULL);
796 
797  if (fs->tx_sent == 0) {
798  /*fprintf(stderr, "fast_flows_retransmit: tx sent == 0\n");
799 
800  fprintf(stderr, "fast_flows_retransmit: "
801  "old_avail=%u new_avail=%u head=%u tx_next_seq=%u old_head=%u "
802  "old_sent=%u old_pos=%u new_pos=%u\n", old_avail, new_avail,
803  fs->tx_head, fs->tx_next_seq, old_head, old_sent, old_pos,
804  fs->tx_next_pos);*/
805  goto out;
806  }
807 
808 
809  flow_reset_retransmit(fs);
810  new_avail = tcp_txavail(fs, NULL);
811 
812  /* fprintf(stderr, "fast_flows_retransmit: "
813  "old_avail=%u new_avail=%u head=%u tx_next_seq=%u old_head=%u "
814  "old_sent=%u old_pos=%u new_pos=%u\n", old_avail, new_avail,
815  fs->tx_head, fs->tx_next_seq, old_head, old_sent, old_pos,
816  fs->tx_next_pos);*/
817 
818  /* update queue manager */
819  if (new_avail > old_avail) {
820  if (qman_set(&ctx->qman, flow_id, fs->tx_rate, new_avail - old_avail,
821  TCP_MSS, QMAN_SET_RATE | QMAN_SET_MAXCHUNK | QMAN_ADD_AVAIL) != 0)
822  {
823  fprintf(stderr, "flast_flows_bump: qman_set 1 failed, UNEXPECTED\n");
824  abort();
825  }
826  }
827 
828 out:
829  fs_unlock(fs);
830  return;
831 }
832 
833 /* read `len` bytes from position `pos` in cirucular transmit buffer */
834 static void flow_tx_read(struct flextcp_pl_flowst *fs, uint32_t pos,
835  uint16_t len, void *dst)
836 {
837  uint32_t part;
838 
839  if (LIKELY(pos + len <= fs->tx_len)) {
840  dma_read(fs->tx_base + pos, len, dst);
841  } else {
842  part = fs->tx_len - pos;
843  dma_read(fs->tx_base + pos, part, dst);
844  dma_read(fs->tx_base, len - part, (uint8_t *) dst + part);
845  }
846 }
847 
848 /* write `len` bytes to position `pos` in cirucular receive buffer */
849 static void flow_rx_write(struct flextcp_pl_flowst *fs, uint32_t pos,
850  uint16_t len, const void *src)
851 {
852  uint32_t part;
853  uint64_t rx_base = fs->rx_base_sp & FLEXNIC_PL_FLOWST_RX_MASK;
854 
855  if (LIKELY(pos + len <= fs->rx_len)) {
856  dma_write(rx_base + pos, len, src);
857  } else {
858  part = fs->rx_len - pos;
859  dma_write(rx_base + pos, part, src);
860  dma_write(rx_base, len - part, (const uint8_t *) src + part);
861  }
862 }
863 
864 #ifdef FLEXNIC_PL_OOO_RECV
865 static void flow_rx_seq_write(struct flextcp_pl_flowst *fs, uint32_t seq,
866  uint16_t len, const void *src)
867 {
868  uint32_t diff = seq - fs->rx_next_seq;
869  uint32_t pos = fs->rx_next_pos + diff;
870  if (pos >= fs->rx_len)
871  pos -= fs->rx_len;
872  assert(pos < fs->rx_len);
873  flow_rx_write(fs, pos, len, src);
874 }
875 #endif
876 
877 static void flow_tx_segment(struct dataplane_context *ctx,
878  struct network_buf_handle *nbh, struct flextcp_pl_flowst *fs,
879  uint32_t seq, uint32_t ack, uint32_t rxwnd, uint16_t payload,
880  uint32_t payload_pos, uint32_t ts_echo, uint32_t ts_my, uint8_t fin)
881 {
882  uint16_t hdrs_len, optlen, fin_fl;
883  struct pkt_tcp *p = network_buf_buf(nbh);
884  struct tcp_timestamp_opt *opt_ts;
885 
886  /* calculate header length depending on options */
887  optlen = (sizeof(*opt_ts) + 3) & ~3;
888  hdrs_len = sizeof(*p) + optlen;
889 
890  /* fill headers */
891  p->eth.dest = fs->remote_mac;
892  memcpy(&p->eth.src, &eth_addr, ETH_ADDR_LEN);
893  p->eth.type = t_beui16(ETH_TYPE_IP);
894 
895  IPH_VHL_SET(&p->ip, 4, 5);
896  p->ip._tos = 0;
897  p->ip.len = t_beui16(hdrs_len - offsetof(struct pkt_tcp, ip) + payload);
898  p->ip.id = t_beui16(3); /* TODO: not sure why we have 3 here */
899  p->ip.offset = t_beui16(0);
900  p->ip.ttl = 0xff;
901  p->ip.proto = IP_PROTO_TCP;
902  p->ip.chksum = 0;
903  p->ip.src = fs->local_ip;
904  p->ip.dest = fs->remote_ip;
905 
906  /* mark as ECN capable if flow marked so */
907  if ((fs->rx_base_sp & FLEXNIC_PL_FLOWST_ECN) == FLEXNIC_PL_FLOWST_ECN) {
908  IPH_ECN_SET(&p->ip, IP_ECN_ECT0);
909  }
910 
911  fin_fl = (fin ? TCP_FIN : 0);
912 
913  p->tcp.src = fs->local_port;
914  p->tcp.dest = fs->remote_port;
915  p->tcp.seqno = t_beui32(seq);
916  p->tcp.ackno = t_beui32(ack);
917  TCPH_HDRLEN_FLAGS_SET(&p->tcp, 5 + optlen / 4, TCP_PSH | TCP_ACK | fin_fl);
918  p->tcp.wnd = t_beui16(MIN(0xFFFF, rxwnd));
919  p->tcp.chksum = 0;
920  p->tcp.urgp = t_beui16(0);
921 
922  /* fill in timestamp option */
923  memset(p + 1, 0, optlen);
924  opt_ts = (struct tcp_timestamp_opt *) (p + 1);
925  opt_ts->kind = TCP_OPT_TIMESTAMP;
926  opt_ts->length = sizeof(*opt_ts);
927  opt_ts->ts_val = t_beui32(ts_my);
928  opt_ts->ts_ecr = t_beui32(ts_echo);
929 
930  /* add payload if requested */
931  if (payload > 0) {
932  flow_tx_read(fs, payload_pos, payload, (uint8_t *) p + hdrs_len);
933  }
934 
935  /* checksums */
936  tcp_checksums(nbh, p, fs->local_ip, fs->remote_ip, hdrs_len - offsetof(struct
937  pkt_tcp, tcp) + payload);
938 
939 #ifdef FLEXNIC_TRACING
940  struct flextcp_pl_trev_txseg te_txseg = {
941  .local_ip = f_beui32(p->ip.src),
942  .remote_ip = f_beui32(p->ip.dest),
943  .local_port = f_beui16(p->tcp.src),
944  .remote_port = f_beui16(p->tcp.dest),
945 
946  .flow_seq = seq,
947  .flow_ack = ack,
948  .flow_flags = TCPH_FLAGS(&p->tcp),
949  .flow_len = payload,
950  };
951  trace_event(FLEXNIC_PL_TREV_TXSEG, sizeof(te_txseg), &te_txseg);
952 #endif
953 
954  tx_send(ctx, nbh, 0, hdrs_len + payload);
955 }
956 
957 static void flow_tx_ack(struct dataplane_context *ctx, uint32_t seq,
958  uint32_t ack, uint32_t rxwnd, uint32_t echots, uint32_t myts,
959  struct network_buf_handle *nbh, struct tcp_timestamp_opt *ts_opt)
960 {
961  struct pkt_tcp *p;
962  struct eth_addr eth;
963  ip_addr_t ip;
964  beui16_t port;
965  uint16_t hdrlen;
966  uint16_t ecn_flags = 0;
967 
968  p = network_buf_bufoff(nbh);
969 
970 #if PL_DEBUG_TCPACK
971  fprintf(stderr, "FLOW local=%08x:%05u remote=%08x:%05u ACK: seq=%u ack=%u\n",
972  f_beui32(p->ip.dest), f_beui16(p->tcp.dest),
973  f_beui32(p->ip.src), f_beui16(p->tcp.src), seq, ack);
974 #endif
975 
976  /* swap addresses */
977  eth = p->eth.src;
978  p->eth.src = p->eth.dest;
979  p->eth.dest = eth;
980  ip = p->ip.src;
981  p->ip.src = p->ip.dest;
982  p->ip.dest = ip;
983  port = p->tcp.src;
984  p->tcp.src = p->tcp.dest;
985  p->tcp.dest = port;
986 
987  hdrlen = sizeof(*p) + (TCPH_HDRLEN(&p->tcp) - 5) * 4;
988 
989  /* If ECN flagged, set TCP response flag */
990  if (IPH_ECN(&p->ip) == IP_ECN_CE) {
991  ecn_flags = TCP_ECE;
992  }
993 
994  /* mark ACKs as ECN in-capable */
995  IPH_ECN_SET(&p->ip, IP_ECN_NONE);
996 
997  /* change TCP header to ACK */
998  p->tcp.seqno = t_beui32(seq);
999  p->tcp.ackno = t_beui32(ack);
1000  TCPH_HDRLEN_FLAGS_SET(&p->tcp, TCPH_HDRLEN(&p->tcp), TCP_ACK | ecn_flags);
1001  p->tcp.wnd = t_beui16(MIN(0xFFFF, rxwnd));
1002  p->tcp.urgp = t_beui16(0);
1003 
1004  /* fill in timestamp option */
1005  ts_opt->ts_val = t_beui32(myts);
1006  ts_opt->ts_ecr = t_beui32(echots);
1007 
1008  p->ip.len = t_beui16(hdrlen - offsetof(struct pkt_tcp, ip));
1009  p->ip.ttl = 0xff;
1010 
1011  /* checksums */
1012  tcp_checksums(nbh, p, p->ip.src, p->ip.dest, hdrlen - offsetof(struct
1013  pkt_tcp, tcp));
1014 
1015 #ifdef FLEXNIC_TRACING
1016  struct flextcp_pl_trev_txack te_txack = {
1017  .local_ip = f_beui32(p->ip.src),
1018  .remote_ip = f_beui32(p->ip.dest),
1019  .local_port = f_beui16(p->tcp.src),
1020  .remote_port = f_beui16(p->tcp.dest),
1021 
1022  .flow_seq = seq,
1023  .flow_ack = ack,
1024  .flow_flags = TCPH_FLAGS(&p->tcp),
1025  };
1026  trace_event(FLEXNIC_PL_TREV_TXACK, sizeof(te_txack), &te_txack);
1027 #endif
1028 
1029  tx_send(ctx, nbh, network_buf_off(nbh), hdrlen);
1030 }
1031 
1032 static void flow_reset_retransmit(struct flextcp_pl_flowst *fs)
1033 {
1034  uint32_t x;
1035 
1036  /* reset flow state as if we never transmitted those segments */
1037  fs->rx_dupack_cnt = 0;
1038 
1039  fs->tx_next_seq -= fs->tx_sent;
1040  if (fs->tx_next_pos >= fs->tx_sent) {
1041  fs->tx_next_pos -= fs->tx_sent;
1042  } else {
1043  x = fs->tx_sent - fs->tx_next_pos;
1044  fs->tx_next_pos = fs->tx_len - x;
1045  }
1046  fs->tx_avail += fs->tx_sent;
1047  fs->rx_remote_avail += fs->tx_sent;
1048  fs->tx_sent = 0;
1049 
1050  /* cut rate by half if first drop in control interval */
1051  if (fs->cnt_tx_drops == 0) {
1052  fs->tx_rate /= 2;
1053  }
1054 
1055  fs->cnt_tx_drops++;
1056 }
1057 
1058 static inline void tcp_checksums(struct network_buf_handle *nbh,
1059  struct pkt_tcp *p, beui32_t ip_s, beui32_t ip_d, uint16_t l3_paylen)
1060 {
1061  p->ip.chksum = 0;
1062  if (config.fp_xsumoffload) {
1063  p->tcp.chksum = tx_xsum_enable(nbh, &p->ip, ip_s, ip_d, l3_paylen);
1064  } else {
1065  p->tcp.chksum = 0;
1066  p->ip.chksum = rte_ipv4_cksum((void *) &p->ip);
1067  p->tcp.chksum = rte_ipv4_udptcp_cksum((void *) &p->ip, (void *) &p->tcp);
1068  }
1069 }
1070 
1071 void fast_flows_kernelxsums(struct network_buf_handle *nbh,
1072  struct pkt_tcp *p)
1073 {
1074  tcp_checksums(nbh, p, p->ip.src, p->ip.dest,
1075  f_beui16(p->ip.len) - sizeof(p->ip));
1076 }
1077 
1078 static inline uint32_t flow_hash(struct flow_key *k)
1079 {
1080  return crc32c_sse42_u32(k->local_port.x | (((uint32_t) k->remote_port.x) << 16),
1081  crc32c_sse42_u64(k->local_ip.x | (((uint64_t) k->remote_ip.x) << 32), 0));
1082 }
1083 
1084 void fast_flows_packet_fss(struct dataplane_context *ctx,
1085  struct network_buf_handle **nbhs, void **fss, uint16_t n)
1086 {
1087  uint32_t hashes[n];
1088  uint32_t h, k, j, eh, fid, ffid;
1089  uint16_t i;
1090  struct pkt_tcp *p;
1091  struct flow_key key;
1092  struct flextcp_pl_flowhte *e;
1093  struct flextcp_pl_flowst *fs;
1094 
1095  /* calculate hashes and prefetch hash table buckets */
1096  for (i = 0; i < n; i++) {
1097  p = network_buf_bufoff(nbhs[i]);
1098 
1099  key.local_ip = p->ip.dest;
1100  key.remote_ip = p->ip.src;
1101  key.local_port = p->tcp.dest;
1102  key.remote_port = p->tcp.src;
1103  h = flow_hash(&key);
1104 
1105  rte_prefetch0(&fp_state->flowht[h % FLEXNIC_PL_FLOWHT_ENTRIES]);
1106  rte_prefetch0(&fp_state->flowht[(h + 3) % FLEXNIC_PL_FLOWHT_ENTRIES]);
1107  hashes[i] = h;
1108  }
1109 
1110  /* prefetch flow state for buckets with matching hashes
1111  * (usually 1 per packet, except in case of collisions) */
1112  for (i = 0; i < n; i++) {
1113  h = hashes[i];
1114  for (j = 0; j < FLEXNIC_PL_FLOWHT_NBSZ; j++) {
1115  k = (h + j) % FLEXNIC_PL_FLOWHT_ENTRIES;
1116  e = &fp_state->flowht[k];
1117 
1118  ffid = e->flow_id;
1119  MEM_BARRIER();
1120  eh = e->flow_hash;
1121 
1122  fid = ffid & ((1 << FLEXNIC_PL_FLOWHTE_POSSHIFT) - 1);
1123  if ((ffid & FLEXNIC_PL_FLOWHTE_VALID) == 0 || eh != h) {
1124  continue;
1125  }
1126 
1127  rte_prefetch0(&fp_state->flowst[fid]);
1128  }
1129  }
1130 
1131  /* finish hash table lookup by checking 5-tuple in flow state */
1132  for (i = 0; i < n; i++) {
1133  p = network_buf_bufoff(nbhs[i]);
1134  fss[i] = NULL;
1135  h = hashes[i];
1136 
1137  for (j = 0; j < FLEXNIC_PL_FLOWHT_NBSZ; j++) {
1138  k = (h + j) % FLEXNIC_PL_FLOWHT_ENTRIES;
1139  e = &fp_state->flowht[k];
1140 
1141  ffid = e->flow_id;
1142  MEM_BARRIER();
1143  eh = e->flow_hash;
1144 
1145  fid = ffid & ((1 << FLEXNIC_PL_FLOWHTE_POSSHIFT) - 1);
1146  if ((ffid & FLEXNIC_PL_FLOWHTE_VALID) == 0 || eh != h) {
1147  continue;
1148  }
1149 
1150  MEM_BARRIER();
1151  fs = &fp_state->flowst[fid];
1152  if ((fs->local_ip.x == p->ip.dest.x) &
1153  (fs->remote_ip.x == p->ip.src.x) &
1154  (fs->local_port.x == p->tcp.dest.x) &
1155  (fs->remote_port.x == p->tcp.src.x))
1156  {
1157  rte_prefetch0((uint8_t *) fs + 64);
1158  fss[i] = &fp_state->flowst[fid];
1159  break;
1160  }
1161  }
1162  }
1163 }
uint32_t tx_next_seq
Definition: tas_memif.h:300
uint32_t tx_len
Definition: tas_memif.h:246
uint16_t bump_seq
Definition: tas_memif.h:263
uint16_t db_id
Definition: tas_memif.h:258
uint32_t tx_avail
Definition: tas_memif.h:293
uint32_t rx_len
Definition: tas_memif.h:244
uint32_t rx_remote_avail
Definition: tas_memif.h:281
Definition: qman.c:57
struct tcp_timestamp_opt * ts
Definition: tcp_common.h:230
uint32_t rx_dupack_cnt
Definition: tas_memif.h:283
uint64_t rx_base_sp
Definition: tas_memif.h:239
struct eth_addr remote_mac
Definition: tas_memif.h:255
uint32_t tx_next_ts
Definition: tas_memif.h:302
uint32_t rtt_est
Definition: tas_memif.h:315
Definition: utils.h:44
uint32_t tx_rate
Definition: tas_memif.h:305
Definition: utils.h:45
uint32_t fp_xsumoffload
Definition: config.h:119
uint32_t rx_next_seq
Definition: tas_memif.h:279
uint16_t cnt_rx_acks
Definition: tas_memif.h:309
uint16_t flow_group
Definition: tas_memif.h:261
uint64_t opaque
Definition: tas_memif.h:236
uint64_t tx_base
Definition: tas_memif.h:241
uint16_t cnt_tx_drops
Definition: tas_memif.h:307
uint32_t cnt_rx_ack_bytes
Definition: tas_memif.h:311
uint32_t tx_next_pos
Definition: tas_memif.h:298
uint32_t rx_avail
Definition: tas_memif.h:274
uint32_t tx_sent
Definition: tas_memif.h:296
uint32_t rx_next_pos
Definition: tas_memif.h:277
uint32_t cnt_rx_ecn_bytes
Definition: tas_memif.h:313