31 #include <tas_memif.h> 32 #include <packet_defs.h> 34 #include <utils_timeout.h> 35 #include <utils_sync.h> 38 #include <rte_config.h> 39 #include <rte_hash_crc.h> 41 #define PKTBUF_SIZE 1536 53 static int adminq_init(
void);
54 static int adminq_init_core(uint16_t core);
55 static inline int rxq_poll(
void);
56 static inline void process_packet(
const void *buf, uint16_t len,
57 uint32_t fn_core, uint16_t flow_group);
58 static inline volatile struct flextcp_pl_ktx *ktx_try_alloc(uint32_t core,
62 static inline int flow_slot_alloc(uint32_t h, uint32_t *i, uint32_t *d);
65 static void flow_id_alloc_init(
void);
66 static int flow_id_alloc(uint32_t *fid);
67 static void flow_id_free(uint32_t flow_id);
69 struct flow_id_item flow_id_items[FLEXNIC_PL_FLOWST_NUM];
72 static uint32_t fn_cores;
76 static uint32_t rxq_len;
77 static uint32_t *rxq_tail;
78 static uint32_t rxq_next;
82 static uint32_t txq_len;
83 static uint32_t *txq_tail;
87 rte_hash_crc_init_alg();
96 fprintf(stderr,
"nicif_init: pktmem_init failed\n");
101 flow_id_alloc_init();
104 fprintf(stderr,
"nicif_init: initializing admin queue failed\n");
116 for (i = 0; i < 512; i++) {
123 ret += (x == -1 ? 0 : 1);
131 uint32_t rxq_len, uint64_t *txq_base, uint32_t txq_len,
int evfd)
137 if (appid >= FLEXNIC_PL_APPST_NUM) {
138 fprintf(stderr,
"nicif_appctx_add: app id too high (%u, max=%u)\n", appid,
139 FLEXNIC_PL_APPST_NUM);
143 if (ast->
ctx_num + 1 >= FLEXNIC_PL_APPST_CTX_NUM) {
144 fprintf(stderr,
"nicif_appctx_add: too many contexts in app\n");
149 for (i = 0; i < tas_info->
cores_num; i++) {
150 actx = &fp_state->appctx[i][db];
151 actx->appst_id = appid;
152 actx->rx_base = rxq_base[i];
153 actx->tx_base = txq_base[i];
154 actx->rx_avail = rxq_len;
160 for (i = 0; i < tas_info->
cores_num; i++) {
161 actx = &fp_state->appctx[i][db];
162 actx->tx_len = txq_len;
163 actx->rx_len = rxq_len;
176 uint16_t port_local, uint32_t ip_remote, uint16_t port_remote,
177 uint64_t rx_base, uint32_t rx_len, uint64_t tx_base, uint32_t tx_len,
178 uint32_t remote_seq, uint32_t local_seq, uint64_t app_opaque,
179 uint32_t flags, uint32_t rate, uint32_t fn_core, uint16_t flow_group,
183 beui32_t lip = t_beui32(ip_local), rip = t_beui32(ip_remote);
184 beui16_t lp = t_beui16(port_local), rp = t_beui16(port_remote);
185 uint32_t i, d, f_id, hash;
189 if (flow_id_alloc(&f_id) != 0) {
190 fprintf(stderr,
"nicif_connection_add: allocating flow state\n");
195 hash = flow_hash(lip, lp, rip, rp);
196 if (flow_slot_alloc(hash, &i, &d) != 0) {
198 fprintf(stderr,
"nicif_connection_add: allocating slot failed\n");
201 assert(i < FLEXNIC_PL_FLOWHT_ENTRIES);
202 assert(d < FLEXNIC_PL_FLOWHT_NBSZ);
205 rx_base |= FLEXNIC_PL_FLOWST_ECN;
208 fs = &fp_state->flowst[f_id];
214 memcpy(&fs->
remote_mac, &mac_remote, ETH_ADDR_LEN);
220 fs->remote_port = rp;
241 hte[i].flow_hash = hash;
243 hte[i].flow_id = FLEXNIC_PL_FLOWHTE_VALID |
244 (d << FLEXNIC_PL_FLOWHTE_POSSHIFT) | f_id;
251 int *tx_closed,
int *rx_closed)
255 util_spin_lock(&fs->
lock);
261 *rx_closed = !!(fs->
rx_base_sp & FLEXNIC_PL_FLOWST_RXFIN);
262 *tx_closed = !!(fs->
rx_base_sp & FLEXNIC_PL_FLOWST_TXFIN) &&
265 util_spin_unlock(&fs->
lock);
267 flow_slot_clear(f_id, fs->local_ip, fs->local_port, fs->remote_ip,
280 fp_state->flowst[f_id].
db_id = dst_db;
290 if (f_id >= FLEXNIC_PL_FLOWST_NUM) {
291 fprintf(stderr,
"nicif_connection_stats: bad flow id\n");
295 fs = &fp_state->flowst[f_id];
318 if (f_id >= FLEXNIC_PL_FLOWST_NUM) {
319 fprintf(stderr,
"nicif_connection_stats: bad flow id\n");
323 fs = &fp_state->flowst[f_id];
335 uint16_t core = fp_state->flow_group_steering[flow_group];
337 if ((ktx = ktx_try_alloc(core, &buf, &tail)) == NULL) {
340 txq_tail[core] = tail;
342 ktx->msg.connretran.flow_id = f_id;
344 ktx->type = FLEXTCP_PL_KTX_CONNRETRAN;
346 notify_fastpath_core(core);
357 if ((ktx = ktx_try_alloc(0, &buf, opaque)) == NULL) {
361 ktx->msg.packet.addr = buf->addr;
362 ktx->msg.packet.len = len;
370 uint32_t tail = (opaque == 0 ? txq_len - 1 : opaque - 1);
374 ktx->type = (!no_ts ? FLEXTCP_PL_KTX_PACKET : FLEXTCP_PL_KTX_PACKET_NOTS);
375 txq_tail[0] = opaque;
377 notify_fastpath_core(0);
380 static int adminq_init(
void)
387 rxq_bufs = calloc(fn_cores,
sizeof(*rxq_bufs));
388 rxq_base = calloc(fn_cores,
sizeof(*rxq_base));
389 rxq_tail = calloc(fn_cores,
sizeof(*rxq_tail));
390 txq_bufs = calloc(fn_cores,
sizeof(*txq_bufs));
391 txq_base = calloc(fn_cores,
sizeof(*txq_base));
392 txq_tail = calloc(fn_cores,
sizeof(*txq_tail));
393 if (rxq_bufs == NULL || rxq_base == NULL || rxq_tail == NULL ||
394 txq_bufs == NULL || txq_base == NULL || txq_tail == NULL)
396 fprintf(stderr,
"adminq_init: queue state alloc failed\n");
402 for (i = 0; i < fn_cores; i++) {
403 if (adminq_init_core(i) != 0)
410 static int adminq_init_core(uint16_t core)
413 uintptr_t off_bufs, off_rx, off_tx;
414 size_t i, sz_bufs, sz_rx, sz_tx;
416 if ((rxq_bufs[core] = calloc(config.
nic_rx_len,
sizeof(**rxq_bufs)))
419 fprintf(stderr,
"adminq_init: calloc rx bufs failed\n");
422 if ((txq_bufs[core] = calloc(config.
nic_tx_len,
sizeof(**txq_bufs)))
425 fprintf(stderr,
"adminq_init: calloc tx bufs failed\n");
426 free(rxq_bufs[core]);
433 fprintf(stderr,
"adminq_init: packetmem_alloc bufs failed\n");
434 free(txq_bufs[core]);
435 free(rxq_bufs[core]);
441 fprintf(stderr,
"adminq_init: packetmem_alloc tx failed\n");
443 free(txq_bufs[core]);
444 free(rxq_bufs[core]);
449 fprintf(stderr,
"adminq_init: packetmem_alloc tx failed\n");
452 free(txq_bufs[core]);
453 free(rxq_bufs[core]);
458 ((uint8_t *) tas_shm + off_rx);
460 ((uint8_t *) tas_shm + off_tx);
462 memset((
void *) rxq_base[core], 0, sz_rx);
463 memset((
void *) txq_base[core], 0, sz_tx);
465 for (i = 0; i < rxq_len; i++) {
466 rxq_bufs[core][i].addr = off_bufs;
467 rxq_bufs[core][i].buf = (uint8_t *) tas_shm + off_bufs;
468 rxq_base[core][i].addr = off_bufs;
469 off_bufs += PKTBUF_SIZE;
471 for (i = 0; i < txq_len; i++) {
472 txq_bufs[core][i].addr = off_bufs;
473 txq_bufs[core][i].buf = (uint8_t *) tas_shm + off_bufs;
474 off_bufs += PKTBUF_SIZE;
477 fp_state->kctx[core].rx_base = off_rx;
478 fp_state->kctx[core].tx_base = off_tx;
480 fp_state->kctx[core].tx_len = sz_tx;
481 fp_state->kctx[core].rx_len = sz_rx;
485 static inline int rxq_poll(
void)
487 uint32_t old_tail, tail, core;
494 old_tail = tail = rxq_tail[core];
495 krx = &rxq_base[core][tail];
496 buf = &rxq_bufs[core][tail];
497 rxq_next = (core + 1) % fn_cores;
501 if (type == FLEXTCP_PL_KRX_INVALID) {
507 if (tail == rxq_len) {
514 case FLEXTCP_PL_KRX_PACKET:
515 process_packet(buf->buf, krx->msg.packet.len, krx->msg.packet.fn_core,
516 krx->msg.packet.flow_group);
520 fprintf(stderr,
"rxq_poll: unknown rx type 0x%x old %x len %x\n", type,
525 rxq_tail[core] = tail;
530 static inline void process_packet(
const void *buf, uint16_t len,
531 uint32_t fn_core, uint16_t flow_group)
533 const struct eth_hdr *eth = buf;
538 if (f_beui16(eth->type) == ETH_TYPE_ARP) {
539 if (len <
sizeof(
struct pkt_arp)) {
540 fprintf(stderr,
"process_packet: short arp packet\n");
545 }
else if (f_beui16(eth->type) == ETH_TYPE_IP) {
546 if (len <
sizeof(*eth) +
sizeof(*ip)) {
547 fprintf(stderr,
"process_packet: short ip packet\n");
551 if (ip->proto == IP_PROTO_TCP) {
552 if (len <
sizeof(*eth) +
sizeof(*ip) +
sizeof(*tcp)) {
553 fprintf(stderr,
"process_packet: short tcp packet\n");
557 to_kni = !!
tcp_packet(buf, len, fn_core, flow_group);
565 static inline volatile struct flextcp_pl_ktx *ktx_try_alloc(uint32_t core,
568 uint32_t tail = txq_tail[core];
570 struct nic_buffer *buf = &txq_bufs[core][tail];
573 if (ktx->type != 0) {
579 if (tail == rxq_len) {
597 } __attribute__((packed)) hk =
598 { .lip = lip, .rip = rip, .lp = lp, .rp = rp };
600 return rte_hash_crc(&hk,
sizeof(hk), 0);
603 static inline int flow_slot_alloc(uint32_t h, uint32_t *pi, uint32_t *pd)
605 uint32_t j, i, l, k, d;
609 j = h % FLEXNIC_PL_FLOWHT_ENTRIES;
610 l = (j + FLEXNIC_PL_FLOWHT_NBSZ) % FLEXNIC_PL_FLOWHT_ENTRIES;
614 for (i = j; i != l; i = (i + 1) % FLEXNIC_PL_FLOWHT_ENTRIES) {
615 if ((hte[i].flow_id & FLEXNIC_PL_FLOWHTE_VALID) == 0) {
624 k = (l + 4 * FLEXNIC_PL_FLOWHT_NBSZ) % FLEXNIC_PL_FLOWHT_ENTRIES;
626 for (; i != k; i = (i + 1) % FLEXNIC_PL_FLOWHT_ENTRIES) {
627 if ((hte[i].flow_id & FLEXNIC_PL_FLOWHTE_VALID) == 0) {
634 fprintf(stderr,
"flow_slot_alloc: no empty slot found\n");
641 while ((j > l || (i < j || i >= l)) && (j < l || (i >= l && i < j))) {
645 i = (k - FLEXNIC_PL_FLOWHT_NBSZ) % FLEXNIC_PL_FLOWHT_ENTRIES;
646 for (; i != k; i = (i + 1) % FLEXNIC_PL_FLOWHT_ENTRIES) {
647 assert((hte[i].flow_id & FLEXNIC_PL_FLOWHTE_VALID) != 0);
650 d = (hte[i].flow_id >> FLEXNIC_PL_FLOWHTE_POSSHIFT) &
651 (FLEXNIC_PL_FLOWHT_NBSZ - 1);
652 d = FLEXNIC_PL_FLOWHT_NBSZ - 1 - d;
655 if ((k - i) % FLEXNIC_PL_FLOWHT_ENTRIES <= d) {
662 fprintf(stderr,
"flow_slot_alloc: no element could be moved\n");
667 assert((hte[k].flow_id & FLEXNIC_PL_FLOWHTE_VALID) == 0);
668 d = (hte[i].flow_id >> FLEXNIC_PL_FLOWHTE_POSSHIFT) &
669 (FLEXNIC_PL_FLOWHT_NBSZ - 1);
672 hte[k].flow_hash = hte[i].flow_hash;
674 hte[k].flow_id = FLEXNIC_PL_FLOWHTE_VALID |
675 (d << FLEXNIC_PL_FLOWHTE_POSSHIFT) |
676 (((1 << FLEXNIC_PL_FLOWHTE_POSSHIFT) - 1) & hte[i].flow_id);
685 *pd = (i - j) % FLEXNIC_PL_FLOWHT_ENTRIES;
692 uint32_t h, k, j, ffid, eh;
695 h = flow_hash(lip, lp, rip, rp);
697 for (j = 0; j < FLEXNIC_PL_FLOWHT_NBSZ; j++) {
698 k = (h + j) % FLEXNIC_PL_FLOWHT_ENTRIES;
699 e = &fp_state->flowht[k];
705 if ((ffid & FLEXNIC_PL_FLOWHTE_VALID) == 0 || eh != h) {
709 if ((ffid & ((1 << FLEXNIC_PL_FLOWHTE_POSSHIFT) - 1)) == f_id) {
710 e->flow_id &= ~FLEXNIC_PL_FLOWHTE_VALID;
715 fprintf(stderr,
"flow_slot_clear: table entry not found\n");
719 static void flow_id_alloc_init(
void)
723 for (i = 0; i < FLEXNIC_PL_FLOWST_NUM; i++) {
724 it = &flow_id_items[i];
729 flow_id_freelist = it;
737 static int flow_id_alloc(uint32_t *fid)
744 flow_id_freelist = it->next;
749 static void flow_id_free(uint32_t flow_id)
752 it->next = flow_id_freelist;
753 flow_id_freelist = it;
int nicif_connection_setrate(uint32_t f_id, uint32_t rate)
#define FLEXNIC_FLAG_READY
void kni_packet(const void *pkt, uint16_t len)
int nicif_connection_add(uint32_t db, uint64_t mac_remote, uint32_t ip_local, uint16_t port_local, uint32_t ip_remote, uint16_t port_remote, uint64_t rx_base, uint32_t rx_len, uint64_t tx_base, uint32_t tx_len, uint32_t remote_seq, uint32_t local_seq, uint64_t app_opaque, uint32_t flags, uint32_t rate, uint32_t fn_core, uint16_t flow_group, uint32_t *pf_id)
void nicif_connection_free(uint32_t f_id)
int tcp_packet(const void *pkt, uint16_t len, uint32_t fn_core, uint16_t flow_group)
void arp_packet(const void *pkt, uint16_t len)
int nicif_connection_move(uint32_t dst_db, uint32_t f_id)
struct eth_addr remote_mac
uint16_t ctx_ids[FLEXNIC_PL_APPST_CTX_NUM]
int packetmem_alloc(size_t length, uintptr_t *off, struct packetmem_handle **handle)
int nicif_connection_retransmit(uint32_t f_id, uint16_t flow_group)
unsigned nicif_poll(void)
int nicif_connection_stats(uint32_t f_id, struct nicif_connection_stats *p_stats)
void packetmem_free(struct packetmem_handle *handle)
int nicif_tx_alloc(uint16_t len, void **pbuf, uint32_t *opaque)
int nicif_connection_disable(uint32_t f_id, uint32_t *tx_seq, uint32_t *rx_seq, int *tx_closed, int *rx_closed)
uint32_t cnt_rx_ack_bytes
int nicif_appctx_add(uint16_t appid, uint32_t db, uint64_t *rxq_base, uint32_t rxq_len, uint64_t *txq_base, uint32_t txq_len, int evfd)
void nicif_tx_send(uint32_t opaque, int no_ts)
uint32_t cnt_rx_ecn_bytes