TAS
TCP Acceleration as an OS Service
internal.h
1 /*
2  * Copyright 2019 University of Washington, Max Planck Institute for
3  * Software Systems, and The University of Texas at Austin
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be
14  * included in all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23  */
24 
25 #ifndef INTERNAL_H_
26 #define INTERNAL_H_
27 
32 #include <stdint.h>
33 
34 #include <utils_nbqueue.h>
35 #include <utils_timeout.h>
36 
37 #include <tas_memif.h>
38 
39 struct config_route;
40 struct connection;
41 struct kernel_statistics;
42 struct listener;
43 struct timeout;
44 enum timeout_type;
45 
46 extern struct timeout_manager timeout_mgr;
47 extern struct kernel_statistics kstats;
48 extern uint32_t cur_ts;
49 extern int kernel_notifyfd;
50 
52  struct nbqueue_el el;
53  struct nbqueue *q;
54  int notify_fd;
55  int32_t status;
56  void *ptr;
57 };
58 
61  uint64_t drops;
63  uint64_t kernel_rexmit;
65  uint64_t ecn_marked;
67  uint64_t acks;
68 };
69 
71 enum timeout_type {
73  TO_ARP_REQ,
75  TO_TCP_HANDSHAKE,
77  TO_TCP_RETRANSMIT,
79  TO_TCP_CLOSED,
80 };
81 
82 /*****************************************************************************/
90 int nicif_init(void);
91 
93 unsigned nicif_poll(void);
94 
108 int nicif_appctx_add(uint16_t appid, uint32_t db, uint64_t *rxq_base,
109  uint32_t rxq_len, uint64_t *txq_base, uint32_t txq_len, int evfd);
110 
114  NICIF_CONN_ECN = (1 << 2),
115 };
116 
141 int nicif_connection_add(uint32_t db, uint64_t mac_remote, uint32_t ip_local,
142  uint16_t port_local, uint32_t ip_remote, uint16_t port_remote,
143  uint64_t rx_base, uint32_t rx_len, uint64_t tx_base, uint32_t tx_len,
144  uint32_t remote_seq, uint32_t local_seq, uint64_t app_opaque,
145  uint32_t flags, uint32_t rate, uint32_t fn_core, uint16_t flow_group,
146  uint32_t *pf_id);
147 
159 int nicif_connection_disable(uint32_t f_id, uint32_t *tx_seq, uint32_t *rx_seq,
160  int *tx_closed, int *rx_closed);
161 
167 void nicif_connection_free(uint32_t f_id);
168 
177 int nicif_connection_move(uint32_t dst_db, uint32_t f_id);
178 
185  uint16_t c_drops;
187  uint16_t c_acks;
189  uint32_t c_ackb;
191  uint32_t c_ecnb;
193  int txp;
195  uint32_t rtt;
196 };
197 
206 int nicif_connection_stats(uint32_t f_id,
207  struct nicif_connection_stats *p_stats);
208 
217 int nicif_connection_setrate(uint32_t f_id, uint32_t rate);
218 
227 int nicif_connection_retransmit(uint32_t f_id, uint16_t core);
228 
241 int nicif_tx_alloc(uint16_t len, void **buf, uint32_t *opaque);
242 
251 void nicif_tx_send(uint32_t opaque, int no_ts);
252 
255 /*****************************************************************************/
264 struct packetmem_handle;
265 
267 int packetmem_init(void);
268 
280 int packetmem_alloc(size_t length, uintptr_t *off,
281  struct packetmem_handle **handle);
282 
290 void packetmem_free(struct packetmem_handle *handle);
291 
294 /*****************************************************************************/
304 int appif_init(void);
305 
307 unsigned appif_poll(void);
308 
315 void appif_conn_opened(struct connection *c, int status);
316 
323 void appif_conn_closed(struct connection *c, int status);
324 
332 void appif_listen_newconn(struct listener *l, uint32_t remote_ip,
333  uint16_t remote_port);
334 
341 void appif_accept_conn(struct connection *c, int status);
342 
345 /*****************************************************************************/
368 };
369 
373  uint32_t ecn_rate;
375  uint32_t window;
378 };
379 
383  uint32_t unproc_acks;
385  uint32_t unproc_ackb;
387  uint32_t unproc_ecnb;
389  uint32_t unproc_drops;
390 
392  uint32_t ecn_rate;
394  uint32_t act_rate;
397 };
398 
402  uint32_t rtt_prev;
404  int32_t rtt_diff;
406  uint32_t hai_cnt;
408  uint32_t act_rate;
410  uint32_t last_ts;
413 };
414 
416 struct connection {
422  uint64_t opaque;
424  struct app_context *ctx;
430  uint32_t db_id;
442  uint8_t *rx_buf;
444  uint8_t *tx_buf;
446  uint32_t rx_len;
448  uint32_t tx_len;
456  uint64_t remote_mac;
458  uint32_t remote_ip;
460  uint32_t local_ip;
462  uint16_t remote_port;
464  uint16_t local_port;
472  enum connection_status status;
474  uint32_t remote_seq;
476  uint32_t local_seq;
478  uint32_t syn_ts;
486  uint32_t timeout;
488  struct timeout to;
492  int to_armed;
500  uint32_t cc_last_ts;
502  uint32_t cc_rtt;
504  uint16_t cc_last_drops;
506  uint16_t cc_last_acks;
508  uint32_t cc_last_ackb;
510  uint32_t cc_last_ecnb;
511 
513  uint32_t cc_rate;
515  uint32_t cc_rexmits;
517  union {
519  struct connection_cc_dctcp_win dctcp_win;
521  struct connection_cc_timely timely;
523  struct connection_cc_dctcp_rate dctcp_rate;
524  } cc;
526  uint32_t cnt_tx_pending;
528  uint32_t ts_tx_pending;
536  struct nicif_completion comp;
538  uint32_t flow_id;
540  uint32_t fn_core;
542  uint32_t flags;
544  uint16_t flow_group;
545 };
546 
548 struct listener {
554  uint64_t opaque;
556  struct app_context *ctx;
560  uint32_t db_id;
568  uint32_t backlog_len;
570  uint32_t backlog_pos;
572  uint32_t backlog_used;
574  void **backlog_ptrs;
576  uint32_t *backlog_cores;
578  uint16_t *backlog_fgs;
584  uint16_t port;
586  uint32_t flags;
587 };
588 
590 extern struct connection *tcp_conns;
591 
593 int tcp_init(void);
594 
596 void tcp_poll(void);
597 
614 int tcp_open(struct app_context *ctx, uint64_t opaque, uint32_t remote_ip,
615  uint16_t remote_port, uint32_t db_id, struct connection **conn);
616 
631 int tcp_listen(struct app_context *ctx, uint64_t opaque, uint16_t local_port,
632  uint32_t backlog, int reuseport, struct listener **listen);
633 
644 int tcp_accept(struct app_context *ctx, uint64_t opaque,
645  struct listener *listen, uint32_t db_id);
646 
657 int tcp_packet(const void *pkt, uint16_t len, uint32_t fn_core,
658  uint16_t flow_group);
659 
665 int tcp_close(struct connection *conn);
666 
672 void tcp_destroy(struct connection *conn);
673 
680 void tcp_timeout(struct timeout *to, enum timeout_type type);
681 
684 /*****************************************************************************/
692 int cc_init(void);
693 
699 unsigned cc_poll(uint32_t cur_ts);
700 
701 uint32_t cc_next_ts(uint32_t cur_ts);
702 
708 void cc_conn_init(struct connection *conn);
709 
715 void cc_conn_remove(struct connection *conn);
716 
719 /*****************************************************************************/
727 int arp_init(void);
728 
743 int arp_request(struct nicif_completion *comp, uint32_t ip, uint64_t *mac);
744 
751 void arp_packet(const void *pkt, uint16_t len);
752 
759 void arp_timeout(struct timeout *to, enum timeout_type type);
760 
763 /*****************************************************************************/
771 int routing_init(void);
772 
785 int routing_resolve(struct nicif_completion *comp, uint32_t ip, uint64_t *mac);
786 
789 /*****************************************************************************/
800 int kni_init(void);
801 
803 void kni_packet(const void *pkt, uint16_t len);
804 
806 unsigned kni_poll(void);
807 
810 #endif // ndef INTERNAL_H_
uint32_t flags
Definition: internal.h:542
void ** backlog_ptrs
Definition: internal.h:574
int tcp_init(void)
Definition: tcp.c:111
void tcp_destroy(struct connection *conn)
Definition: tcp.c:428
int to_armed
Definition: internal.h:492
struct packetmem_handle * rx_handle
Definition: internal.h:438
uint32_t db_id
Definition: internal.h:430
void appif_conn_closed(struct connection *c, int status)
Definition: appif_ctx.c:100
int to_attempts
Definition: internal.h:490
int nicif_connection_setrate(uint32_t f_id, uint32_t rate)
Definition: nicif.c:314
uint64_t ecn_marked
Definition: internal.h:65
uint64_t remote_mac
Definition: internal.h:456
unsigned kni_poll(void)
Definition: kni.c:135
void kni_packet(const void *pkt, uint16_t len)
Definition: kni.c:109
uint16_t local_port
Definition: internal.h:464
unsigned cc_poll(uint32_t cur_ts)
Definition: cc.c:86
uint64_t acks
Definition: internal.h:67
nicif_connection_flags
Definition: internal.h:112
struct nicif_completion comp
Definition: internal.h:536
uint32_t timeout_type
Time and type. Type is stored in the 4 most significant bits, and the time in the 28 least significan...
Definition: utils_timeout.h:43
uint32_t fn_core
Definition: internal.h:540
void tcp_poll(void)
Definition: tcp.c:123
int arp_request(struct nicif_completion *comp, uint32_t ip, uint64_t *mac)
Definition: arp.c:81
struct app_context * new_ctx
Definition: internal.h:426
int nicif_connection_add(uint32_t db, uint64_t mac_remote, uint32_t ip_local, uint16_t port_local, uint32_t ip_remote, uint16_t port_remote, uint64_t rx_base, uint32_t rx_len, uint64_t tx_base, uint32_t tx_len, uint32_t remote_seq, uint32_t local_seq, uint64_t app_opaque, uint32_t flags, uint32_t rate, uint32_t fn_core, uint16_t flow_group, uint32_t *pf_id)
Definition: nicif.c:175
int nicif_init(void)
Definition: nicif.c:85
uint16_t remote_port
Definition: internal.h:462
void nicif_connection_free(uint32_t f_id)
Definition: nicif.c:272
int tcp_packet(const void *pkt, uint16_t len, uint32_t fn_core, uint16_t flow_group)
Definition: tcp.c:350
void arp_packet(const void *pkt, uint16_t len)
Definition: arp.c:138
uint32_t local_ip
Definition: internal.h:460
uint32_t flags
Definition: internal.h:586
int packetmem_init(void)
Definition: packetmem.c:44
struct app_context * ctx
Definition: internal.h:424
uint64_t kernel_rexmit
Definition: internal.h:63
uint32_t cc_last_ackb
Definition: internal.h:508
uint32_t flow_id
Definition: internal.h:538
void tcp_timeout(struct timeout *to, enum timeout_type type)
Definition: tcp.c:434
uint64_t drops
Definition: internal.h:61
int tcp_listen(struct app_context *ctx, uint64_t opaque, uint16_t local_port, uint32_t backlog, int reuseport, struct listener **listen)
Definition: tcp.c:211
struct listener * app_next
Definition: internal.h:558
int nicif_connection_move(uint32_t dst_db, uint32_t f_id)
Definition: nicif.c:278
uint32_t remote_ip
Definition: internal.h:458
void appif_listen_newconn(struct listener *l, uint32_t remote_ip, uint16_t remote_port)
Definition: appif_ctx.c:143
uint16_t cc_last_acks
Definition: internal.h:506
uint32_t local_seq
Definition: internal.h:476
uint8_t * tx_buf
Definition: internal.h:444
struct connection * tcp_conns
uint16_t cc_last_drops
Definition: internal.h:504
uint32_t cc_rtt
Definition: internal.h:502
void arp_timeout(struct timeout *to, enum timeout_type type)
Definition: arp.c:209
int tcp_accept(struct app_context *ctx, uint64_t opaque, struct listener *listen, uint32_t db_id)
Definition: tcp.c:322
uint32_t cc_last_ts
Definition: internal.h:500
uint32_t rx_len
Definition: internal.h:446
int packetmem_alloc(size_t length, uintptr_t *off, struct packetmem_handle **handle)
Definition: packetmem.c:61
uint32_t backlog_pos
Definition: internal.h:570
uint16_t flow_group
Definition: internal.h:544
struct app_context * ctx
Definition: internal.h:556
struct connection * ht_next
Definition: internal.h:534
struct timeout to
Definition: internal.h:488
int cc_init(void)
Definition: cc.c:35
int tcp_open(struct app_context *ctx, uint64_t opaque, uint32_t remote_ip, uint16_t remote_port, uint32_t db_id, struct connection **conn)
Definition: tcp.c:147
int nicif_connection_retransmit(uint32_t f_id, uint16_t core)
Definition: nicif.c:330
uint32_t db_id
Definition: internal.h:560
int appif_init(void)
Definition: appif.c:100
unsigned nicif_poll(void)
Definition: nicif.c:111
int nicif_connection_stats(uint32_t f_id, struct nicif_connection_stats *p_stats)
Definition: nicif.c:285
uint64_t opaque
Definition: internal.h:422
uint32_t backlog_len
Definition: internal.h:568
uint32_t backlog_used
Definition: internal.h:572
uint64_t opaque
Definition: internal.h:554
void packetmem_free(struct packetmem_handle *handle)
Definition: packetmem.c:113
connection_status
Definition: internal.h:353
uint16_t port
Definition: internal.h:584
int kni_init(void)
Definition: kni.c:67
uint32_t cnt_tx_pending
Definition: internal.h:526
uint32_t ts_tx_pending
Definition: internal.h:528
int tcp_close(struct connection *conn)
Definition: tcp.c:392
void cc_conn_init(struct connection *conn)
Definition: cc.c:175
int nicif_tx_alloc(uint16_t len, void **buf, uint32_t *opaque)
Definition: nicif.c:352
uint32_t cc_rexmits
Definition: internal.h:515
void appif_conn_opened(struct connection *c, int status)
Definition: appif_ctx.c:57
int nicif_connection_disable(uint32_t f_id, uint32_t *tx_seq, uint32_t *rx_seq, int *tx_closed, int *rx_closed)
Definition: nicif.c:250
struct connection * cc_next
Definition: internal.h:530
uint32_t timeout
Definition: internal.h:486
uint32_t tx_len
Definition: internal.h:448
struct packetmem_handle * tx_handle
Definition: internal.h:440
void appif_accept_conn(struct connection *c, int status)
Definition: appif_ctx.c:173
void cc_conn_remove(struct connection *conn)
Definition: cc.c:209
uint32_t remote_seq
Definition: internal.h:474
int nicif_appctx_add(uint16_t appid, uint32_t db, uint64_t *rxq_base, uint32_t rxq_len, uint64_t *txq_base, uint32_t txq_len, int evfd)
Definition: nicif.c:130
uint32_t syn_ts
Definition: internal.h:478
struct connection * wait_conns
Definition: internal.h:582
int arp_init(void)
Definition: arp.c:58
uint32_t cc_last_ecnb
Definition: internal.h:510
uint32_t * backlog_cores
Definition: internal.h:576
uint32_t cc_rate
Definition: internal.h:513
void nicif_tx_send(uint32_t opaque, int no_ts)
Definition: nicif.c:368
struct connection * app_next
Definition: internal.h:428
int routing_init(void)
Definition: routing.c:48
unsigned appif_poll(void)
Definition: appif.c:130
uint8_t * rx_buf
Definition: internal.h:442
uint16_t * backlog_fgs
Definition: internal.h:578
int routing_resolve(struct nicif_completion *comp, uint32_t ip, uint64_t *mac)
Definition: routing.c:89