TAS
TCP Acceleration as an OS Service
kernel.c
1 /*
2  * Copyright 2019 University of Washington, Max Planck Institute for
3  * Software Systems, and The University of Texas at Austin
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be
14  * included in all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23  */
24 
25 #include <inttypes.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <sys/epoll.h>
29 #include <sys/eventfd.h>
30 #include <unistd.h>
31 #include <errno.h>
32 
33 #include <utils.h>
34 #include <tas.h>
35 #include "internal.h"
36 
37 static void slowpath_block(uint32_t cur_ts);
38 static void timeout_trigger(struct timeout *to, uint8_t type, void *opaque);
39 static void signal_tas_ready(void);
40 void flexnic_loadmon(uint32_t cur_ts);
41 
42 struct timeout_manager timeout_mgr;
43 static int exited = 0;
44 struct kernel_statistics kstats;
45 uint32_t cur_ts;
46 int kernel_notifyfd = 0;
47 static int epfd;
48 
49 int slowpath_main(void)
50 {
51  struct notify_blockstate nbs;
52  uint32_t last_print = 0;
53  uint32_t loadmon_ts = 0;
54 
55  kernel_notifyfd = eventfd(0, EFD_NONBLOCK);
56  assert(kernel_notifyfd != -1);
57 
58  struct epoll_event ev = {
59  .events = EPOLLIN,
60  .data.fd = kernel_notifyfd,
61  };
62 
63  epfd = epoll_create1(0);
64  assert(epfd != -1);
65 
66  int r = epoll_ctl(epfd, EPOLL_CTL_ADD, kernel_notifyfd, &ev);
67  assert(r == 0);
68 
69  /* initialize timers for timeouts */
70  if (util_timeout_init(&timeout_mgr, timeout_trigger, NULL)) {
71  fprintf(stderr, "timeout_init failed\n");
72  return EXIT_FAILURE;
73  }
74 
75  /* initialize kni */
76  if (kni_init()) {
77  fprintf(stderr, "kni_init failed\n");
78  return EXIT_FAILURE;
79  }
80 
81  /* initialize routing subsystem */
82  if (routing_init()) {
83  fprintf(stderr, "routing_init failed\n");
84  return EXIT_FAILURE;
85  }
86 
87  /* connect to NIC */
88  if (nicif_init()) {
89  fprintf(stderr, "nicif_init failed\n");
90  return EXIT_FAILURE;
91  }
92 
93  /* initialize CC */
94  if (cc_init()) {
95  fprintf(stderr, "cc_init failed\n");
96  return EXIT_FAILURE;
97  }
98 
99  /* prepare application interface */
100  if (appif_init()) {
101  fprintf(stderr, "appif_init failed\n");
102  return EXIT_FAILURE;
103  }
104 
105  if (arp_init()) {
106  fprintf(stderr, "arp_init failed\n");
107  return EXIT_FAILURE;
108  }
109 
110 
111  if (tcp_init()) {
112  fprintf(stderr, "tcp_init failed\n");
113  return EXIT_FAILURE;
114  }
115 
116  signal_tas_ready();
117 
118  notify_canblock_reset(&nbs);
119  while (exited == 0) {
120  unsigned n = 0;
121 
122  cur_ts = util_timeout_time_us();
123  n += nicif_poll();
124  n += cc_poll(cur_ts);
125  n += appif_poll();
126  n += kni_poll();
127  tcp_poll();
128  util_timeout_poll_ts(&timeout_mgr, cur_ts);
129 
130  if (config.fp_autoscale && cur_ts - loadmon_ts >= 10000) {
131  flexnic_loadmon(cur_ts);
132  loadmon_ts = cur_ts;
133  }
134 
135  if (notify_canblock(&nbs, n != 0, util_rdtsc())) {
136  slowpath_block(cur_ts);
137  notify_canblock_reset(&nbs);
138  }
139 
140  if (cur_ts - last_print >= 1000000) {
141  if (!config.quiet) {
142  printf("stats: drops=%"PRIu64" k_rexmit=%"PRIu64" ecn=%"PRIu64" acks=%"
143  PRIu64"\n", kstats.drops, kstats.kernel_rexmit, kstats.ecn_marked,
144  kstats.acks);
145  fflush(stdout);
146  }
147  last_print = cur_ts;
148  }
149  }
150 
151  return EXIT_SUCCESS;
152 }
153 
154 static void slowpath_block(uint32_t cur_ts)
155 {
156  int n, i, ret, timeout_ms;
157  struct epoll_event event[2];
158  uint64_t val;
159  uint32_t cc_timeout = cc_next_ts(cur_ts),
160  util_timeout = util_timeout_next(&timeout_mgr, cur_ts),
161  timeout_us;
162 
163  if(cc_timeout != -1U && util_timeout != -1U) {
164  timeout_us = MIN(cc_timeout, util_timeout);
165  } else if(cc_timeout != -1U) {
166  timeout_us = util_timeout;
167  } else {
168  timeout_us = cc_timeout;
169  }
170  if(timeout_us != -1U) {
171  timeout_ms = timeout_us / 1000;
172  } else {
173  timeout_ms = -1;
174  }
175 
176  // Deal with load management
177  if(timeout_ms == -1 || timeout_ms > 1000) {
178  timeout_ms = 10;
179  }
180 
181 again:
182  n = epoll_wait(epfd, event, 2, timeout_ms);
183  if(n == -1 && errno == EINTR) {
184  /* To support attaching GDB */
185  goto again;
186  } else if (n == -1) {
187  perror("slowpath_block: epoll_wait failed");
188  abort();
189  }
190 
191  for(i = 0; i < n; i++) {
192  assert(event[i].data.fd == kernel_notifyfd);
193  ret = read(kernel_notifyfd, &val, sizeof(uint64_t));
194  if ((ret > 0 && ret != sizeof(uint64_t)) ||
195  (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK))
196  {
197  perror("slowpath_block: read failed");
198  abort();
199  }
200  }
201 }
202 
203 static void timeout_trigger(struct timeout *to, uint8_t type, void *opaque)
204 {
205  switch (type) {
206  case TO_ARP_REQ:
207  arp_timeout(to, type);
208  break;
209 
210  case TO_TCP_HANDSHAKE:
211  case TO_TCP_RETRANSMIT:
212  case TO_TCP_CLOSED:
213  tcp_timeout(to, type);
214  break;
215 
216  default:
217  fprintf(stderr, "Unknown timeout type: %u\n", type);
218  abort();
219  }
220 }
221 
222 static void signal_tas_ready(void)
223 {
224  uint64_t x;
225 
226  printf("TAS ready\n");
227  fflush(stdout);
228 
229  x = 1;
230  if (config.ready_fd >= 0 &&
231  write(config.ready_fd, &x, sizeof(x)) < 0)
232  {
233  perror("TAS signal: ready fd write failed");
234  /* proceeed */
235  }
236 }
uint32_t fp_autoscale
Definition: config.h:121
int tcp_init(void)
Definition: tcp.c:111
uint64_t ecn_marked
Definition: internal.h:65
unsigned kni_poll(void)
Definition: kni.c:135
unsigned cc_poll(uint32_t cur_ts)
Definition: cc.c:86
uint64_t acks
Definition: internal.h:67
void tcp_poll(void)
Definition: tcp.c:123
int nicif_init(void)
Definition: nicif.c:85
uint64_t kernel_rexmit
Definition: internal.h:63
void tcp_timeout(struct timeout *to, enum timeout_type type)
Definition: tcp.c:434
uint64_t drops
Definition: internal.h:61
uint32_t util_timeout_time_us(void)
Definition: timeout.c:72
void arp_timeout(struct timeout *to, enum timeout_type type)
Definition: arp.c:209
int cc_init(void)
Definition: cc.c:35
int util_timeout_init(struct timeout_manager *mgr, void(*handler)(struct timeout *, uint8_t, void *), void *handler_opaque)
Definition: timeout.c:62
int appif_init(void)
Definition: appif.c:100
unsigned nicif_poll(void)
Definition: nicif.c:111
void util_timeout_poll_ts(struct timeout_manager *mgr, uint32_t cur_ts)
Definition: timeout.c:84
int kni_init(void)
Definition: kni.c:67
int ready_fd
Definition: config.h:133
int arp_init(void)
Definition: arp.c:58
int routing_init(void)
Definition: routing.c:48
unsigned appif_poll(void)
Definition: appif.c:130
static int epfd
Definition: appif.c:80