34 #include <rte_config.h> 35 #include <rte_malloc.h> 36 #include <rte_cycles.h> 42 #define dprintf(...) do { } while (0) 44 #define FLAG_INSKIPLIST 1 45 #define FLAG_INNOLIMITL 2 48 #define SKIPLIST_BITS 3 50 #define IDXLIST_INVAL (-1U) 52 #define RNG_SEED 0x12345678 53 #define TIMESTAMP_BITS 32 54 #define TIMESTAMP_MASK 0xFFFFFFFF 70 } __attribute__((packed));
71 STATIC_ASSERT((
sizeof(
struct queue) == 32), queue_size);
75 static inline void set_impl(
struct qman_thread *t, uint32_t
id, uint32_t
rate,
79 static inline void queue_activate_nolimit(
struct qman_thread *t,
80 struct queue *q, uint32_t idx);
81 static inline unsigned poll_nolimit(
struct qman_thread *t, uint32_t cur_ts,
82 unsigned num,
unsigned *q_ids, uint16_t *q_bytes);
85 static inline void queue_activate_skiplist(
struct qman_thread *t,
86 struct queue *q, uint32_t idx);
87 static inline unsigned poll_skiplist(
struct qman_thread *t, uint32_t cur_ts,
88 unsigned num,
unsigned *q_ids, uint16_t *q_bytes);
89 static inline uint8_t queue_level(
struct qman_thread *t);
91 static inline void queue_fire(
struct qman_thread *t,
92 struct queue *q, uint32_t idx,
unsigned *q_id, uint16_t *q_bytes);
95 static inline uint32_t timestamp(
void);
96 static inline int timestamp_lessthaneq(
struct qman_thread *t, uint32_t a,
98 static inline int64_t rel_time(uint32_t cur_ts, uint32_t ts_in);
106 if ((t->queues = calloc(1,
sizeof(*t->queues) * FLEXNIC_NUM_QMQUEUES))
109 fprintf(stderr,
"qman_thread_init: queues malloc failed\n");
113 for (i = 0; i < QMAN_SKIPLIST_LEVELS; i++) {
114 t->head_idx[i] = IDXLIST_INVAL;
116 t->nolimit_head_idx = t->nolimit_tail_idx = IDXLIST_INVAL;
117 utils_rng_init(&t->rng, RNG_SEED * ctx->id + ctx->id);
120 t->ts_real = timestamp();
125 uint32_t qman_timestamp(uint64_t cycles)
127 static uint64_t freq = 0;
130 freq = rte_get_tsc_hz();
132 cycles *= 1000000ULL;
137 uint32_t qman_next_ts(
struct qman_thread *t, uint32_t cur_ts)
139 uint32_t ts = timestamp();
140 uint32_t ret_ts = t->ts_virtual + (ts - t->ts_real);
142 if(t->nolimit_head_idx != IDXLIST_INVAL) {
144 fprintf(stderr,
"QMan nolimit has work\n");
148 uint32_t idx = t->head_idx[0];
149 if(idx != IDXLIST_INVAL) {
150 struct queue *q = &t->queues[idx];
152 if(timestamp_lessthaneq(t, q->
next_ts, ret_ts)) {
157 return rel_time(ret_ts, q->
next_ts) / 1000;
165 int qman_poll(
struct qman_thread *t,
unsigned num,
unsigned *q_ids,
169 uint32_t ts = timestamp();
172 if (t->nolimit_first) {
173 x = poll_nolimit(t, ts, num, q_ids, q_bytes);
174 y = poll_skiplist(t, ts, num - x, q_ids + x, q_bytes + x);
176 x = poll_skiplist(t, ts, num, q_ids, q_bytes);
177 y = poll_nolimit(t, ts, num - x, q_ids + x, q_bytes + x);
179 t->nolimit_first = !t->nolimit_first;
187 #ifdef FLEXNIC_TRACE_QMAN 192 trace_event(FLEXNIC_TRACE_EV_QMSET,
sizeof(evt), &evt);
195 dprintf(
"qman_set: id=%u rate=%u avail=%u max_chunk=%u qidx=%u tid=%u\n",
196 id, rate, avail, max_chunk, qidx, tid);
198 if (
id >= FLEXNIC_NUM_QMQUEUES) {
199 fprintf(stderr,
"qman_set: invalid queue id: %u >= %u\n",
id,
200 FLEXNIC_NUM_QMQUEUES);
204 set_impl(t,
id, rate, avail, max_chunk, flags);
210 static void inline set_impl(
struct qman_thread *t, uint32_t idx, uint32_t rate,
211 uint32_t avail, uint16_t max_chunk, uint8_t flags)
213 struct queue *q = &t->queues[idx];
216 if ((flags & QMAN_SET_RATE) != 0) {
220 if ((flags & QMAN_SET_MAXCHUNK) != 0) {
224 if ((flags & QMAN_SET_AVAIL) != 0) {
227 }
else if ((flags & QMAN_ADD_AVAIL) != 0) {
232 dprintf(
"set_impl: t=%p q=%p idx=%u avail=%u rate=%u qflags=%x flags=%x\n", t, q, idx, q->
avail, q->
rate, q->
flags, flags);
234 if (new_avail && q->
avail > 0
235 && ((q->
flags & (FLAG_INSKIPLIST | FLAG_INNOLIMITL)) == 0)) {
236 queue_activate(t, q, idx);
244 static inline void queue_activate_nolimit(
struct qman_thread *t,
245 struct queue *q, uint32_t idx)
247 struct queue *q_tail;
249 assert((q->
flags & (FLAG_INSKIPLIST | FLAG_INNOLIMITL)) == 0);
251 dprintf(
"queue_activate_nolimit: t=%p q=%p avail=%u rate=%u flags=%x\n", t, q, q->
avail, q->
rate, q->
flags);
253 q->
flags |= FLAG_INNOLIMITL;
255 if (t->nolimit_tail_idx == IDXLIST_INVAL) {
256 t->nolimit_head_idx = t->nolimit_tail_idx = idx;
260 q_tail = &t->queues[t->nolimit_tail_idx];
262 t->nolimit_tail_idx = idx;
266 static inline unsigned poll_nolimit(
struct qman_thread *t, uint32_t cur_ts,
267 unsigned num,
unsigned *q_ids, uint16_t *q_bytes)
273 for (cnt = 0; cnt < num && t->nolimit_head_idx != IDXLIST_INVAL;) {
274 idx = t->nolimit_head_idx;
279 t->nolimit_tail_idx = IDXLIST_INVAL;
281 q->
flags &= ~FLAG_INNOLIMITL;
282 dprintf(
"poll_nolimit: t=%p q=%p idx=%u avail=%u rate=%u flags=%x\n", t, q, idx, q->
avail, q->
rate, q->
flags);
284 queue_fire(t, q, idx, q_ids + cnt, q_bytes + cnt);
298 return t->ts_virtual + ((uint64_t) bytes * 8 * 1000000) / q->
rate;
302 static inline void queue_activate_skiplist(
struct qman_thread *t,
303 struct queue *q, uint32_t q_idx)
307 uint32_t preds[QMAN_SKIPLIST_LEVELS];
308 uint32_t pred, idx, ts, max_ts;
310 assert((q->
flags & (FLAG_INSKIPLIST | FLAG_INNOLIMITL)) == 0);
312 dprintf(
"queue_activate_skiplist: t=%p q=%p idx=%u avail=%u rate=%u flags=%x ts_virt=%u next_ts=%u\n", t, q, q_idx, q->
avail, q->
rate, q->
flags,
320 max_ts = queue_new_ts(t, q, q->
max_chunk);
321 if (timestamp_lessthaneq(t, ts, t->ts_virtual)) {
322 ts = q->
next_ts = t->ts_virtual;
323 }
else if (!timestamp_lessthaneq(t, ts, max_ts)) {
329 pred = IDXLIST_INVAL;
330 for (l = QMAN_SKIPLIST_LEVELS - 1; l >= 0; l--) {
331 idx = (pred != IDXLIST_INVAL ? pred : t->head_idx[l]);
332 while (idx != IDXLIST_INVAL &&
333 timestamp_lessthaneq(t, t->queues[idx].
next_ts, ts))
339 dprintf(
" pred[%u] = %d\n", l, pred);
343 level = queue_level(t);
344 dprintf(
" level = %u\n", level);
347 for (l = QMAN_SKIPLIST_LEVELS - 1; l >= 0; l--) {
352 if (idx != IDXLIST_INVAL) {
357 t->head_idx[l] = q_idx;
362 q->
flags |= FLAG_INSKIPLIST;
366 static inline unsigned poll_skiplist(
struct qman_thread *t, uint32_t cur_ts,
367 unsigned num,
unsigned *q_ids, uint16_t *q_bytes)
370 uint32_t idx, max_vts;
375 max_vts = t->ts_virtual + (cur_ts - t->ts_real);
377 for (cnt = 0; cnt < num;) {
378 idx = t->head_idx[0];
381 if (idx == IDXLIST_INVAL) {
382 t->ts_virtual = max_vts;
389 dprintf(
"poll_skiplist: next_ts=%u vts=%u rts=%u max_vts=%u cur_ts=%u\n",
390 q->
next_ts, t->ts_virtual, t->ts_real, max_vts, cur_ts);
391 if (!timestamp_lessthaneq(t, q->
next_ts, max_vts)) {
392 t->ts_virtual = max_vts;
397 for (l = 0; l < QMAN_SKIPLIST_LEVELS && t->head_idx[l] == idx; l++) {
400 assert((q->
flags & FLAG_INSKIPLIST) != 0);
401 q->
flags &= ~FLAG_INSKIPLIST;
406 dprintf(
"poll_skiplist: t=%p q=%p idx=%u avail=%u rate=%u flags=%x\n", t, q, idx, q->
avail, q->
rate, q->
flags);
409 queue_fire(t, q, idx, q_ids + cnt, q_bytes + cnt);
416 idx = t->head_idx[0];
417 if (idx != IDXLIST_INVAL &&
418 timestamp_lessthaneq(t, t->queues[idx].
next_ts, max_vts))
420 t->ts_virtual = t->queues[idx].
next_ts;
422 t->ts_virtual = max_vts;
431 static inline uint8_t queue_level(
struct qman_thread *t)
433 uint8_t x = (__builtin_ffs(utils_rng_gen32(&t->rng)) - 1) / SKIPLIST_BITS;
434 return (x < QMAN_SKIPLIST_LEVELS ? x : QMAN_SKIPLIST_LEVELS - 1);
439 static inline void queue_fire(
struct qman_thread *t,
440 struct queue *q, uint32_t idx,
unsigned *q_id, uint16_t *q_bytes)
444 assert(q->
avail > 0);
449 dprintf(
"queue_fire: t=%p q=%p idx=%u gidx=%u bytes=%u avail=%u rate=%u\n", t, q, idx, idx, bytes, q->
avail, q->
rate);
451 q->
next_ts = queue_new_ts(t, q, bytes);
455 queue_activate(t, q, idx);
461 #ifdef FLEXNIC_TRACE_QMAN 463 .id = *q_id, .bytes = bytes,
465 trace_event(FLEXNIC_TRACE_EV_QMEVT,
sizeof(evt), &evt);
473 queue_activate_nolimit(t, q, idx);
475 queue_activate_skiplist(t, q, idx);
479 static inline uint32_t timestamp(
void)
481 static uint64_t freq = 0;
482 uint64_t cycles = rte_get_tsc_cycles();
485 freq = rte_get_tsc_hz();
487 cycles *= 1000000000ULL;
493 static inline int64_t rel_time(uint32_t cur_ts, uint32_t ts_in)
496 const uint64_t middle = (1ULL << (TIMESTAMP_BITS - 1));
499 if (cur_ts < middle) {
501 start = (cur_ts - middle) & TIMESTAMP_MASK;
502 end = (1ULL << TIMESTAMP_BITS);
503 if (start <= ts && ts < end) {
505 return ts - start - middle;
510 }
else if (cur_ts == middle) {
516 end = ((cur_ts + middle) & TIMESTAMP_MASK) + 1;
517 if (start <= cur_ts && ts < end) {
519 return ts + ((1ULL << TIMESTAMP_BITS) - cur_ts);
527 static inline int timestamp_lessthaneq(
struct qman_thread *t, uint32_t a,
530 return rel_time(t->ts_virtual, a) <= rel_time(t->ts_virtual, b);
uint32_t next_idxs[QMAN_SKIPLIST_LEVELS]