1 /*
2 * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch_helpers.h>
8 #include <assert.h>
9 #include <common/debug.h>
10 #include <errno.h>
11 #include <stdbool.h>
12 #include <stddef.h>
13 #include <string.h>
14
15 #include "ivc.h"
16
17 /*
18 * IVC channel reset protocol.
19 *
20 * Each end uses its tx_channel.state to indicate its synchronization state.
21 */
22 enum {
23 /*
24 * This value is zero for backwards compatibility with services that
25 * assume channels to be initially zeroed. Such channels are in an
26 * initially valid state, but cannot be asynchronously reset, and must
27 * maintain a valid state at all times.
28 *
29 * The transmitting end can enter the established state from the sync or
30 * ack state when it observes the receiving endpoint in the ack or
31 * established state, indicating that has cleared the counters in our
32 * rx_channel.
33 */
34 ivc_state_established = U(0),
35
36 /*
37 * If an endpoint is observed in the sync state, the remote endpoint is
38 * allowed to clear the counters it owns asynchronously with respect to
39 * the current endpoint. Therefore, the current endpoint is no longer
40 * allowed to communicate.
41 */
42 ivc_state_sync = U(1),
43
44 /*
45 * When the transmitting end observes the receiving end in the sync
46 * state, it can clear the w_count and r_count and transition to the ack
47 * state. If the remote endpoint observes us in the ack state, it can
48 * return to the established state once it has cleared its counters.
49 */
50 ivc_state_ack = U(2)
51 };
52
53 /*
54 * This structure is divided into two-cache aligned parts, the first is only
55 * written through the tx_channel pointer, while the second is only written
56 * through the rx_channel pointer. This delineates ownership of the cache lines,
57 * which is critical to performance and necessary in non-cache coherent
58 * implementations.
59 */
60 struct ivc_channel_header {
61 struct {
62 /* fields owned by the transmitting end */
63 uint32_t w_count;
64 uint32_t state;
65 uint32_t w_rsvd[IVC_CHHDR_TX_FIELDS - 2];
66 };
67 struct {
68 /* fields owned by the receiving end */
69 uint32_t r_count;
70 uint32_t r_rsvd[IVC_CHHDR_RX_FIELDS - 1];
71 };
72 };
73
ivc_channel_empty(const struct ivc * ivc,volatile const struct ivc_channel_header * ch)74 static inline bool ivc_channel_empty(const struct ivc *ivc,
75 volatile const struct ivc_channel_header *ch)
76 {
77 /*
78 * This function performs multiple checks on the same values with
79 * security implications, so sample the counters' current values in
80 * shared memory to ensure that these checks use the same values.
81 */
82 uint32_t wr_count = ch->w_count;
83 uint32_t rd_count = ch->r_count;
84 bool ret = false;
85
86 (void)ivc;
87
88 /*
89 * Perform an over-full check to prevent denial of service attacks where
90 * a server could be easily fooled into believing that there's an
91 * extremely large number of frames ready, since receivers are not
92 * expected to check for full or over-full conditions.
93 *
94 * Although the channel isn't empty, this is an invalid case caused by
95 * a potentially malicious peer, so returning empty is safer, because it
96 * gives the impression that the channel has gone silent.
97 */
98 if (((wr_count - rd_count) > ivc->nframes) || (wr_count == rd_count)) {
99 ret = true;
100 }
101
102 return ret;
103 }
104
ivc_channel_full(const struct ivc * ivc,volatile const struct ivc_channel_header * ch)105 static inline bool ivc_channel_full(const struct ivc *ivc,
106 volatile const struct ivc_channel_header *ch)
107 {
108 uint32_t wr_count = ch->w_count;
109 uint32_t rd_count = ch->r_count;
110
111 (void)ivc;
112
113 /*
114 * Invalid cases where the counters indicate that the queue is over
115 * capacity also appear full.
116 */
117 return ((wr_count - rd_count) >= ivc->nframes);
118 }
119
ivc_channel_avail_count(const struct ivc * ivc,volatile const struct ivc_channel_header * ch)120 static inline uint32_t ivc_channel_avail_count(const struct ivc *ivc,
121 volatile const struct ivc_channel_header *ch)
122 {
123 uint32_t wr_count = ch->w_count;
124 uint32_t rd_count = ch->r_count;
125
126 (void)ivc;
127
128 /*
129 * This function isn't expected to be used in scenarios where an
130 * over-full situation can lead to denial of service attacks. See the
131 * comment in ivc_channel_empty() for an explanation about special
132 * over-full considerations.
133 */
134 return (wr_count - rd_count);
135 }
136
ivc_advance_tx(struct ivc * ivc)137 static inline void ivc_advance_tx(struct ivc *ivc)
138 {
139 ivc->tx_channel->w_count++;
140
141 if (ivc->w_pos == (ivc->nframes - (uint32_t)1U)) {
142 ivc->w_pos = 0U;
143 } else {
144 ivc->w_pos++;
145 }
146 }
147
ivc_advance_rx(struct ivc * ivc)148 static inline void ivc_advance_rx(struct ivc *ivc)
149 {
150 ivc->rx_channel->r_count++;
151
152 if (ivc->r_pos == (ivc->nframes - (uint32_t)1U)) {
153 ivc->r_pos = 0U;
154 } else {
155 ivc->r_pos++;
156 }
157 }
158
ivc_check_read(const struct ivc * ivc)159 static inline int32_t ivc_check_read(const struct ivc *ivc)
160 {
161 /*
162 * tx_channel->state is set locally, so it is not synchronized with
163 * state from the remote peer. The remote peer cannot reset its
164 * transmit counters until we've acknowledged its synchronization
165 * request, so no additional synchronization is required because an
166 * asynchronous transition of rx_channel->state to ivc_state_ack is not
167 * allowed.
168 */
169 if (ivc->tx_channel->state != ivc_state_established) {
170 return -ECONNRESET;
171 }
172
173 /*
174 * Avoid unnecessary invalidations when performing repeated accesses to
175 * an IVC channel by checking the old queue pointers first.
176 * Synchronization is only necessary when these pointers indicate empty
177 * or full.
178 */
179 if (!ivc_channel_empty(ivc, ivc->rx_channel)) {
180 return 0;
181 }
182
183 return ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
184 }
185
ivc_check_write(const struct ivc * ivc)186 static inline int32_t ivc_check_write(const struct ivc *ivc)
187 {
188 if (ivc->tx_channel->state != ivc_state_established) {
189 return -ECONNRESET;
190 }
191
192 if (!ivc_channel_full(ivc, ivc->tx_channel)) {
193 return 0;
194 }
195
196 return ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
197 }
198
tegra_ivc_can_read(const struct ivc * ivc)199 bool tegra_ivc_can_read(const struct ivc *ivc)
200 {
201 return ivc_check_read(ivc) == 0;
202 }
203
tegra_ivc_can_write(const struct ivc * ivc)204 bool tegra_ivc_can_write(const struct ivc *ivc)
205 {
206 return ivc_check_write(ivc) == 0;
207 }
208
tegra_ivc_tx_empty(const struct ivc * ivc)209 bool tegra_ivc_tx_empty(const struct ivc *ivc)
210 {
211 return ivc_channel_empty(ivc, ivc->tx_channel);
212 }
213
calc_frame_offset(uint32_t frame_index,uint32_t frame_size,uint32_t frame_offset)214 static inline uintptr_t calc_frame_offset(uint32_t frame_index,
215 uint32_t frame_size, uint32_t frame_offset)
216 {
217 return ((uintptr_t)frame_index * (uintptr_t)frame_size) +
218 (uintptr_t)frame_offset;
219 }
220
ivc_frame_pointer(const struct ivc * ivc,volatile const struct ivc_channel_header * ch,uint32_t frame)221 static void *ivc_frame_pointer(const struct ivc *ivc,
222 volatile const struct ivc_channel_header *ch,
223 uint32_t frame)
224 {
225 assert(frame < ivc->nframes);
226 return (void *)((uintptr_t)(&ch[1]) +
227 calc_frame_offset(frame, ivc->frame_size, 0));
228 }
229
tegra_ivc_read(struct ivc * ivc,void * buf,size_t max_read)230 int32_t tegra_ivc_read(struct ivc *ivc, void *buf, size_t max_read)
231 {
232 const void *src;
233 int32_t result;
234
235 if (buf == NULL) {
236 return -EINVAL;
237 }
238
239 if (max_read > ivc->frame_size) {
240 return -E2BIG;
241 }
242
243 result = ivc_check_read(ivc);
244 if (result != 0) {
245 return result;
246 }
247
248 /*
249 * Order observation of w_pos potentially indicating new data before
250 * data read.
251 */
252 dmbish();
253
254 src = ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
255
256 (void)memcpy(buf, src, max_read);
257
258 ivc_advance_rx(ivc);
259
260 /*
261 * Ensure our write to r_pos occurs before our read from w_pos.
262 */
263 dmbish();
264
265 /*
266 * Notify only upon transition from full to non-full.
267 * The available count can only asynchronously increase, so the
268 * worst possible side-effect will be a spurious notification.
269 */
270 if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) {
271 ivc->notify(ivc);
272 }
273
274 return (int32_t)max_read;
275 }
276
277 /* directly peek at the next frame rx'ed */
tegra_ivc_read_get_next_frame(const struct ivc * ivc)278 void *tegra_ivc_read_get_next_frame(const struct ivc *ivc)
279 {
280 if (ivc_check_read(ivc) != 0) {
281 return NULL;
282 }
283
284 /*
285 * Order observation of w_pos potentially indicating new data before
286 * data read.
287 */
288 dmbld();
289
290 return ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
291 }
292
tegra_ivc_read_advance(struct ivc * ivc)293 int32_t tegra_ivc_read_advance(struct ivc *ivc)
294 {
295 /*
296 * No read barriers or synchronization here: the caller is expected to
297 * have already observed the channel non-empty. This check is just to
298 * catch programming errors.
299 */
300 int32_t result = ivc_check_read(ivc);
301 if (result != 0) {
302 return result;
303 }
304
305 ivc_advance_rx(ivc);
306
307 /*
308 * Ensure our write to r_pos occurs before our read from w_pos.
309 */
310 dmbish();
311
312 /*
313 * Notify only upon transition from full to non-full.
314 * The available count can only asynchronously increase, so the
315 * worst possible side-effect will be a spurious notification.
316 */
317 if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) {
318 ivc->notify(ivc);
319 }
320
321 return 0;
322 }
323
tegra_ivc_write(struct ivc * ivc,const void * buf,size_t size)324 int32_t tegra_ivc_write(struct ivc *ivc, const void *buf, size_t size)
325 {
326 void *p;
327 int32_t result;
328
329 if ((buf == NULL) || (ivc == NULL)) {
330 return -EINVAL;
331 }
332
333 if (size > ivc->frame_size) {
334 return -E2BIG;
335 }
336
337 result = ivc_check_write(ivc);
338 if (result != 0) {
339 return result;
340 }
341
342 p = ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
343
344 (void)memset(p, 0, ivc->frame_size);
345 (void)memcpy(p, buf, size);
346
347 /*
348 * Ensure that updated data is visible before the w_pos counter
349 * indicates that it is ready.
350 */
351 dmbst();
352
353 ivc_advance_tx(ivc);
354
355 /*
356 * Ensure our write to w_pos occurs before our read from r_pos.
357 */
358 dmbish();
359
360 /*
361 * Notify only upon transition from empty to non-empty.
362 * The available count can only asynchronously decrease, so the
363 * worst possible side-effect will be a spurious notification.
364 */
365 if (ivc_channel_avail_count(ivc, ivc->tx_channel) == 1U) {
366 ivc->notify(ivc);
367 }
368
369 return (int32_t)size;
370 }
371
372 /* directly poke at the next frame to be tx'ed */
tegra_ivc_write_get_next_frame(const struct ivc * ivc)373 void *tegra_ivc_write_get_next_frame(const struct ivc *ivc)
374 {
375 if (ivc_check_write(ivc) != 0) {
376 return NULL;
377 }
378
379 return ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
380 }
381
382 /* advance the tx buffer */
tegra_ivc_write_advance(struct ivc * ivc)383 int32_t tegra_ivc_write_advance(struct ivc *ivc)
384 {
385 int32_t result = ivc_check_write(ivc);
386
387 if (result != 0) {
388 return result;
389 }
390
391 /*
392 * Order any possible stores to the frame before update of w_pos.
393 */
394 dmbst();
395
396 ivc_advance_tx(ivc);
397
398 /*
399 * Ensure our write to w_pos occurs before our read from r_pos.
400 */
401 dmbish();
402
403 /*
404 * Notify only upon transition from empty to non-empty.
405 * The available count can only asynchronously decrease, so the
406 * worst possible side-effect will be a spurious notification.
407 */
408 if (ivc_channel_avail_count(ivc, ivc->tx_channel) == (uint32_t)1U) {
409 ivc->notify(ivc);
410 }
411
412 return 0;
413 }
414
tegra_ivc_channel_reset(const struct ivc * ivc)415 void tegra_ivc_channel_reset(const struct ivc *ivc)
416 {
417 ivc->tx_channel->state = ivc_state_sync;
418 ivc->notify(ivc);
419 }
420
421 /*
422 * ===============================================================
423 * IVC State Transition Table - see tegra_ivc_channel_notified()
424 * ===============================================================
425 *
426 * local remote action
427 * ----- ------ -----------------------------------
428 * SYNC EST <none>
429 * SYNC ACK reset counters; move to EST; notify
430 * SYNC SYNC reset counters; move to ACK; notify
431 * ACK EST move to EST; notify
432 * ACK ACK move to EST; notify
433 * ACK SYNC reset counters; move to ACK; notify
434 * EST EST <none>
435 * EST ACK <none>
436 * EST SYNC reset counters; move to ACK; notify
437 *
438 * ===============================================================
439 */
tegra_ivc_channel_notified(struct ivc * ivc)440 int32_t tegra_ivc_channel_notified(struct ivc *ivc)
441 {
442 uint32_t peer_state;
443
444 /* Copy the receiver's state out of shared memory. */
445 peer_state = ivc->rx_channel->state;
446
447 if (peer_state == (uint32_t)ivc_state_sync) {
448 /*
449 * Order observation of ivc_state_sync before stores clearing
450 * tx_channel.
451 */
452 dmbld();
453
454 /*
455 * Reset tx_channel counters. The remote end is in the SYNC
456 * state and won't make progress until we change our state,
457 * so the counters are not in use at this time.
458 */
459 ivc->tx_channel->w_count = 0U;
460 ivc->rx_channel->r_count = 0U;
461
462 ivc->w_pos = 0U;
463 ivc->r_pos = 0U;
464
465 /*
466 * Ensure that counters appear cleared before new state can be
467 * observed.
468 */
469 dmbst();
470
471 /*
472 * Move to ACK state. We have just cleared our counters, so it
473 * is now safe for the remote end to start using these values.
474 */
475 ivc->tx_channel->state = ivc_state_ack;
476
477 /*
478 * Notify remote end to observe state transition.
479 */
480 ivc->notify(ivc);
481
482 } else if ((ivc->tx_channel->state == (uint32_t)ivc_state_sync) &&
483 (peer_state == (uint32_t)ivc_state_ack)) {
484 /*
485 * Order observation of ivc_state_sync before stores clearing
486 * tx_channel.
487 */
488 dmbld();
489
490 /*
491 * Reset tx_channel counters. The remote end is in the ACK
492 * state and won't make progress until we change our state,
493 * so the counters are not in use at this time.
494 */
495 ivc->tx_channel->w_count = 0U;
496 ivc->rx_channel->r_count = 0U;
497
498 ivc->w_pos = 0U;
499 ivc->r_pos = 0U;
500
501 /*
502 * Ensure that counters appear cleared before new state can be
503 * observed.
504 */
505 dmbst();
506
507 /*
508 * Move to ESTABLISHED state. We know that the remote end has
509 * already cleared its counters, so it is safe to start
510 * writing/reading on this channel.
511 */
512 ivc->tx_channel->state = ivc_state_established;
513
514 /*
515 * Notify remote end to observe state transition.
516 */
517 ivc->notify(ivc);
518
519 } else if (ivc->tx_channel->state == (uint32_t)ivc_state_ack) {
520 /*
521 * At this point, we have observed the peer to be in either
522 * the ACK or ESTABLISHED state. Next, order observation of
523 * peer state before storing to tx_channel.
524 */
525 dmbld();
526
527 /*
528 * Move to ESTABLISHED state. We know that we have previously
529 * cleared our counters, and we know that the remote end has
530 * cleared its counters, so it is safe to start writing/reading
531 * on this channel.
532 */
533 ivc->tx_channel->state = ivc_state_established;
534
535 /*
536 * Notify remote end to observe state transition.
537 */
538 ivc->notify(ivc);
539
540 } else {
541 /*
542 * There is no need to handle any further action. Either the
543 * channel is already fully established, or we are waiting for
544 * the remote end to catch up with our current state. Refer
545 * to the diagram in "IVC State Transition Table" above.
546 */
547 }
548
549 return ((ivc->tx_channel->state == (uint32_t)ivc_state_established) ? 0 : -EAGAIN);
550 }
551
tegra_ivc_align(size_t size)552 size_t tegra_ivc_align(size_t size)
553 {
554 return (size + (IVC_ALIGN - 1U)) & ~(IVC_ALIGN - 1U);
555 }
556
tegra_ivc_total_queue_size(size_t queue_size)557 size_t tegra_ivc_total_queue_size(size_t queue_size)
558 {
559 if ((queue_size & (IVC_ALIGN - 1U)) != 0U) {
560 ERROR("queue_size (%d) must be %d-byte aligned\n",
561 (int32_t)queue_size, IVC_ALIGN);
562 return 0;
563 }
564 return queue_size + sizeof(struct ivc_channel_header);
565 }
566
check_ivc_params(uintptr_t queue_base1,uintptr_t queue_base2,uint32_t nframes,uint32_t frame_size)567 static int32_t check_ivc_params(uintptr_t queue_base1, uintptr_t queue_base2,
568 uint32_t nframes, uint32_t frame_size)
569 {
570 assert((offsetof(struct ivc_channel_header, w_count)
571 & (IVC_ALIGN - 1U)) == 0U);
572 assert((offsetof(struct ivc_channel_header, r_count)
573 & (IVC_ALIGN - 1U)) == 0U);
574 assert((sizeof(struct ivc_channel_header) & (IVC_ALIGN - 1U)) == 0U);
575
576 if (((uint64_t)nframes * (uint64_t)frame_size) >= 0x100000000ULL) {
577 ERROR("nframes * frame_size overflows\n");
578 return -EINVAL;
579 }
580
581 /*
582 * The headers must at least be aligned enough for counters
583 * to be accessed atomically.
584 */
585 if ((queue_base1 & (IVC_ALIGN - 1U)) != 0U) {
586 ERROR("ivc channel start not aligned: %lx\n", queue_base1);
587 return -EINVAL;
588 }
589 if ((queue_base2 & (IVC_ALIGN - 1U)) != 0U) {
590 ERROR("ivc channel start not aligned: %lx\n", queue_base2);
591 return -EINVAL;
592 }
593
594 if ((frame_size & (IVC_ALIGN - 1U)) != 0U) {
595 ERROR("frame size not adequately aligned: %u\n",
596 frame_size);
597 return -EINVAL;
598 }
599
600 if (queue_base1 < queue_base2) {
601 if ((queue_base1 + ((uint64_t)frame_size * nframes)) > queue_base2) {
602 ERROR("queue regions overlap: %lx + %x, %x\n",
603 queue_base1, frame_size,
604 frame_size * nframes);
605 return -EINVAL;
606 }
607 } else {
608 if ((queue_base2 + ((uint64_t)frame_size * nframes)) > queue_base1) {
609 ERROR("queue regions overlap: %lx + %x, %x\n",
610 queue_base2, frame_size,
611 frame_size * nframes);
612 return -EINVAL;
613 }
614 }
615
616 return 0;
617 }
618
tegra_ivc_init(struct ivc * ivc,uintptr_t rx_base,uintptr_t tx_base,uint32_t nframes,uint32_t frame_size,ivc_notify_function notify)619 int32_t tegra_ivc_init(struct ivc *ivc, uintptr_t rx_base, uintptr_t tx_base,
620 uint32_t nframes, uint32_t frame_size,
621 ivc_notify_function notify)
622 {
623 int32_t result;
624
625 /* sanity check input params */
626 if ((ivc == NULL) || (notify == NULL)) {
627 return -EINVAL;
628 }
629
630 result = check_ivc_params(rx_base, tx_base, nframes, frame_size);
631 if (result != 0) {
632 return result;
633 }
634
635 /*
636 * All sizes that can be returned by communication functions should
637 * fit in a 32-bit integer.
638 */
639 if (frame_size > (1u << 31)) {
640 return -E2BIG;
641 }
642
643 ivc->rx_channel = (struct ivc_channel_header *)rx_base;
644 ivc->tx_channel = (struct ivc_channel_header *)tx_base;
645 ivc->notify = notify;
646 ivc->frame_size = frame_size;
647 ivc->nframes = nframes;
648 ivc->w_pos = 0U;
649 ivc->r_pos = 0U;
650
651 INFO("%s: done\n", __func__);
652
653 return 0;
654 }
655