1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
4 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
5 */
6
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/sched/signal.h>
10 #include <linux/types.h>
11 #include <linux/errno.h>
12 #include <linux/cdev.h>
13 #include <linux/fs.h>
14 #include <linux/device.h>
15 #include <linux/mm.h>
16 #include <linux/highmem.h>
17 #include <linux/pagemap.h>
18 #include <linux/bug.h>
19 #include <linux/completion.h>
20 #include <linux/list.h>
21 #include <linux/of.h>
22 #include <linux/platform_device.h>
23 #include <linux/compat.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/rcupdate.h>
26 #include <linux/delay.h>
27 #include <linux/slab.h>
28 #include <linux/interrupt.h>
29 #include <linux/io.h>
30 #include <linux/uaccess.h>
31 #include <soc/bcm2835/raspberrypi-firmware.h>
32
33 #include "vchiq_core.h"
34 #include "vchiq_ioctl.h"
35 #include "vchiq_arm.h"
36 #include "vchiq_debugfs.h"
37 #include "vchiq_connected.h"
38 #include "vchiq_pagelist.h"
39
40 #define DEVICE_NAME "vchiq"
41
42 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
43
44 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
45
46 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
47 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
48
49 #define BELL0 0x00
50 #define BELL2 0x08
51
52 #define ARM_DS_ACTIVE BIT(2)
53
54 /* Override the default prefix, which would be vchiq_arm (from the filename) */
55 #undef MODULE_PARAM_PREFIX
56 #define MODULE_PARAM_PREFIX DEVICE_NAME "."
57
58 #define KEEPALIVE_VER 1
59 #define KEEPALIVE_VER_MIN KEEPALIVE_VER
60
61 /* Run time control of log level, based on KERN_XXX level. */
62 int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT;
63 int vchiq_susp_log_level = VCHIQ_LOG_ERROR;
64
65 DEFINE_SPINLOCK(msg_queue_spinlock);
66 struct vchiq_state g_state;
67
68 static struct platform_device *bcm2835_camera;
69 static struct platform_device *bcm2835_audio;
70
71 struct vchiq_drvdata {
72 const unsigned int cache_line_size;
73 struct rpi_firmware *fw;
74 };
75
76 static struct vchiq_drvdata bcm2835_drvdata = {
77 .cache_line_size = 32,
78 };
79
80 static struct vchiq_drvdata bcm2836_drvdata = {
81 .cache_line_size = 64,
82 };
83
84 struct vchiq_arm_state {
85 /* Keepalive-related data */
86 struct task_struct *ka_thread;
87 struct completion ka_evt;
88 atomic_t ka_use_count;
89 atomic_t ka_use_ack_count;
90 atomic_t ka_release_count;
91
92 rwlock_t susp_res_lock;
93
94 struct vchiq_state *state;
95
96 /*
97 * Global use count for videocore.
98 * This is equal to the sum of the use counts for all services. When
99 * this hits zero the videocore suspend procedure will be initiated.
100 */
101 int videocore_use_count;
102
103 /*
104 * Use count to track requests from videocore peer.
105 * This use count is not associated with a service, so needs to be
106 * tracked separately with the state.
107 */
108 int peer_use_count;
109
110 /*
111 * Flag to indicate that the first vchiq connect has made it through.
112 * This means that both sides should be fully ready, and we should
113 * be able to suspend after this point.
114 */
115 int first_connect;
116 };
117
118 struct vchiq_2835_state {
119 int inited;
120 struct vchiq_arm_state arm_state;
121 };
122
123 struct vchiq_pagelist_info {
124 struct pagelist *pagelist;
125 size_t pagelist_buffer_size;
126 dma_addr_t dma_addr;
127 enum dma_data_direction dma_dir;
128 unsigned int num_pages;
129 unsigned int pages_need_release;
130 struct page **pages;
131 struct scatterlist *scatterlist;
132 unsigned int scatterlist_mapped;
133 };
134
135 static void __iomem *g_regs;
136 /* This value is the size of the L2 cache lines as understood by the
137 * VPU firmware, which determines the required alignment of the
138 * offsets/sizes in pagelists.
139 *
140 * Modern VPU firmware looks for a DT "cache-line-size" property in
141 * the VCHIQ node and will overwrite it with the actual L2 cache size,
142 * which the kernel must then respect. That property was rejected
143 * upstream, so we have to use the VPU firmware's compatibility value
144 * of 32.
145 */
146 static unsigned int g_cache_line_size = 32;
147 static unsigned int g_fragments_size;
148 static char *g_fragments_base;
149 static char *g_free_fragments;
150 static struct semaphore g_free_fragments_sema;
151 static struct device *g_dev;
152
153 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
154
155 static enum vchiq_status
156 vchiq_blocking_bulk_transfer(unsigned int handle, void *data,
157 unsigned int size, enum vchiq_bulk_dir dir);
158
159 static irqreturn_t
vchiq_doorbell_irq(int irq,void * dev_id)160 vchiq_doorbell_irq(int irq, void *dev_id)
161 {
162 struct vchiq_state *state = dev_id;
163 irqreturn_t ret = IRQ_NONE;
164 unsigned int status;
165
166 /* Read (and clear) the doorbell */
167 status = readl(g_regs + BELL0);
168
169 if (status & ARM_DS_ACTIVE) { /* Was the doorbell rung? */
170 remote_event_pollall(state);
171 ret = IRQ_HANDLED;
172 }
173
174 return ret;
175 }
176
177 static void
cleanup_pagelistinfo(struct vchiq_pagelist_info * pagelistinfo)178 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
179 {
180 if (pagelistinfo->scatterlist_mapped) {
181 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
182 pagelistinfo->num_pages, pagelistinfo->dma_dir);
183 }
184
185 if (pagelistinfo->pages_need_release)
186 unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages);
187
188 dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
189 pagelistinfo->pagelist, pagelistinfo->dma_addr);
190 }
191
192 /* There is a potential problem with partial cache lines (pages?)
193 * at the ends of the block when reading. If the CPU accessed anything in
194 * the same line (page?) then it may have pulled old data into the cache,
195 * obscuring the new data underneath. We can solve this by transferring the
196 * partial cache lines separately, and allowing the ARM to copy into the
197 * cached area.
198 */
199
200 static struct vchiq_pagelist_info *
create_pagelist(char * buf,char __user * ubuf,size_t count,unsigned short type)201 create_pagelist(char *buf, char __user *ubuf,
202 size_t count, unsigned short type)
203 {
204 struct pagelist *pagelist;
205 struct vchiq_pagelist_info *pagelistinfo;
206 struct page **pages;
207 u32 *addrs;
208 unsigned int num_pages, offset, i, k;
209 int actual_pages;
210 size_t pagelist_size;
211 struct scatterlist *scatterlist, *sg;
212 int dma_buffers;
213 dma_addr_t dma_addr;
214
215 if (count >= INT_MAX - PAGE_SIZE)
216 return NULL;
217
218 if (buf)
219 offset = (uintptr_t)buf & (PAGE_SIZE - 1);
220 else
221 offset = (uintptr_t)ubuf & (PAGE_SIZE - 1);
222 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
223
224 if ((size_t)num_pages > (SIZE_MAX - sizeof(struct pagelist) -
225 sizeof(struct vchiq_pagelist_info)) /
226 (sizeof(u32) + sizeof(pages[0]) +
227 sizeof(struct scatterlist)))
228 return NULL;
229
230 pagelist_size = sizeof(struct pagelist) +
231 (num_pages * sizeof(u32)) +
232 (num_pages * sizeof(pages[0]) +
233 (num_pages * sizeof(struct scatterlist))) +
234 sizeof(struct vchiq_pagelist_info);
235
236 /* Allocate enough storage to hold the page pointers and the page
237 * list
238 */
239 pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
240 GFP_KERNEL);
241
242 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
243
244 if (!pagelist)
245 return NULL;
246
247 addrs = pagelist->addrs;
248 pages = (struct page **)(addrs + num_pages);
249 scatterlist = (struct scatterlist *)(pages + num_pages);
250 pagelistinfo = (struct vchiq_pagelist_info *)
251 (scatterlist + num_pages);
252
253 pagelist->length = count;
254 pagelist->type = type;
255 pagelist->offset = offset;
256
257 /* Populate the fields of the pagelistinfo structure */
258 pagelistinfo->pagelist = pagelist;
259 pagelistinfo->pagelist_buffer_size = pagelist_size;
260 pagelistinfo->dma_addr = dma_addr;
261 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ?
262 DMA_TO_DEVICE : DMA_FROM_DEVICE;
263 pagelistinfo->num_pages = num_pages;
264 pagelistinfo->pages_need_release = 0;
265 pagelistinfo->pages = pages;
266 pagelistinfo->scatterlist = scatterlist;
267 pagelistinfo->scatterlist_mapped = 0;
268
269 if (buf) {
270 unsigned long length = count;
271 unsigned int off = offset;
272
273 for (actual_pages = 0; actual_pages < num_pages;
274 actual_pages++) {
275 struct page *pg =
276 vmalloc_to_page((buf +
277 (actual_pages * PAGE_SIZE)));
278 size_t bytes = PAGE_SIZE - off;
279
280 if (!pg) {
281 cleanup_pagelistinfo(pagelistinfo);
282 return NULL;
283 }
284
285 if (bytes > length)
286 bytes = length;
287 pages[actual_pages] = pg;
288 length -= bytes;
289 off = 0;
290 }
291 /* do not try and release vmalloc pages */
292 } else {
293 actual_pages = pin_user_pages_fast((unsigned long)ubuf & PAGE_MASK, num_pages,
294 type == PAGELIST_READ, pages);
295
296 if (actual_pages != num_pages) {
297 vchiq_log_info(vchiq_arm_log_level,
298 "%s - only %d/%d pages locked",
299 __func__, actual_pages, num_pages);
300
301 /* This is probably due to the process being killed */
302 if (actual_pages > 0)
303 unpin_user_pages(pages, actual_pages);
304 cleanup_pagelistinfo(pagelistinfo);
305 return NULL;
306 }
307 /* release user pages */
308 pagelistinfo->pages_need_release = 1;
309 }
310
311 /*
312 * Initialize the scatterlist so that the magic cookie
313 * is filled if debugging is enabled
314 */
315 sg_init_table(scatterlist, num_pages);
316 /* Now set the pages for each scatterlist */
317 for (i = 0; i < num_pages; i++) {
318 unsigned int len = PAGE_SIZE - offset;
319
320 if (len > count)
321 len = count;
322 sg_set_page(scatterlist + i, pages[i], len, offset);
323 offset = 0;
324 count -= len;
325 }
326
327 dma_buffers = dma_map_sg(g_dev,
328 scatterlist,
329 num_pages,
330 pagelistinfo->dma_dir);
331
332 if (dma_buffers == 0) {
333 cleanup_pagelistinfo(pagelistinfo);
334 return NULL;
335 }
336
337 pagelistinfo->scatterlist_mapped = 1;
338
339 /* Combine adjacent blocks for performance */
340 k = 0;
341 for_each_sg(scatterlist, sg, dma_buffers, i) {
342 u32 len = sg_dma_len(sg);
343 u32 addr = sg_dma_address(sg);
344
345 /* Note: addrs is the address + page_count - 1
346 * The firmware expects blocks after the first to be page-
347 * aligned and a multiple of the page size
348 */
349 WARN_ON(len == 0);
350 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
351 WARN_ON(i && (addr & ~PAGE_MASK));
352 if (k > 0 &&
353 ((addrs[k - 1] & PAGE_MASK) +
354 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
355 == (addr & PAGE_MASK))
356 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
357 else
358 addrs[k++] = (addr & PAGE_MASK) |
359 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
360 }
361
362 /* Partial cache lines (fragments) require special measures */
363 if ((type == PAGELIST_READ) &&
364 ((pagelist->offset & (g_cache_line_size - 1)) ||
365 ((pagelist->offset + pagelist->length) &
366 (g_cache_line_size - 1)))) {
367 char *fragments;
368
369 if (down_interruptible(&g_free_fragments_sema)) {
370 cleanup_pagelistinfo(pagelistinfo);
371 return NULL;
372 }
373
374 WARN_ON(!g_free_fragments);
375
376 down(&g_free_fragments_mutex);
377 fragments = g_free_fragments;
378 WARN_ON(!fragments);
379 g_free_fragments = *(char **)g_free_fragments;
380 up(&g_free_fragments_mutex);
381 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
382 (fragments - g_fragments_base) / g_fragments_size;
383 }
384
385 return pagelistinfo;
386 }
387
388 static void
free_pagelist(struct vchiq_pagelist_info * pagelistinfo,int actual)389 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
390 int actual)
391 {
392 struct pagelist *pagelist = pagelistinfo->pagelist;
393 struct page **pages = pagelistinfo->pages;
394 unsigned int num_pages = pagelistinfo->num_pages;
395
396 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
397 __func__, pagelistinfo->pagelist, actual);
398
399 /*
400 * NOTE: dma_unmap_sg must be called before the
401 * cpu can touch any of the data/pages.
402 */
403 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
404 pagelistinfo->num_pages, pagelistinfo->dma_dir);
405 pagelistinfo->scatterlist_mapped = 0;
406
407 /* Deal with any partial cache lines (fragments) */
408 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
409 char *fragments = g_fragments_base +
410 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
411 g_fragments_size;
412 int head_bytes, tail_bytes;
413
414 head_bytes = (g_cache_line_size - pagelist->offset) &
415 (g_cache_line_size - 1);
416 tail_bytes = (pagelist->offset + actual) &
417 (g_cache_line_size - 1);
418
419 if ((actual >= 0) && (head_bytes != 0)) {
420 if (head_bytes > actual)
421 head_bytes = actual;
422
423 memcpy((char *)kmap(pages[0]) +
424 pagelist->offset,
425 fragments,
426 head_bytes);
427 kunmap(pages[0]);
428 }
429 if ((actual >= 0) && (head_bytes < actual) &&
430 (tail_bytes != 0)) {
431 memcpy((char *)kmap(pages[num_pages - 1]) +
432 ((pagelist->offset + actual) &
433 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
434 fragments + g_cache_line_size,
435 tail_bytes);
436 kunmap(pages[num_pages - 1]);
437 }
438
439 down(&g_free_fragments_mutex);
440 *(char **)fragments = g_free_fragments;
441 g_free_fragments = fragments;
442 up(&g_free_fragments_mutex);
443 up(&g_free_fragments_sema);
444 }
445
446 /* Need to mark all the pages dirty. */
447 if (pagelist->type != PAGELIST_WRITE &&
448 pagelistinfo->pages_need_release) {
449 unsigned int i;
450
451 for (i = 0; i < num_pages; i++)
452 set_page_dirty(pages[i]);
453 }
454
455 cleanup_pagelistinfo(pagelistinfo);
456 }
457
vchiq_platform_init(struct platform_device * pdev,struct vchiq_state * state)458 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
459 {
460 struct device *dev = &pdev->dev;
461 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
462 struct rpi_firmware *fw = drvdata->fw;
463 struct vchiq_slot_zero *vchiq_slot_zero;
464 void *slot_mem;
465 dma_addr_t slot_phys;
466 u32 channelbase;
467 int slot_mem_size, frag_mem_size;
468 int err, irq, i;
469
470 /*
471 * VCHI messages between the CPU and firmware use
472 * 32-bit bus addresses.
473 */
474 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
475
476 if (err < 0)
477 return err;
478
479 g_cache_line_size = drvdata->cache_line_size;
480 g_fragments_size = 2 * g_cache_line_size;
481
482 /* Allocate space for the channels in coherent memory */
483 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
484 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
485
486 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
487 &slot_phys, GFP_KERNEL);
488 if (!slot_mem) {
489 dev_err(dev, "could not allocate DMA memory\n");
490 return -ENOMEM;
491 }
492
493 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
494
495 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
496 if (!vchiq_slot_zero)
497 return -EINVAL;
498
499 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
500 (int)slot_phys + slot_mem_size;
501 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
502 MAX_FRAGMENTS;
503
504 g_fragments_base = (char *)slot_mem + slot_mem_size;
505
506 g_free_fragments = g_fragments_base;
507 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
508 *(char **)&g_fragments_base[i * g_fragments_size] =
509 &g_fragments_base[(i + 1) * g_fragments_size];
510 }
511 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
512 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
513
514 err = vchiq_init_state(state, vchiq_slot_zero);
515 if (err)
516 return err;
517
518 g_regs = devm_platform_ioremap_resource(pdev, 0);
519 if (IS_ERR(g_regs))
520 return PTR_ERR(g_regs);
521
522 irq = platform_get_irq(pdev, 0);
523 if (irq <= 0)
524 return irq;
525
526 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
527 "VCHIQ doorbell", state);
528 if (err) {
529 dev_err(dev, "failed to register irq=%d\n", irq);
530 return err;
531 }
532
533 /* Send the base address of the slots to VideoCore */
534 channelbase = slot_phys;
535 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
536 &channelbase, sizeof(channelbase));
537 if (err || channelbase) {
538 dev_err(dev, "failed to set channelbase\n");
539 return err ? : -ENXIO;
540 }
541
542 g_dev = dev;
543 vchiq_log_info(vchiq_arm_log_level, "vchiq_init - done (slots %pK, phys %pad)",
544 vchiq_slot_zero, &slot_phys);
545
546 vchiq_call_connected_callbacks();
547
548 return 0;
549 }
550
551 static void
vchiq_arm_init_state(struct vchiq_state * state,struct vchiq_arm_state * arm_state)552 vchiq_arm_init_state(struct vchiq_state *state,
553 struct vchiq_arm_state *arm_state)
554 {
555 if (arm_state) {
556 rwlock_init(&arm_state->susp_res_lock);
557
558 init_completion(&arm_state->ka_evt);
559 atomic_set(&arm_state->ka_use_count, 0);
560 atomic_set(&arm_state->ka_use_ack_count, 0);
561 atomic_set(&arm_state->ka_release_count, 0);
562
563 arm_state->state = state;
564 arm_state->first_connect = 0;
565 }
566 }
567
568 int
vchiq_platform_init_state(struct vchiq_state * state)569 vchiq_platform_init_state(struct vchiq_state *state)
570 {
571 struct vchiq_2835_state *platform_state;
572
573 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
574 if (!state->platform_state)
575 return -ENOMEM;
576
577 platform_state = (struct vchiq_2835_state *)state->platform_state;
578
579 platform_state->inited = 1;
580 vchiq_arm_init_state(state, &platform_state->arm_state);
581
582 return 0;
583 }
584
585 struct vchiq_arm_state*
vchiq_platform_get_arm_state(struct vchiq_state * state)586 vchiq_platform_get_arm_state(struct vchiq_state *state)
587 {
588 struct vchiq_2835_state *platform_state;
589
590 platform_state = (struct vchiq_2835_state *)state->platform_state;
591
592 WARN_ON_ONCE(!platform_state->inited);
593
594 return &platform_state->arm_state;
595 }
596
597 void
remote_event_signal(struct remote_event * event)598 remote_event_signal(struct remote_event *event)
599 {
600 wmb();
601
602 event->fired = 1;
603
604 dsb(sy); /* data barrier operation */
605
606 if (event->armed)
607 writel(0, g_regs + BELL2); /* trigger vc interrupt */
608 }
609
610 int
vchiq_prepare_bulk_data(struct vchiq_bulk * bulk,void * offset,void __user * uoffset,int size,int dir)611 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset,
612 void __user *uoffset, int size, int dir)
613 {
614 struct vchiq_pagelist_info *pagelistinfo;
615
616 pagelistinfo = create_pagelist(offset, uoffset, size,
617 (dir == VCHIQ_BULK_RECEIVE)
618 ? PAGELIST_READ
619 : PAGELIST_WRITE);
620
621 if (!pagelistinfo)
622 return -ENOMEM;
623
624 bulk->data = pagelistinfo->dma_addr;
625
626 /*
627 * Store the pagelistinfo address in remote_data,
628 * which isn't used by the slave.
629 */
630 bulk->remote_data = pagelistinfo;
631
632 return 0;
633 }
634
635 void
vchiq_complete_bulk(struct vchiq_bulk * bulk)636 vchiq_complete_bulk(struct vchiq_bulk *bulk)
637 {
638 if (bulk && bulk->remote_data && bulk->actual)
639 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
640 bulk->actual);
641 }
642
vchiq_dump_platform_state(void * dump_context)643 int vchiq_dump_platform_state(void *dump_context)
644 {
645 char buf[80];
646 int len;
647
648 len = snprintf(buf, sizeof(buf), " Platform: 2835 (VC master)");
649 return vchiq_dump(dump_context, buf, len + 1);
650 }
651
652 #define VCHIQ_INIT_RETRIES 10
vchiq_initialise(struct vchiq_instance ** instance_out)653 int vchiq_initialise(struct vchiq_instance **instance_out)
654 {
655 struct vchiq_state *state;
656 struct vchiq_instance *instance = NULL;
657 int i, ret;
658
659 /*
660 * VideoCore may not be ready due to boot up timing.
661 * It may never be ready if kernel and firmware are mismatched,so don't
662 * block forever.
663 */
664 for (i = 0; i < VCHIQ_INIT_RETRIES; i++) {
665 state = vchiq_get_state();
666 if (state)
667 break;
668 usleep_range(500, 600);
669 }
670 if (i == VCHIQ_INIT_RETRIES) {
671 vchiq_log_error(vchiq_core_log_level, "%s: videocore not initialized\n", __func__);
672 ret = -ENOTCONN;
673 goto failed;
674 } else if (i > 0) {
675 vchiq_log_warning(vchiq_core_log_level,
676 "%s: videocore initialized after %d retries\n", __func__, i);
677 }
678
679 instance = kzalloc(sizeof(*instance), GFP_KERNEL);
680 if (!instance) {
681 vchiq_log_error(vchiq_core_log_level,
682 "%s: error allocating vchiq instance\n", __func__);
683 ret = -ENOMEM;
684 goto failed;
685 }
686
687 instance->connected = 0;
688 instance->state = state;
689 mutex_init(&instance->bulk_waiter_list_mutex);
690 INIT_LIST_HEAD(&instance->bulk_waiter_list);
691
692 *instance_out = instance;
693
694 ret = 0;
695
696 failed:
697 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, ret);
698
699 return ret;
700 }
701 EXPORT_SYMBOL(vchiq_initialise);
702
free_bulk_waiter(struct vchiq_instance * instance)703 void free_bulk_waiter(struct vchiq_instance *instance)
704 {
705 struct bulk_waiter_node *waiter, *next;
706
707 list_for_each_entry_safe(waiter, next,
708 &instance->bulk_waiter_list, list) {
709 list_del(&waiter->list);
710 vchiq_log_info(vchiq_arm_log_level, "bulk_waiter - cleaned up %pK for pid %d",
711 waiter, waiter->pid);
712 kfree(waiter);
713 }
714 }
715
vchiq_shutdown(struct vchiq_instance * instance)716 enum vchiq_status vchiq_shutdown(struct vchiq_instance *instance)
717 {
718 enum vchiq_status status = VCHIQ_SUCCESS;
719 struct vchiq_state *state = instance->state;
720
721 if (mutex_lock_killable(&state->mutex))
722 return VCHIQ_RETRY;
723
724 /* Remove all services */
725 vchiq_shutdown_internal(state, instance);
726
727 mutex_unlock(&state->mutex);
728
729 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
730
731 free_bulk_waiter(instance);
732 kfree(instance);
733
734 return status;
735 }
736 EXPORT_SYMBOL(vchiq_shutdown);
737
vchiq_is_connected(struct vchiq_instance * instance)738 static int vchiq_is_connected(struct vchiq_instance *instance)
739 {
740 return instance->connected;
741 }
742
vchiq_connect(struct vchiq_instance * instance)743 enum vchiq_status vchiq_connect(struct vchiq_instance *instance)
744 {
745 enum vchiq_status status;
746 struct vchiq_state *state = instance->state;
747
748 if (mutex_lock_killable(&state->mutex)) {
749 vchiq_log_trace(vchiq_core_log_level, "%s: call to mutex_lock failed", __func__);
750 status = VCHIQ_RETRY;
751 goto failed;
752 }
753 status = vchiq_connect_internal(state, instance);
754
755 if (status == VCHIQ_SUCCESS)
756 instance->connected = 1;
757
758 mutex_unlock(&state->mutex);
759
760 failed:
761 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
762
763 return status;
764 }
765 EXPORT_SYMBOL(vchiq_connect);
766
767 static enum vchiq_status
vchiq_add_service(struct vchiq_instance * instance,const struct vchiq_service_params_kernel * params,unsigned int * phandle)768 vchiq_add_service(struct vchiq_instance *instance,
769 const struct vchiq_service_params_kernel *params,
770 unsigned int *phandle)
771 {
772 enum vchiq_status status;
773 struct vchiq_state *state = instance->state;
774 struct vchiq_service *service = NULL;
775 int srvstate;
776
777 *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
778
779 srvstate = vchiq_is_connected(instance)
780 ? VCHIQ_SRVSTATE_LISTENING
781 : VCHIQ_SRVSTATE_HIDDEN;
782
783 service = vchiq_add_service_internal(state, params, srvstate, instance, NULL);
784
785 if (service) {
786 *phandle = service->handle;
787 status = VCHIQ_SUCCESS;
788 } else {
789 status = VCHIQ_ERROR;
790 }
791
792 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
793
794 return status;
795 }
796
797 enum vchiq_status
vchiq_open_service(struct vchiq_instance * instance,const struct vchiq_service_params_kernel * params,unsigned int * phandle)798 vchiq_open_service(struct vchiq_instance *instance,
799 const struct vchiq_service_params_kernel *params,
800 unsigned int *phandle)
801 {
802 enum vchiq_status status = VCHIQ_ERROR;
803 struct vchiq_state *state = instance->state;
804 struct vchiq_service *service = NULL;
805
806 *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
807
808 if (!vchiq_is_connected(instance))
809 goto failed;
810
811 service = vchiq_add_service_internal(state, params, VCHIQ_SRVSTATE_OPENING, instance, NULL);
812
813 if (service) {
814 *phandle = service->handle;
815 status = vchiq_open_service_internal(service, current->pid);
816 if (status != VCHIQ_SUCCESS) {
817 vchiq_remove_service(service->handle);
818 *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
819 }
820 }
821
822 failed:
823 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
824
825 return status;
826 }
827 EXPORT_SYMBOL(vchiq_open_service);
828
829 enum vchiq_status
vchiq_bulk_transmit(unsigned int handle,const void * data,unsigned int size,void * userdata,enum vchiq_bulk_mode mode)830 vchiq_bulk_transmit(unsigned int handle, const void *data, unsigned int size,
831 void *userdata, enum vchiq_bulk_mode mode)
832 {
833 enum vchiq_status status;
834
835 while (1) {
836 switch (mode) {
837 case VCHIQ_BULK_MODE_NOCALLBACK:
838 case VCHIQ_BULK_MODE_CALLBACK:
839 status = vchiq_bulk_transfer(handle,
840 (void *)data, NULL,
841 size, userdata, mode,
842 VCHIQ_BULK_TRANSMIT);
843 break;
844 case VCHIQ_BULK_MODE_BLOCKING:
845 status = vchiq_blocking_bulk_transfer(handle, (void *)data, size,
846 VCHIQ_BULK_TRANSMIT);
847 break;
848 default:
849 return VCHIQ_ERROR;
850 }
851
852 /*
853 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
854 * to implement a retry mechanism since this function is
855 * supposed to block until queued
856 */
857 if (status != VCHIQ_RETRY)
858 break;
859
860 msleep(1);
861 }
862
863 return status;
864 }
865 EXPORT_SYMBOL(vchiq_bulk_transmit);
866
vchiq_bulk_receive(unsigned int handle,void * data,unsigned int size,void * userdata,enum vchiq_bulk_mode mode)867 enum vchiq_status vchiq_bulk_receive(unsigned int handle, void *data,
868 unsigned int size, void *userdata,
869 enum vchiq_bulk_mode mode)
870 {
871 enum vchiq_status status;
872
873 while (1) {
874 switch (mode) {
875 case VCHIQ_BULK_MODE_NOCALLBACK:
876 case VCHIQ_BULK_MODE_CALLBACK:
877 status = vchiq_bulk_transfer(handle, data, NULL,
878 size, userdata,
879 mode, VCHIQ_BULK_RECEIVE);
880 break;
881 case VCHIQ_BULK_MODE_BLOCKING:
882 status = vchiq_blocking_bulk_transfer(handle, (void *)data, size,
883 VCHIQ_BULK_RECEIVE);
884 break;
885 default:
886 return VCHIQ_ERROR;
887 }
888
889 /*
890 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
891 * to implement a retry mechanism since this function is
892 * supposed to block until queued
893 */
894 if (status != VCHIQ_RETRY)
895 break;
896
897 msleep(1);
898 }
899
900 return status;
901 }
902 EXPORT_SYMBOL(vchiq_bulk_receive);
903
904 static enum vchiq_status
vchiq_blocking_bulk_transfer(unsigned int handle,void * data,unsigned int size,enum vchiq_bulk_dir dir)905 vchiq_blocking_bulk_transfer(unsigned int handle, void *data, unsigned int size,
906 enum vchiq_bulk_dir dir)
907 {
908 struct vchiq_instance *instance;
909 struct vchiq_service *service;
910 enum vchiq_status status;
911 struct bulk_waiter_node *waiter = NULL;
912 bool found = false;
913
914 service = find_service_by_handle(handle);
915 if (!service)
916 return VCHIQ_ERROR;
917
918 instance = service->instance;
919
920 vchiq_service_put(service);
921
922 mutex_lock(&instance->bulk_waiter_list_mutex);
923 list_for_each_entry(waiter, &instance->bulk_waiter_list, list) {
924 if (waiter->pid == current->pid) {
925 list_del(&waiter->list);
926 found = true;
927 break;
928 }
929 }
930 mutex_unlock(&instance->bulk_waiter_list_mutex);
931
932 if (found) {
933 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;
934
935 if (bulk) {
936 /* This thread has an outstanding bulk transfer. */
937 /* FIXME: why compare a dma address to a pointer? */
938 if ((bulk->data != (dma_addr_t)(uintptr_t)data) || (bulk->size != size)) {
939 /*
940 * This is not a retry of the previous one.
941 * Cancel the signal when the transfer completes.
942 */
943 spin_lock(&bulk_waiter_spinlock);
944 bulk->userdata = NULL;
945 spin_unlock(&bulk_waiter_spinlock);
946 }
947 }
948 } else {
949 waiter = kzalloc(sizeof(*waiter), GFP_KERNEL);
950 if (!waiter) {
951 vchiq_log_error(vchiq_core_log_level, "%s - out of memory", __func__);
952 return VCHIQ_ERROR;
953 }
954 }
955
956 status = vchiq_bulk_transfer(handle, data, NULL, size,
957 &waiter->bulk_waiter,
958 VCHIQ_BULK_MODE_BLOCKING, dir);
959 if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) || !waiter->bulk_waiter.bulk) {
960 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;
961
962 if (bulk) {
963 /* Cancel the signal when the transfer completes. */
964 spin_lock(&bulk_waiter_spinlock);
965 bulk->userdata = NULL;
966 spin_unlock(&bulk_waiter_spinlock);
967 }
968 kfree(waiter);
969 } else {
970 waiter->pid = current->pid;
971 mutex_lock(&instance->bulk_waiter_list_mutex);
972 list_add(&waiter->list, &instance->bulk_waiter_list);
973 mutex_unlock(&instance->bulk_waiter_list_mutex);
974 vchiq_log_info(vchiq_arm_log_level, "saved bulk_waiter %pK for pid %d", waiter,
975 current->pid);
976 }
977
978 return status;
979 }
980
981 static enum vchiq_status
add_completion(struct vchiq_instance * instance,enum vchiq_reason reason,struct vchiq_header * header,struct user_service * user_service,void * bulk_userdata)982 add_completion(struct vchiq_instance *instance, enum vchiq_reason reason,
983 struct vchiq_header *header, struct user_service *user_service,
984 void *bulk_userdata)
985 {
986 struct vchiq_completion_data_kernel *completion;
987 int insert;
988
989 DEBUG_INITIALISE(g_state.local);
990
991 insert = instance->completion_insert;
992 while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
993 /* Out of space - wait for the client */
994 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
995 vchiq_log_trace(vchiq_arm_log_level, "%s - completion queue full", __func__);
996 DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
997 if (wait_for_completion_interruptible(&instance->remove_event)) {
998 vchiq_log_info(vchiq_arm_log_level, "service_callback interrupted");
999 return VCHIQ_RETRY;
1000 } else if (instance->closing) {
1001 vchiq_log_info(vchiq_arm_log_level, "service_callback closing");
1002 return VCHIQ_SUCCESS;
1003 }
1004 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1005 }
1006
1007 completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];
1008
1009 completion->header = header;
1010 completion->reason = reason;
1011 /* N.B. service_userdata is updated while processing AWAIT_COMPLETION */
1012 completion->service_userdata = user_service->service;
1013 completion->bulk_userdata = bulk_userdata;
1014
1015 if (reason == VCHIQ_SERVICE_CLOSED) {
1016 /*
1017 * Take an extra reference, to be held until
1018 * this CLOSED notification is delivered.
1019 */
1020 vchiq_service_get(user_service->service);
1021 if (instance->use_close_delivered)
1022 user_service->close_pending = 1;
1023 }
1024
1025 /*
1026 * A write barrier is needed here to ensure that the entire completion
1027 * record is written out before the insert point.
1028 */
1029 wmb();
1030
1031 if (reason == VCHIQ_MESSAGE_AVAILABLE)
1032 user_service->message_available_pos = insert;
1033
1034 insert++;
1035 instance->completion_insert = insert;
1036
1037 complete(&instance->insert_event);
1038
1039 return VCHIQ_SUCCESS;
1040 }
1041
1042 enum vchiq_status
service_callback(enum vchiq_reason reason,struct vchiq_header * header,unsigned int handle,void * bulk_userdata)1043 service_callback(enum vchiq_reason reason, struct vchiq_header *header,
1044 unsigned int handle, void *bulk_userdata)
1045 {
1046 /*
1047 * How do we ensure the callback goes to the right client?
1048 * The service_user data points to a user_service record
1049 * containing the original callback and the user state structure, which
1050 * contains a circular buffer for completion records.
1051 */
1052 struct user_service *user_service;
1053 struct vchiq_service *service;
1054 struct vchiq_instance *instance;
1055 bool skip_completion = false;
1056
1057 DEBUG_INITIALISE(g_state.local);
1058
1059 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1060
1061 service = handle_to_service(handle);
1062 if (WARN_ON(!service))
1063 return VCHIQ_SUCCESS;
1064
1065 user_service = (struct user_service *)service->base.userdata;
1066 instance = user_service->instance;
1067
1068 if (!instance || instance->closing)
1069 return VCHIQ_SUCCESS;
1070
1071 vchiq_log_trace(vchiq_arm_log_level,
1072 "%s - service %lx(%d,%p), reason %d, header %lx, instance %lx, bulk_userdata %lx",
1073 __func__, (unsigned long)user_service, service->localport,
1074 user_service->userdata, reason, (unsigned long)header,
1075 (unsigned long)instance, (unsigned long)bulk_userdata);
1076
1077 if (header && user_service->is_vchi) {
1078 spin_lock(&msg_queue_spinlock);
1079 while (user_service->msg_insert ==
1080 (user_service->msg_remove + MSG_QUEUE_SIZE)) {
1081 spin_unlock(&msg_queue_spinlock);
1082 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1083 DEBUG_COUNT(MSG_QUEUE_FULL_COUNT);
1084 vchiq_log_trace(vchiq_arm_log_level, "%s - msg queue full", __func__);
1085 /*
1086 * If there is no MESSAGE_AVAILABLE in the completion
1087 * queue, add one
1088 */
1089 if ((user_service->message_available_pos -
1090 instance->completion_remove) < 0) {
1091 enum vchiq_status status;
1092
1093 vchiq_log_info(vchiq_arm_log_level,
1094 "Inserting extra MESSAGE_AVAILABLE");
1095 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1096 status = add_completion(instance, reason, NULL, user_service,
1097 bulk_userdata);
1098 if (status != VCHIQ_SUCCESS) {
1099 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1100 return status;
1101 }
1102 }
1103
1104 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1105 if (wait_for_completion_interruptible(&user_service->remove_event)) {
1106 vchiq_log_info(vchiq_arm_log_level, "%s interrupted", __func__);
1107 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1108 return VCHIQ_RETRY;
1109 } else if (instance->closing) {
1110 vchiq_log_info(vchiq_arm_log_level, "%s closing", __func__);
1111 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1112 return VCHIQ_ERROR;
1113 }
1114 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1115 spin_lock(&msg_queue_spinlock);
1116 }
1117
1118 user_service->msg_queue[user_service->msg_insert &
1119 (MSG_QUEUE_SIZE - 1)] = header;
1120 user_service->msg_insert++;
1121
1122 /*
1123 * If there is a thread waiting in DEQUEUE_MESSAGE, or if
1124 * there is a MESSAGE_AVAILABLE in the completion queue then
1125 * bypass the completion queue.
1126 */
1127 if (((user_service->message_available_pos -
1128 instance->completion_remove) >= 0) ||
1129 user_service->dequeue_pending) {
1130 user_service->dequeue_pending = 0;
1131 skip_completion = true;
1132 }
1133
1134 spin_unlock(&msg_queue_spinlock);
1135 complete(&user_service->insert_event);
1136
1137 header = NULL;
1138 }
1139 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1140
1141 if (skip_completion)
1142 return VCHIQ_SUCCESS;
1143
1144 return add_completion(instance, reason, header, user_service,
1145 bulk_userdata);
1146 }
1147
vchiq_dump(void * dump_context,const char * str,int len)1148 int vchiq_dump(void *dump_context, const char *str, int len)
1149 {
1150 struct dump_context *context = (struct dump_context *)dump_context;
1151 int copy_bytes;
1152
1153 if (context->actual >= context->space)
1154 return 0;
1155
1156 if (context->offset > 0) {
1157 int skip_bytes = min_t(int, len, context->offset);
1158
1159 str += skip_bytes;
1160 len -= skip_bytes;
1161 context->offset -= skip_bytes;
1162 if (context->offset > 0)
1163 return 0;
1164 }
1165 copy_bytes = min_t(int, len, context->space - context->actual);
1166 if (copy_bytes == 0)
1167 return 0;
1168 if (copy_to_user(context->buf + context->actual, str,
1169 copy_bytes))
1170 return -EFAULT;
1171 context->actual += copy_bytes;
1172 len -= copy_bytes;
1173
1174 /*
1175 * If the terminating NUL is included in the length, then it
1176 * marks the end of a line and should be replaced with a
1177 * carriage return.
1178 */
1179 if ((len == 0) && (str[copy_bytes - 1] == '\0')) {
1180 char cr = '\n';
1181
1182 if (copy_to_user(context->buf + context->actual - 1,
1183 &cr, 1))
1184 return -EFAULT;
1185 }
1186 return 0;
1187 }
1188
vchiq_dump_platform_instances(void * dump_context)1189 int vchiq_dump_platform_instances(void *dump_context)
1190 {
1191 struct vchiq_state *state = vchiq_get_state();
1192 char buf[80];
1193 int len;
1194 int i;
1195
1196 /*
1197 * There is no list of instances, so instead scan all services,
1198 * marking those that have been dumped.
1199 */
1200
1201 rcu_read_lock();
1202 for (i = 0; i < state->unused_service; i++) {
1203 struct vchiq_service *service;
1204 struct vchiq_instance *instance;
1205
1206 service = rcu_dereference(state->services[i]);
1207 if (!service || service->base.callback != service_callback)
1208 continue;
1209
1210 instance = service->instance;
1211 if (instance)
1212 instance->mark = 0;
1213 }
1214 rcu_read_unlock();
1215
1216 for (i = 0; i < state->unused_service; i++) {
1217 struct vchiq_service *service;
1218 struct vchiq_instance *instance;
1219 int err;
1220
1221 rcu_read_lock();
1222 service = rcu_dereference(state->services[i]);
1223 if (!service || service->base.callback != service_callback) {
1224 rcu_read_unlock();
1225 continue;
1226 }
1227
1228 instance = service->instance;
1229 if (!instance || instance->mark) {
1230 rcu_read_unlock();
1231 continue;
1232 }
1233 rcu_read_unlock();
1234
1235 len = snprintf(buf, sizeof(buf),
1236 "Instance %pK: pid %d,%s completions %d/%d",
1237 instance, instance->pid,
1238 instance->connected ? " connected, " :
1239 "",
1240 instance->completion_insert -
1241 instance->completion_remove,
1242 MAX_COMPLETIONS);
1243 err = vchiq_dump(dump_context, buf, len + 1);
1244 if (err)
1245 return err;
1246 instance->mark = 1;
1247 }
1248 return 0;
1249 }
1250
vchiq_dump_platform_service_state(void * dump_context,struct vchiq_service * service)1251 int vchiq_dump_platform_service_state(void *dump_context,
1252 struct vchiq_service *service)
1253 {
1254 struct user_service *user_service =
1255 (struct user_service *)service->base.userdata;
1256 char buf[80];
1257 int len;
1258
1259 len = scnprintf(buf, sizeof(buf), " instance %pK", service->instance);
1260
1261 if ((service->base.callback == service_callback) && user_service->is_vchi) {
1262 len += scnprintf(buf + len, sizeof(buf) - len, ", %d/%d messages",
1263 user_service->msg_insert - user_service->msg_remove,
1264 MSG_QUEUE_SIZE);
1265
1266 if (user_service->dequeue_pending)
1267 len += scnprintf(buf + len, sizeof(buf) - len,
1268 " (dequeue pending)");
1269 }
1270
1271 return vchiq_dump(dump_context, buf, len + 1);
1272 }
1273
1274 struct vchiq_state *
vchiq_get_state(void)1275 vchiq_get_state(void)
1276 {
1277 if (!g_state.remote)
1278 pr_err("%s: g_state.remote == NULL\n", __func__);
1279 else if (g_state.remote->initialised != 1)
1280 pr_notice("%s: g_state.remote->initialised != 1 (%d)\n",
1281 __func__, g_state.remote->initialised);
1282
1283 return (g_state.remote &&
1284 (g_state.remote->initialised == 1)) ? &g_state : NULL;
1285 }
1286
1287 /*
1288 * Autosuspend related functionality
1289 */
1290
1291 static enum vchiq_status
vchiq_keepalive_vchiq_callback(enum vchiq_reason reason,struct vchiq_header * header,unsigned int service_user,void * bulk_user)1292 vchiq_keepalive_vchiq_callback(enum vchiq_reason reason,
1293 struct vchiq_header *header,
1294 unsigned int service_user, void *bulk_user)
1295 {
1296 vchiq_log_error(vchiq_susp_log_level, "%s callback reason %d", __func__, reason);
1297 return 0;
1298 }
1299
1300 static int
vchiq_keepalive_thread_func(void * v)1301 vchiq_keepalive_thread_func(void *v)
1302 {
1303 struct vchiq_state *state = (struct vchiq_state *)v;
1304 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1305
1306 enum vchiq_status status;
1307 struct vchiq_instance *instance;
1308 unsigned int ka_handle;
1309 int ret;
1310
1311 struct vchiq_service_params_kernel params = {
1312 .fourcc = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'),
1313 .callback = vchiq_keepalive_vchiq_callback,
1314 .version = KEEPALIVE_VER,
1315 .version_min = KEEPALIVE_VER_MIN
1316 };
1317
1318 ret = vchiq_initialise(&instance);
1319 if (ret) {
1320 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_initialise failed %d", __func__,
1321 ret);
1322 goto exit;
1323 }
1324
1325 status = vchiq_connect(instance);
1326 if (status != VCHIQ_SUCCESS) {
1327 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_connect failed %d", __func__,
1328 status);
1329 goto shutdown;
1330 }
1331
1332 status = vchiq_add_service(instance, ¶ms, &ka_handle);
1333 if (status != VCHIQ_SUCCESS) {
1334 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_open_service failed %d", __func__,
1335 status);
1336 goto shutdown;
1337 }
1338
1339 while (1) {
1340 long rc = 0, uc = 0;
1341
1342 if (wait_for_completion_interruptible(&arm_state->ka_evt)) {
1343 vchiq_log_error(vchiq_susp_log_level, "%s interrupted", __func__);
1344 flush_signals(current);
1345 continue;
1346 }
1347
1348 /*
1349 * read and clear counters. Do release_count then use_count to
1350 * prevent getting more releases than uses
1351 */
1352 rc = atomic_xchg(&arm_state->ka_release_count, 0);
1353 uc = atomic_xchg(&arm_state->ka_use_count, 0);
1354
1355 /*
1356 * Call use/release service the requisite number of times.
1357 * Process use before release so use counts don't go negative
1358 */
1359 while (uc--) {
1360 atomic_inc(&arm_state->ka_use_ack_count);
1361 status = vchiq_use_service(ka_handle);
1362 if (status != VCHIQ_SUCCESS) {
1363 vchiq_log_error(vchiq_susp_log_level,
1364 "%s vchiq_use_service error %d", __func__, status);
1365 }
1366 }
1367 while (rc--) {
1368 status = vchiq_release_service(ka_handle);
1369 if (status != VCHIQ_SUCCESS) {
1370 vchiq_log_error(vchiq_susp_log_level,
1371 "%s vchiq_release_service error %d", __func__,
1372 status);
1373 }
1374 }
1375 }
1376
1377 shutdown:
1378 vchiq_shutdown(instance);
1379 exit:
1380 return 0;
1381 }
1382
1383 int
vchiq_use_internal(struct vchiq_state * state,struct vchiq_service * service,enum USE_TYPE_E use_type)1384 vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service,
1385 enum USE_TYPE_E use_type)
1386 {
1387 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1388 int ret = 0;
1389 char entity[16];
1390 int *entity_uc;
1391 int local_uc;
1392
1393 if (!arm_state) {
1394 ret = -EINVAL;
1395 goto out;
1396 }
1397
1398 if (use_type == USE_TYPE_VCHIQ) {
1399 sprintf(entity, "VCHIQ: ");
1400 entity_uc = &arm_state->peer_use_count;
1401 } else if (service) {
1402 sprintf(entity, "%c%c%c%c:%03d",
1403 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
1404 service->client_id);
1405 entity_uc = &service->service_use_count;
1406 } else {
1407 vchiq_log_error(vchiq_susp_log_level, "%s null service ptr", __func__);
1408 ret = -EINVAL;
1409 goto out;
1410 }
1411
1412 write_lock_bh(&arm_state->susp_res_lock);
1413 local_uc = ++arm_state->videocore_use_count;
1414 ++(*entity_uc);
1415
1416 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity,
1417 *entity_uc, local_uc);
1418
1419 write_unlock_bh(&arm_state->susp_res_lock);
1420
1421 if (!ret) {
1422 enum vchiq_status status = VCHIQ_SUCCESS;
1423 long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0);
1424
1425 while (ack_cnt && (status == VCHIQ_SUCCESS)) {
1426 /* Send the use notify to videocore */
1427 status = vchiq_send_remote_use_active(state);
1428 if (status == VCHIQ_SUCCESS)
1429 ack_cnt--;
1430 else
1431 atomic_add(ack_cnt, &arm_state->ka_use_ack_count);
1432 }
1433 }
1434
1435 out:
1436 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
1437 return ret;
1438 }
1439
1440 int
vchiq_release_internal(struct vchiq_state * state,struct vchiq_service * service)1441 vchiq_release_internal(struct vchiq_state *state, struct vchiq_service *service)
1442 {
1443 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1444 int ret = 0;
1445 char entity[16];
1446 int *entity_uc;
1447
1448 if (!arm_state) {
1449 ret = -EINVAL;
1450 goto out;
1451 }
1452
1453 if (service) {
1454 sprintf(entity, "%c%c%c%c:%03d",
1455 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
1456 service->client_id);
1457 entity_uc = &service->service_use_count;
1458 } else {
1459 sprintf(entity, "PEER: ");
1460 entity_uc = &arm_state->peer_use_count;
1461 }
1462
1463 write_lock_bh(&arm_state->susp_res_lock);
1464 if (!arm_state->videocore_use_count || !(*entity_uc)) {
1465 /* Don't use BUG_ON - don't allow user thread to crash kernel */
1466 WARN_ON(!arm_state->videocore_use_count);
1467 WARN_ON(!(*entity_uc));
1468 ret = -EINVAL;
1469 goto unlock;
1470 }
1471 --arm_state->videocore_use_count;
1472 --(*entity_uc);
1473
1474 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity,
1475 *entity_uc, arm_state->videocore_use_count);
1476
1477 unlock:
1478 write_unlock_bh(&arm_state->susp_res_lock);
1479
1480 out:
1481 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
1482 return ret;
1483 }
1484
1485 void
vchiq_on_remote_use(struct vchiq_state * state)1486 vchiq_on_remote_use(struct vchiq_state *state)
1487 {
1488 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1489
1490 atomic_inc(&arm_state->ka_use_count);
1491 complete(&arm_state->ka_evt);
1492 }
1493
1494 void
vchiq_on_remote_release(struct vchiq_state * state)1495 vchiq_on_remote_release(struct vchiq_state *state)
1496 {
1497 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1498
1499 atomic_inc(&arm_state->ka_release_count);
1500 complete(&arm_state->ka_evt);
1501 }
1502
1503 int
vchiq_use_service_internal(struct vchiq_service * service)1504 vchiq_use_service_internal(struct vchiq_service *service)
1505 {
1506 return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
1507 }
1508
1509 int
vchiq_release_service_internal(struct vchiq_service * service)1510 vchiq_release_service_internal(struct vchiq_service *service)
1511 {
1512 return vchiq_release_internal(service->state, service);
1513 }
1514
1515 struct vchiq_debugfs_node *
vchiq_instance_get_debugfs_node(struct vchiq_instance * instance)1516 vchiq_instance_get_debugfs_node(struct vchiq_instance *instance)
1517 {
1518 return &instance->debugfs_node;
1519 }
1520
1521 int
vchiq_instance_get_use_count(struct vchiq_instance * instance)1522 vchiq_instance_get_use_count(struct vchiq_instance *instance)
1523 {
1524 struct vchiq_service *service;
1525 int use_count = 0, i;
1526
1527 i = 0;
1528 rcu_read_lock();
1529 while ((service = __next_service_by_instance(instance->state,
1530 instance, &i)))
1531 use_count += service->service_use_count;
1532 rcu_read_unlock();
1533 return use_count;
1534 }
1535
1536 int
vchiq_instance_get_pid(struct vchiq_instance * instance)1537 vchiq_instance_get_pid(struct vchiq_instance *instance)
1538 {
1539 return instance->pid;
1540 }
1541
1542 int
vchiq_instance_get_trace(struct vchiq_instance * instance)1543 vchiq_instance_get_trace(struct vchiq_instance *instance)
1544 {
1545 return instance->trace;
1546 }
1547
1548 void
vchiq_instance_set_trace(struct vchiq_instance * instance,int trace)1549 vchiq_instance_set_trace(struct vchiq_instance *instance, int trace)
1550 {
1551 struct vchiq_service *service;
1552 int i;
1553
1554 i = 0;
1555 rcu_read_lock();
1556 while ((service = __next_service_by_instance(instance->state,
1557 instance, &i)))
1558 service->trace = trace;
1559 rcu_read_unlock();
1560 instance->trace = (trace != 0);
1561 }
1562
1563 enum vchiq_status
vchiq_use_service(unsigned int handle)1564 vchiq_use_service(unsigned int handle)
1565 {
1566 enum vchiq_status ret = VCHIQ_ERROR;
1567 struct vchiq_service *service = find_service_by_handle(handle);
1568
1569 if (service) {
1570 ret = vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
1571 vchiq_service_put(service);
1572 }
1573 return ret;
1574 }
1575 EXPORT_SYMBOL(vchiq_use_service);
1576
1577 enum vchiq_status
vchiq_release_service(unsigned int handle)1578 vchiq_release_service(unsigned int handle)
1579 {
1580 enum vchiq_status ret = VCHIQ_ERROR;
1581 struct vchiq_service *service = find_service_by_handle(handle);
1582
1583 if (service) {
1584 ret = vchiq_release_internal(service->state, service);
1585 vchiq_service_put(service);
1586 }
1587 return ret;
1588 }
1589 EXPORT_SYMBOL(vchiq_release_service);
1590
1591 struct service_data_struct {
1592 int fourcc;
1593 int clientid;
1594 int use_count;
1595 };
1596
1597 void
vchiq_dump_service_use_state(struct vchiq_state * state)1598 vchiq_dump_service_use_state(struct vchiq_state *state)
1599 {
1600 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1601 struct service_data_struct *service_data;
1602 int i, found = 0;
1603 /*
1604 * If there's more than 64 services, only dump ones with
1605 * non-zero counts
1606 */
1607 int only_nonzero = 0;
1608 static const char *nz = "<-- preventing suspend";
1609
1610 int peer_count;
1611 int vc_use_count;
1612 int active_services;
1613
1614 if (!arm_state)
1615 return;
1616
1617 service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data),
1618 GFP_KERNEL);
1619 if (!service_data)
1620 return;
1621
1622 read_lock_bh(&arm_state->susp_res_lock);
1623 peer_count = arm_state->peer_use_count;
1624 vc_use_count = arm_state->videocore_use_count;
1625 active_services = state->unused_service;
1626 if (active_services > MAX_SERVICES)
1627 only_nonzero = 1;
1628
1629 rcu_read_lock();
1630 for (i = 0; i < active_services; i++) {
1631 struct vchiq_service *service_ptr =
1632 rcu_dereference(state->services[i]);
1633
1634 if (!service_ptr)
1635 continue;
1636
1637 if (only_nonzero && !service_ptr->service_use_count)
1638 continue;
1639
1640 if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE)
1641 continue;
1642
1643 service_data[found].fourcc = service_ptr->base.fourcc;
1644 service_data[found].clientid = service_ptr->client_id;
1645 service_data[found].use_count = service_ptr->service_use_count;
1646 found++;
1647 if (found >= MAX_SERVICES)
1648 break;
1649 }
1650 rcu_read_unlock();
1651
1652 read_unlock_bh(&arm_state->susp_res_lock);
1653
1654 if (only_nonzero)
1655 vchiq_log_warning(vchiq_susp_log_level, "Too many active services (%d). Only dumping up to first %d services with non-zero use-count",
1656 active_services, found);
1657
1658 for (i = 0; i < found; i++) {
1659 vchiq_log_warning(vchiq_susp_log_level, "----- %c%c%c%c:%d service count %d %s",
1660 VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc),
1661 service_data[i].clientid, service_data[i].use_count,
1662 service_data[i].use_count ? nz : "");
1663 }
1664 vchiq_log_warning(vchiq_susp_log_level, "----- VCHIQ use count count %d", peer_count);
1665 vchiq_log_warning(vchiq_susp_log_level, "--- Overall vchiq instance use count %d",
1666 vc_use_count);
1667
1668 kfree(service_data);
1669 }
1670
1671 enum vchiq_status
vchiq_check_service(struct vchiq_service * service)1672 vchiq_check_service(struct vchiq_service *service)
1673 {
1674 struct vchiq_arm_state *arm_state;
1675 enum vchiq_status ret = VCHIQ_ERROR;
1676
1677 if (!service || !service->state)
1678 goto out;
1679
1680 arm_state = vchiq_platform_get_arm_state(service->state);
1681
1682 read_lock_bh(&arm_state->susp_res_lock);
1683 if (service->service_use_count)
1684 ret = VCHIQ_SUCCESS;
1685 read_unlock_bh(&arm_state->susp_res_lock);
1686
1687 if (ret == VCHIQ_ERROR) {
1688 vchiq_log_error(vchiq_susp_log_level,
1689 "%s ERROR - %c%c%c%c:%d service count %d, state count %d", __func__,
1690 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), service->client_id,
1691 service->service_use_count, arm_state->videocore_use_count);
1692 vchiq_dump_service_use_state(service->state);
1693 }
1694 out:
1695 return ret;
1696 }
1697
vchiq_platform_conn_state_changed(struct vchiq_state * state,enum vchiq_connstate oldstate,enum vchiq_connstate newstate)1698 void vchiq_platform_conn_state_changed(struct vchiq_state *state,
1699 enum vchiq_connstate oldstate,
1700 enum vchiq_connstate newstate)
1701 {
1702 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1703 char threadname[16];
1704
1705 vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id,
1706 get_conn_state_name(oldstate), get_conn_state_name(newstate));
1707 if (state->conn_state != VCHIQ_CONNSTATE_CONNECTED)
1708 return;
1709
1710 write_lock_bh(&arm_state->susp_res_lock);
1711 if (arm_state->first_connect) {
1712 write_unlock_bh(&arm_state->susp_res_lock);
1713 return;
1714 }
1715
1716 arm_state->first_connect = 1;
1717 write_unlock_bh(&arm_state->susp_res_lock);
1718 snprintf(threadname, sizeof(threadname), "vchiq-keep/%d",
1719 state->id);
1720 arm_state->ka_thread = kthread_create(&vchiq_keepalive_thread_func,
1721 (void *)state,
1722 threadname);
1723 if (IS_ERR(arm_state->ka_thread)) {
1724 vchiq_log_error(vchiq_susp_log_level,
1725 "vchiq: FATAL: couldn't create thread %s",
1726 threadname);
1727 } else {
1728 wake_up_process(arm_state->ka_thread);
1729 }
1730 }
1731
1732 static const struct of_device_id vchiq_of_match[] = {
1733 { .compatible = "brcm,bcm2835-vchiq", .data = &bcm2835_drvdata },
1734 { .compatible = "brcm,bcm2836-vchiq", .data = &bcm2836_drvdata },
1735 {},
1736 };
1737 MODULE_DEVICE_TABLE(of, vchiq_of_match);
1738
1739 static struct platform_device *
vchiq_register_child(struct platform_device * pdev,const char * name)1740 vchiq_register_child(struct platform_device *pdev, const char *name)
1741 {
1742 struct platform_device_info pdevinfo;
1743 struct platform_device *child;
1744
1745 memset(&pdevinfo, 0, sizeof(pdevinfo));
1746
1747 pdevinfo.parent = &pdev->dev;
1748 pdevinfo.name = name;
1749 pdevinfo.id = PLATFORM_DEVID_NONE;
1750 pdevinfo.dma_mask = DMA_BIT_MASK(32);
1751
1752 child = platform_device_register_full(&pdevinfo);
1753 if (IS_ERR(child)) {
1754 dev_warn(&pdev->dev, "%s not registered\n", name);
1755 child = NULL;
1756 }
1757
1758 return child;
1759 }
1760
vchiq_probe(struct platform_device * pdev)1761 static int vchiq_probe(struct platform_device *pdev)
1762 {
1763 struct device_node *fw_node;
1764 const struct of_device_id *of_id;
1765 struct vchiq_drvdata *drvdata;
1766 int err;
1767
1768 of_id = of_match_node(vchiq_of_match, pdev->dev.of_node);
1769 drvdata = (struct vchiq_drvdata *)of_id->data;
1770 if (!drvdata)
1771 return -EINVAL;
1772
1773 fw_node = of_find_compatible_node(NULL, NULL,
1774 "raspberrypi,bcm2835-firmware");
1775 if (!fw_node) {
1776 dev_err(&pdev->dev, "Missing firmware node\n");
1777 return -ENOENT;
1778 }
1779
1780 drvdata->fw = devm_rpi_firmware_get(&pdev->dev, fw_node);
1781 of_node_put(fw_node);
1782 if (!drvdata->fw)
1783 return -EPROBE_DEFER;
1784
1785 platform_set_drvdata(pdev, drvdata);
1786
1787 err = vchiq_platform_init(pdev, &g_state);
1788 if (err)
1789 goto failed_platform_init;
1790
1791 vchiq_debugfs_init();
1792
1793 vchiq_log_info(vchiq_arm_log_level,
1794 "vchiq: platform initialised - version %d (min %d)",
1795 VCHIQ_VERSION, VCHIQ_VERSION_MIN);
1796
1797 /*
1798 * Simply exit on error since the function handles cleanup in
1799 * cases of failure.
1800 */
1801 err = vchiq_register_chrdev(&pdev->dev);
1802 if (err) {
1803 vchiq_log_warning(vchiq_arm_log_level,
1804 "Failed to initialize vchiq cdev");
1805 goto error_exit;
1806 }
1807
1808 bcm2835_camera = vchiq_register_child(pdev, "bcm2835-camera");
1809 bcm2835_audio = vchiq_register_child(pdev, "bcm2835_audio");
1810
1811 return 0;
1812
1813 failed_platform_init:
1814 vchiq_log_warning(vchiq_arm_log_level, "could not initialize vchiq platform");
1815 error_exit:
1816 return err;
1817 }
1818
vchiq_remove(struct platform_device * pdev)1819 static int vchiq_remove(struct platform_device *pdev)
1820 {
1821 platform_device_unregister(bcm2835_audio);
1822 platform_device_unregister(bcm2835_camera);
1823 vchiq_debugfs_deinit();
1824 vchiq_deregister_chrdev();
1825
1826 return 0;
1827 }
1828
1829 static struct platform_driver vchiq_driver = {
1830 .driver = {
1831 .name = "bcm2835_vchiq",
1832 .of_match_table = vchiq_of_match,
1833 },
1834 .probe = vchiq_probe,
1835 .remove = vchiq_remove,
1836 };
1837
vchiq_driver_init(void)1838 static int __init vchiq_driver_init(void)
1839 {
1840 int ret;
1841
1842 ret = platform_driver_register(&vchiq_driver);
1843 if (ret)
1844 pr_err("Failed to register vchiq driver\n");
1845
1846 return ret;
1847 }
1848 module_init(vchiq_driver_init);
1849
vchiq_driver_exit(void)1850 static void __exit vchiq_driver_exit(void)
1851 {
1852 platform_driver_unregister(&vchiq_driver);
1853 }
1854 module_exit(vchiq_driver_exit);
1855
1856 MODULE_LICENSE("Dual BSD/GPL");
1857 MODULE_DESCRIPTION("Videocore VCHIQ driver");
1858 MODULE_AUTHOR("Broadcom Corporation");
1859