Lines Matching refs:ipc_imem
23 static int ipc_imem_msg_send_device_sleep(struct iosm_imem *ipc_imem, u32 state) in ipc_imem_msg_send_device_sleep() argument
30 ipc_imem->device_sleep = state; in ipc_imem_msg_send_device_sleep()
32 return ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol, in ipc_imem_msg_send_device_sleep()
36 static bool ipc_imem_dl_skb_alloc(struct iosm_imem *ipc_imem, in ipc_imem_dl_skb_alloc() argument
43 return ipc_protocol_dl_td_prepare(ipc_imem->ipc_protocol, pipe); in ipc_imem_dl_skb_alloc()
49 static int ipc_imem_tq_td_alloc_timer(struct iosm_imem *ipc_imem, int arg, in ipc_imem_tq_td_alloc_timer() argument
57 struct ipc_pipe *pipe = &ipc_imem->channels[i].dl_pipe; in ipc_imem_tq_td_alloc_timer()
62 while (ipc_imem_dl_skb_alloc(ipc_imem, pipe)) in ipc_imem_tq_td_alloc_timer()
70 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_tq_td_alloc_timer()
74 ipc_imem->hrtimer_period = in ipc_imem_tq_td_alloc_timer()
76 if (!hrtimer_active(&ipc_imem->td_alloc_timer)) in ipc_imem_tq_td_alloc_timer()
77 hrtimer_start(&ipc_imem->td_alloc_timer, in ipc_imem_tq_td_alloc_timer()
78 ipc_imem->hrtimer_period, in ipc_imem_tq_td_alloc_timer()
86 struct iosm_imem *ipc_imem = in ipc_imem_td_alloc_timer_cb() local
89 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_alloc_timer, 0, NULL, in ipc_imem_td_alloc_timer_cb()
95 static int ipc_imem_tq_fast_update_timer_cb(struct iosm_imem *ipc_imem, int arg, in ipc_imem_tq_fast_update_timer_cb() argument
98 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_tq_fast_update_timer_cb()
107 struct iosm_imem *ipc_imem = in ipc_imem_fast_update_timer_cb() local
110 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_fast_update_timer_cb, 0, in ipc_imem_fast_update_timer_cb()
115 static int ipc_imem_setup_cp_mux_cap_init(struct iosm_imem *ipc_imem, in ipc_imem_setup_cp_mux_cap_init() argument
118 ipc_mmio_update_cp_capability(ipc_imem->mmio); in ipc_imem_setup_cp_mux_cap_init()
120 if (!ipc_imem->mmio->has_mux_lite) { in ipc_imem_setup_cp_mux_cap_init()
121 dev_err(ipc_imem->dev, "Failed to get Mux capability."); in ipc_imem_setup_cp_mux_cap_init()
127 cfg->ul_flow = (ipc_imem->mmio->has_ul_flow_credit == 1) ? in ipc_imem_setup_cp_mux_cap_init()
140 void ipc_imem_msg_send_feature_set(struct iosm_imem *ipc_imem, in ipc_imem_msg_send_feature_set() argument
147 ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol, in ipc_imem_msg_send_feature_set()
151 ipc_protocol_msg_send(ipc_imem->ipc_protocol, in ipc_imem_msg_send_feature_set()
155 void ipc_imem_td_update_timer_start(struct iosm_imem *ipc_imem) in ipc_imem_td_update_timer_start() argument
158 if (!ipc_imem->enter_runtime || ipc_imem->td_update_timer_suspended) { in ipc_imem_td_update_timer_start()
160 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_td_update_timer_start()
165 if (!hrtimer_active(&ipc_imem->tdupdate_timer)) { in ipc_imem_td_update_timer_start()
166 ipc_imem->hrtimer_period = in ipc_imem_td_update_timer_start()
168 if (!hrtimer_active(&ipc_imem->tdupdate_timer)) in ipc_imem_td_update_timer_start()
169 hrtimer_start(&ipc_imem->tdupdate_timer, in ipc_imem_td_update_timer_start()
170 ipc_imem->hrtimer_period, in ipc_imem_td_update_timer_start()
181 bool ipc_imem_ul_write_td(struct iosm_imem *ipc_imem) in ipc_imem_ul_write_td() argument
191 for (i = 0; i < ipc_imem->nr_of_channels; i++) { in ipc_imem_ul_write_td()
192 channel = &ipc_imem->channels[i]; in ipc_imem_ul_write_td()
205 ipc_protocol_ul_td_send(ipc_imem->ipc_protocol, in ipc_imem_ul_write_td()
209 ipc_protocol_ul_td_send(ipc_imem->ipc_protocol, in ipc_imem_ul_write_td()
217 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_ul_write_td()
224 void ipc_imem_ipc_init_check(struct iosm_imem *ipc_imem) in ipc_imem_ipc_init_check() argument
228 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT; in ipc_imem_ipc_init_check()
231 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC, in ipc_imem_ipc_init_check()
235 if (ipc_mmio_get_ipc_state(ipc_imem->mmio) == in ipc_imem_ipc_init_check()
236 ipc_imem->ipc_requested_state) { in ipc_imem_ipc_init_check()
238 ipc_mmio_config(ipc_imem->mmio); in ipc_imem_ipc_init_check()
241 ipc_imem->ipc_requested_state = in ipc_imem_ipc_init_check()
243 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC, in ipc_imem_ipc_init_check()
252 dev_err(ipc_imem->dev, "%s: ipc_status(%d) ne. IPC_MEM_DEVICE_IPC_INIT", in ipc_imem_ipc_init_check()
253 ipc_imem_phase_get_string(ipc_imem->phase), in ipc_imem_ipc_init_check()
254 ipc_mmio_get_ipc_state(ipc_imem->mmio)); in ipc_imem_ipc_init_check()
256 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_TIMEOUT); in ipc_imem_ipc_init_check()
260 static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem, in ipc_imem_dl_skb_process() argument
273 ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, in ipc_imem_dl_skb_process()
276 wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port, in ipc_imem_dl_skb_process()
282 ipc_mux_dl_decode(ipc_imem->mux, skb); in ipc_imem_dl_skb_process()
285 dev_err(ipc_imem->dev, "Invalid channel type"); in ipc_imem_dl_skb_process()
291 static void ipc_imem_dl_pipe_process(struct iosm_imem *ipc_imem, in ipc_imem_dl_pipe_process() argument
302 ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head, in ipc_imem_dl_pipe_process()
315 skb = ipc_protocol_dl_td_process(ipc_imem->ipc_protocol, pipe); in ipc_imem_dl_pipe_process()
318 ipc_imem_dl_skb_process(ipc_imem, pipe, skb); in ipc_imem_dl_pipe_process()
322 while (ipc_imem_dl_skb_alloc(ipc_imem, pipe)) in ipc_imem_dl_pipe_process()
327 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_dl_pipe_process()
332 ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer); in ipc_imem_dl_pipe_process()
340 ipc_imem->hrtimer_period = in ipc_imem_dl_pipe_process()
342 hrtimer_start(&ipc_imem->fast_update_timer, in ipc_imem_dl_pipe_process()
343 ipc_imem->hrtimer_period, HRTIMER_MODE_REL); in ipc_imem_dl_pipe_process()
346 if (ipc_imem->app_notify_dl_pend) in ipc_imem_dl_pipe_process()
347 complete(&ipc_imem->dl_pend_sem); in ipc_imem_dl_pipe_process()
351 static void ipc_imem_ul_pipe_process(struct iosm_imem *ipc_imem, in ipc_imem_ul_pipe_process() argument
362 ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head, in ipc_imem_ul_pipe_process()
374 skb = ipc_protocol_ul_td_process(ipc_imem->ipc_protocol, pipe); in ipc_imem_ul_pipe_process()
388 ipc_mux_ul_encoded_process(ipc_imem->mux, skb); in ipc_imem_ul_pipe_process()
390 dev_err(ipc_imem->dev, in ipc_imem_ul_pipe_process()
394 ipc_pcie_kfree_skb(ipc_imem->pcie, skb); in ipc_imem_ul_pipe_process()
400 ipc_mux_check_n_restart_tx(ipc_imem->mux); in ipc_imem_ul_pipe_process()
402 if (ipc_imem->app_notify_ul_pend) in ipc_imem_ul_pipe_process()
403 complete(&ipc_imem->ul_pend_sem); in ipc_imem_ul_pipe_process()
407 static void ipc_imem_rom_irq_exec(struct iosm_imem *ipc_imem) in ipc_imem_rom_irq_exec() argument
411 channel = ipc_imem->ipc_devlink->devlink_sio.channel; in ipc_imem_rom_irq_exec()
412 ipc_imem->rom_exit_code = ipc_mmio_get_rom_exit_code(ipc_imem->mmio); in ipc_imem_rom_irq_exec()
417 static int ipc_imem_tq_td_update_timer_cb(struct iosm_imem *ipc_imem, int arg, in ipc_imem_tq_td_update_timer_cb() argument
420 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, in ipc_imem_tq_td_update_timer_cb()
426 static void ipc_imem_slp_control_exec(struct iosm_imem *ipc_imem) in ipc_imem_slp_control_exec() argument
429 if (ipc_protocol_pm_dev_sleep_handle(ipc_imem->ipc_protocol) && in ipc_imem_slp_control_exec()
430 hrtimer_active(&ipc_imem->tdupdate_timer)) { in ipc_imem_slp_control_exec()
432 ipc_imem_tq_td_update_timer_cb(ipc_imem, 0, NULL, 0); in ipc_imem_slp_control_exec()
434 ipc_imem_hrtimer_stop(&ipc_imem->tdupdate_timer); in ipc_imem_slp_control_exec()
436 ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer); in ipc_imem_slp_control_exec()
441 static int ipc_imem_tq_startup_timer_cb(struct iosm_imem *ipc_imem, int arg, in ipc_imem_tq_startup_timer_cb() argument
445 if (ipc_imem_phase_update(ipc_imem) != IPC_P_RUN) in ipc_imem_tq_startup_timer_cb()
448 if (ipc_mmio_get_ipc_state(ipc_imem->mmio) == in ipc_imem_tq_startup_timer_cb()
450 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT; in ipc_imem_tq_startup_timer_cb()
452 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC, in ipc_imem_tq_startup_timer_cb()
455 ipc_imem->hrtimer_period = ktime_set(0, 100 * 1000UL * 1000ULL); in ipc_imem_tq_startup_timer_cb()
457 if (!hrtimer_active(&ipc_imem->startup_timer)) in ipc_imem_tq_startup_timer_cb()
458 hrtimer_start(&ipc_imem->startup_timer, in ipc_imem_tq_startup_timer_cb()
459 ipc_imem->hrtimer_period, in ipc_imem_tq_startup_timer_cb()
461 } else if (ipc_mmio_get_ipc_state(ipc_imem->mmio) == in ipc_imem_tq_startup_timer_cb()
464 ipc_imem_hrtimer_stop(&ipc_imem->startup_timer); in ipc_imem_tq_startup_timer_cb()
467 ipc_mmio_config(ipc_imem->mmio); in ipc_imem_tq_startup_timer_cb()
468 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_RUNNING; in ipc_imem_tq_startup_timer_cb()
469 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC, in ipc_imem_tq_startup_timer_cb()
479 struct iosm_imem *ipc_imem = in ipc_imem_startup_timer_cb() local
482 if (ktime_to_ns(ipc_imem->hrtimer_period)) { in ipc_imem_startup_timer_cb()
483 hrtimer_forward_now(&ipc_imem->startup_timer, in ipc_imem_startup_timer_cb()
484 ipc_imem->hrtimer_period); in ipc_imem_startup_timer_cb()
488 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_startup_timer_cb, 0, in ipc_imem_startup_timer_cb()
495 ipc_imem_get_exec_stage_buffered(struct iosm_imem *ipc_imem) in ipc_imem_get_exec_stage_buffered() argument
497 return (ipc_imem->phase == IPC_P_RUN && in ipc_imem_get_exec_stage_buffered()
498 ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) ? in ipc_imem_get_exec_stage_buffered()
499 ipc_protocol_get_ap_exec_stage(ipc_imem->ipc_protocol) : in ipc_imem_get_exec_stage_buffered()
500 ipc_mmio_get_exec_stage(ipc_imem->mmio); in ipc_imem_get_exec_stage_buffered()
504 static int ipc_imem_send_mdm_rdy_cb(struct iosm_imem *ipc_imem, int arg, in ipc_imem_send_mdm_rdy_cb() argument
508 ipc_imem_get_exec_stage_buffered(ipc_imem); in ipc_imem_send_mdm_rdy_cb()
511 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY); in ipc_imem_send_mdm_rdy_cb()
523 struct iosm_imem *ipc_imem; in ipc_imem_run_state_worker() local
526 ipc_imem = container_of(instance, struct iosm_imem, run_state_worker); in ipc_imem_run_state_worker()
528 if (ipc_imem->phase != IPC_P_RUN) { in ipc_imem_run_state_worker()
529 dev_err(ipc_imem->dev, in ipc_imem_run_state_worker()
534 if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag)) in ipc_imem_run_state_worker()
535 ipc_devlink_deinit(ipc_imem->ipc_devlink); in ipc_imem_run_state_worker()
537 if (!ipc_imem_setup_cp_mux_cap_init(ipc_imem, &mux_cfg)) in ipc_imem_run_state_worker()
538 ipc_imem->mux = ipc_mux_init(&mux_cfg, ipc_imem); in ipc_imem_run_state_worker()
540 ipc_imem_wwan_channel_init(ipc_imem, mux_cfg.protocol); in ipc_imem_run_state_worker()
541 if (ipc_imem->mux) in ipc_imem_run_state_worker()
542 ipc_imem->mux->wwan = ipc_imem->wwan; in ipc_imem_run_state_worker()
546 ipc_imem->ipc_port[ctrl_chl_idx] = NULL; in ipc_imem_run_state_worker()
548 ipc_imem_channel_init(ipc_imem, IPC_CTYPE_CTRL, in ipc_imem_run_state_worker()
551 ipc_imem->ipc_port[ctrl_chl_idx] = in ipc_imem_run_state_worker()
552 ipc_port_init(ipc_imem, chnl_cfg_port); in ipc_imem_run_state_worker()
558 ipc_task_queue_send_task(ipc_imem, ipc_imem_send_mdm_rdy_cb, 0, NULL, 0, in ipc_imem_run_state_worker()
564 set_bit(FULLY_FUNCTIONAL, &ipc_imem->flag); in ipc_imem_run_state_worker()
570 static void ipc_imem_handle_irq(struct iosm_imem *ipc_imem, int irq) in ipc_imem_handle_irq() argument
579 ipc_imem->ev_irq_pending[irq] = false; in ipc_imem_handle_irq()
582 old_phase = ipc_imem->phase; in ipc_imem_handle_irq()
585 dev_dbg(ipc_imem->dev, in ipc_imem_handle_irq()
592 phase = ipc_imem_phase_update(ipc_imem); in ipc_imem_handle_irq()
596 if (!ipc_imem->enter_runtime) { in ipc_imem_handle_irq()
598 ipc_imem->enter_runtime = 1; in ipc_imem_handle_irq()
603 ipc_imem_msg_send_device_sleep(ipc_imem, in ipc_imem_handle_irq()
604 ipc_imem->device_sleep); in ipc_imem_handle_irq()
606 ipc_imem_msg_send_feature_set(ipc_imem, in ipc_imem_handle_irq()
612 ipc_protocol_get_ipc_status(ipc_imem->ipc_protocol); in ipc_imem_handle_irq()
615 if (ipc_imem->ipc_status != curr_ipc_status) { in ipc_imem_handle_irq()
616 ipc_imem->ipc_status = curr_ipc_status; in ipc_imem_handle_irq()
618 if (ipc_imem->ipc_status == in ipc_imem_handle_irq()
620 schedule_work(&ipc_imem->run_state_worker); in ipc_imem_handle_irq()
625 ipc_imem_slp_control_exec(ipc_imem); in ipc_imem_handle_irq()
631 dev_err(ipc_imem->dev, "confused phase %s", in ipc_imem_handle_irq()
647 ipc_imem_rom_irq_exec(ipc_imem); in ipc_imem_handle_irq()
655 ipc_protocol_msg_process(ipc_imem, irq); in ipc_imem_handle_irq()
659 struct ipc_pipe *ul_pipe = &ipc_imem->channels[i].ul_pipe; in ipc_imem_handle_irq()
660 struct ipc_pipe *dl_pipe = &ipc_imem->channels[i].dl_pipe; in ipc_imem_handle_irq()
664 ipc_imem_dl_pipe_process(ipc_imem, dl_pipe); in ipc_imem_handle_irq()
671 ipc_imem_ul_pipe_process(ipc_imem, ul_pipe); in ipc_imem_handle_irq()
675 if (ipc_mux_ul_data_encode(ipc_imem->mux)) in ipc_imem_handle_irq()
676 ipc_imem_td_update_timer_start(ipc_imem); in ipc_imem_handle_irq()
681 ul_pending |= ipc_imem_ul_write_td(ipc_imem); in ipc_imem_handle_irq()
685 ipc_imem->hrtimer_period = in ipc_imem_handle_irq()
687 if (!hrtimer_active(&ipc_imem->tdupdate_timer)) in ipc_imem_handle_irq()
688 hrtimer_start(&ipc_imem->tdupdate_timer, in ipc_imem_handle_irq()
689 ipc_imem->hrtimer_period, in ipc_imem_handle_irq()
698 ipc_imem->ipc_requested_state == IPC_MEM_DEVICE_IPC_RUNNING && in ipc_imem_handle_irq()
699 ipc_mmio_get_ipc_state(ipc_imem->mmio) == in ipc_imem_handle_irq()
701 complete(&ipc_imem->ipc_devlink->devlink_sio.channel->ul_sem); in ipc_imem_handle_irq()
705 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_DONT_CARE; in ipc_imem_handle_irq()
708 ipc_imem->hrtimer_period = in ipc_imem_handle_irq()
710 if (!hrtimer_active(&ipc_imem->td_alloc_timer)) in ipc_imem_handle_irq()
711 hrtimer_start(&ipc_imem->td_alloc_timer, in ipc_imem_handle_irq()
712 ipc_imem->hrtimer_period, in ipc_imem_handle_irq()
718 static int ipc_imem_tq_irq_cb(struct iosm_imem *ipc_imem, int arg, void *msg, in ipc_imem_tq_irq_cb() argument
721 ipc_imem_handle_irq(ipc_imem, arg); in ipc_imem_tq_irq_cb()
726 void ipc_imem_ul_send(struct iosm_imem *ipc_imem) in ipc_imem_ul_send() argument
729 if (ipc_imem_ul_write_td(ipc_imem)) in ipc_imem_ul_send()
730 ipc_imem_td_update_timer_start(ipc_imem); in ipc_imem_ul_send()
734 static enum ipc_phase ipc_imem_phase_update_check(struct iosm_imem *ipc_imem, in ipc_imem_phase_update_check() argument
739 if (ipc_imem->phase != IPC_P_ROM) { in ipc_imem_phase_update_check()
741 ipc_uevent_send(ipc_imem->dev, UEVENT_ROM_READY); in ipc_imem_phase_update_check()
744 ipc_imem->phase = IPC_P_ROM; in ipc_imem_phase_update_check()
748 ipc_imem->phase = IPC_P_PSI; in ipc_imem_phase_update_check()
752 ipc_imem->phase = IPC_P_EBL; in ipc_imem_phase_update_check()
756 if (ipc_imem->phase != IPC_P_RUN && in ipc_imem_phase_update_check()
757 ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) { in ipc_imem_phase_update_check()
758 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY); in ipc_imem_phase_update_check()
760 ipc_imem->phase = IPC_P_RUN; in ipc_imem_phase_update_check()
764 if (ipc_imem->phase != IPC_P_CRASH) in ipc_imem_phase_update_check()
765 ipc_uevent_send(ipc_imem->dev, UEVENT_CRASH); in ipc_imem_phase_update_check()
767 ipc_imem->phase = IPC_P_CRASH; in ipc_imem_phase_update_check()
771 if (ipc_imem->phase != IPC_P_CD_READY) in ipc_imem_phase_update_check()
772 ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY); in ipc_imem_phase_update_check()
773 ipc_imem->phase = IPC_P_CD_READY; in ipc_imem_phase_update_check()
780 ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN); in ipc_imem_phase_update_check()
784 return ipc_imem->phase; in ipc_imem_phase_update_check()
788 static bool ipc_imem_pipe_open(struct iosm_imem *ipc_imem, in ipc_imem_pipe_open() argument
795 if (ipc_protocol_msg_send(ipc_imem->ipc_protocol, in ipc_imem_pipe_open()
803 static int ipc_imem_tq_pipe_td_alloc(struct iosm_imem *ipc_imem, int arg, in ipc_imem_tq_pipe_td_alloc() argument
811 processed |= ipc_imem_dl_skb_alloc(ipc_imem, dl_pipe); in ipc_imem_tq_pipe_td_alloc()
817 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, arg); in ipc_imem_tq_pipe_td_alloc()
825 struct iosm_imem *ipc_imem = in ipc_imem_td_update_timer_cb() local
828 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_update_timer_cb, 0, in ipc_imem_td_update_timer_cb()
834 enum ipc_phase ipc_imem_phase_update(struct iosm_imem *ipc_imem) in ipc_imem_phase_update() argument
837 ipc_imem_get_exec_stage_buffered(ipc_imem); in ipc_imem_phase_update()
839 return ipc_imem->phase == IPC_P_OFF_REQ ? in ipc_imem_phase_update()
840 ipc_imem->phase : in ipc_imem_phase_update()
841 ipc_imem_phase_update_check(ipc_imem, exec_stage); in ipc_imem_phase_update()
876 void ipc_imem_pipe_close(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe) in ipc_imem_pipe_close() argument
881 ipc_protocol_msg_send(ipc_imem->ipc_protocol, IPC_MSG_PREP_PIPE_CLOSE, in ipc_imem_pipe_close()
884 ipc_imem_pipe_cleanup(ipc_imem, pipe); in ipc_imem_pipe_close()
887 void ipc_imem_channel_close(struct iosm_imem *ipc_imem, int channel_id) in ipc_imem_channel_close() argument
891 if (channel_id < 0 || channel_id >= ipc_imem->nr_of_channels) { in ipc_imem_channel_close()
892 dev_err(ipc_imem->dev, "invalid channel id %d", channel_id); in ipc_imem_channel_close()
896 channel = &ipc_imem->channels[channel_id]; in ipc_imem_channel_close()
899 dev_err(ipc_imem->dev, "ch[%d]: invalid channel state %d", in ipc_imem_channel_close()
909 if (ipc_imem->phase == IPC_P_RUN) { in ipc_imem_channel_close()
910 ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe); in ipc_imem_channel_close()
911 ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe); in ipc_imem_channel_close()
914 ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe); in ipc_imem_channel_close()
915 ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe); in ipc_imem_channel_close()
921 struct ipc_mem_channel *ipc_imem_channel_open(struct iosm_imem *ipc_imem, in ipc_imem_channel_open() argument
927 dev_err(ipc_imem->dev, "invalid channel ID: %d", channel_id); in ipc_imem_channel_open()
931 channel = &ipc_imem->channels[channel_id]; in ipc_imem_channel_open()
935 if (!ipc_imem_pipe_open(ipc_imem, &channel->ul_pipe)) in ipc_imem_channel_open()
938 if (!ipc_imem_pipe_open(ipc_imem, &channel->dl_pipe)) in ipc_imem_channel_open()
942 if (ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_pipe_td_alloc, db_id, in ipc_imem_channel_open()
944 dev_err(ipc_imem->dev, "td allocation failed : %d", channel_id); in ipc_imem_channel_open()
951 ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe); in ipc_imem_channel_open()
953 ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe); in ipc_imem_channel_open()
959 void ipc_imem_pm_suspend(struct iosm_imem *ipc_imem) in ipc_imem_pm_suspend() argument
961 ipc_protocol_suspend(ipc_imem->ipc_protocol); in ipc_imem_pm_suspend()
964 void ipc_imem_pm_s2idle_sleep(struct iosm_imem *ipc_imem, bool sleep) in ipc_imem_pm_s2idle_sleep() argument
966 ipc_protocol_s2idle_sleep(ipc_imem->ipc_protocol, sleep); in ipc_imem_pm_s2idle_sleep()
969 void ipc_imem_pm_resume(struct iosm_imem *ipc_imem) in ipc_imem_pm_resume() argument
973 if (ipc_protocol_resume(ipc_imem->ipc_protocol)) { in ipc_imem_pm_resume()
974 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio); in ipc_imem_pm_resume()
975 ipc_imem_phase_update_check(ipc_imem, stage); in ipc_imem_pm_resume()
985 int ipc_imem_channel_alloc(struct iosm_imem *ipc_imem, int index, in ipc_imem_channel_alloc() argument
992 for (i = 0; i < ipc_imem->nr_of_channels; i++) { in ipc_imem_channel_alloc()
993 channel = &ipc_imem->channels[i]; in ipc_imem_channel_alloc()
998 if (i >= ipc_imem->nr_of_channels) { in ipc_imem_channel_alloc()
999 dev_dbg(ipc_imem->dev, in ipc_imem_channel_alloc()
1005 if (ipc_imem->channels[i].state != IMEM_CHANNEL_FREE) { in ipc_imem_channel_alloc()
1006 dev_dbg(ipc_imem->dev, "channel is in use"); in ipc_imem_channel_alloc()
1020 void ipc_imem_channel_init(struct iosm_imem *ipc_imem, enum ipc_ctype ctype, in ipc_imem_channel_init() argument
1027 dev_err(ipc_imem->dev, "invalid pipe: ul_pipe=%d, dl_pipe=%d", in ipc_imem_channel_init()
1032 if (ipc_imem->nr_of_channels >= IPC_MEM_MAX_CHANNELS) { in ipc_imem_channel_init()
1033 dev_err(ipc_imem->dev, "too many channels"); in ipc_imem_channel_init()
1037 channel = &ipc_imem->channels[ipc_imem->nr_of_channels]; in ipc_imem_channel_init()
1038 channel->channel_id = ipc_imem->nr_of_channels; in ipc_imem_channel_init()
1043 ipc_imem->nr_of_channels++; in ipc_imem_channel_init()
1045 ipc_imem_channel_update(ipc_imem, channel->channel_id, chnl_cfg, in ipc_imem_channel_init()
1053 void ipc_imem_channel_update(struct iosm_imem *ipc_imem, int id, in ipc_imem_channel_update() argument
1058 if (id < 0 || id >= ipc_imem->nr_of_channels) { in ipc_imem_channel_update()
1059 dev_err(ipc_imem->dev, "invalid channel id %d", id); in ipc_imem_channel_update()
1063 channel = &ipc_imem->channels[id]; in ipc_imem_channel_update()
1067 dev_err(ipc_imem->dev, "invalid channel state %d", in ipc_imem_channel_update()
1093 static void ipc_imem_channel_reset(struct iosm_imem *ipc_imem) in ipc_imem_channel_reset() argument
1097 for (i = 0; i < ipc_imem->nr_of_channels; i++) { in ipc_imem_channel_reset()
1100 channel = &ipc_imem->channels[i]; in ipc_imem_channel_reset()
1102 ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe); in ipc_imem_channel_reset()
1103 ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe); in ipc_imem_channel_reset()
1109 void ipc_imem_pipe_cleanup(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe) in ipc_imem_pipe_cleanup() argument
1120 ipc_pcie_kfree_skb(ipc_imem->pcie, skb); in ipc_imem_pipe_cleanup()
1122 ipc_protocol_pipe_cleanup(ipc_imem->ipc_protocol, pipe); in ipc_imem_pipe_cleanup()
1126 static void ipc_imem_device_ipc_uninit(struct iosm_imem *ipc_imem) in ipc_imem_device_ipc_uninit() argument
1134 if (ipc_pcie_check_data_link_active(ipc_imem->pcie)) { in ipc_imem_device_ipc_uninit()
1139 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC, in ipc_imem_device_ipc_uninit()
1141 ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio); in ipc_imem_device_ipc_uninit()
1151 ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio); in ipc_imem_device_ipc_uninit()
1156 void ipc_imem_cleanup(struct iosm_imem *ipc_imem) in ipc_imem_cleanup() argument
1158 ipc_imem->phase = IPC_P_OFF_REQ; in ipc_imem_cleanup()
1161 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_NOT_READY); in ipc_imem_cleanup()
1163 hrtimer_cancel(&ipc_imem->td_alloc_timer); in ipc_imem_cleanup()
1164 hrtimer_cancel(&ipc_imem->tdupdate_timer); in ipc_imem_cleanup()
1165 hrtimer_cancel(&ipc_imem->fast_update_timer); in ipc_imem_cleanup()
1166 hrtimer_cancel(&ipc_imem->startup_timer); in ipc_imem_cleanup()
1169 cancel_work_sync(&ipc_imem->run_state_worker); in ipc_imem_cleanup()
1171 if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) { in ipc_imem_cleanup()
1172 ipc_mux_deinit(ipc_imem->mux); in ipc_imem_cleanup()
1173 ipc_wwan_deinit(ipc_imem->wwan); in ipc_imem_cleanup()
1174 ipc_port_deinit(ipc_imem->ipc_port); in ipc_imem_cleanup()
1177 if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag)) in ipc_imem_cleanup()
1178 ipc_devlink_deinit(ipc_imem->ipc_devlink); in ipc_imem_cleanup()
1180 ipc_imem_device_ipc_uninit(ipc_imem); in ipc_imem_cleanup()
1181 ipc_imem_channel_reset(ipc_imem); in ipc_imem_cleanup()
1183 ipc_protocol_deinit(ipc_imem->ipc_protocol); in ipc_imem_cleanup()
1184 ipc_task_deinit(ipc_imem->ipc_task); in ipc_imem_cleanup()
1186 kfree(ipc_imem->ipc_task); in ipc_imem_cleanup()
1187 kfree(ipc_imem->mmio); in ipc_imem_cleanup()
1189 ipc_imem->phase = IPC_P_OFF; in ipc_imem_cleanup()
1197 static int ipc_imem_config(struct iosm_imem *ipc_imem) in ipc_imem_config() argument
1202 init_completion(&ipc_imem->ul_pend_sem); in ipc_imem_config()
1204 init_completion(&ipc_imem->dl_pend_sem); in ipc_imem_config()
1207 ipc_imem->ipc_status = IPC_MEM_DEVICE_IPC_UNINIT; in ipc_imem_config()
1208 ipc_imem->enter_runtime = 0; in ipc_imem_config()
1210 phase = ipc_imem_phase_update(ipc_imem); in ipc_imem_config()
1215 ipc_imem->hrtimer_period = ktime_set(0, 1000 * 1000 * 1000ULL); in ipc_imem_config()
1217 if (!hrtimer_active(&ipc_imem->startup_timer)) in ipc_imem_config()
1218 hrtimer_start(&ipc_imem->startup_timer, in ipc_imem_config()
1219 ipc_imem->hrtimer_period, in ipc_imem_config()
1227 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_UNINIT; in ipc_imem_config()
1230 if (ipc_imem->ipc_requested_state == in ipc_imem_config()
1231 ipc_mmio_get_ipc_state(ipc_imem->mmio)) { in ipc_imem_config()
1232 ipc_imem_ipc_init_check(ipc_imem); in ipc_imem_config()
1236 dev_err(ipc_imem->dev, in ipc_imem_config()
1238 ipc_mmio_get_ipc_state(ipc_imem->mmio)); in ipc_imem_config()
1242 dev_dbg(ipc_imem->dev, in ipc_imem_config()
1247 dev_err(ipc_imem->dev, "unexpected operation phase %d", phase); in ipc_imem_config()
1251 complete(&ipc_imem->dl_pend_sem); in ipc_imem_config()
1252 complete(&ipc_imem->ul_pend_sem); in ipc_imem_config()
1253 ipc_imem->phase = IPC_P_OFF; in ipc_imem_config()
1261 struct iosm_imem *ipc_imem = kzalloc(sizeof(*pcie->imem), GFP_KERNEL); in ipc_imem_init() local
1264 if (!ipc_imem) in ipc_imem_init()
1268 ipc_imem->pcie = pcie; in ipc_imem_init()
1269 ipc_imem->dev = dev; in ipc_imem_init()
1271 ipc_imem->pci_device_id = device_id; in ipc_imem_init()
1273 ipc_imem->cp_version = 0; in ipc_imem_init()
1274 ipc_imem->device_sleep = IPC_HOST_SLEEP_ENTER_SLEEP; in ipc_imem_init()
1277 ipc_imem->nr_of_channels = 0; in ipc_imem_init()
1280 ipc_imem->mmio = ipc_mmio_init(mmio, ipc_imem->dev); in ipc_imem_init()
1281 if (!ipc_imem->mmio) { in ipc_imem_init()
1282 dev_err(ipc_imem->dev, "failed to initialize mmio region"); in ipc_imem_init()
1286 ipc_imem->ipc_task = kzalloc(sizeof(*ipc_imem->ipc_task), in ipc_imem_init()
1290 if (!ipc_imem->ipc_task) in ipc_imem_init()
1293 if (ipc_task_init(ipc_imem->ipc_task)) in ipc_imem_init()
1296 ipc_imem->ipc_task->dev = ipc_imem->dev; in ipc_imem_init()
1298 INIT_WORK(&ipc_imem->run_state_worker, ipc_imem_run_state_worker); in ipc_imem_init()
1300 ipc_imem->ipc_protocol = ipc_protocol_init(ipc_imem); in ipc_imem_init()
1302 if (!ipc_imem->ipc_protocol) in ipc_imem_init()
1306 ipc_imem->phase = IPC_P_OFF; in ipc_imem_init()
1308 hrtimer_init(&ipc_imem->startup_timer, CLOCK_MONOTONIC, in ipc_imem_init()
1310 ipc_imem->startup_timer.function = ipc_imem_startup_timer_cb; in ipc_imem_init()
1312 hrtimer_init(&ipc_imem->tdupdate_timer, CLOCK_MONOTONIC, in ipc_imem_init()
1314 ipc_imem->tdupdate_timer.function = ipc_imem_td_update_timer_cb; in ipc_imem_init()
1316 hrtimer_init(&ipc_imem->fast_update_timer, CLOCK_MONOTONIC, in ipc_imem_init()
1318 ipc_imem->fast_update_timer.function = ipc_imem_fast_update_timer_cb; in ipc_imem_init()
1320 hrtimer_init(&ipc_imem->td_alloc_timer, CLOCK_MONOTONIC, in ipc_imem_init()
1322 ipc_imem->td_alloc_timer.function = ipc_imem_td_alloc_timer_cb; in ipc_imem_init()
1324 if (ipc_imem_config(ipc_imem)) { in ipc_imem_init()
1325 dev_err(ipc_imem->dev, "failed to initialize the imem"); in ipc_imem_init()
1329 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio); in ipc_imem_init()
1332 ipc_imem->ipc_devlink = ipc_devlink_init(ipc_imem); in ipc_imem_init()
1333 if (!ipc_imem->ipc_devlink) { in ipc_imem_init()
1334 dev_err(ipc_imem->dev, "Devlink register failed"); in ipc_imem_init()
1338 if (ipc_flash_link_establish(ipc_imem)) in ipc_imem_init()
1341 set_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag); in ipc_imem_init()
1343 return ipc_imem; in ipc_imem_init()
1345 ipc_devlink_deinit(ipc_imem->ipc_devlink); in ipc_imem_init()
1347 hrtimer_cancel(&ipc_imem->td_alloc_timer); in ipc_imem_init()
1348 hrtimer_cancel(&ipc_imem->fast_update_timer); in ipc_imem_init()
1349 hrtimer_cancel(&ipc_imem->tdupdate_timer); in ipc_imem_init()
1350 hrtimer_cancel(&ipc_imem->startup_timer); in ipc_imem_init()
1352 cancel_work_sync(&ipc_imem->run_state_worker); in ipc_imem_init()
1353 ipc_task_deinit(ipc_imem->ipc_task); in ipc_imem_init()
1355 kfree(ipc_imem->ipc_task); in ipc_imem_init()
1357 kfree(ipc_imem->mmio); in ipc_imem_init()
1359 kfree(ipc_imem); in ipc_imem_init()
1363 void ipc_imem_irq_process(struct iosm_imem *ipc_imem, int irq) in ipc_imem_irq_process() argument
1366 if (ipc_imem && !ipc_imem->ev_irq_pending[irq]) { in ipc_imem_irq_process()
1367 ipc_imem->ev_irq_pending[irq] = true; in ipc_imem_irq_process()
1368 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_irq_cb, irq, in ipc_imem_irq_process()
1373 void ipc_imem_td_update_timer_suspend(struct iosm_imem *ipc_imem, bool suspend) in ipc_imem_td_update_timer_suspend() argument
1375 ipc_imem->td_update_timer_suspended = suspend; in ipc_imem_td_update_timer_suspend()
1381 static int ipc_imem_devlink_trigger_chip_info_cb(struct iosm_imem *ipc_imem, in ipc_imem_devlink_trigger_chip_info_cb() argument
1391 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio); in ipc_imem_devlink_trigger_chip_info_cb()
1393 dev_err(ipc_imem->dev, in ipc_imem_devlink_trigger_chip_info_cb()
1398 size = ipc_imem->mmio->chip_info_size; in ipc_imem_devlink_trigger_chip_info_cb()
1402 skb = ipc_pcie_alloc_local_skb(ipc_imem->pcie, GFP_ATOMIC, size); in ipc_imem_devlink_trigger_chip_info_cb()
1404 dev_err(ipc_imem->dev, "exhausted skbuf kernel DL memory"); in ipc_imem_devlink_trigger_chip_info_cb()
1409 ipc_mmio_copy_chip_info(ipc_imem->mmio, skb_put(skb, size), size); in ipc_imem_devlink_trigger_chip_info_cb()
1411 dev_dbg(ipc_imem->dev, "execution_stage[%X] eq. BOOT", stage); in ipc_imem_devlink_trigger_chip_info_cb()
1412 ipc_imem->phase = ipc_imem_phase_update(ipc_imem); in ipc_imem_devlink_trigger_chip_info_cb()
1413 ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb); in ipc_imem_devlink_trigger_chip_info_cb()
1419 int ipc_imem_devlink_trigger_chip_info(struct iosm_imem *ipc_imem) in ipc_imem_devlink_trigger_chip_info() argument
1421 return ipc_task_queue_send_task(ipc_imem, in ipc_imem_devlink_trigger_chip_info()