Lines Matching refs:psock
403 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, in sk_msg_recvmsg() argument
411 msg_rx = sk_psock_peek_msg(psock); in sk_msg_recvmsg()
459 msg_rx = sk_psock_next_msg(psock, msg_rx); in sk_msg_recvmsg()
467 msg_rx = sk_psock_dequeue_msg(psock); in sk_msg_recvmsg()
470 msg_rx = sk_psock_peek_msg(psock); in sk_msg_recvmsg()
479 struct sk_psock *psock; in sk_msg_is_readable() local
483 psock = sk_psock(sk); in sk_msg_is_readable()
484 if (likely(psock)) in sk_msg_is_readable()
485 empty = list_empty(&psock->ingress_msg); in sk_msg_is_readable()
512 struct sk_psock *psock, in sk_psock_skb_ingress_enqueue() argument
535 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress_enqueue()
536 sk_psock_data_ready(sk, psock); in sk_psock_skb_ingress_enqueue()
540 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
543 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_skb_ingress() argument
546 struct sock *sk = psock->sk; in sk_psock_skb_ingress()
555 return sk_psock_skb_ingress_self(psock, skb, off, len); in sk_psock_skb_ingress()
567 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg); in sk_psock_skb_ingress()
577 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_skb_ingress_self() argument
581 struct sock *sk = psock->sk; in sk_psock_skb_ingress_self()
588 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg); in sk_psock_skb_ingress_self()
594 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_handle_skb() argument
598 if (!sock_writeable(psock->sk)) in sk_psock_handle_skb()
600 return skb_send_sock(psock->sk, skb, off, len); in sk_psock_handle_skb()
602 return sk_psock_skb_ingress(psock, skb, off, len); in sk_psock_handle_skb()
605 static void sk_psock_skb_state(struct sk_psock *psock, in sk_psock_skb_state() argument
610 spin_lock_bh(&psock->ingress_lock); in sk_psock_skb_state()
611 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_skb_state()
616 sock_drop(psock->sk, skb); in sk_psock_skb_state()
618 spin_unlock_bh(&psock->ingress_lock); in sk_psock_skb_state()
623 struct sk_psock *psock = container_of(work, struct sk_psock, work); in sk_psock_backlog() local
624 struct sk_psock_work_state *state = &psock->work_state; in sk_psock_backlog()
630 mutex_lock(&psock->work_mutex); in sk_psock_backlog()
632 spin_lock_bh(&psock->ingress_lock); in sk_psock_backlog()
637 spin_unlock_bh(&psock->ingress_lock); in sk_psock_backlog()
642 while ((skb = skb_dequeue(&psock->ingress_skb))) { in sk_psock_backlog()
656 if (!sock_flag(psock->sk, SOCK_DEAD)) in sk_psock_backlog()
657 ret = sk_psock_handle_skb(psock, skb, off, in sk_psock_backlog()
661 sk_psock_skb_state(psock, state, skb, in sk_psock_backlog()
666 sk_psock_report_error(psock, ret ? -ret : EPIPE); in sk_psock_backlog()
667 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_backlog()
668 sock_drop(psock->sk, skb); in sk_psock_backlog()
679 mutex_unlock(&psock->work_mutex); in sk_psock_backlog()
684 struct sk_psock *psock; in sk_psock_init() local
690 psock = ERR_PTR(-EBUSY); in sk_psock_init()
694 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node); in sk_psock_init()
695 if (!psock) { in sk_psock_init()
696 psock = ERR_PTR(-ENOMEM); in sk_psock_init()
701 psock->sk = sk; in sk_psock_init()
702 psock->eval = __SK_NONE; in sk_psock_init()
703 psock->sk_proto = prot; in sk_psock_init()
704 psock->saved_unhash = prot->unhash; in sk_psock_init()
705 psock->saved_close = prot->close; in sk_psock_init()
706 psock->saved_write_space = sk->sk_write_space; in sk_psock_init()
708 INIT_LIST_HEAD(&psock->link); in sk_psock_init()
709 spin_lock_init(&psock->link_lock); in sk_psock_init()
711 INIT_WORK(&psock->work, sk_psock_backlog); in sk_psock_init()
712 mutex_init(&psock->work_mutex); in sk_psock_init()
713 INIT_LIST_HEAD(&psock->ingress_msg); in sk_psock_init()
714 spin_lock_init(&psock->ingress_lock); in sk_psock_init()
715 skb_queue_head_init(&psock->ingress_skb); in sk_psock_init()
717 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_init()
718 refcount_set(&psock->refcnt, 1); in sk_psock_init()
720 rcu_assign_sk_user_data_nocopy(sk, psock); in sk_psock_init()
725 return psock; in sk_psock_init()
729 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock) in sk_psock_link_pop() argument
733 spin_lock_bh(&psock->link_lock); in sk_psock_link_pop()
734 link = list_first_entry_or_null(&psock->link, struct sk_psock_link, in sk_psock_link_pop()
738 spin_unlock_bh(&psock->link_lock); in sk_psock_link_pop()
742 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) in __sk_psock_purge_ingress_msg() argument
746 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
748 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
753 static void __sk_psock_zap_ingress(struct sk_psock *psock) in __sk_psock_zap_ingress() argument
757 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) { in __sk_psock_zap_ingress()
759 sock_drop(psock->sk, skb); in __sk_psock_zap_ingress()
761 kfree_skb(psock->work_state.skb); in __sk_psock_zap_ingress()
765 psock->work_state.skb = NULL; in __sk_psock_zap_ingress()
766 __sk_psock_purge_ingress_msg(psock); in __sk_psock_zap_ingress()
769 static void sk_psock_link_destroy(struct sk_psock *psock) in sk_psock_link_destroy() argument
773 list_for_each_entry_safe(link, tmp, &psock->link, list) { in sk_psock_link_destroy()
779 void sk_psock_stop(struct sk_psock *psock, bool wait) in sk_psock_stop() argument
781 spin_lock_bh(&psock->ingress_lock); in sk_psock_stop()
782 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_stop()
783 sk_psock_cork_free(psock); in sk_psock_stop()
784 __sk_psock_zap_ingress(psock); in sk_psock_stop()
785 spin_unlock_bh(&psock->ingress_lock); in sk_psock_stop()
788 cancel_work_sync(&psock->work); in sk_psock_stop()
791 static void sk_psock_done_strp(struct sk_psock *psock);
795 struct sk_psock *psock = container_of(to_rcu_work(work), in sk_psock_destroy() local
799 sk_psock_done_strp(psock); in sk_psock_destroy()
801 cancel_work_sync(&psock->work); in sk_psock_destroy()
802 mutex_destroy(&psock->work_mutex); in sk_psock_destroy()
804 psock_progs_drop(&psock->progs); in sk_psock_destroy()
806 sk_psock_link_destroy(psock); in sk_psock_destroy()
807 sk_psock_cork_free(psock); in sk_psock_destroy()
809 if (psock->sk_redir) in sk_psock_destroy()
810 sock_put(psock->sk_redir); in sk_psock_destroy()
811 sock_put(psock->sk); in sk_psock_destroy()
812 kfree(psock); in sk_psock_destroy()
815 void sk_psock_drop(struct sock *sk, struct sk_psock *psock) in sk_psock_drop() argument
818 sk_psock_restore_proto(sk, psock); in sk_psock_drop()
820 if (psock->progs.stream_parser) in sk_psock_drop()
821 sk_psock_stop_strp(sk, psock); in sk_psock_drop()
822 else if (psock->progs.stream_verdict || psock->progs.skb_verdict) in sk_psock_drop()
823 sk_psock_stop_verdict(sk, psock); in sk_psock_drop()
826 sk_psock_stop(psock, false); in sk_psock_drop()
828 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy); in sk_psock_drop()
829 queue_rcu_work(system_wq, &psock->rwork); in sk_psock_drop()
846 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, in sk_psock_msg_verdict() argument
853 prog = READ_ONCE(psock->progs.msg_parser); in sk_psock_msg_verdict()
863 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
865 if (psock->sk_redir) in sk_psock_msg_verdict()
866 sock_put(psock->sk_redir); in sk_psock_msg_verdict()
867 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()
868 if (!psock->sk_redir) { in sk_psock_msg_verdict()
872 sock_hold(psock->sk_redir); in sk_psock_msg_verdict()
932 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_tls_strp_read() argument
938 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_tls_strp_read()
940 skb->sk = psock->sk; in sk_psock_tls_strp_read()
947 sk_psock_tls_verdict_apply(skb, psock, ret); in sk_psock_tls_strp_read()
953 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_verdict_apply() argument
963 sk_other = psock->sk; in sk_psock_verdict_apply()
965 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
978 if (skb_queue_empty(&psock->ingress_skb)) { in sk_psock_verdict_apply()
987 err = sk_psock_skb_ingress_self(psock, skb, off, len); in sk_psock_verdict_apply()
990 spin_lock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
991 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
992 skb_queue_tail(&psock->ingress_skb, skb); in sk_psock_verdict_apply()
993 schedule_work(&psock->work); in sk_psock_verdict_apply()
996 spin_unlock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
1004 err = sk_psock_skb_redirect(psock, skb); in sk_psock_verdict_apply()
1009 sock_drop(psock->sk, skb); in sk_psock_verdict_apply()
1017 struct sk_psock *psock; in sk_psock_write_space() local
1021 psock = sk_psock(sk); in sk_psock_write_space()
1022 if (likely(psock)) { in sk_psock_write_space()
1023 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_write_space()
1024 schedule_work(&psock->work); in sk_psock_write_space()
1025 write_space = psock->saved_write_space; in sk_psock_write_space()
1035 struct sk_psock *psock; in sk_psock_strp_read() local
1042 psock = sk_psock(sk); in sk_psock_strp_read()
1043 if (unlikely(!psock)) { in sk_psock_strp_read()
1047 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_strp_read()
1058 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_strp_read()
1070 struct sk_psock *psock = container_of(strp, struct sk_psock, strp); in sk_psock_strp_parse() local
1075 prog = READ_ONCE(psock->progs.stream_parser); in sk_psock_strp_parse()
1077 skb->sk = psock->sk; in sk_psock_strp_parse()
1088 struct sk_psock *psock; in sk_psock_strp_data_ready() local
1091 psock = sk_psock(sk); in sk_psock_strp_data_ready()
1092 if (likely(psock)) { in sk_psock_strp_data_ready()
1094 psock->saved_data_ready(sk); in sk_psock_strp_data_ready()
1097 strp_data_ready(&psock->strp); in sk_psock_strp_data_ready()
1104 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_init_strp() argument
1112 return strp_init(&psock->strp, sk, &cb); in sk_psock_init_strp()
1115 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_start_strp() argument
1117 if (psock->saved_data_ready) in sk_psock_start_strp()
1120 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_strp()
1125 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_strp() argument
1127 psock_set_prog(&psock->progs.stream_parser, NULL); in sk_psock_stop_strp()
1129 if (!psock->saved_data_ready) in sk_psock_stop_strp()
1132 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_strp()
1133 psock->saved_data_ready = NULL; in sk_psock_stop_strp()
1134 strp_stop(&psock->strp); in sk_psock_stop_strp()
1137 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1140 if (psock->progs.stream_parser) in sk_psock_done_strp()
1141 strp_done(&psock->strp); in sk_psock_done_strp()
1144 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1153 struct sk_psock *psock; in sk_psock_verdict_recv() local
1166 psock = sk_psock(sk); in sk_psock_verdict_recv()
1167 if (unlikely(!psock)) { in sk_psock_verdict_recv()
1172 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_verdict_recv()
1174 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_verdict_recv()
1183 if (sk_psock_verdict_apply(psock, skb, ret) < 0) in sk_psock_verdict_recv()
1205 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_start_verdict() argument
1207 if (psock->saved_data_ready) in sk_psock_start_verdict()
1210 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_verdict()
1215 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_verdict() argument
1217 psock_set_prog(&psock->progs.stream_verdict, NULL); in sk_psock_stop_verdict()
1218 psock_set_prog(&psock->progs.skb_verdict, NULL); in sk_psock_stop_verdict()
1220 if (!psock->saved_data_ready) in sk_psock_stop_verdict()
1223 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_verdict()
1224 psock->saved_data_ready = NULL; in sk_psock_stop_verdict()