1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 #include <linux/prctl.h>
4 #include "sched.h"
5 
6 /*
7  * A simple wrapper around refcount. An allocated sched_core_cookie's
8  * address is used to compute the cookie of the task.
9  */
10 struct sched_core_cookie {
11 	refcount_t refcnt;
12 };
13 
sched_core_alloc_cookie(void)14 static unsigned long sched_core_alloc_cookie(void)
15 {
16 	struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL);
17 	if (!ck)
18 		return 0;
19 
20 	refcount_set(&ck->refcnt, 1);
21 	sched_core_get();
22 
23 	return (unsigned long)ck;
24 }
25 
sched_core_put_cookie(unsigned long cookie)26 static void sched_core_put_cookie(unsigned long cookie)
27 {
28 	struct sched_core_cookie *ptr = (void *)cookie;
29 
30 	if (ptr && refcount_dec_and_test(&ptr->refcnt)) {
31 		kfree(ptr);
32 		sched_core_put();
33 	}
34 }
35 
sched_core_get_cookie(unsigned long cookie)36 static unsigned long sched_core_get_cookie(unsigned long cookie)
37 {
38 	struct sched_core_cookie *ptr = (void *)cookie;
39 
40 	if (ptr)
41 		refcount_inc(&ptr->refcnt);
42 
43 	return cookie;
44 }
45 
46 /*
47  * sched_core_update_cookie - replace the cookie on a task
48  * @p: the task to update
49  * @cookie: the new cookie
50  *
51  * Effectively exchange the task cookie; caller is responsible for lifetimes on
52  * both ends.
53  *
54  * Returns: the old cookie
55  */
sched_core_update_cookie(struct task_struct * p,unsigned long cookie)56 static unsigned long sched_core_update_cookie(struct task_struct *p,
57 					      unsigned long cookie)
58 {
59 	unsigned long old_cookie;
60 	struct rq_flags rf;
61 	struct rq *rq;
62 	bool enqueued;
63 
64 	rq = task_rq_lock(p, &rf);
65 
66 	/*
67 	 * Since creating a cookie implies sched_core_get(), and we cannot set
68 	 * a cookie until after we've created it, similarly, we cannot destroy
69 	 * a cookie until after we've removed it, we must have core scheduling
70 	 * enabled here.
71 	 */
72 	SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));
73 
74 	enqueued = sched_core_enqueued(p);
75 	if (enqueued)
76 		sched_core_dequeue(rq, p);
77 
78 	old_cookie = p->core_cookie;
79 	p->core_cookie = cookie;
80 
81 	if (enqueued)
82 		sched_core_enqueue(rq, p);
83 
84 	/*
85 	 * If task is currently running, it may not be compatible anymore after
86 	 * the cookie change, so enter the scheduler on its CPU to schedule it
87 	 * away.
88 	 */
89 	if (task_running(rq, p))
90 		resched_curr(rq);
91 
92 	task_rq_unlock(rq, p, &rf);
93 
94 	return old_cookie;
95 }
96 
sched_core_clone_cookie(struct task_struct * p)97 static unsigned long sched_core_clone_cookie(struct task_struct *p)
98 {
99 	unsigned long cookie, flags;
100 
101 	raw_spin_lock_irqsave(&p->pi_lock, flags);
102 	cookie = sched_core_get_cookie(p->core_cookie);
103 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
104 
105 	return cookie;
106 }
107 
sched_core_fork(struct task_struct * p)108 void sched_core_fork(struct task_struct *p)
109 {
110 	RB_CLEAR_NODE(&p->core_node);
111 	p->core_cookie = sched_core_clone_cookie(current);
112 }
113 
sched_core_free(struct task_struct * p)114 void sched_core_free(struct task_struct *p)
115 {
116 	sched_core_put_cookie(p->core_cookie);
117 }
118 
__sched_core_set(struct task_struct * p,unsigned long cookie)119 static void __sched_core_set(struct task_struct *p, unsigned long cookie)
120 {
121 	cookie = sched_core_get_cookie(cookie);
122 	cookie = sched_core_update_cookie(p, cookie);
123 	sched_core_put_cookie(cookie);
124 }
125 
126 /* Called from prctl interface: PR_SCHED_CORE */
sched_core_share_pid(unsigned int cmd,pid_t pid,enum pid_type type,unsigned long uaddr)127 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
128 			 unsigned long uaddr)
129 {
130 	unsigned long cookie = 0, id = 0;
131 	struct task_struct *task, *p;
132 	struct pid *grp;
133 	int err = 0;
134 
135 	if (!static_branch_likely(&sched_smt_present))
136 		return -ENODEV;
137 
138 	BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
139 	BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID);
140 	BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID);
141 
142 	if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 ||
143 	    (cmd != PR_SCHED_CORE_GET && uaddr))
144 		return -EINVAL;
145 
146 	rcu_read_lock();
147 	if (pid == 0) {
148 		task = current;
149 	} else {
150 		task = find_task_by_vpid(pid);
151 		if (!task) {
152 			rcu_read_unlock();
153 			return -ESRCH;
154 		}
155 	}
156 	get_task_struct(task);
157 	rcu_read_unlock();
158 
159 	/*
160 	 * Check if this process has the right to modify the specified
161 	 * process. Use the regular "ptrace_may_access()" checks.
162 	 */
163 	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
164 		err = -EPERM;
165 		goto out;
166 	}
167 
168 	switch (cmd) {
169 	case PR_SCHED_CORE_GET:
170 		if (type != PIDTYPE_PID || uaddr & 7) {
171 			err = -EINVAL;
172 			goto out;
173 		}
174 		cookie = sched_core_clone_cookie(task);
175 		if (cookie) {
176 			/* XXX improve ? */
177 			ptr_to_hashval((void *)cookie, &id);
178 		}
179 		err = put_user(id, (u64 __user *)uaddr);
180 		goto out;
181 
182 	case PR_SCHED_CORE_CREATE:
183 		cookie = sched_core_alloc_cookie();
184 		if (!cookie) {
185 			err = -ENOMEM;
186 			goto out;
187 		}
188 		break;
189 
190 	case PR_SCHED_CORE_SHARE_TO:
191 		cookie = sched_core_clone_cookie(current);
192 		break;
193 
194 	case PR_SCHED_CORE_SHARE_FROM:
195 		if (type != PIDTYPE_PID) {
196 			err = -EINVAL;
197 			goto out;
198 		}
199 		cookie = sched_core_clone_cookie(task);
200 		__sched_core_set(current, cookie);
201 		goto out;
202 
203 	default:
204 		err = -EINVAL;
205 		goto out;
206 	};
207 
208 	if (type == PIDTYPE_PID) {
209 		__sched_core_set(task, cookie);
210 		goto out;
211 	}
212 
213 	read_lock(&tasklist_lock);
214 	grp = task_pid_type(task, type);
215 
216 	do_each_pid_thread(grp, type, p) {
217 		if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) {
218 			err = -EPERM;
219 			goto out_tasklist;
220 		}
221 	} while_each_pid_thread(grp, type, p);
222 
223 	do_each_pid_thread(grp, type, p) {
224 		__sched_core_set(p, cookie);
225 	} while_each_pid_thread(grp, type, p);
226 out_tasklist:
227 	read_unlock(&tasklist_lock);
228 
229 out:
230 	sched_core_put_cookie(cookie);
231 	put_task_struct(task);
232 	return err;
233 }
234 
235