1/* $NetBSD: subr_xcall.c,v 1.26 2018/02/07 04:25:09 ozaki-r Exp $ */
2
3/*-
4 * Copyright (c) 2007-2010 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran and Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Cross call support
34 *
35 * Background
36 *
37 * Sometimes it is necessary to modify hardware state that is tied
38 * directly to individual CPUs (such as a CPU's local timer), and
39 * these updates can not be done remotely by another CPU. The LWP
40 * requesting the update may be unable to guarantee that it will be
41 * running on the CPU where the update must occur, when the update
42 * occurs.
43 *
44 * Additionally, it's sometimes necessary to modify per-CPU software
45 * state from a remote CPU. Where these update operations are so
46 * rare or the access to the per-CPU data so frequent that the cost
47 * of using locking or atomic operations to provide coherency is
48 * prohibitive, another way must be found.
49 *
50 * Cross calls help to solve these types of problem by allowing
51 * any CPU in the system to request that an arbitrary function be
52 * executed on any other CPU.
53 *
54 * Implementation
55 *
56 * A slow mechanism for making 'low priority' cross calls is
57 * provided. The function to be executed runs on the remote CPU
58 * within a bound kthread. No queueing is provided, and the
59 * implementation uses global state. The function being called may
60 * block briefly on locks, but in doing so must be careful to not
61 * interfere with other cross calls in the system. The function is
62 * called with thread context and not from a soft interrupt, so it
63 * can ensure that it is not interrupting other code running on the
64 * CPU, and so has exclusive access to the CPU. Since this facility
65 * is heavyweight, it's expected that it will not be used often.
66 *
67 * Cross calls must not allocate memory, as the pagedaemon uses
68 * them (and memory allocation may need to wait on the pagedaemon).
69 *
70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is
71 * also provided. The function to be executed runs on a software
72 * interrupt context, at IPL_SOFTSERIAL level, and is expected to
73 * be very lightweight, e.g. avoid blocking.
74 */
75
76#include <sys/cdefs.h>
77__KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.26 2018/02/07 04:25:09 ozaki-r Exp $");
78
79#include <sys/types.h>
80#include <sys/param.h>
81#include <sys/xcall.h>
82#include <sys/mutex.h>
83#include <sys/condvar.h>
84#include <sys/evcnt.h>
85#include <sys/kthread.h>
86#include <sys/cpu.h>
87
88#ifdef _RUMPKERNEL
89#include "rump_private.h"
90#endif
91
92/* Cross-call state box. */
93typedef struct {
94 kmutex_t xc_lock;
95 kcondvar_t xc_busy;
96 xcfunc_t xc_func;
97 void * xc_arg1;
98 void * xc_arg2;
99 uint64_t xc_headp;
100 uint64_t xc_donep;
101 unsigned int xc_ipl;
102} xc_state_t;
103
104/* Bit indicating high (1) or low (0) priority. */
105#define XC_PRI_BIT (1ULL << 63)
106
107/* Low priority xcall structures. */
108static xc_state_t xc_low_pri __cacheline_aligned;
109
110/* High priority xcall structures. */
111static xc_state_t xc_high_pri __cacheline_aligned;
112static void * xc_sihs[4] __cacheline_aligned;
113
114/* Event counters. */
115static struct evcnt xc_unicast_ev __cacheline_aligned;
116static struct evcnt xc_broadcast_ev __cacheline_aligned;
117
118static void xc_init(void);
119static void xc_thread(void *);
120
121static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *,
122 unsigned int);
123static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *);
124
125/* The internal form of IPL */
126#define XC_IPL_MASK 0xff00
127/*
128 * Assign 0 to XC_IPL_SOFTSERIAL to treat IPL_SOFTSERIAL as the default value
129 * (just XC_HIGHPRI).
130 */
131#define XC_IPL_SOFTSERIAL 0
132#define XC_IPL_SOFTNET 1
133#define XC_IPL_SOFTBIO 2
134#define XC_IPL_SOFTCLOCK 3
135#define XC_IPL_MAX XC_IPL_SOFTCLOCK
136
137CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs));
138
139/*
140 * xc_init:
141 *
142 * Initialize low and high priority cross-call structures.
143 */
144static void
145xc_init(void)
146{
147 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri;
148
149 memset(xclo, 0, sizeof(xc_state_t));
150 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE);
151 cv_init(&xclo->xc_busy, "xclocv");
152
153 memset(xchi, 0, sizeof(xc_state_t));
154 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
155 cv_init(&xchi->xc_busy, "xchicv");
156
157 /* Set up a softint for each IPL_SOFT*. */
158#define SETUP_SOFTINT(xipl, sipl) do { \
159 xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\
160 xc__highpri_intr, NULL); \
161 KASSERT(xc_sihs[(xipl)] != NULL); \
162 } while (0)
163
164 SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL);
165 /*
166 * If a IPL_SOFTXXX have the same value of the previous, we don't use
167 * the IPL (see xc_encode_ipl). So we don't need to allocate a softint
168 * for it.
169 */
170#if IPL_SOFTNET != IPL_SOFTSERIAL
171 SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET);
172#endif
173#if IPL_SOFTBIO != IPL_SOFTNET
174 SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO);
175#endif
176#if IPL_SOFTCLOCK != IPL_SOFTBIO
177 SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK);
178#endif
179
180#undef SETUP_SOFTINT
181
182 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
183 "crosscall", "unicast");
184 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
185 "crosscall", "broadcast");
186}
187
188/*
189 * Encode an IPL to a form that can be embedded into flags of xc_broadcast
190 * or xc_unicast.
191 */
192unsigned int
193xc_encode_ipl(int ipl)
194{
195
196 switch (ipl) {
197 case IPL_SOFTSERIAL:
198 return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK);
199 /* IPL_SOFT* can be the same value (e.g., on sparc or mips). */
200#if IPL_SOFTNET != IPL_SOFTSERIAL
201 case IPL_SOFTNET:
202 return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK);
203#endif
204#if IPL_SOFTBIO != IPL_SOFTNET
205 case IPL_SOFTBIO:
206 return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK);
207#endif
208#if IPL_SOFTCLOCK != IPL_SOFTBIO
209 case IPL_SOFTCLOCK:
210 return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK);
211#endif
212 }
213
214 panic("Invalid IPL: %d", ipl);
215}
216
217/*
218 * Extract an XC_IPL from flags of xc_broadcast or xc_unicast.
219 */
220static inline unsigned int
221xc_extract_ipl(unsigned int flags)
222{
223
224 return __SHIFTOUT(flags, XC_IPL_MASK);
225}
226
227/*
228 * xc_init_cpu:
229 *
230 * Initialize the cross-call subsystem. Called once for each CPU
231 * in the system as they are attached.
232 */
233void
234xc_init_cpu(struct cpu_info *ci)
235{
236 static bool again = false;
237 int error __diagused;
238
239 if (!again) {
240 /* Autoconfiguration will prevent re-entry. */
241 xc_init();
242 again = true;
243 }
244 cv_init(&ci->ci_data.cpu_xcall, "xcall");
245 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread,
246 NULL, NULL, "xcall/%u", ci->ci_index);
247 KASSERT(error == 0);
248}
249
250/*
251 * xc_broadcast:
252 *
253 * Trigger a call on all CPUs in the system.
254 */
255uint64_t
256xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2)
257{
258
259 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
260 ASSERT_SLEEPABLE();
261
262 if ((flags & XC_HIGHPRI) != 0) {
263 int ipl = xc_extract_ipl(flags);
264 return xc_highpri(func, arg1, arg2, NULL, ipl);
265 } else {
266 return xc_lowpri(func, arg1, arg2, NULL);
267 }
268}
269
270/*
271 * xc_unicast:
272 *
273 * Trigger a call on one CPU.
274 */
275uint64_t
276xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2,
277 struct cpu_info *ci)
278{
279
280 KASSERT(ci != NULL);
281 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
282 ASSERT_SLEEPABLE();
283
284 if ((flags & XC_HIGHPRI) != 0) {
285 int ipl = xc_extract_ipl(flags);
286 return xc_highpri(func, arg1, arg2, ci, ipl);
287 } else {
288 return xc_lowpri(func, arg1, arg2, ci);
289 }
290}
291
292/*
293 * xc_wait:
294 *
295 * Wait for a cross call to complete.
296 */
297void
298xc_wait(uint64_t where)
299{
300 xc_state_t *xc;
301
302 KASSERT(!cpu_intr_p() && !cpu_softintr_p());
303 ASSERT_SLEEPABLE();
304
305 /* Determine whether it is high or low priority cross-call. */
306 if ((where & XC_PRI_BIT) != 0) {
307 xc = &xc_high_pri;
308 where &= ~XC_PRI_BIT;
309 } else {
310 xc = &xc_low_pri;
311 }
312
313 /* Fast path, if already done. */
314 if (xc->xc_donep >= where) {
315 return;
316 }
317
318 /* Slow path: block until awoken. */
319 mutex_enter(&xc->xc_lock);
320 while (xc->xc_donep < where) {
321 cv_wait(&xc->xc_busy, &xc->xc_lock);
322 }
323 mutex_exit(&xc->xc_lock);
324}
325
326/*
327 * xc_lowpri:
328 *
329 * Trigger a low priority call on one or more CPUs.
330 */
331static inline uint64_t
332xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
333{
334 xc_state_t *xc = &xc_low_pri;
335 CPU_INFO_ITERATOR cii;
336 uint64_t where;
337
338 mutex_enter(&xc->xc_lock);
339 while (xc->xc_headp != xc->xc_donep) {
340 cv_wait(&xc->xc_busy, &xc->xc_lock);
341 }
342 xc->xc_arg1 = arg1;
343 xc->xc_arg2 = arg2;
344 xc->xc_func = func;
345 if (ci == NULL) {
346 xc_broadcast_ev.ev_count++;
347 for (CPU_INFO_FOREACH(cii, ci)) {
348 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0)
349 continue;
350 xc->xc_headp += 1;
351 ci->ci_data.cpu_xcall_pending = true;
352 cv_signal(&ci->ci_data.cpu_xcall);
353 }
354 } else {
355 xc_unicast_ev.ev_count++;
356 xc->xc_headp += 1;
357 ci->ci_data.cpu_xcall_pending = true;
358 cv_signal(&ci->ci_data.cpu_xcall);
359 }
360 KASSERT(xc->xc_donep < xc->xc_headp);
361 where = xc->xc_headp;
362 mutex_exit(&xc->xc_lock);
363
364 /* Return a low priority ticket. */
365 KASSERT((where & XC_PRI_BIT) == 0);
366 return where;
367}
368
369/*
370 * xc_thread:
371 *
372 * One thread per-CPU to dispatch low priority calls.
373 */
374static void
375xc_thread(void *cookie)
376{
377 struct cpu_info *ci = curcpu();
378 xc_state_t *xc = &xc_low_pri;
379 void *arg1, *arg2;
380 xcfunc_t func;
381
382 mutex_enter(&xc->xc_lock);
383 for (;;) {
384 while (!ci->ci_data.cpu_xcall_pending) {
385 if (xc->xc_headp == xc->xc_donep) {
386 cv_broadcast(&xc->xc_busy);
387 }
388 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock);
389 KASSERT(ci == curcpu());
390 }
391 ci->ci_data.cpu_xcall_pending = false;
392 func = xc->xc_func;
393 arg1 = xc->xc_arg1;
394 arg2 = xc->xc_arg2;
395 mutex_exit(&xc->xc_lock);
396
397 KASSERT(func != NULL);
398 (*func)(arg1, arg2);
399
400 mutex_enter(&xc->xc_lock);
401 xc->xc_donep++;
402 }
403 /* NOTREACHED */
404}
405
406/*
407 * xc_ipi_handler:
408 *
409 * Handler of cross-call IPI.
410 */
411void
412xc_ipi_handler(void)
413{
414 xc_state_t *xc = & xc_high_pri;
415
416 KASSERT(xc->xc_ipl < __arraycount(xc_sihs));
417 KASSERT(xc_sihs[xc->xc_ipl] != NULL);
418
419 /* Executes xc__highpri_intr() via software interrupt. */
420 softint_schedule(xc_sihs[xc->xc_ipl]);
421}
422
423/*
424 * xc__highpri_intr:
425 *
426 * A software interrupt handler for high priority calls.
427 */
428void
429xc__highpri_intr(void *dummy)
430{
431 xc_state_t *xc = &xc_high_pri;
432 void *arg1, *arg2;
433 xcfunc_t func;
434
435 KASSERTMSG(!cpu_intr_p(), "high priority xcall for function %p",
436 xc->xc_func);
437 /*
438 * Lock-less fetch of function and its arguments.
439 * Safe since it cannot change at this point.
440 */
441 KASSERT(xc->xc_donep < xc->xc_headp);
442 func = xc->xc_func;
443 arg1 = xc->xc_arg1;
444 arg2 = xc->xc_arg2;
445
446 KASSERT(func != NULL);
447 (*func)(arg1, arg2);
448
449 /*
450 * Note the request as done, and if we have reached the head,
451 * cross-call has been processed - notify waiters, if any.
452 */
453 mutex_enter(&xc->xc_lock);
454 if (++xc->xc_donep == xc->xc_headp) {
455 cv_broadcast(&xc->xc_busy);
456 }
457 mutex_exit(&xc->xc_lock);
458}
459
460/*
461 * xc_highpri:
462 *
463 * Trigger a high priority call on one or more CPUs.
464 */
465static inline uint64_t
466xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci,
467 unsigned int ipl)
468{
469 xc_state_t *xc = &xc_high_pri;
470 uint64_t where;
471
472 mutex_enter(&xc->xc_lock);
473 while (xc->xc_headp != xc->xc_donep) {
474 cv_wait(&xc->xc_busy, &xc->xc_lock);
475 }
476 xc->xc_func = func;
477 xc->xc_arg1 = arg1;
478 xc->xc_arg2 = arg2;
479 xc->xc_headp += (ci ? 1 : ncpu);
480 xc->xc_ipl = ipl;
481 where = xc->xc_headp;
482 mutex_exit(&xc->xc_lock);
483
484 /*
485 * Send the IPI once lock is released.
486 * Note: it will handle the local CPU case.
487 */
488
489#ifdef _RUMPKERNEL
490 rump_xc_highpri(ci);
491#else
492#ifdef MULTIPROCESSOR
493 kpreempt_disable();
494 if (curcpu() == ci) {
495 /* Unicast: local CPU. */
496 xc_ipi_handler();
497 } else if (ci) {
498 /* Unicast: remote CPU. */
499 xc_send_ipi(ci);
500 } else {
501 /* Broadcast: all, including local. */
502 xc_send_ipi(NULL);
503 xc_ipi_handler();
504 }
505 kpreempt_enable();
506#else
507 KASSERT(ci == NULL || curcpu() == ci);
508 xc_ipi_handler();
509#endif
510#endif
511
512 /* Indicate a high priority ticket. */
513 return (where | XC_PRI_BIT);
514}
515