1/* $NetBSD: rump.c,v 1.334 2019/05/17 03:34:26 ozaki-r Exp $ */
2
3/*
4 * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.334 2019/05/17 03:34:26 ozaki-r Exp $");
30
31#include <sys/systm.h>
32#define ELFSIZE ARCH_ELFSIZE
33
34#include <sys/param.h>
35#include <sys/atomic.h>
36#include <sys/buf.h>
37#include <sys/callout.h>
38#include <sys/conf.h>
39#include <sys/cpu.h>
40#include <sys/device.h>
41#include <sys/evcnt.h>
42#include <sys/event.h>
43#include <sys/exec_elf.h>
44#include <sys/filedesc.h>
45#include <sys/iostat.h>
46#include <sys/kauth.h>
47#include <sys/kcpuset.h>
48#include <sys/kernel.h>
49#include <sys/kmem.h>
50#include <sys/kprintf.h>
51#include <sys/kthread.h>
52#include <sys/ksyms.h>
53#include <sys/msgbuf.h>
54#include <sys/module.h>
55#include <sys/namei.h>
56#include <sys/once.h>
57#include <sys/percpu.h>
58#include <sys/pipe.h>
59#include <sys/pool.h>
60#include <sys/pserialize.h>
61#include <sys/queue.h>
62#include <sys/reboot.h>
63#include <sys/resourcevar.h>
64#include <sys/select.h>
65#include <sys/sysctl.h>
66#include <sys/syscall.h>
67#include <sys/syscallvar.h>
68#include <sys/threadpool.h>
69#include <sys/timetc.h>
70#include <sys/tty.h>
71#include <sys/uidinfo.h>
72#include <sys/vmem.h>
73#include <sys/xcall.h>
74#include <sys/cprng.h>
75#include <sys/rnd.h>
76#include <sys/ktrace.h>
77#include <sys/psref.h>
78
79#include <rump-sys/kern.h>
80#include <rump-sys/dev.h>
81#include <rump-sys/net.h>
82#include <rump-sys/vfs.h>
83
84#include <rump/rumpuser.h>
85
86#include <secmodel/suser/suser.h>
87
88#include <prop/proplib.h>
89
90#include <uvm/uvm_extern.h>
91#include <uvm/uvm_readahead.h>
92
93char machine[] = MACHINE;
94char machine_arch[] = MACHINE_ARCH;
95
96struct proc *initproc;
97
98struct device rump_rootdev = {
99 .dv_class = DV_VIRTUAL
100};
101
102#ifdef RUMP_WITHOUT_THREADS
103int rump_threads = 0;
104#else
105int rump_threads = 1;
106#endif
107
108static void rump_component_addlocal(void);
109static struct lwp *bootlwp;
110
111/* 16k should be enough for std rump needs */
112static char rump_msgbuf[16*1024] __aligned(256);
113
114bool rump_ttycomponent = false;
115
116pool_cache_t pnbuf_cache;
117
118static void
119rump_aiodone_worker(struct work *wk, void *dummy)
120{
121 struct buf *bp = (struct buf *)wk;
122
123 KASSERT(&bp->b_work == wk);
124 bp->b_iodone(bp);
125}
126
127static int rump_inited;
128
129void (*rump_vfs_drainbufs)(int) = (void *)nullop;
130int (*rump_vfs_makeonedevnode)(dev_t, const char *,
131 devmajor_t, devminor_t) = (void *)nullop;
132int (*rump_vfs_makedevnodes)(dev_t, const char *, char,
133 devmajor_t, devminor_t, int) = (void *)nullop;
134int (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop;
135
136rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop;
137rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop;
138
139static void add_linkedin_modules(const struct modinfo *const *, size_t);
140
141static pid_t rspo_wrap_getpid(void) {
142 return rump_sysproxy_hyp_getpid();
143}
144static int rspo_wrap_syscall(int num, void *arg, long *retval) {
145 return rump_sysproxy_hyp_syscall(num, arg, retval);
146}
147static int rspo_wrap_rfork(void *priv, int flag, const char *comm) {
148 return rump_sysproxy_hyp_rfork(priv, flag, comm);
149}
150static void rspo_wrap_lwpexit(void) {
151 rump_sysproxy_hyp_lwpexit();
152}
153static void rspo_wrap_execnotify(const char *comm) {
154 rump_sysproxy_hyp_execnotify(comm);
155}
156static const struct rumpuser_hyperup hyp = {
157 .hyp_schedule = rump_schedule,
158 .hyp_unschedule = rump_unschedule,
159 .hyp_backend_unschedule = rump_user_unschedule,
160 .hyp_backend_schedule = rump_user_schedule,
161 .hyp_lwproc_switch = rump_lwproc_switch,
162 .hyp_lwproc_release = rump_lwproc_releaselwp,
163 .hyp_lwproc_newlwp = rump_lwproc_newlwp,
164 .hyp_lwproc_curlwp = rump_lwproc_curlwp,
165
166 .hyp_getpid = rspo_wrap_getpid,
167 .hyp_syscall = rspo_wrap_syscall,
168 .hyp_lwproc_rfork = rspo_wrap_rfork,
169 .hyp_lwpexit = rspo_wrap_lwpexit,
170 .hyp_execnotify = rspo_wrap_execnotify,
171};
172struct rump_sysproxy_ops rump_sysproxy_ops = {
173 .rspo_copyin = (void *)enxio,
174 .rspo_copyinstr = (void *)enxio,
175 .rspo_copyout = (void *)enxio,
176 .rspo_copyoutstr = (void *)enxio,
177 .rspo_anonmmap = (void *)enxio,
178 .rspo_raise = (void *)enxio,
179 .rspo_fini = (void *)enxio,
180 .rspo_hyp_getpid = (void *)enxio,
181 .rspo_hyp_syscall = (void *)enxio,
182 .rspo_hyp_rfork = (void *)enxio,
183 .rspo_hyp_lwpexit = (void *)enxio,
184 .rspo_hyp_execnotify = (void *)enxio,
185};
186
187int
188rump_daemonize_begin(void)
189{
190
191 if (rump_inited)
192 return EALREADY;
193
194 return rumpuser_daemonize_begin();
195}
196
197int
198rump_daemonize_done(int error)
199{
200
201 return rumpuser_daemonize_done(error);
202}
203
204#ifdef RUMP_USE_CTOR
205
206/* sysctl bootstrap handling */
207struct sysctl_boot_chain sysctl_boot_chain \
208 = LIST_HEAD_INITIALIZER(sysctl_boot_chain);
209__link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */
210
211#else /* RUMP_USE_CTOR */
212
213RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT)
214{
215 __link_set_decl(rump_components, struct rump_component);
216
217 /*
218 * Trick compiler into generating references so that statically
219 * linked rump kernels are generated with the link set symbols.
220 */
221 asm("" :: "r"(__start_link_set_rump_components));
222 asm("" :: "r"(__stop_link_set_rump_components));
223}
224
225#endif /* RUMP_USE_CTOR */
226
227int
228rump_init(void)
229{
230 char buf[256];
231 struct timespec ts;
232 int64_t sec;
233 long nsec;
234 struct lwp *l, *initlwp;
235 int i, numcpu;
236
237 /* not reentrant */
238 if (rump_inited)
239 return 0;
240 else if (rump_inited == -1)
241 panic("rump_init: host process restart required");
242 else
243 rump_inited = 1;
244
245 /* initialize hypervisor */
246 if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) {
247 rumpuser_dprintf("rumpuser init failed\n");
248 return EINVAL;
249 }
250
251 /* init minimal lwp/cpu context */
252 rump_lwproc_init();
253 l = &lwp0;
254 l->l_cpu = l->l_target_cpu = &rump_bootcpu;
255 rump_lwproc_curlwp_set(l);
256
257 /* retrieve env vars which affect the early stage of bootstrap */
258 if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) {
259 rump_threads = *buf != '0';
260 }
261 if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) {
262 if (*buf != '0')
263 boothowto = AB_VERBOSE;
264 }
265
266 if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0)
267 panic("mandatory hypervisor configuration (NCPU) missing");
268 numcpu = strtoll(buf, NULL, 10);
269 if (numcpu < 1) {
270 panic("rump kernels are not lightweight enough for \"%d\" CPUs",
271 numcpu);
272 }
273
274 rump_thread_init();
275 rump_cpus_bootstrap(&numcpu);
276
277 rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec);
278 boottime.tv_sec = sec;
279 boottime.tv_nsec = nsec;
280
281 initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf));
282 aprint_verbose("%s%s", copyright, version);
283
284 rump_intr_init(numcpu);
285
286 rump_tsleep_init();
287
288 rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN);
289 ksyms_init();
290 uvm_init();
291 evcnt_init();
292
293 kcpuset_sysinit();
294 once_init();
295 kernconfig_lock_init();
296 prop_kern_init();
297
298 kmem_init();
299
300 uvm_ra_init();
301 uao_init();
302
303 mutex_obj_init();
304 rw_obj_init();
305 callout_startup();
306
307 kprintf_init();
308 percpu_init();
309 pserialize_init();
310
311 kauth_init();
312
313 secmodel_init();
314 sysctl_init();
315 /*
316 * The above call to sysctl_init() only initializes sysctl nodes
317 * from link sets. Initialize sysctls in case we used ctors.
318 */
319#ifdef RUMP_USE_CTOR
320 {
321 struct sysctl_setup_chain *ssc;
322
323 while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) {
324 LIST_REMOVE(ssc, ssc_entries);
325 ssc->ssc_func(NULL);
326 }
327 }
328#endif /* RUMP_USE_CTOR */
329
330 rnd_init();
331 cprng_init();
332 kern_cprng = cprng_strong_create("kernel", IPL_VM,
333 CPRNG_INIT_ANY|CPRNG_REKEY_ANY);
334
335 rump_hyperentropy_init();
336
337 procinit();
338 proc0_init();
339 uid_init();
340 chgproccnt(0, 1);
341
342 l->l_proc = &proc0;
343 lwp_update_creds(l);
344
345 lwpinit_specificdata();
346 lwp_initspecific(&lwp0);
347
348 /* Must be called after lwpinit_specificdata */
349 psref_init();
350
351 threadpools_init();
352
353 loginit();
354
355 rump_biglock_init();
356
357 rump_scheduler_init(numcpu);
358 /* revert temporary context and schedule a semireal context */
359 rump_lwproc_curlwp_clear(l);
360 initproc = &proc0; /* borrow proc0 before we get initproc started */
361 rump_schedule();
362 bootlwp = curlwp;
363
364 inittimecounter();
365 ntp_init();
366
367#ifdef KTRACE
368 ktrinit();
369#endif
370
371 ts = boottime;
372 tc_setclock(&ts);
373
374 extern krwlock_t exec_lock;
375 rw_init(&exec_lock);
376
377 /* we are mostly go. do per-cpu subsystem init */
378 for (i = 0; i < numcpu; i++) {
379 struct cpu_info *ci = cpu_lookup(i);
380
381 /* attach non-bootstrap CPUs */
382 if (i > 0) {
383 rump_cpu_attach(ci);
384 ncpu++;
385 }
386
387 callout_init_cpu(ci);
388 softint_init(ci);
389 xc_init_cpu(ci);
390 pool_cache_cpu_init(ci);
391 selsysinit(ci);
392 percpu_init_cpu(ci);
393
394 TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
395 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
396
397 aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i);
398 }
399 ncpuonline = ncpu;
400
401 /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */
402 cprng_fast_init();
403
404 mp_online = true;
405
406 /* CPUs are up. allow kernel threads to run */
407 rump_thread_allow(NULL);
408
409 rnd_init_softint();
410
411 kqueue_init();
412 iostat_init();
413 fd_sys_init();
414 module_init();
415 devsw_init();
416 pipe_init();
417 resource_init();
418 procinit_sysctl();
419 time_init();
420 time_init2();
421
422 /* start page baroness */
423 if (rump_threads) {
424 if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL,
425 uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0)
426 panic("pagedaemon create failed");
427 } else
428 uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */
429
430 /* process dso's */
431 rumpuser_dl_bootstrap(add_linkedin_modules,
432 rump_kernelfsym_load, rump_component_load);
433
434 rump_component_addlocal();
435 rump_component_init(RUMP_COMPONENT_KERN);
436
437 /* initialize factions, if present */
438 rump_component_init(RUMP__FACTION_VFS);
439 /* pnbuf_cache is used even without vfs */
440 if (rump_component_count(RUMP__FACTION_VFS) == 0) {
441 pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl",
442 NULL, IPL_NONE, NULL, NULL, NULL);
443 }
444 rump_component_init(RUMP__FACTION_NET);
445 rump_component_init(RUMP__FACTION_DEV);
446 KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1
447 && rump_component_count(RUMP__FACTION_NET) <= 1
448 && rump_component_count(RUMP__FACTION_DEV) <= 1);
449
450 rump_component_init(RUMP_COMPONENT_KERN_VFS);
451
452 /*
453 * if we initialized the tty component above, the tyttymtx is
454 * now initialized. otherwise, we need to initialize it.
455 */
456 if (!rump_ttycomponent)
457 mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM);
458
459 cold = 0;
460
461 /* aieeeedondest */
462 if (rump_threads) {
463 if (workqueue_create(&uvm.aiodone_queue, "aiodoned",
464 rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE))
465 panic("aiodoned");
466 }
467
468 sysctl_finalize();
469
470 module_init_class(MODULE_CLASS_ANY);
471
472 if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME,
473 hostname, MAXHOSTNAMELEN) != 0) {
474 panic("mandatory hypervisor configuration (HOSTNAME) missing");
475 }
476 hostnamelen = strlen(hostname);
477
478 sigemptyset(&sigcantmask);
479
480 if (rump_threads)
481 vmem_rehash_start();
482
483 /*
484 * Create init (proc 1), used to attach implicit threads in rump.
485 * (note: must be done after vfsinit to get cwdi)
486 */
487 initlwp = rump__lwproc_alloclwp(NULL);
488 mutex_enter(proc_lock);
489 initproc = proc_find_raw(1);
490 mutex_exit(proc_lock);
491 if (initproc == NULL)
492 panic("where in the world is initproc?");
493 strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm));
494
495 rump_component_init(RUMP_COMPONENT_POSTINIT);
496
497 /* load syscalls */
498 rump_component_init(RUMP_COMPONENT_SYSCALL);
499
500 /* component inits done */
501 bootlwp = NULL;
502
503 /* open 0/1/2 for init */
504 KASSERT(rump_lwproc_curlwp() == NULL);
505 rump_lwproc_switch(initlwp);
506 rump_consdev_init();
507 rump_lwproc_switch(NULL);
508
509 /* release cpu */
510 rump_unschedule();
511
512 return 0;
513}
514/* historic compat */
515__strong_alias(rump__init,rump_init);
516
517static int compcounter[RUMP_COMPONENT_MAX];
518static int compinited[RUMP_COMPONENT_MAX];
519
520/*
521 * Yea, this is O(n^2), but we're only looking at a handful of components.
522 * Components are always initialized from the thread that called rump_init().
523 */
524static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead);
525
526#ifdef RUMP_USE_CTOR
527struct modinfo_boot_chain modinfo_boot_chain \
528 = LIST_HEAD_INITIALIZER(modinfo_boot_chain);
529
530static void
531rump_component_addlocal(void)
532{
533 struct modinfo_chain *mc;
534
535 while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) {
536 LIST_REMOVE(mc, mc_entries);
537 module_builtin_add(&mc->mc_info, 1, false);
538 }
539}
540
541#else /* RUMP_USE_CTOR */
542
543static void
544rump_component_addlocal(void)
545{
546 __link_set_decl(rump_components, struct rump_component);
547 struct rump_component *const *rc;
548
549 __link_set_foreach(rc, rump_components) {
550 rump_component_load(*rc);
551 }
552}
553#endif /* RUMP_USE_CTOR */
554
555void
556rump_component_load(const struct rump_component *rc_const)
557{
558 struct rump_component *rc, *rc_iter;
559
560 /* time for rump component loading and unloading has passed */
561 if (!cold)
562 return;
563
564 /*
565 * XXX: this is ok since the "const" was removed from the
566 * definition of RUMP_COMPONENT().
567 *
568 * However, to preserve the hypercall interface, the const
569 * remains here. This can be fixed in the next hypercall revision.
570 */
571 rc = __UNCONST(rc_const);
572
573 KASSERT(!rump_inited || curlwp == bootlwp);
574
575 LIST_FOREACH(rc_iter, &rchead, rc_entries) {
576 if (rc_iter == rc)
577 return;
578 }
579
580 LIST_INSERT_HEAD(&rchead, rc, rc_entries);
581 KASSERT(rc->rc_type < RUMP_COMPONENT_MAX);
582 compcounter[rc->rc_type]++;
583}
584
585void
586rump_component_unload(struct rump_component *rc)
587{
588
589 /*
590 * Checking for cold is enough because rump_init() both
591 * flips it and handles component loading.
592 */
593 if (!cold)
594 return;
595
596 LIST_REMOVE(rc, rc_entries);
597}
598
599int
600rump_component_count(enum rump_component_type type)
601{
602
603 KASSERT(curlwp == bootlwp);
604 KASSERT(type < RUMP_COMPONENT_MAX);
605 return compcounter[type];
606}
607
608void
609rump_component_init(enum rump_component_type type)
610{
611 const struct rump_component *rc, *rc_safe;
612
613 KASSERT(curlwp == bootlwp);
614 KASSERT(!compinited[type]);
615 LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) {
616 if (rc->rc_type == type) {
617 rc->rc_init();
618 LIST_REMOVE(rc, rc_entries);
619 }
620 }
621 compinited[type] = 1;
622}
623
624/*
625 * Initialize a module which has already been loaded and linked
626 * with dlopen(). This is fundamentally the same as a builtin module.
627 *
628 * XXX: this interface does not really work in the RUMP_USE_CTOR case,
629 * but I'm not sure it's anything to cry about. In feeling blue,
630 * things could somehow be handled via modinfo_boot_chain.
631 */
632int
633rump_module_init(const struct modinfo * const *mip, size_t nmodinfo)
634{
635
636 return module_builtin_add(mip, nmodinfo, true);
637}
638
639/*
640 * Finish module (flawless victory, fatality!).
641 */
642int
643rump_module_fini(const struct modinfo *mi)
644{
645
646 return module_builtin_remove(mi, true);
647}
648
649/*
650 * Add loaded and linked module to the builtin list. It will
651 * later be initialized with module_init_class().
652 */
653
654static void
655add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo)
656{
657
658 module_builtin_add(mip, nmodinfo, false);
659}
660
661int
662rump_kernelfsym_load(void *symtab, uint64_t symsize,
663 char *strtab, uint64_t strsize)
664{
665 static int inited = 0;
666 Elf64_Ehdr ehdr;
667
668 if (inited)
669 return EBUSY;
670 inited = 1;
671
672 /*
673 * Use 64bit header since it's bigger. Shouldn't make a
674 * difference, since we're passing in all zeroes anyway.
675 */
676 memset(&ehdr, 0, sizeof(ehdr));
677 ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize);
678
679 return 0;
680}
681
682int
683rump_boot_gethowto()
684{
685
686 return boothowto;
687}
688
689void
690rump_boot_sethowto(int howto)
691{
692
693 boothowto = howto;
694}
695
696int
697rump_getversion(void)
698{
699
700 return __NetBSD_Version__;
701}
702/* compat */
703__strong_alias(rump_pub_getversion,rump_getversion);
704
705/*
706 * Note: may be called unscheduled. Not fully safe since no locking
707 * of allevents (currently that's not even available).
708 */
709void
710rump_printevcnts()
711{
712 struct evcnt *ev;
713
714 TAILQ_FOREACH(ev, &allevents, ev_list)
715 rumpuser_dprintf("%s / %s: %" PRIu64 "\n",
716 ev->ev_group, ev->ev_name, ev->ev_count);
717}
718
719/*
720 * If you use this interface ... well ... all bets are off.
721 * The original purpose is for the p2k fs server library to be
722 * able to use the same pid/lid for VOPs as the host kernel.
723 */
724void
725rump_allbetsareoff_setid(pid_t pid, int lid)
726{
727 struct lwp *l = curlwp;
728 struct proc *p = l->l_proc;
729
730 l->l_lid = lid;
731 p->p_pid = pid;
732}
733
734#include <sys/pserialize.h>
735
736static void
737ipiemu(void *a1, void *a2)
738{
739
740 xc__highpri_intr(NULL);
741 pserialize_switchpoint();
742}
743
744void
745rump_xc_highpri(struct cpu_info *ci)
746{
747
748 if (ci)
749 xc_unicast(0, ipiemu, NULL, NULL, ci);
750 else
751 xc_broadcast(0, ipiemu, NULL, NULL);
752}
753
754int
755rump_syscall(int num, void *data, size_t dlen, register_t *retval)
756{
757 struct proc *p;
758 struct emul *e;
759 struct sysent *callp;
760 const int *etrans = NULL;
761 int rv;
762
763 rump_schedule();
764 p = curproc;
765 e = p->p_emul;
766#ifndef __HAVE_MINIMAL_EMUL
767 KASSERT(num > 0 && num < e->e_nsysent);
768#endif
769 callp = e->e_sysent + num;
770
771 rv = sy_invoke(callp, curlwp, data, retval, num);
772
773 /*
774 * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is
775 * an invariant ...
776 */
777#if !defined(__HAVE_MINIMAL_EMUL)
778 etrans = e->e_errno;
779#elif defined(__HAVE_SYSCALL_INTERN)
780 etrans = p->p_emuldata;
781#endif
782
783 if (etrans) {
784 rv = etrans[rv];
785 /*
786 * XXX: small hack since Linux etrans vectors on some
787 * archs contain negative errnos, but rump_syscalls
788 * uses the -1 + errno ABI. Note that these
789 * negative values are always the result of translation,
790 * otherwise the above translation method would not
791 * work very well.
792 */
793 if (rv < 0)
794 rv = -rv;
795 }
796 rump_unschedule();
797
798 return rv;
799}
800
801void
802rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall)
803{
804 struct sysent *callp;
805 size_t i;
806
807 for (i = 0; i < ncall; i++) {
808 callp = rump_sysent + calls[i].ros_num;
809 KASSERT(bootlwp != NULL
810 && callp->sy_call == (sy_call_t *)enosys);
811 callp->sy_call = calls[i].ros_handler;
812 }
813}
814
815struct rump_boot_etfs *ebstart;
816void
817rump_boot_etfs_register(struct rump_boot_etfs *eb)
818{
819
820 /*
821 * Could use atomics, but, since caller would need to synchronize
822 * against calling rump_init() anyway, easier to just specify the
823 * interface as "caller serializes". This solve-by-specification
824 * approach avoids the grey area of using atomics before rump_init()
825 * runs.
826 */
827 eb->_eb_next = ebstart;
828 eb->eb_status = -1;
829 ebstart = eb;
830}
831