1/* $NetBSD: pthread.c,v 1.153 2019/03/05 01:35:52 christos Exp $ */
2
3/*-
4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__RCSID("$NetBSD: pthread.c,v 1.153 2019/03/05 01:35:52 christos Exp $");
34
35#define __EXPOSE_STACK 1
36
37#include <sys/param.h>
38#include <sys/exec_elf.h>
39#include <sys/mman.h>
40#include <sys/lwp.h>
41#include <sys/lwpctl.h>
42#include <sys/resource.h>
43#include <sys/sysctl.h>
44#include <sys/tls.h>
45#include <uvm/uvm_param.h>
46
47#include <assert.h>
48#include <dlfcn.h>
49#include <err.h>
50#include <errno.h>
51#include <lwp.h>
52#include <signal.h>
53#include <stdio.h>
54#include <stdlib.h>
55#include <stddef.h>
56#include <string.h>
57#include <syslog.h>
58#include <ucontext.h>
59#include <unistd.h>
60#include <sched.h>
61
62#include "atexit.h"
63#include "pthread.h"
64#include "pthread_int.h"
65#include "pthread_makelwp.h"
66#include "reentrant.h"
67
68pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
69static rb_tree_t pthread__alltree;
70
71static signed int pthread__cmp(void *, const void *, const void *);
72
73static const rb_tree_ops_t pthread__alltree_ops = {
74 .rbto_compare_nodes = pthread__cmp,
75 .rbto_compare_key = pthread__cmp,
76 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
77 .rbto_context = NULL
78};
79
80static void pthread__create_tramp(void *);
81static void pthread__initthread(pthread_t);
82static void pthread__scrubthread(pthread_t, char *, int);
83static void pthread__initmain(pthread_t *);
84static void pthread__fork_callback(void);
85static void pthread__reap(pthread_t);
86static void pthread__child_callback(void);
87static void pthread__start(void);
88
89void pthread__init(void);
90
91int pthread__started;
92int __uselibcstub = 1;
93pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
94pthread_queue_t pthread__deadqueue;
95pthread_queue_t pthread__allqueue;
96
97static pthread_attr_t pthread_default_attr;
98static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
99
100enum {
101 DIAGASSERT_ABORT = 1<<0,
102 DIAGASSERT_STDERR = 1<<1,
103 DIAGASSERT_SYSLOG = 1<<2
104};
105
106static int pthread__diagassert;
107
108int pthread__concurrency;
109int pthread__nspins;
110int pthread__unpark_max = PTHREAD__UNPARK_MAX;
111int pthread__dbg; /* set by libpthread_dbg if active */
112
113/*
114 * We have to initialize the pthread_stack* variables here because
115 * mutexes are used before pthread_init() and thus pthread__initmain()
116 * are called. Since mutexes only save the stack pointer and not a
117 * pointer to the thread data, it is safe to change the mapping from
118 * stack pointer to thread data afterwards.
119 */
120size_t pthread__stacksize;
121size_t pthread__guardsize;
122size_t pthread__pagesize;
123static struct __pthread_st *pthread__main;
124static size_t __pthread_st_size;
125
126int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
127
128__strong_alias(__libc_thr_self,pthread_self)
129__strong_alias(__libc_thr_create,pthread_create)
130__strong_alias(__libc_thr_exit,pthread_exit)
131__strong_alias(__libc_thr_errno,pthread__errno)
132__strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
133__strong_alias(__libc_thr_equal,pthread_equal)
134__strong_alias(__libc_thr_init,pthread__init)
135
136/*
137 * Static library kludge. Place a reference to a symbol any library
138 * file which does not already have a reference here.
139 */
140extern int pthread__cancel_stub_binder;
141
142void *pthread__static_lib_binder[] = {
143 &pthread__cancel_stub_binder,
144 pthread_cond_init,
145 pthread_mutex_init,
146 pthread_rwlock_init,
147 pthread_barrier_init,
148 pthread_key_create,
149 pthread_setspecific,
150};
151
152#define NHASHLOCK 64
153
154static union hashlock {
155 pthread_mutex_t mutex;
156 char pad[64];
157} hashlocks[NHASHLOCK] __aligned(64);
158
159/*
160 * This needs to be started by the library loading code, before main()
161 * gets to run, for various things that use the state of the initial thread
162 * to work properly (thread-specific data is an application-visible example;
163 * spinlock counts for mutexes is an internal example).
164 */
165void
166pthread__init(void)
167{
168 pthread_t first;
169 char *p;
170 int i;
171 int mib[2];
172 unsigned int value;
173 size_t len;
174 extern int __isthreaded;
175
176 /*
177 * Allocate pthread_keys descriptors before
178 * reseting __uselibcstub because otherwise
179 * malloc() will call pthread_keys_create()
180 * while pthread_keys descriptors are not
181 * yet allocated.
182 */
183 pthread__main = pthread_tsd_init(&__pthread_st_size);
184 if (pthread__main == NULL)
185 err(EXIT_FAILURE, "Cannot allocate pthread storage");
186
187 __uselibcstub = 0;
188
189 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
190 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
191
192 mib[0] = CTL_VM;
193 mib[1] = VM_THREAD_GUARD_SIZE;
194 len = sizeof(value);
195 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
196 pthread__guardsize = value;
197 else
198 pthread__guardsize = pthread__pagesize;
199
200 /* Initialize locks first; they're needed elsewhere. */
201 pthread__lockprim_init();
202 for (i = 0; i < NHASHLOCK; i++) {
203 pthread_mutex_init(&hashlocks[i].mutex, NULL);
204 }
205
206 /* Fetch parameters. */
207 i = (int)_lwp_unpark_all(NULL, 0, NULL);
208 if (i == -1)
209 err(EXIT_FAILURE, "_lwp_unpark_all");
210 if (i < pthread__unpark_max)
211 pthread__unpark_max = i;
212
213 /* Basic data structure setup */
214 pthread_attr_init(&pthread_default_attr);
215 PTQ_INIT(&pthread__allqueue);
216 PTQ_INIT(&pthread__deadqueue);
217
218 rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
219
220 /* Create the thread structure corresponding to main() */
221 pthread__initmain(&first);
222 pthread__initthread(first);
223 pthread__scrubthread(first, NULL, 0);
224
225 first->pt_lid = _lwp_self();
226 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
227 (void)rb_tree_insert_node(&pthread__alltree, first);
228
229 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
230 err(EXIT_FAILURE, "_lwp_ctl");
231 }
232
233 /* Start subsystems */
234 PTHREAD_MD_INIT
235
236 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
237 switch (*p) {
238 case 'a':
239 pthread__diagassert |= DIAGASSERT_ABORT;
240 break;
241 case 'A':
242 pthread__diagassert &= ~DIAGASSERT_ABORT;
243 break;
244 case 'e':
245 pthread__diagassert |= DIAGASSERT_STDERR;
246 break;
247 case 'E':
248 pthread__diagassert &= ~DIAGASSERT_STDERR;
249 break;
250 case 'l':
251 pthread__diagassert |= DIAGASSERT_SYSLOG;
252 break;
253 case 'L':
254 pthread__diagassert &= ~DIAGASSERT_SYSLOG;
255 break;
256 }
257 }
258
259 /* Tell libc that we're here and it should role-play accordingly. */
260 pthread_atfork(NULL, NULL, pthread__fork_callback);
261 __isthreaded = 1;
262}
263
264static void
265pthread__fork_callback(void)
266{
267 struct __pthread_st *self = pthread__self();
268
269 /* lwpctl state is not copied across fork. */
270 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
271 err(EXIT_FAILURE, "_lwp_ctl");
272 }
273 self->pt_lid = _lwp_self();
274}
275
276static void
277pthread__child_callback(void)
278{
279
280 /*
281 * Clean up data structures that a forked child process might
282 * trip over. Note that if threads have been created (causing
283 * this handler to be registered) the standards say that the
284 * child will trigger undefined behavior if it makes any
285 * pthread_* calls (or any other calls that aren't
286 * async-signal-safe), so we don't really have to clean up
287 * much. Anything that permits some pthread_* calls to work is
288 * merely being polite.
289 */
290 pthread__started = 0;
291}
292
293static void
294pthread__start(void)
295{
296
297 /*
298 * Per-process timers are cleared by fork(); despite the
299 * various restrictions on fork() and threads, it's legal to
300 * fork() before creating any threads.
301 */
302 pthread_atfork(NULL, NULL, pthread__child_callback);
303}
304
305
306/* General-purpose thread data structure sanitization. */
307/* ARGSUSED */
308static void
309pthread__initthread(pthread_t t)
310{
311
312 t->pt_self = t;
313 t->pt_magic = PT_MAGIC;
314 t->pt_willpark = 0;
315 t->pt_unpark = 0;
316 t->pt_nwaiters = 0;
317 t->pt_sleepobj = NULL;
318 t->pt_signalled = 0;
319 t->pt_havespecific = 0;
320 t->pt_early = NULL;
321 t->pt_lwpctl = &pthread__dummy_lwpctl;
322 t->pt_blocking = 0;
323 t->pt_droplock = NULL;
324
325 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
326 pthread_mutex_init(&t->pt_lock, NULL);
327 PTQ_INIT(&t->pt_cleanup_stack);
328 pthread_cond_init(&t->pt_joiners, NULL);
329}
330
331static void
332pthread__scrubthread(pthread_t t, char *name, int flags)
333{
334
335 t->pt_state = PT_STATE_RUNNING;
336 t->pt_exitval = NULL;
337 t->pt_flags = flags;
338 t->pt_cancel = 0;
339 t->pt_errno = 0;
340 t->pt_name = name;
341 t->pt_lid = 0;
342}
343
344static int
345pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
346{
347 void *stackbase, *stackbase2, *redzone;
348 size_t stacksize, guardsize;
349 bool allocated;
350
351 if (attr != NULL) {
352 pthread_attr_getstack(attr, &stackbase, &stacksize);
353 pthread_attr_getguardsize(attr, &guardsize);
354 } else {
355 stackbase = NULL;
356 stacksize = 0;
357 guardsize = pthread__guardsize;
358 }
359 if (stacksize == 0)
360 stacksize = pthread__stacksize;
361
362 if (newthread->pt_stack_allocated) {
363 if (stackbase == NULL &&
364 newthread->pt_stack.ss_size == stacksize &&
365 newthread->pt_guardsize == guardsize)
366 return 0;
367 stackbase2 = newthread->pt_stack.ss_sp;
368#ifndef __MACHINE_STACK_GROWS_UP
369 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
370#endif
371 munmap(stackbase2,
372 newthread->pt_stack.ss_size + newthread->pt_guardsize);
373 newthread->pt_stack.ss_sp = NULL;
374 newthread->pt_stack.ss_size = 0;
375 newthread->pt_guardsize = 0;
376 newthread->pt_stack_allocated = false;
377 }
378
379 newthread->pt_stack_allocated = false;
380
381 if (stackbase == NULL) {
382 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
383 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
384 stackbase = mmap(NULL, stacksize + guardsize,
385 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
386 if (stackbase == MAP_FAILED)
387 return ENOMEM;
388 allocated = true;
389 } else {
390 allocated = false;
391 }
392#ifdef __MACHINE_STACK_GROWS_UP
393 redzone = (char *)stackbase + stacksize;
394 stackbase2 = (char *)stackbase;
395#else
396 redzone = (char *)stackbase;
397 stackbase2 = (char *)stackbase + guardsize;
398#endif
399 if (allocated && guardsize &&
400 mprotect(redzone, guardsize, PROT_NONE) == -1) {
401 munmap(stackbase, stacksize + guardsize);
402 return EPERM;
403 }
404 newthread->pt_stack.ss_size = stacksize;
405 newthread->pt_stack.ss_sp = stackbase2;
406 newthread->pt_guardsize = guardsize;
407 newthread->pt_stack_allocated = allocated;
408 return 0;
409}
410
411int
412pthread_create(pthread_t *thread, const pthread_attr_t *attr,
413 void *(*startfunc)(void *), void *arg)
414{
415 pthread_t newthread;
416 pthread_attr_t nattr;
417 struct pthread_attr_private *p;
418 char * volatile name;
419 unsigned long flag;
420 void *private_area;
421 int ret;
422
423 if (__predict_false(__uselibcstub)) {
424 pthread__errorfunc(__FILE__, __LINE__, __func__,
425 "pthread_create() requires linking with -lpthread");
426 return __libc_thr_create_stub(thread, attr, startfunc, arg);
427 }
428
429 /*
430 * It's okay to check this without a lock because there can
431 * only be one thread before it becomes true.
432 */
433 if (pthread__started == 0) {
434 pthread__start();
435 pthread__started = 1;
436 }
437
438 if (attr == NULL)
439 nattr = pthread_default_attr;
440 else if (attr->pta_magic == PT_ATTR_MAGIC)
441 nattr = *attr;
442 else
443 return EINVAL;
444
445 /* Fetch misc. attributes from the attr structure. */
446 name = NULL;
447 if ((p = nattr.pta_private) != NULL)
448 if (p->ptap_name[0] != '\0')
449 if ((name = strdup(p->ptap_name)) == NULL)
450 return ENOMEM;
451
452 newthread = NULL;
453
454 /*
455 * Try to reclaim a dead thread.
456 */
457 if (!PTQ_EMPTY(&pthread__deadqueue)) {
458 pthread_mutex_lock(&pthread__deadqueue_lock);
459 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
460 /* Still running? */
461 if (newthread->pt_lwpctl->lc_curcpu ==
462 LWPCTL_CPU_EXITED ||
463 (_lwp_kill(newthread->pt_lid, 0) == -1 &&
464 errno == ESRCH))
465 break;
466 }
467 if (newthread)
468 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
469 pthread_mutex_unlock(&pthread__deadqueue_lock);
470#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
471 if (newthread && newthread->pt_tls) {
472 _rtld_tls_free(newthread->pt_tls);
473 newthread->pt_tls = NULL;
474 }
475#endif
476 }
477
478 /*
479 * If necessary set up a stack, allocate space for a pthread_st,
480 * and initialize it.
481 */
482 if (newthread == NULL) {
483 newthread = calloc(1, __pthread_st_size);
484 if (newthread == NULL) {
485 free(name);
486 return ENOMEM;
487 }
488 newthread->pt_stack_allocated = false;
489
490 if (pthread__getstack(newthread, attr)) {
491 free(newthread);
492 free(name);
493 return ENOMEM;
494 }
495
496#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
497 newthread->pt_tls = NULL;
498#endif
499
500 /* Add to list of all threads. */
501 pthread_rwlock_wrlock(&pthread__alltree_lock);
502 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
503 (void)rb_tree_insert_node(&pthread__alltree, newthread);
504 pthread_rwlock_unlock(&pthread__alltree_lock);
505
506 /* Will be reset by the thread upon exit. */
507 pthread__initthread(newthread);
508 } else {
509 if (pthread__getstack(newthread, attr)) {
510 pthread_mutex_lock(&pthread__deadqueue_lock);
511 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
512 pthread_mutex_unlock(&pthread__deadqueue_lock);
513 return ENOMEM;
514 }
515 }
516
517 /*
518 * Create the new LWP.
519 */
520 pthread__scrubthread(newthread, name, nattr.pta_flags);
521 newthread->pt_func = startfunc;
522 newthread->pt_arg = arg;
523#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
524 private_area = newthread->pt_tls = _rtld_tls_allocate();
525 newthread->pt_tls->tcb_pthread = newthread;
526#else
527 private_area = newthread;
528#endif
529
530 flag = LWP_DETACHED;
531 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
532 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
533 flag |= LWP_SUSPENDED;
534
535 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
536 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
537 flag, &newthread->pt_lid);
538 if (ret != 0) {
539 ret = errno;
540 pthread_mutex_lock(&newthread->pt_lock);
541 /* Will unlock and free name. */
542 pthread__reap(newthread);
543 return ret;
544 }
545
546 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
547 if (p != NULL) {
548 (void)pthread_setschedparam(newthread, p->ptap_policy,
549 &p->ptap_sp);
550 }
551 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
552 (void)_lwp_continue(newthread->pt_lid);
553 }
554 }
555
556 *thread = newthread;
557
558 return 0;
559}
560
561
562__dead static void
563pthread__create_tramp(void *cookie)
564{
565 pthread_t self;
566 void *retval;
567
568 self = cookie;
569
570 /*
571 * Throw away some stack in a feeble attempt to reduce cache
572 * thrash. May help for SMT processors. XXX We should not
573 * be allocating stacks on fixed 2MB boundaries. Needs a
574 * thread register or decent thread local storage.
575 *
576 * Note that we may race with the kernel in _lwp_create(),
577 * and so pt_lid can be unset at this point, but we don't
578 * care.
579 */
580 (void)alloca(((unsigned)self->pt_lid & 7) << 8);
581
582 if (self->pt_name != NULL) {
583 pthread_mutex_lock(&self->pt_lock);
584 if (self->pt_name != NULL)
585 (void)_lwp_setname(0, self->pt_name);
586 pthread_mutex_unlock(&self->pt_lock);
587 }
588
589 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
590 err(EXIT_FAILURE, "_lwp_ctl");
591 }
592
593 retval = (*self->pt_func)(self->pt_arg);
594
595 pthread_exit(retval);
596
597 /*NOTREACHED*/
598 pthread__abort();
599}
600
601int
602pthread_suspend_np(pthread_t thread)
603{
604 pthread_t self;
605
606 self = pthread__self();
607 if (self == thread) {
608 return EDEADLK;
609 }
610 if (pthread__find(thread) != 0)
611 return ESRCH;
612 if (_lwp_suspend(thread->pt_lid) == 0)
613 return 0;
614 return errno;
615}
616
617int
618pthread_resume_np(pthread_t thread)
619{
620
621 if (pthread__find(thread) != 0)
622 return ESRCH;
623 if (_lwp_continue(thread->pt_lid) == 0)
624 return 0;
625 return errno;
626}
627
628void
629pthread_exit(void *retval)
630{
631 pthread_t self;
632 struct pt_clean_t *cleanup;
633 char *name;
634
635 if (__predict_false(__uselibcstub)) {
636 __libc_thr_exit_stub(retval);
637 goto out;
638 }
639
640 self = pthread__self();
641
642 /* Disable cancellability. */
643 pthread_mutex_lock(&self->pt_lock);
644 self->pt_flags |= PT_FLAG_CS_DISABLED;
645 self->pt_cancel = 0;
646
647 /* Call any cancellation cleanup handlers */
648 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
649 pthread_mutex_unlock(&self->pt_lock);
650 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
651 cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
652 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
653 (*cleanup->ptc_cleanup)(cleanup->ptc_arg);
654 }
655 pthread_mutex_lock(&self->pt_lock);
656 }
657
658 pthread_mutex_unlock(&self->pt_lock);
659 __cxa_thread_run_atexit();
660 pthread_mutex_lock(&self->pt_lock);
661
662 /* Perform cleanup of thread-specific data */
663 pthread__destroy_tsd(self);
664
665 /* Signal our exit. */
666 self->pt_exitval = retval;
667 if (self->pt_flags & PT_FLAG_DETACHED) {
668 self->pt_state = PT_STATE_DEAD;
669 name = self->pt_name;
670 self->pt_name = NULL;
671 pthread_mutex_unlock(&self->pt_lock);
672 if (name != NULL)
673 free(name);
674 pthread_mutex_lock(&pthread__deadqueue_lock);
675 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq);
676 pthread_mutex_unlock(&pthread__deadqueue_lock);
677 _lwp_exit();
678 } else {
679 self->pt_state = PT_STATE_ZOMBIE;
680 pthread_cond_broadcast(&self->pt_joiners);
681 pthread_mutex_unlock(&self->pt_lock);
682 /* Note: name will be freed by the joiner. */
683 _lwp_exit();
684 }
685
686out:
687 /*NOTREACHED*/
688 pthread__abort();
689 exit(1);
690}
691
692
693int
694pthread_join(pthread_t thread, void **valptr)
695{
696 pthread_t self;
697 int error;
698
699 self = pthread__self();
700
701 if (pthread__find(thread) != 0)
702 return ESRCH;
703
704 if (thread->pt_magic != PT_MAGIC)
705 return EINVAL;
706
707 if (thread == self)
708 return EDEADLK;
709
710 self->pt_droplock = &thread->pt_lock;
711 pthread_mutex_lock(&thread->pt_lock);
712 for (;;) {
713 if (thread->pt_state == PT_STATE_ZOMBIE)
714 break;
715 if (thread->pt_state == PT_STATE_DEAD) {
716 pthread_mutex_unlock(&thread->pt_lock);
717 self->pt_droplock = NULL;
718 return ESRCH;
719 }
720 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
721 pthread_mutex_unlock(&thread->pt_lock);
722 self->pt_droplock = NULL;
723 return EINVAL;
724 }
725 error = pthread_cond_wait(&thread->pt_joiners,
726 &thread->pt_lock);
727 if (error != 0) {
728 pthread__errorfunc(__FILE__, __LINE__,
729 __func__, "unexpected return from cond_wait()");
730 }
731
732 }
733 pthread__testcancel(self);
734 if (valptr != NULL)
735 *valptr = thread->pt_exitval;
736 /* pthread__reap() will drop the lock. */
737 pthread__reap(thread);
738 self->pt_droplock = NULL;
739
740 return 0;
741}
742
743static void
744pthread__reap(pthread_t thread)
745{
746 char *name;
747
748 name = thread->pt_name;
749 thread->pt_name = NULL;
750 thread->pt_state = PT_STATE_DEAD;
751 pthread_mutex_unlock(&thread->pt_lock);
752
753 pthread_mutex_lock(&pthread__deadqueue_lock);
754 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
755 pthread_mutex_unlock(&pthread__deadqueue_lock);
756
757 if (name != NULL)
758 free(name);
759}
760
761int
762pthread_equal(pthread_t t1, pthread_t t2)
763{
764 if (__predict_false(__uselibcstub))
765 return __libc_thr_equal_stub(t1, t2);
766
767 /* Nothing special here. */
768 return (t1 == t2);
769}
770
771
772int
773pthread_detach(pthread_t thread)
774{
775
776 if (pthread__find(thread) != 0)
777 return ESRCH;
778
779 if (thread->pt_magic != PT_MAGIC)
780 return EINVAL;
781
782 pthread_mutex_lock(&thread->pt_lock);
783 thread->pt_flags |= PT_FLAG_DETACHED;
784 if (thread->pt_state == PT_STATE_ZOMBIE) {
785 /* pthread__reap() will drop the lock. */
786 pthread__reap(thread);
787 } else {
788 /*
789 * Not valid for threads to be waiting in
790 * pthread_join() (there are intractable
791 * sync issues from the application
792 * perspective), but give those threads
793 * a chance anyway.
794 */
795 pthread_cond_broadcast(&thread->pt_joiners);
796 pthread_mutex_unlock(&thread->pt_lock);
797 }
798
799 return 0;
800}
801
802
803int
804pthread_getname_np(pthread_t thread, char *name, size_t len)
805{
806
807 if (pthread__find(thread) != 0)
808 return ESRCH;
809
810 if (thread->pt_magic != PT_MAGIC)
811 return EINVAL;
812
813 pthread_mutex_lock(&thread->pt_lock);
814 if (thread->pt_name == NULL)
815 name[0] = '\0';
816 else
817 strlcpy(name, thread->pt_name, len);
818 pthread_mutex_unlock(&thread->pt_lock);
819
820 return 0;
821}
822
823
824int
825pthread_setname_np(pthread_t thread, const char *name, void *arg)
826{
827 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
828 int namelen;
829
830 if (pthread__find(thread) != 0)
831 return ESRCH;
832
833 if (thread->pt_magic != PT_MAGIC)
834 return EINVAL;
835
836 namelen = snprintf(newname, sizeof(newname), name, arg);
837 if (namelen >= PTHREAD_MAX_NAMELEN_NP)
838 return EINVAL;
839
840 cp = strdup(newname);
841 if (cp == NULL)
842 return ENOMEM;
843
844 pthread_mutex_lock(&thread->pt_lock);
845 oldname = thread->pt_name;
846 thread->pt_name = cp;
847 (void)_lwp_setname(thread->pt_lid, cp);
848 pthread_mutex_unlock(&thread->pt_lock);
849
850 if (oldname != NULL)
851 free(oldname);
852
853 return 0;
854}
855
856
857
858/*
859 * XXX There should be a way for applications to use the efficent
860 * inline version, but there are opacity/namespace issues.
861 */
862pthread_t
863pthread_self(void)
864{
865 if (__predict_false(__uselibcstub))
866 return (pthread_t)__libc_thr_self_stub();
867
868 return pthread__self();
869}
870
871
872int
873pthread_cancel(pthread_t thread)
874{
875
876 if (pthread__find(thread) != 0)
877 return ESRCH;
878 pthread_mutex_lock(&thread->pt_lock);
879 thread->pt_flags |= PT_FLAG_CS_PENDING;
880 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) {
881 thread->pt_cancel = 1;
882 pthread_mutex_unlock(&thread->pt_lock);
883 _lwp_wakeup(thread->pt_lid);
884 } else
885 pthread_mutex_unlock(&thread->pt_lock);
886
887 return 0;
888}
889
890
891int
892pthread_setcancelstate(int state, int *oldstate)
893{
894 pthread_t self;
895 int retval;
896
897 if (__predict_false(__uselibcstub))
898 return __libc_thr_setcancelstate_stub(state, oldstate);
899
900 self = pthread__self();
901 retval = 0;
902
903 pthread_mutex_lock(&self->pt_lock);
904
905 if (oldstate != NULL) {
906 if (self->pt_flags & PT_FLAG_CS_DISABLED)
907 *oldstate = PTHREAD_CANCEL_DISABLE;
908 else
909 *oldstate = PTHREAD_CANCEL_ENABLE;
910 }
911
912 if (state == PTHREAD_CANCEL_DISABLE) {
913 self->pt_flags |= PT_FLAG_CS_DISABLED;
914 if (self->pt_cancel) {
915 self->pt_flags |= PT_FLAG_CS_PENDING;
916 self->pt_cancel = 0;
917 }
918 } else if (state == PTHREAD_CANCEL_ENABLE) {
919 self->pt_flags &= ~PT_FLAG_CS_DISABLED;
920 /*
921 * If a cancellation was requested while cancellation
922 * was disabled, note that fact for future
923 * cancellation tests.
924 */
925 if (self->pt_flags & PT_FLAG_CS_PENDING) {
926 self->pt_cancel = 1;
927 /* This is not a deferred cancellation point. */
928 if (self->pt_flags & PT_FLAG_CS_ASYNC) {
929 pthread_mutex_unlock(&self->pt_lock);
930 pthread__cancelled();
931 }
932 }
933 } else
934 retval = EINVAL;
935
936 pthread_mutex_unlock(&self->pt_lock);
937
938 return retval;
939}
940
941
942int
943pthread_setcanceltype(int type, int *oldtype)
944{
945 pthread_t self;
946 int retval;
947
948 self = pthread__self();
949 retval = 0;
950
951 pthread_mutex_lock(&self->pt_lock);
952
953 if (oldtype != NULL) {
954 if (self->pt_flags & PT_FLAG_CS_ASYNC)
955 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
956 else
957 *oldtype = PTHREAD_CANCEL_DEFERRED;
958 }
959
960 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
961 self->pt_flags |= PT_FLAG_CS_ASYNC;
962 if (self->pt_cancel) {
963 pthread_mutex_unlock(&self->pt_lock);
964 pthread__cancelled();
965 }
966 } else if (type == PTHREAD_CANCEL_DEFERRED)
967 self->pt_flags &= ~PT_FLAG_CS_ASYNC;
968 else
969 retval = EINVAL;
970
971 pthread_mutex_unlock(&self->pt_lock);
972
973 return retval;
974}
975
976
977void
978pthread_testcancel(void)
979{
980 pthread_t self;
981
982 self = pthread__self();
983 if (self->pt_cancel)
984 pthread__cancelled();
985}
986
987
988/*
989 * POSIX requires that certain functions return an error rather than
990 * invoking undefined behavior even when handed completely bogus
991 * pthread_t values, e.g. stack garbage.
992 */
993int
994pthread__find(pthread_t id)
995{
996 pthread_t target;
997 int error;
998
999 pthread_rwlock_rdlock(&pthread__alltree_lock);
1000 target = rb_tree_find_node(&pthread__alltree, id);
1001 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
1002 pthread_rwlock_unlock(&pthread__alltree_lock);
1003
1004 return error;
1005}
1006
1007
1008void
1009pthread__testcancel(pthread_t self)
1010{
1011
1012 if (self->pt_cancel)
1013 pthread__cancelled();
1014}
1015
1016
1017void
1018pthread__cancelled(void)
1019{
1020 pthread_mutex_t *droplock;
1021 pthread_t self;
1022
1023 self = pthread__self();
1024 droplock = self->pt_droplock;
1025 self->pt_droplock = NULL;
1026
1027 if (droplock != NULL && pthread_mutex_held_np(droplock))
1028 pthread_mutex_unlock(droplock);
1029
1030 pthread_exit(PTHREAD_CANCELED);
1031}
1032
1033
1034void
1035pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
1036{
1037 pthread_t self;
1038 struct pt_clean_t *entry;
1039
1040 self = pthread__self();
1041 entry = store;
1042 entry->ptc_cleanup = cleanup;
1043 entry->ptc_arg = arg;
1044 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
1045}
1046
1047
1048void
1049pthread__cleanup_pop(int ex, void *store)
1050{
1051 pthread_t self;
1052 struct pt_clean_t *entry;
1053
1054 self = pthread__self();
1055 entry = store;
1056
1057 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
1058 if (ex)
1059 (*entry->ptc_cleanup)(entry->ptc_arg);
1060}
1061
1062
1063int *
1064pthread__errno(void)
1065{
1066 pthread_t self;
1067
1068 if (__predict_false(__uselibcstub)) {
1069 pthread__errorfunc(__FILE__, __LINE__, __func__,
1070 "pthread__errno() requires linking with -lpthread");
1071 return __libc_thr_errno_stub();
1072 }
1073
1074 self = pthread__self();
1075
1076 return &(self->pt_errno);
1077}
1078
1079ssize_t _sys_write(int, const void *, size_t);
1080
1081void
1082pthread__assertfunc(const char *file, int line, const char *function,
1083 const char *expr)
1084{
1085 char buf[1024];
1086 int len;
1087
1088 /*
1089 * snprintf should not acquire any locks, or we could
1090 * end up deadlocked if the assert caller held locks.
1091 */
1092 len = snprintf(buf, 1024,
1093 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
1094 expr, file, line,
1095 function ? ", function \"" : "",
1096 function ? function : "",
1097 function ? "\"" : "");
1098
1099 _sys_write(STDERR_FILENO, buf, (size_t)len);
1100 (void)kill(getpid(), SIGABRT);
1101
1102 _exit(1);
1103}
1104
1105
1106void
1107pthread__errorfunc(const char *file, int line, const char *function,
1108 const char *msg)
1109{
1110 char buf[1024];
1111 size_t len;
1112
1113 if (pthread__diagassert == 0)
1114 return;
1115
1116 /*
1117 * snprintf should not acquire any locks, or we could
1118 * end up deadlocked if the assert caller held locks.
1119 */
1120 len = snprintf(buf, 1024,
1121 "%s: Error detected by libpthread: %s.\n"
1122 "Detected by file \"%s\", line %d%s%s%s.\n"
1123 "See pthread(3) for information.\n",
1124 getprogname(), msg, file, line,
1125 function ? ", function \"" : "",
1126 function ? function : "",
1127 function ? "\"" : "");
1128
1129 if (pthread__diagassert & DIAGASSERT_STDERR)
1130 _sys_write(STDERR_FILENO, buf, len);
1131
1132 if (pthread__diagassert & DIAGASSERT_SYSLOG)
1133 syslog(LOG_DEBUG | LOG_USER, "%s", buf);
1134
1135 if (pthread__diagassert & DIAGASSERT_ABORT) {
1136 (void)kill(getpid(), SIGABRT);
1137 _exit(1);
1138 }
1139}
1140
1141/*
1142 * Thread park/unpark operations. The kernel operations are
1143 * modelled after a brief description from "Multithreading in
1144 * the Solaris Operating Environment":
1145 *
1146 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
1147 */
1148
1149#define OOPS(msg) \
1150 pthread__errorfunc(__FILE__, __LINE__, __func__, msg)
1151
1152int
1153pthread__park(pthread_t self, pthread_mutex_t *lock,
1154 pthread_queue_t *queue, const struct timespec *abstime,
1155 int cancelpt, const void *hint)
1156{
1157 int rv, error;
1158 void *obj;
1159
1160 /*
1161 * For non-interlocked release of mutexes we need a store
1162 * barrier before incrementing pt_blocking away from zero.
1163 * This is provided by pthread_mutex_unlock().
1164 */
1165 self->pt_willpark = 1;
1166 pthread_mutex_unlock(lock);
1167 self->pt_willpark = 0;
1168 self->pt_blocking++;
1169
1170 /*
1171 * Wait until we are awoken by a pending unpark operation,
1172 * a signal, an unpark posted after we have gone asleep,
1173 * or an expired timeout.
1174 *
1175 * It is fine to test the value of pt_sleepobj without
1176 * holding any locks, because:
1177 *
1178 * o Only the blocking thread (this thread) ever sets them
1179 * to a non-NULL value.
1180 *
1181 * o Other threads may set them NULL, but if they do so they
1182 * must also make this thread return from _lwp_park.
1183 *
1184 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
1185 * calls and all make use of spinlocks in the kernel. So
1186 * these system calls act as full memory barriers, and will
1187 * ensure that the calling CPU's store buffers are drained.
1188 * In combination with the spinlock release before unpark,
1189 * this means that modification of pt_sleepobj/onq by another
1190 * thread will become globally visible before that thread
1191 * schedules an unpark operation on this thread.
1192 *
1193 * Note: the test in the while() statement dodges the park op if
1194 * we have already been awoken, unless there is another thread to
1195 * awaken. This saves a syscall - if we were already awakened,
1196 * the next call to _lwp_park() would need to return early in order
1197 * to eat the previous wakeup.
1198 */
1199 rv = 0;
1200 do {
1201 /*
1202 * If we deferred unparking a thread, arrange to
1203 * have _lwp_park() restart it before blocking.
1204 */
1205 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
1206 __UNCONST(abstime), self->pt_unpark, hint, hint);
1207 self->pt_unpark = 0;
1208 if (error != 0) {
1209 switch (rv = errno) {
1210 case EINTR:
1211 case EALREADY:
1212 rv = 0;
1213 break;
1214 case ETIMEDOUT:
1215 break;
1216 default:
1217 OOPS("_lwp_park failed");
1218 break;
1219 }
1220 }
1221 /* Check for cancellation. */
1222 if (cancelpt && self->pt_cancel)
1223 rv = EINTR;
1224 } while (self->pt_sleepobj != NULL && rv == 0);
1225
1226 /*
1227 * If we have been awoken early but are still on the queue,
1228 * then remove ourself. Again, it's safe to do the test
1229 * without holding any locks.
1230 */
1231 if (__predict_false(self->pt_sleepobj != NULL)) {
1232 pthread_mutex_lock(lock);
1233 if ((obj = self->pt_sleepobj) != NULL) {
1234 PTQ_REMOVE(queue, self, pt_sleep);
1235 self->pt_sleepobj = NULL;
1236 if (obj != NULL && self->pt_early != NULL)
1237 (*self->pt_early)(obj);
1238 }
1239 pthread_mutex_unlock(lock);
1240 }
1241 self->pt_early = NULL;
1242 self->pt_blocking--;
1243 membar_sync();
1244
1245 return rv;
1246}
1247
1248void
1249pthread__unpark(pthread_queue_t *queue, pthread_t self,
1250 pthread_mutex_t *interlock)
1251{
1252 pthread_t target;
1253 u_int max;
1254 size_t nwaiters;
1255
1256 max = pthread__unpark_max;
1257 nwaiters = self->pt_nwaiters;
1258 target = PTQ_FIRST(queue);
1259 if (nwaiters == max) {
1260 /* Overflow. */
1261 (void)_lwp_unpark_all(self->pt_waiters, nwaiters,
1262 __UNVOLATILE(&interlock->ptm_waiters));
1263 nwaiters = 0;
1264 }
1265 target->pt_sleepobj = NULL;
1266 self->pt_waiters[nwaiters++] = target->pt_lid;
1267 PTQ_REMOVE(queue, target, pt_sleep);
1268 self->pt_nwaiters = nwaiters;
1269 pthread__mutex_deferwake(self, interlock);
1270}
1271
1272void
1273pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
1274 pthread_mutex_t *interlock)
1275{
1276 pthread_t target;
1277 u_int max;
1278 size_t nwaiters;
1279
1280 max = pthread__unpark_max;
1281 nwaiters = self->pt_nwaiters;
1282 PTQ_FOREACH(target, queue, pt_sleep) {
1283 if (nwaiters == max) {
1284 /* Overflow. */
1285 (void)_lwp_unpark_all(self->pt_waiters, nwaiters,
1286 __UNVOLATILE(&interlock->ptm_waiters));
1287 nwaiters = 0;
1288 }
1289 target->pt_sleepobj = NULL;
1290 self->pt_waiters[nwaiters++] = target->pt_lid;
1291 }
1292 self->pt_nwaiters = nwaiters;
1293 PTQ_INIT(queue);
1294 pthread__mutex_deferwake(self, interlock);
1295}
1296
1297#undef OOPS
1298
1299static void
1300pthread__initmainstack(void)
1301{
1302 struct rlimit slimit;
1303 const AuxInfo *aux;
1304 size_t size, len;
1305 int mib[2];
1306 unsigned int value;
1307
1308 _DIAGASSERT(_dlauxinfo() != NULL);
1309
1310 if (getrlimit(RLIMIT_STACK, &slimit) == -1)
1311 err(EXIT_FAILURE,
1312 "Couldn't get stack resource consumption limits");
1313 size = slimit.rlim_cur;
1314 pthread__main->pt_stack.ss_size = size;
1315 pthread__main->pt_guardsize = pthread__pagesize;
1316
1317 mib[0] = CTL_VM;
1318 mib[1] = VM_GUARD_SIZE;
1319 len = sizeof(value);
1320 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
1321 pthread__main->pt_guardsize = value;
1322
1323 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
1324 if (aux->a_type == AT_STACKBASE) {
1325#ifdef __MACHINE_STACK_GROWS_UP
1326 pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
1327#else
1328 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
1329#endif
1330 break;
1331 }
1332 }
1333 pthread__copy_tsd(pthread__main);
1334}
1335
1336/*
1337 * Set up the slightly special stack for the "initial" thread, which
1338 * runs on the normal system stack, and thus gets slightly different
1339 * treatment.
1340 */
1341static void
1342pthread__initmain(pthread_t *newt)
1343{
1344 char *value;
1345
1346 pthread__initmainstack();
1347
1348 value = pthread__getenv("PTHREAD_STACKSIZE");
1349 if (value != NULL) {
1350 pthread__stacksize = atoi(value) * 1024;
1351 if (pthread__stacksize > pthread__main->pt_stack.ss_size)
1352 pthread__stacksize = pthread__main->pt_stack.ss_size;
1353 }
1354 if (pthread__stacksize == 0)
1355 pthread__stacksize = pthread__main->pt_stack.ss_size;
1356 pthread__stacksize += pthread__pagesize - 1;
1357 pthread__stacksize &= ~(pthread__pagesize - 1);
1358 if (pthread__stacksize < 4 * pthread__pagesize)
1359 errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
1360 4 * pthread__pagesize / 1024);
1361
1362 *newt = pthread__main;
1363#if defined(_PTHREAD_GETTCB_EXT)
1364 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
1365#elif defined(__HAVE___LWP_GETTCB_FAST)
1366 pthread__main->pt_tls = __lwp_gettcb_fast();
1367#else
1368 pthread__main->pt_tls = _lwp_getprivate();
1369#endif
1370 pthread__main->pt_tls->tcb_pthread = pthread__main;
1371}
1372
1373static signed int
1374/*ARGSUSED*/
1375pthread__cmp(void *ctx, const void *n1, const void *n2)
1376{
1377 const uintptr_t p1 = (const uintptr_t)n1;
1378 const uintptr_t p2 = (const uintptr_t)n2;
1379
1380 if (p1 < p2)
1381 return -1;
1382 if (p1 > p2)
1383 return 1;
1384 return 0;
1385}
1386
1387/* Because getenv() wants to use locks. */
1388char *
1389pthread__getenv(const char *name)
1390{
1391 extern char **environ;
1392 size_t l_name, offset;
1393
1394 if (issetugid())
1395 return (NULL);
1396
1397 l_name = strlen(name);
1398 for (offset = 0; environ[offset] != NULL; offset++) {
1399 if (strncmp(name, environ[offset], l_name) == 0 &&
1400 environ[offset][l_name] == '=') {
1401 return environ[offset] + l_name + 1;
1402 }
1403 }
1404
1405 return NULL;
1406}
1407
1408pthread_mutex_t *
1409pthread__hashlock(volatile const void *p)
1410{
1411 uintptr_t v;
1412
1413 v = (uintptr_t)p;
1414 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
1415}
1416
1417int
1418pthread__checkpri(int pri)
1419{
1420 static int havepri;
1421 static long min, max;
1422
1423 if (!havepri) {
1424 min = sysconf(_SC_SCHED_PRI_MIN);
1425 max = sysconf(_SC_SCHED_PRI_MAX);
1426 havepri = 1;
1427 }
1428 return (pri < min || pri > max) ? EINVAL : 0;
1429}
1430