1/* $NetBSD: lock.h,v 1.28 2017/09/16 23:55:45 christos Exp $ */
2
3/*-
4 * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Machine-dependent spin lock operations.
34 */
35
36#ifndef _X86_LOCK_H_
37#define _X86_LOCK_H_
38
39#include <sys/param.h>
40
41static __inline int
42__SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
43{
44 return *__ptr == __SIMPLELOCK_LOCKED;
45}
46
47static __inline int
48__SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
49{
50 return *__ptr == __SIMPLELOCK_UNLOCKED;
51}
52
53static __inline void
54__cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
55{
56
57 *__ptr = __SIMPLELOCK_LOCKED;
58}
59
60static __inline void
61__cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
62{
63
64 *__ptr = __SIMPLELOCK_UNLOCKED;
65}
66
67#ifdef _HARDKERNEL
68# include <machine/cpufunc.h>
69# define SPINLOCK_SPIN_HOOK /* nothing */
70# ifdef SPINLOCK_BACKOFF_HOOK
71# undef SPINLOCK_BACKOFF_HOOK
72# endif
73# define SPINLOCK_BACKOFF_HOOK x86_pause()
74# define SPINLOCK_INLINE
75#else /* !_HARDKERNEL */
76# define SPINLOCK_BODY
77# define SPINLOCK_INLINE static __inline __unused
78#endif /* _HARDKERNEL */
79
80SPINLOCK_INLINE void __cpu_simple_lock_init(__cpu_simple_lock_t *);
81SPINLOCK_INLINE void __cpu_simple_lock(__cpu_simple_lock_t *);
82SPINLOCK_INLINE int __cpu_simple_lock_try(__cpu_simple_lock_t *);
83SPINLOCK_INLINE void __cpu_simple_unlock(__cpu_simple_lock_t *);
84
85#ifdef SPINLOCK_BODY
86SPINLOCK_INLINE void
87__cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
88{
89
90 *lockp = __SIMPLELOCK_UNLOCKED;
91 __insn_barrier();
92}
93
94SPINLOCK_INLINE int
95__cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
96{
97 uint8_t val;
98
99 val = __SIMPLELOCK_LOCKED;
100 __asm volatile ("xchgb %0,(%2)" :
101 "=qQ" (val)
102 :"0" (val), "r" (lockp));
103 __insn_barrier();
104 return val == __SIMPLELOCK_UNLOCKED;
105}
106
107SPINLOCK_INLINE void
108__cpu_simple_lock(__cpu_simple_lock_t *lockp)
109{
110
111 while (!__cpu_simple_lock_try(lockp))
112 /* nothing */;
113 __insn_barrier();
114}
115
116/*
117 * Note on x86 memory ordering
118 *
119 * When releasing a lock we must ensure that no stores or loads from within
120 * the critical section are re-ordered by the CPU to occur outside of it:
121 * they must have completed and be visible to other processors once the lock
122 * has been released.
123 *
124 * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
125 * back) memory region. In that case, memory ordering on x86 platforms
126 * looks like this:
127 *
128 * i386 All loads/stores occur in instruction sequence.
129 *
130 * i486 All loads/stores occur in instruction sequence. In
131 * Pentium exceptional circumstances, loads can be re-ordered around
132 * stores, but for the purposes of releasing a lock it does
133 * not matter. Stores may not be immediately visible to other
134 * processors as they can be buffered. However, since the
135 * stores are buffered in order the lock release will always be
136 * the last operation in the critical section that becomes
137 * visible to other CPUs.
138 *
139 * Pentium Pro The "Intel 64 and IA-32 Architectures Software Developer's
140 * onwards Manual" volume 3A (order number 248966) says that (1) "Reads
141 * can be carried out speculatively and in any order" and (2)
142 * "Reads can pass buffered stores, but the processor is
143 * self-consistent.". This would be a problem for the below,
144 * and would mandate a locked instruction cycle or load fence
145 * before releasing the simple lock.
146 *
147 * The "Intel Pentium 4 Processor Optimization" guide (order
148 * number 253668-022US) says: "Loads can be moved before stores
149 * that occurred earlier in the program if they are not
150 * predicted to load from the same linear address.". This is
151 * not a problem since the only loads that can be re-ordered
152 * take place once the lock has been released via a store.
153 *
154 * The above two documents seem to contradict each other,
155 * however with the exception of early steppings of the Pentium
156 * Pro, the second document is closer to the truth: a store
157 * will always act as a load fence for all loads that precede
158 * the store in instruction order.
159 *
160 * Again, note that stores can be buffered and will not always
161 * become immediately visible to other CPUs: they are however
162 * buffered in order.
163 *
164 * AMD64 Stores occur in order and are buffered. Loads can be
165 * reordered, however stores act as load fences, meaning that
166 * loads can not be reordered around stores.
167 */
168SPINLOCK_INLINE void
169__cpu_simple_unlock(__cpu_simple_lock_t *lockp)
170{
171
172 __insn_barrier();
173 *lockp = __SIMPLELOCK_UNLOCKED;
174}
175
176#endif /* SPINLOCK_BODY */
177
178#endif /* _X86_LOCK_H_ */
179