1/* $NetBSD: atomic.S,v 1.23 2018/07/18 13:39:36 bouyer Exp $ */
2
3/*-
4 * Copyright (c) 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe, and by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include <sys/param.h>
33#include <machine/asm.h>
34/*
35 * __HAVE_ constants should not be in <machine/types.h>
36 * because we can't use them from assembly. OTOH we
37 * only need __HAVE_ATOMIC64_OPS here, and we don't.
38 */
39#ifdef _KERNEL
40#define ALIAS(f, t) STRONG_ALIAS(f,t)
41#else
42#define ALIAS(f, t) WEAK_ALIAS(f,t)
43#endif
44
45#ifdef _HARDKERNEL
46#include "opt_xen.h"
47#define LOCK(n) .Lpatch ## n: lock
48#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
49#else
50#define LOCK(n) lock
51#define ENDLABEL(a) /* nothing */
52#endif
53
54 .text
55
56ENTRY(_atomic_add_32)
57 movl 4(%esp), %edx
58 movl 8(%esp), %eax
59 LOCK(1)
60 addl %eax, (%edx)
61 ret
62END(_atomic_add_32)
63
64ENTRY(_atomic_add_32_nv)
65 movl 4(%esp), %edx
66 movl 8(%esp), %eax
67 movl %eax, %ecx
68 LOCK(2)
69 xaddl %eax, (%edx)
70 addl %ecx, %eax
71 ret
72END(_atomic_add_32_nv)
73
74ENTRY(_atomic_and_32)
75 movl 4(%esp), %edx
76 movl 8(%esp), %eax
77 LOCK(3)
78 andl %eax, (%edx)
79 ret
80END(_atomic_and_32)
81
82ENTRY(_atomic_and_32_nv)
83 movl 4(%esp), %edx
84 movl (%edx), %eax
850:
86 movl %eax, %ecx
87 andl 8(%esp), %ecx
88 LOCK(4)
89 cmpxchgl %ecx, (%edx)
90 jnz 1f
91 movl %ecx, %eax
92 ret
931:
94 jmp 0b
95END(_atomic_and_32_nv)
96
97ENTRY(_atomic_dec_32)
98 movl 4(%esp), %edx
99 LOCK(5)
100 decl (%edx)
101 ret
102END(_atomic_dec_32)
103
104ENTRY(_atomic_dec_32_nv)
105 movl 4(%esp), %edx
106 movl $-1, %eax
107 LOCK(6)
108 xaddl %eax, (%edx)
109 decl %eax
110 ret
111END(_atomic_dec_32_nv)
112
113ENTRY(_atomic_inc_32)
114 movl 4(%esp), %edx
115 LOCK(7)
116 incl (%edx)
117 ret
118END(_atomic_inc_32)
119
120ENTRY(_atomic_inc_32_nv)
121 movl 4(%esp), %edx
122 movl $1, %eax
123 LOCK(8)
124 xaddl %eax, (%edx)
125 incl %eax
126 ret
127END(_atomic_inc_32_nv)
128
129ENTRY(_atomic_or_32)
130 movl 4(%esp), %edx
131 movl 8(%esp), %eax
132 LOCK(9)
133 orl %eax, (%edx)
134 ret
135END(_atomic_or_32)
136
137ENTRY(_atomic_or_32_nv)
138 movl 4(%esp), %edx
139 movl (%edx), %eax
1400:
141 movl %eax, %ecx
142 orl 8(%esp), %ecx
143 LOCK(10)
144 cmpxchgl %ecx, (%edx)
145 jnz 1f
146 movl %ecx, %eax
147 ret
1481:
149 jmp 0b
150END(_atomic_or_32_nv)
151
152ENTRY(_atomic_swap_32)
153 movl 4(%esp), %edx
154 movl 8(%esp), %eax
155 xchgl %eax, (%edx)
156 ret
157END(_atomic_swap_32)
158
159ENTRY(_atomic_cas_32)
160 movl 4(%esp), %edx
161 movl 8(%esp), %eax
162 movl 12(%esp), %ecx
163 LOCK(12)
164 cmpxchgl %ecx, (%edx)
165 /* %eax now contains the old value */
166 ret
167END(_atomic_cas_32)
168
169ENTRY(_atomic_cas_32_ni)
170 movl 4(%esp), %edx
171 movl 8(%esp), %eax
172 movl 12(%esp), %ecx
173 cmpxchgl %ecx, (%edx)
174 /* %eax now contains the old value */
175 ret
176END(_atomic_cas_32_ni)
177
178ENTRY(_membar_consumer)
179 LOCK(13)
180 addl $0, -4(%esp)
181 ret
182END(_membar_consumer)
183ENDLABEL(membar_consumer_end)
184
185ENTRY(_membar_producer)
186 /* A store is enough */
187 movl $0, -4(%esp)
188 ret
189END(_membar_producer)
190ENDLABEL(membar_producer_end)
191
192ENTRY(_membar_sync)
193 LOCK(14)
194 addl $0, -4(%esp)
195 ret
196END(_membar_sync)
197ENDLABEL(membar_sync_end)
198
199#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL)
200#ifdef XEN
201STRONG_ALIAS(_atomic_cas_64,_atomic_cas_cx8)
202#else
203ENTRY(_atomic_cas_64)
204#ifdef _HARDKERNEL
205 pushf
206 cli
207#endif /* _HARDKERNEL */
208 pushl %edi
209 pushl %ebx
210 movl 12(%esp), %edi
211 movl 16(%esp), %eax
212 movl 20(%esp), %edx
213 movl 24(%esp), %ebx
214 movl 28(%esp), %ecx
215 cmpl 0(%edi), %eax
216 jne 2f
217 cmpl 4(%edi), %edx
218 jne 2f
219 movl %ebx, 0(%edi)
220 movl %ecx, 4(%edi)
2211:
222 popl %ebx
223 popl %edi
224#ifdef _HARDKERNEL
225 popf
226#endif /* _HARDKERNEL */
227 ret
2282:
229 movl 0(%edi), %eax
230 movl 4(%edi), %edx
231 jmp 1b
232END(_atomic_cas_64)
233ENDLABEL(_atomic_cas_64_end)
234#endif /* !XEN */
235
236ENTRY(_atomic_cas_cx8)
237 pushl %edi
238 pushl %ebx
239 movl 12(%esp), %edi
240 movl 16(%esp), %eax
241 movl 20(%esp), %edx
242 movl 24(%esp), %ebx
243 movl 28(%esp), %ecx
244 LOCK(15)
245 cmpxchg8b (%edi)
246 popl %ebx
247 popl %edi
248 ret
249#ifdef _HARDKERNEL
250#ifdef GPROF
251 .space 16, 0x90
252#else
253 .space 32, 0x90
254#endif
255#endif /* _HARDKERNEL */
256END(_atomic_cas_cx8)
257ENDLABEL(_atomic_cas_cx8_end)
258#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */
259
260#ifdef _HARDKERNEL
261ENTRY(sse2_lfence)
262 lfence
263 ret
264END(sse2_lfence)
265ENDLABEL(sse2_lfence_end)
266
267ENTRY(sse2_mfence)
268 mfence
269 ret
270END(sse2_mfence)
271ENDLABEL(sse2_mfence_end)
272
273atomic_lockpatch:
274 .globl atomic_lockpatch
275 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5
276 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10
277 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0
278#endif /* _HARDKERNEL */
279
280ALIAS(atomic_add_32,_atomic_add_32)
281ALIAS(atomic_add_int,_atomic_add_32)
282ALIAS(atomic_add_long,_atomic_add_32)
283ALIAS(atomic_add_ptr,_atomic_add_32)
284
285ALIAS(atomic_add_32_nv,_atomic_add_32_nv)
286ALIAS(atomic_add_int_nv,_atomic_add_32_nv)
287ALIAS(atomic_add_long_nv,_atomic_add_32_nv)
288ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv)
289
290ALIAS(atomic_and_32,_atomic_and_32)
291ALIAS(atomic_and_uint,_atomic_and_32)
292ALIAS(atomic_and_ulong,_atomic_and_32)
293ALIAS(atomic_and_ptr,_atomic_and_32)
294
295ALIAS(atomic_and_32_nv,_atomic_and_32_nv)
296ALIAS(atomic_and_uint_nv,_atomic_and_32_nv)
297ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv)
298ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv)
299
300ALIAS(atomic_dec_32,_atomic_dec_32)
301ALIAS(atomic_dec_uint,_atomic_dec_32)
302ALIAS(atomic_dec_ulong,_atomic_dec_32)
303ALIAS(atomic_dec_ptr,_atomic_dec_32)
304
305ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv)
306ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv)
307ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv)
308ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv)
309
310ALIAS(atomic_inc_32,_atomic_inc_32)
311ALIAS(atomic_inc_uint,_atomic_inc_32)
312ALIAS(atomic_inc_ulong,_atomic_inc_32)
313ALIAS(atomic_inc_ptr,_atomic_inc_32)
314
315ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv)
316ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv)
317ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv)
318ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv)
319
320ALIAS(atomic_or_32,_atomic_or_32)
321ALIAS(atomic_or_uint,_atomic_or_32)
322ALIAS(atomic_or_ulong,_atomic_or_32)
323ALIAS(atomic_or_ptr,_atomic_or_32)
324
325ALIAS(atomic_or_32_nv,_atomic_or_32_nv)
326ALIAS(atomic_or_uint_nv,_atomic_or_32_nv)
327ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv)
328ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv)
329
330ALIAS(atomic_swap_32,_atomic_swap_32)
331ALIAS(atomic_swap_uint,_atomic_swap_32)
332ALIAS(atomic_swap_ulong,_atomic_swap_32)
333ALIAS(atomic_swap_ptr,_atomic_swap_32)
334
335ALIAS(atomic_cas_32,_atomic_cas_32)
336ALIAS(atomic_cas_uint,_atomic_cas_32)
337ALIAS(atomic_cas_ulong,_atomic_cas_32)
338ALIAS(atomic_cas_ptr,_atomic_cas_32)
339
340ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni)
341ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni)
342ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni)
343ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni)
344
345#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL)
346ALIAS(atomic_cas_64,_atomic_cas_64)
347ALIAS(atomic_cas_64_ni,_atomic_cas_64)
348ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64)
349#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */
350
351ALIAS(membar_consumer,_membar_consumer)
352ALIAS(membar_producer,_membar_producer)
353ALIAS(membar_enter,_membar_consumer)
354ALIAS(membar_exit,_membar_producer)
355ALIAS(membar_sync,_membar_sync)
356
357STRONG_ALIAS(_atomic_add_int,_atomic_add_32)
358STRONG_ALIAS(_atomic_add_long,_atomic_add_32)
359STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32)
360
361STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv)
362STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv)
363STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv)
364
365STRONG_ALIAS(_atomic_and_uint,_atomic_and_32)
366STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32)
367STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32)
368
369STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv)
370STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv)
371STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv)
372
373STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32)
374STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32)
375STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32)
376
377STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv)
378STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv)
379STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv)
380
381STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32)
382STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32)
383STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32)
384
385STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv)
386STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv)
387STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv)
388
389STRONG_ALIAS(_atomic_or_uint,_atomic_or_32)
390STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32)
391STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32)
392
393STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv)
394STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv)
395STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv)
396
397STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32)
398STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32)
399STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32)
400
401STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32)
402STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32)
403STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32)
404
405STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni)
406STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni)
407STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni)
408
409STRONG_ALIAS(_membar_enter,_membar_consumer)
410STRONG_ALIAS(_membar_exit,_membar_producer)
411