1/* x86-64 out-of-line atomic helpers, called by cc1-emitted code for
  2 * __atomic_* builtins (see cmd/dev/cc/qbe.c's mkatomicfn). every
  3 * memory order argument is treated as seq_cst: a conservative choice
  4 * that is always standards-conformant, only ever slower than what a
  5 * weaker order would allow. matches the real gcc/clang libatomic
  6 * fallback abi (out-of-line calls for sizes the compiler does not
  7 * inline directly), size-suffixed per operation, so any future
  8 * inlining work can drop straight in without changing callers
  9 */
 10
 11.text
 12
 13.globl __atomic_load_4
 14__atomic_load_4:
 15	movl (%rdi), %eax
 16	ret
 17
 18.globl __atomic_load_8
 19__atomic_load_8:
 20	movq (%rdi), %rax
 21	ret
 22
 23/* plain store already has release ordering on x86-64; mfence upgrades
 24 * it to full seq_cst, ordering it against a later seq_cst load
 25 */
 26.globl __atomic_store_4
 27__atomic_store_4:
 28	movl %esi, (%rdi)
 29	mfence
 30	ret
 31
 32.globl __atomic_store_8
 33__atomic_store_8:
 34	movq %rsi, (%rdi)
 35	mfence
 36	ret
 37
 38/* xchg with a memory operand is implicitly locked, no explicit lock
 39 * prefix needed
 40 */
 41.globl __atomic_exchange_4
 42__atomic_exchange_4:
 43	movl %esi, %eax
 44	xchg %eax, (%rdi)
 45	ret
 46
 47.globl __atomic_exchange_8
 48__atomic_exchange_8:
 49	movq %rsi, %rax
 50	xchg %rax, (%rdi)
 51	ret
 52
 53.globl __atomic_fetch_add_4
 54__atomic_fetch_add_4:
 55	movl %esi, %eax
 56	lock xadd %eax, (%rdi)
 57	ret
 58
 59.globl __atomic_fetch_add_8
 60__atomic_fetch_add_8:
 61	movq %rsi, %rax
 62	lock xadd %rax, (%rdi)
 63	ret
 64
 65.globl __atomic_fetch_sub_4
 66__atomic_fetch_sub_4:
 67	neg %esi
 68	movl %esi, %eax
 69	lock xadd %eax, (%rdi)
 70	ret
 71
 72.globl __atomic_fetch_sub_8
 73__atomic_fetch_sub_8:
 74	neg %rsi
 75	movq %rsi, %rax
 76	lock xadd %rax, (%rdi)
 77	ret
 78
 79/* no single locked fetch-and-op instruction exists for and/or/xor:
 80 * load, compute the new value, lock cmpxchg it in, retry on failure
 81 * (another cpu changed *ptr in between). on a successful cmpxchg eax
 82 * is left unmodified, which is exactly the old value fetch-and-op
 83 * must return; on failure eax is reloaded with the real current
 84 * value by the instruction itself, which is exactly what the retry
 85 * needs
 86 */
 87.globl __atomic_fetch_and_4
 88__atomic_fetch_and_4:
 89	movl (%rdi), %eax
 90.Lretry_and_4:
 91	movl %eax, %ecx
 92	andl %esi, %ecx
 93	lock cmpxchg %ecx, (%rdi)
 94	jnz .Lretry_and_4
 95	ret
 96
 97.globl __atomic_fetch_and_8
 98__atomic_fetch_and_8:
 99	movq (%rdi), %rax
100.Lretry_and_8:
101	movq %rax, %rcx
102	andq %rsi, %rcx
103	lock cmpxchg %rcx, (%rdi)
104	jnz .Lretry_and_8
105	ret
106
107.globl __atomic_fetch_or_4
108__atomic_fetch_or_4:
109	movl (%rdi), %eax
110.Lretry_or_4:
111	movl %eax, %ecx
112	orl %esi, %ecx
113	lock cmpxchg %ecx, (%rdi)
114	jnz .Lretry_or_4
115	ret
116
117.globl __atomic_fetch_or_8
118__atomic_fetch_or_8:
119	movq (%rdi), %rax
120.Lretry_or_8:
121	movq %rax, %rcx
122	orq %rsi, %rcx
123	lock cmpxchg %rcx, (%rdi)
124	jnz .Lretry_or_8
125	ret
126
127.globl __atomic_fetch_xor_4
128__atomic_fetch_xor_4:
129	movl (%rdi), %eax
130.Lretry_xor_4:
131	movl %eax, %ecx
132	xor %esi, %ecx
133	lock cmpxchg %ecx, (%rdi)
134	jnz .Lretry_xor_4
135	ret
136
137.globl __atomic_fetch_xor_8
138__atomic_fetch_xor_8:
139	movq (%rdi), %rax
140.Lretry_xor_8:
141	movq %rax, %rcx
142	xor %rsi, %rcx
143	lock cmpxchg %rcx, (%rdi)
144	jnz .Lretry_xor_8
145	ret
146
147/* __atomic_compare_exchange_N(ptr, expected, desired): the accumulator
148 * is loaded from *expected before cmpxchg, exactly matching the
149 * instructions own semantics (compare accumulator against *ptr; on
150 * match store desired and set zf; on mismatch load the accumulators
151 * own register with *ptrs real value and clear zf), so success needs
152 * no extra work at all and failure only needs writing the reloaded
153 * accumulator back out to *expected
154 */
155.globl __atomic_compare_exchange_4
156__atomic_compare_exchange_4:
157	movl (%rsi), %eax
158	lock cmpxchg %edx, (%rdi)
159	jz .Lcas4_ok
160	movl %eax, (%rsi)
161	movl $0, %eax
162	ret
163.Lcas4_ok:
164	movl $1, %eax
165	ret
166
167.globl __atomic_compare_exchange_8
168__atomic_compare_exchange_8:
169	movq (%rsi), %rax
170	lock cmpxchg %rdx, (%rdi)
171	jz .Lcas8_ok
172	movq %rax, (%rsi)
173	movl $0, %eax
174	ret
175.Lcas8_ok:
176	movl $1, %eax
177	ret