spirv_std/arch/
atomics.rs

1#[cfg(target_arch = "spirv")]
2use core::arch::asm;
3
4use crate::{Float, Integer, Number, SignedInteger, UnsignedInteger};
5
6/// Atomically load through `ptr` using the given `SEMANTICS`. All subparts of
7/// the value that is loaded are read atomically with respect to all other
8/// atomic accesses to it within `SCOPE`.
9#[spirv_std_macros::gpu_only]
10#[doc(alias = "OpAtomicLoad")]
11#[inline]
12pub unsafe fn atomic_load<N: Number, const SCOPE: u32, const SEMANTICS: u32>(ptr: &N) -> N {
13    unsafe {
14        let mut result = N::default();
15
16        asm! {
17            "%u32 = OpTypeInt 32 0",
18            "%scope = OpConstant %u32 {scope}",
19            "%semantics = OpConstant %u32 {semantics}",
20            "%result = OpAtomicLoad _ {ptr} %scope %semantics",
21            "OpStore {result} %result",
22            scope = const SCOPE,
23            semantics = const SEMANTICS,
24            ptr = in(reg) ptr,
25            result = in(reg) &mut result
26        }
27
28        result
29    }
30}
31
32/// Atomically store through `ptr` using the given `SEMANTICS`. All subparts of
33/// `value` are written atomically with respect to all other atomic accesses to
34/// it within `SCOPE`.
35#[spirv_std_macros::gpu_only]
36#[doc(alias = "OpAtomicStore")]
37#[inline]
38pub unsafe fn atomic_store<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
39    ptr: &mut N,
40    value: N,
41) {
42    unsafe {
43        asm! {
44            "%u32 = OpTypeInt 32 0",
45            "%scope = OpConstant %u32 {scope}",
46            "%semantics = OpConstant %u32 {semantics}",
47            "%value = OpLoad _ {value}",
48            "OpAtomicStore {ptr} %scope %semantics %value",
49            scope = const SCOPE,
50            semantics = const SEMANTICS,
51            ptr = in(reg) ptr,
52            value = in(reg) &value
53        }
54    }
55}
56
57/// Perform the following steps atomically with respect to any other atomic
58/// accesses within `SCOPE` to the same location:
59///
60/// 1. Load through `ptr` to get the original value,
61/// 2. Get a new value from copying `value`, and
62/// 3. Store the new value back through `ptr`.
63///
64/// The result is the original value.
65#[spirv_std_macros::gpu_only]
66#[doc(alias = "OpAtomicExchange")]
67#[inline]
68pub unsafe fn atomic_exchange<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
69    ptr: &mut N,
70    value: N,
71) -> N {
72    unsafe {
73        let mut old = N::default();
74
75        asm! {
76            "%u32 = OpTypeInt 32 0",
77            "%scope = OpConstant %u32 {scope}",
78            "%semantics = OpConstant %u32 {semantics}",
79            "%value = OpLoad _ {value}",
80            "%old = OpAtomicExchange _ {ptr} %scope %semantics %value",
81            "OpStore {old} %old",
82            scope = const SCOPE,
83            semantics = const SEMANTICS,
84            ptr = in(reg) ptr,
85            old = in(reg) &mut old,
86            value = in(reg) &value
87        }
88
89        old
90    }
91}
92
93/// Perform the following steps atomically with respect to any other atomic
94/// accesses within `SCOPE` to the same location:
95///
96/// 1. Load through `ptr` to get the original value
97/// 2. Get a new value from `value` only if the original value equals
98///    `comparator`, and
99/// 3. Store the new value back through `ptr`, only if the original value
100///    equaled `comparator`.
101///
102/// The result is the original value.
103#[spirv_std_macros::gpu_only]
104#[doc(alias = "OpAtomicCompareExchange")]
105#[inline]
106pub unsafe fn atomic_compare_exchange<
107    I: Integer,
108    const SCOPE: u32,
109    const EQUAL: u32,
110    const UNEQUAL: u32,
111>(
112    ptr: &mut I,
113    value: I,
114    comparator: I,
115) -> I {
116    unsafe {
117        let mut old = I::default();
118
119        asm! {
120            "%u32 = OpTypeInt 32 0",
121            "%scope = OpConstant %u32 {scope}",
122            "%equal = OpConstant %u32 {equal}",
123            "%unequal = OpConstant %u32 {unequal}",
124            "%value = OpLoad _ {value}",
125            "%comparator = OpLoad _ {comparator}",
126            "%old = OpAtomicCompareExchange _ {ptr} %scope %equal %unequal %value %comparator",
127            "OpStore {old} %old",
128            scope = const SCOPE,
129            equal = const EQUAL,
130            unequal = const UNEQUAL,
131            ptr = in(reg) ptr,
132            value = in(reg) &value,
133            comparator = in(reg) &comparator,
134            old = in(reg) &mut old,
135        }
136
137        old
138    }
139}
140
141/// Perform the following steps atomically with respect to any other atomic
142/// accesses within `SCOPE` to the same location:
143///
144/// 1. Load through `ptr` to get an original value,
145/// 2. Get a new value through integer addition of 1 to original value, and
146/// 3. Store the new value back through `ptr`.
147///
148/// The result is the original value.
149#[spirv_std_macros::gpu_only]
150#[doc(alias = "OpAtomicIIncrement")]
151#[inline]
152pub unsafe fn atomic_i_increment<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
153    ptr: &mut I,
154) -> I {
155    unsafe {
156        let mut old = I::default();
157
158        asm! {
159            "%u32 = OpTypeInt 32 0",
160            "%scope = OpConstant %u32 {scope}",
161            "%semantics = OpConstant %u32 {semantics}",
162            "%old = OpAtomicIIncrement _ {ptr} %scope %semantics",
163            "OpStore {old} %old",
164            scope = const SCOPE,
165            semantics = const SEMANTICS,
166            ptr = in(reg) ptr,
167            old = in(reg) &mut old
168        }
169
170        old
171    }
172}
173
174/// Perform the following steps atomically with respect to any other atomic
175/// accesses within `SCOPE` to the same location:
176///
177/// 1) load through `ptr` to get an original value,
178/// 2) get a new value through integer subtraction of 1 from original value, and
179/// 3) store the new value back through `ptr`.
180///
181/// The result is the original value.
182#[spirv_std_macros::gpu_only]
183#[doc(alias = "OpAtomicIDecrement")]
184#[inline]
185pub unsafe fn atomic_i_decrement<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
186    ptr: &mut I,
187) -> I {
188    unsafe {
189        let mut old = I::default();
190
191        asm! {
192            "%u32 = OpTypeInt 32 0",
193            "%scope = OpConstant %u32 {scope}",
194            "%semantics = OpConstant %u32 {semantics}",
195            "%old = OpAtomicIDecrement _ {ptr} %scope %semantics",
196            "OpStore {old} %old",
197            scope = const SCOPE,
198            semantics = const SEMANTICS,
199            ptr = in(reg) ptr,
200            old = in(reg) &mut old
201        }
202
203        old
204    }
205}
206
207/// Perform the following steps atomically with respect to any other atomic
208/// accesses within `SCOPE` to the same location:
209///
210/// 1) load through `ptr` to get an original value,
211/// 2) get a new value by integer addition of original value and `value`, and
212/// 3) store the new value back through `ptr`.
213///
214/// The result is the Original Value.
215#[spirv_std_macros::gpu_only]
216#[doc(alias = "OpAtomicIAdd")]
217#[inline]
218pub unsafe fn atomic_i_add<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
219    ptr: &mut I,
220    value: I,
221) -> I {
222    unsafe {
223        let mut old = I::default();
224
225        asm! {
226            "%u32 = OpTypeInt 32 0",
227            "%scope = OpConstant %u32 {scope}",
228            "%semantics = OpConstant %u32 {semantics}",
229            "%value = OpLoad _ {value}",
230            "%old = OpAtomicIAdd _ {ptr} %scope %semantics %value",
231            "OpStore {old} %old",
232            scope = const SCOPE,
233            semantics = const SEMANTICS,
234            ptr = in(reg) ptr,
235            old = in(reg) &mut old,
236            value = in(reg) &value
237        }
238
239        old
240    }
241}
242
243/// Perform the following steps atomically with respect to any other atomic
244/// accesses within `SCOPE` to the same location:
245///
246/// 1) load through `ptr` to get an original value,
247/// 2) get a new value by integer subtraction of original value and `value`, and
248/// 3) store the new value back through `ptr`.
249///
250/// The result is the Original Value.
251#[spirv_std_macros::gpu_only]
252#[doc(alias = "OpAtomicISub")]
253#[inline]
254pub unsafe fn atomic_i_sub<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
255    ptr: &mut I,
256    value: I,
257) -> I {
258    unsafe {
259        let mut old = I::default();
260
261        asm! {
262            "%u32 = OpTypeInt 32 0",
263            "%scope = OpConstant %u32 {scope}",
264            "%semantics = OpConstant %u32 {semantics}",
265            "%value = OpLoad _ {value}",
266            "%old = OpAtomicISub _ {ptr} %scope %semantics %value",
267            "OpStore {old} %old",
268            scope = const SCOPE,
269            semantics = const SEMANTICS,
270            ptr = in(reg) ptr,
271            old = in(reg) &mut old,
272            value = in(reg) &value
273        }
274
275        old
276    }
277}
278
279/// Perform the following steps atomically with respect to any other atomic
280/// accesses within Scope to the same location:
281///
282/// 1. Load through `ptr` to get an original value,
283/// 2. Get a new value by finding the smallest signed integer of original value
284///    and `value`, and
285/// 3. Store the new value back through `ptr`.
286///
287/// The result is the original value.
288#[spirv_std_macros::gpu_only]
289#[doc(alias = "OpAtomicSMin")]
290#[inline]
291pub unsafe fn atomic_s_min<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
292    ptr: &mut S,
293    value: S,
294) -> S {
295    unsafe {
296        let mut old = S::default();
297
298        asm! {
299            "%u32 = OpTypeInt 32 0",
300            "%scope = OpConstant %u32 {scope}",
301            "%semantics = OpConstant %u32 {semantics}",
302            "%value = OpLoad _ {value}",
303            "%old = OpAtomicSMin _ {ptr} %scope %semantics %value",
304            "OpStore {old} %old",
305            scope = const SCOPE,
306            semantics = const SEMANTICS,
307            ptr = in(reg) ptr,
308            old = in(reg) &mut old,
309            value = in(reg) &value
310        }
311
312        old
313    }
314}
315
316/// Perform the following steps atomically with respect to any other atomic
317/// accesses within Scope to the same location:
318///
319/// 1. Load through `ptr` to get an original value,
320/// 2. Get a new value by finding the smallest unsigned integer of original
321///    value and `value`, and
322/// 3. Store the new value back through `ptr`.
323///
324/// The result is the original value.
325#[spirv_std_macros::gpu_only]
326#[doc(alias = "OpAtomicUMin")]
327#[inline]
328pub unsafe fn atomic_u_min<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
329    ptr: &mut U,
330    value: U,
331) -> U {
332    unsafe {
333        let mut old = U::default();
334
335        asm! {
336            "%u32 = OpTypeInt 32 0",
337            "%scope = OpConstant %u32 {scope}",
338            "%semantics = OpConstant %u32 {semantics}",
339            "%value = OpLoad _ {value}",
340            "%old = OpAtomicUMin _ {ptr} %scope %semantics %value",
341            "OpStore {old} %old",
342            scope = const SCOPE,
343            semantics = const SEMANTICS,
344            ptr = in(reg) ptr,
345            old = in(reg) &mut old,
346            value = in(reg) &value
347        }
348
349        old
350    }
351}
352
353/// Perform the following steps atomically with respect to any other atomic
354/// accesses within Scope to the same location:
355///
356/// 1. Load through `ptr` to get an original value,
357/// 2. Get a new value by finding the largest signed integer of original value
358///    and `value`, and
359/// 3. Store the new value back through `ptr`.
360///
361/// The result is the original value.
362#[spirv_std_macros::gpu_only]
363#[doc(alias = "OpAtomicSMax")]
364#[inline]
365pub unsafe fn atomic_s_max<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
366    ptr: &mut S,
367    value: S,
368) -> S {
369    unsafe {
370        let mut old = S::default();
371
372        asm! {
373            "%u32 = OpTypeInt 32 0",
374            "%scope = OpConstant %u32 {scope}",
375            "%semantics = OpConstant %u32 {semantics}",
376            "%value = OpLoad _ {value}",
377            "%old = OpAtomicSMax _ {ptr} %scope %semantics %value",
378            "OpStore {old} %old",
379            scope = const SCOPE,
380            semantics = const SEMANTICS,
381            ptr = in(reg) ptr,
382            old = in(reg) &mut old,
383            value = in(reg) &value
384        }
385
386        old
387    }
388}
389
390/// Perform the following steps atomically with respect to any other atomic
391/// accesses within Scope to the same location:
392///
393/// 1. Load through `ptr` to get an original value,
394/// 2. Get a new value by finding the largest unsigned integer of original
395///    value and `value`, and
396/// 3. Store the new value back through `ptr`.
397///
398/// The result is the original value.
399#[spirv_std_macros::gpu_only]
400#[doc(alias = "OpAtomicUMax")]
401#[inline]
402pub unsafe fn atomic_u_max<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
403    ptr: &mut U,
404    value: U,
405) -> U {
406    unsafe {
407        let mut old = U::default();
408
409        asm! {
410            "%u32 = OpTypeInt 32 0",
411            "%scope = OpConstant %u32 {scope}",
412            "%semantics = OpConstant %u32 {semantics}",
413            "%value = OpLoad _ {value}",
414            "%old = OpAtomicUMax _ {ptr} %scope %semantics %value",
415            "OpStore {old} %old",
416            scope = const SCOPE,
417            semantics = const SEMANTICS,
418            ptr = in(reg) ptr,
419            old = in(reg) &mut old,
420            value = in(reg) &value
421        }
422
423        old
424    }
425}
426
427/// Perform the following steps atomically with respect to any other atomic
428/// accesses within Scope to the same location:
429///
430/// 1. Load through `ptr` to get an original value,
431/// 2. Get a new value by the bitwise AND of the original value and `value`, and
432/// 3. Store the new value back through `ptr`.
433///
434/// The result is the original value.
435#[spirv_std_macros::gpu_only]
436#[doc(alias = "OpAtomicAnd")]
437#[inline]
438pub unsafe fn atomic_and<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
439    ptr: &mut I,
440    value: I,
441) -> I {
442    unsafe {
443        let mut old = I::default();
444
445        asm! {
446            "%u32 = OpTypeInt 32 0",
447            "%scope = OpConstant %u32 {scope}",
448            "%semantics = OpConstant %u32 {semantics}",
449            "%value = OpLoad _ {value}",
450            "%old = OpAtomicAnd _ {ptr} %scope %semantics %value",
451            "OpStore {old} %old",
452            scope = const SCOPE,
453            semantics = const SEMANTICS,
454            ptr = in(reg) ptr,
455            old = in(reg) &mut old,
456            value = in(reg) &value
457        }
458
459        old
460    }
461}
462
463/// Perform the following steps atomically with respect to any other atomic
464/// accesses within Scope to the same location:
465///
466/// 1. Load through `ptr` to get an original value,
467/// 2. Get a new value by the bitwise OR of the original value and `value`, and
468/// 3. Store the new value back through `ptr`.
469///
470/// The result is the original value.
471#[spirv_std_macros::gpu_only]
472#[doc(alias = "OpAtomicOr")]
473#[inline]
474pub unsafe fn atomic_or<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
475    ptr: &mut I,
476    value: I,
477) -> I {
478    unsafe {
479        let mut old = I::default();
480
481        asm! {
482            "%u32 = OpTypeInt 32 0",
483            "%scope = OpConstant %u32 {scope}",
484            "%semantics = OpConstant %u32 {semantics}",
485            "%value = OpLoad _ {value}",
486            "%old = OpAtomicOr _ {ptr} %scope %semantics %value",
487            "OpStore {old} %old",
488            scope = const SCOPE,
489            semantics = const SEMANTICS,
490            ptr = in(reg) ptr,
491            old = in(reg) &mut old,
492            value = in(reg) &value
493        }
494
495        old
496    }
497}
498
499/// Perform the following steps atomically with respect to any other atomic
500/// accesses within Scope to the same location:
501///
502/// 1. Load through `ptr` to get an original value,
503/// 2. Get a new value by the bitwise XOR of the original value and `value`, and
504/// 3. Store the new value back through `ptr`.
505///
506/// The result is the original value.
507#[spirv_std_macros::gpu_only]
508#[doc(alias = "OpAtomicXor")]
509#[inline]
510pub unsafe fn atomic_xor<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
511    ptr: &mut I,
512    value: I,
513) -> I {
514    unsafe {
515        let mut old = I::default();
516
517        asm! {
518            "%u32 = OpTypeInt 32 0",
519            "%scope = OpConstant %u32 {scope}",
520            "%semantics = OpConstant %u32 {semantics}",
521            "%value = OpLoad _ {value}",
522            "%old = OpAtomicXor _ {ptr} %scope %semantics %value",
523            "OpStore {old} %old",
524            scope = const SCOPE,
525            semantics = const SEMANTICS,
526            ptr = in(reg) ptr,
527            old = in(reg) &mut old,
528            value = in(reg) &value
529        }
530
531        old
532    }
533}
534
535/// Perform the following steps atomically with respect to any other atomic
536/// accesses within Scope to the same location:
537///
538/// 1. Load through `ptr` to get an original value,
539/// 2. Get a new value by finding the smallest signed integer of original value
540///    and `value`, and
541/// 3. Store the new value back through `ptr`.
542///
543/// The result is the original value.
544#[spirv_std_macros::gpu_only]
545#[doc(alias = "OpAtomicFMinEXT")]
546#[inline]
547pub unsafe fn atomic_f_min<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
548    ptr: &mut F,
549    value: F,
550) -> F {
551    unsafe {
552        let mut old = F::default();
553
554        asm! {
555            "%u32 = OpTypeInt 32 0",
556            "%scope = OpConstant %u32 {scope}",
557            "%semantics = OpConstant %u32 {semantics}",
558            "%value = OpLoad _ {value}",
559            "%old = OpAtomicFMinEXT _ {ptr} %scope %semantics %value",
560            "OpStore {old} %old",
561            scope = const SCOPE,
562            semantics = const SEMANTICS,
563            ptr = in(reg) ptr,
564            old = in(reg) &mut old,
565            value = in(reg) &value
566        }
567
568        old
569    }
570}
571
572/// Perform the following steps atomically with respect to any other atomic
573/// accesses within Scope to the same location:
574///
575/// 1. Load through `ptr` to get an original value,
576/// 2. Get a new value by finding the largest signed integer of original value
577///    and `value`, and
578/// 3. Store the new value back through `ptr`.
579///
580/// The result is the original value.
581#[spirv_std_macros::gpu_only]
582#[doc(alias = "OpAtomicFMaxEXT")]
583#[inline]
584pub unsafe fn atomic_f_max<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
585    ptr: &mut F,
586    value: F,
587) -> F {
588    unsafe {
589        let mut old = F::default();
590
591        asm! {
592            "%u32 = OpTypeInt 32 0",
593            "%scope = OpConstant %u32 {scope}",
594            "%semantics = OpConstant %u32 {semantics}",
595            "%value = OpLoad _ {value}",
596            "%old = OpAtomicFMaxEXT _ {ptr} %scope %semantics %value",
597            "OpStore {old} %old",
598            scope = const SCOPE,
599            semantics = const SEMANTICS,
600            ptr = in(reg) ptr,
601            old = in(reg) &mut old,
602            value = in(reg) &value
603        }
604
605        old
606    }
607}
608
609/// Perform the following steps atomically with respect to any other atomic
610/// accesses within `SCOPE` to the same location:
611///
612/// 1) load through `ptr` to get an original value,
613/// 2) get a new value by integer addition of original value and `value`, and
614/// 3) store the new value back through `ptr`.
615///
616/// The result is the Original Value.
617#[spirv_std_macros::gpu_only]
618#[doc(alias = "OpAtomicFAddEXT")]
619#[inline]
620pub unsafe fn atomic_f_add<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
621    ptr: &mut F,
622    value: F,
623) -> F {
624    unsafe {
625        let mut old = F::default();
626
627        asm! {
628            "%u32 = OpTypeInt 32 0",
629            "%scope = OpConstant %u32 {scope}",
630            "%semantics = OpConstant %u32 {semantics}",
631            "%value = OpLoad _ {value}",
632            "%old = OpAtomicFAddEXT _ {ptr} %scope %semantics %value",
633            "OpStore {old} %old",
634            scope = const SCOPE,
635            semantics = const SEMANTICS,
636            ptr = in(reg) ptr,
637            old = in(reg) &mut old,
638            value = in(reg) &value
639        }
640
641        old
642    }
643}
spirv_std/arch/atomics.rs

spirv_std/arch/
atomics.rs
