spirv_std/arch/
atomics.rs

1#[cfg(target_arch = "spirv")]
2use core::arch::asm;
3
4use crate::{
5    float::Float,
6    integer::{Integer, SignedInteger, UnsignedInteger},
7    number::Number,
8};
9
10/// Atomically load through `ptr` using the given `SEMANTICS`. All subparts of
11/// the value that is loaded are read atomically with respect to all other
12/// atomic accesses to it within `SCOPE`.
13#[spirv_std_macros::gpu_only]
14#[doc(alias = "OpAtomicLoad")]
15#[inline]
16pub unsafe fn atomic_load<N: Number, const SCOPE: u32, const SEMANTICS: u32>(ptr: &N) -> N {
17    unsafe {
18        let mut result = N::default();
19
20        asm! {
21            "%u32 = OpTypeInt 32 0",
22            "%scope = OpConstant %u32 {scope}",
23            "%semantics = OpConstant %u32 {semantics}",
24            "%result = OpAtomicLoad _ {ptr} %scope %semantics",
25            "OpStore {result} %result",
26            scope = const SCOPE,
27            semantics = const SEMANTICS,
28            ptr = in(reg) ptr,
29            result = in(reg) &mut result
30        }
31
32        result
33    }
34}
35
36/// Atomically store through `ptr` using the given `SEMANTICS`. All subparts of
37/// `value` are written atomically with respect to all other atomic accesses to
38/// it within `SCOPE`.
39#[spirv_std_macros::gpu_only]
40#[doc(alias = "OpAtomicStore")]
41#[inline]
42pub unsafe fn atomic_store<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
43    ptr: &mut N,
44    value: N,
45) {
46    unsafe {
47        asm! {
48            "%u32 = OpTypeInt 32 0",
49            "%scope = OpConstant %u32 {scope}",
50            "%semantics = OpConstant %u32 {semantics}",
51            "%value = OpLoad _ {value}",
52            "OpAtomicStore {ptr} %scope %semantics %value",
53            scope = const SCOPE,
54            semantics = const SEMANTICS,
55            ptr = in(reg) ptr,
56            value = in(reg) &value
57        }
58    }
59}
60
61/// Perform the following steps atomically with respect to any other atomic
62/// accesses within `SCOPE` to the same location:
63///
64/// 1. Load through `ptr` to get the original value,
65/// 2. Get a new value from copying `value`, and
66/// 3. Store the new value back through `ptr`.
67///
68/// The result is the original value.
69#[spirv_std_macros::gpu_only]
70#[doc(alias = "OpAtomicExchange")]
71#[inline]
72pub unsafe fn atomic_exchange<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
73    ptr: &mut N,
74    value: N,
75) -> N {
76    unsafe {
77        let mut old = N::default();
78
79        asm! {
80            "%u32 = OpTypeInt 32 0",
81            "%scope = OpConstant %u32 {scope}",
82            "%semantics = OpConstant %u32 {semantics}",
83            "%value = OpLoad _ {value}",
84            "%old = OpAtomicExchange _ {ptr} %scope %semantics %value",
85            "OpStore {old} %old",
86            scope = const SCOPE,
87            semantics = const SEMANTICS,
88            ptr = in(reg) ptr,
89            old = in(reg) &mut old,
90            value = in(reg) &value
91        }
92
93        old
94    }
95}
96
97/// Perform the following steps atomically with respect to any other atomic
98/// accesses within `SCOPE` to the same location:
99///
100/// 1. Load through `ptr` to get the original value
101/// 2. Get a new value from `value` only if the original value equals
102///    `comparator`, and
103/// 3. Store the new value back through `ptr`, only if the original value
104///    equaled `comparator`.
105///
106/// The result is the original value.
107#[spirv_std_macros::gpu_only]
108#[doc(alias = "OpAtomicCompareExchange")]
109#[inline]
110pub unsafe fn atomic_compare_exchange<
111    I: Integer,
112    const SCOPE: u32,
113    const EQUAL: u32,
114    const UNEQUAL: u32,
115>(
116    ptr: &mut I,
117    value: I,
118    comparator: I,
119) -> I {
120    unsafe {
121        let mut old = I::default();
122
123        asm! {
124            "%u32 = OpTypeInt 32 0",
125            "%scope = OpConstant %u32 {scope}",
126            "%equal = OpConstant %u32 {equal}",
127            "%unequal = OpConstant %u32 {unequal}",
128            "%value = OpLoad _ {value}",
129            "%comparator = OpLoad _ {comparator}",
130            "%old = OpAtomicCompareExchange _ {ptr} %scope %equal %unequal %value %comparator",
131            "OpStore {old} %old",
132            scope = const SCOPE,
133            equal = const EQUAL,
134            unequal = const UNEQUAL,
135            ptr = in(reg) ptr,
136            value = in(reg) &value,
137            comparator = in(reg) &comparator,
138            old = in(reg) &mut old,
139        }
140
141        old
142    }
143}
144
145/// Perform the following steps atomically with respect to any other atomic
146/// accesses within `SCOPE` to the same location:
147///
148/// 1. Load through `ptr` to get an original value,
149/// 2. Get a new value through integer addition of 1 to original value, and
150/// 3. Store the new value back through `ptr`.
151///
152/// The result is the original value.
153#[spirv_std_macros::gpu_only]
154#[doc(alias = "OpAtomicIIncrement")]
155#[inline]
156pub unsafe fn atomic_i_increment<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
157    ptr: &mut I,
158) -> I {
159    unsafe {
160        let mut old = I::default();
161
162        asm! {
163            "%u32 = OpTypeInt 32 0",
164            "%scope = OpConstant %u32 {scope}",
165            "%semantics = OpConstant %u32 {semantics}",
166            "%old = OpAtomicIIncrement _ {ptr} %scope %semantics",
167            "OpStore {old} %old",
168            scope = const SCOPE,
169            semantics = const SEMANTICS,
170            ptr = in(reg) ptr,
171            old = in(reg) &mut old
172        }
173
174        old
175    }
176}
177
178/// Perform the following steps atomically with respect to any other atomic
179/// accesses within `SCOPE` to the same location:
180///
181/// 1) load through `ptr` to get an original value,
182/// 2) get a new value through integer subtraction of 1 from original value, and
183/// 3) store the new value back through `ptr`.
184///
185/// The result is the original value.
186#[spirv_std_macros::gpu_only]
187#[doc(alias = "OpAtomicIDecrement")]
188#[inline]
189pub unsafe fn atomic_i_decrement<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
190    ptr: &mut I,
191) -> I {
192    unsafe {
193        let mut old = I::default();
194
195        asm! {
196            "%u32 = OpTypeInt 32 0",
197            "%scope = OpConstant %u32 {scope}",
198            "%semantics = OpConstant %u32 {semantics}",
199            "%old = OpAtomicIDecrement _ {ptr} %scope %semantics",
200            "OpStore {old} %old",
201            scope = const SCOPE,
202            semantics = const SEMANTICS,
203            ptr = in(reg) ptr,
204            old = in(reg) &mut old
205        }
206
207        old
208    }
209}
210
211/// Perform the following steps atomically with respect to any other atomic
212/// accesses within `SCOPE` to the same location:
213///
214/// 1) load through `ptr` to get an original value,
215/// 2) get a new value by integer addition of original value and `value`, and
216/// 3) store the new value back through `ptr`.
217///
218/// The result is the Original Value.
219#[spirv_std_macros::gpu_only]
220#[doc(alias = "OpAtomicIAdd")]
221#[inline]
222pub unsafe fn atomic_i_add<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
223    ptr: &mut I,
224    value: I,
225) -> I {
226    unsafe {
227        let mut old = I::default();
228
229        asm! {
230            "%u32 = OpTypeInt 32 0",
231            "%scope = OpConstant %u32 {scope}",
232            "%semantics = OpConstant %u32 {semantics}",
233            "%value = OpLoad _ {value}",
234            "%old = OpAtomicIAdd _ {ptr} %scope %semantics %value",
235            "OpStore {old} %old",
236            scope = const SCOPE,
237            semantics = const SEMANTICS,
238            ptr = in(reg) ptr,
239            old = in(reg) &mut old,
240            value = in(reg) &value
241        }
242
243        old
244    }
245}
246
247/// Perform the following steps atomically with respect to any other atomic
248/// accesses within `SCOPE` to the same location:
249///
250/// 1) load through `ptr` to get an original value,
251/// 2) get a new value by integer subtraction of original value and `value`, and
252/// 3) store the new value back through `ptr`.
253///
254/// The result is the Original Value.
255#[spirv_std_macros::gpu_only]
256#[doc(alias = "OpAtomicISub")]
257#[inline]
258pub unsafe fn atomic_i_sub<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
259    ptr: &mut I,
260    value: I,
261) -> I {
262    unsafe {
263        let mut old = I::default();
264
265        asm! {
266            "%u32 = OpTypeInt 32 0",
267            "%scope = OpConstant %u32 {scope}",
268            "%semantics = OpConstant %u32 {semantics}",
269            "%value = OpLoad _ {value}",
270            "%old = OpAtomicISub _ {ptr} %scope %semantics %value",
271            "OpStore {old} %old",
272            scope = const SCOPE,
273            semantics = const SEMANTICS,
274            ptr = in(reg) ptr,
275            old = in(reg) &mut old,
276            value = in(reg) &value
277        }
278
279        old
280    }
281}
282
283/// Perform the following steps atomically with respect to any other atomic
284/// accesses within Scope to the same location:
285///
286/// 1. Load through `ptr` to get an original value,
287/// 2. Get a new value by finding the smallest signed integer of original value
288///    and `value`, and
289/// 3. Store the new value back through `ptr`.
290///
291/// The result is the original value.
292#[spirv_std_macros::gpu_only]
293#[doc(alias = "OpAtomicSMin")]
294#[inline]
295pub unsafe fn atomic_s_min<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
296    ptr: &mut S,
297    value: S,
298) -> S {
299    unsafe {
300        let mut old = S::default();
301
302        asm! {
303            "%u32 = OpTypeInt 32 0",
304            "%scope = OpConstant %u32 {scope}",
305            "%semantics = OpConstant %u32 {semantics}",
306            "%value = OpLoad _ {value}",
307            "%old = OpAtomicSMin _ {ptr} %scope %semantics %value",
308            "OpStore {old} %old",
309            scope = const SCOPE,
310            semantics = const SEMANTICS,
311            ptr = in(reg) ptr,
312            old = in(reg) &mut old,
313            value = in(reg) &value
314        }
315
316        old
317    }
318}
319
320/// Perform the following steps atomically with respect to any other atomic
321/// accesses within Scope to the same location:
322///
323/// 1. Load through `ptr` to get an original value,
324/// 2. Get a new value by finding the smallest unsigned integer of original
325///    value and `value`, and
326/// 3. Store the new value back through `ptr`.
327///
328/// The result is the original value.
329#[spirv_std_macros::gpu_only]
330#[doc(alias = "OpAtomicUMin")]
331#[inline]
332pub unsafe fn atomic_u_min<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
333    ptr: &mut U,
334    value: U,
335) -> U {
336    unsafe {
337        let mut old = U::default();
338
339        asm! {
340            "%u32 = OpTypeInt 32 0",
341            "%scope = OpConstant %u32 {scope}",
342            "%semantics = OpConstant %u32 {semantics}",
343            "%value = OpLoad _ {value}",
344            "%old = OpAtomicUMin _ {ptr} %scope %semantics %value",
345            "OpStore {old} %old",
346            scope = const SCOPE,
347            semantics = const SEMANTICS,
348            ptr = in(reg) ptr,
349            old = in(reg) &mut old,
350            value = in(reg) &value
351        }
352
353        old
354    }
355}
356
357/// Perform the following steps atomically with respect to any other atomic
358/// accesses within Scope to the same location:
359///
360/// 1. Load through `ptr` to get an original value,
361/// 2. Get a new value by finding the largest signed integer of original value
362///    and `value`, and
363/// 3. Store the new value back through `ptr`.
364///
365/// The result is the original value.
366#[spirv_std_macros::gpu_only]
367#[doc(alias = "OpAtomicSMax")]
368#[inline]
369pub unsafe fn atomic_s_max<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
370    ptr: &mut S,
371    value: S,
372) -> S {
373    unsafe {
374        let mut old = S::default();
375
376        asm! {
377            "%u32 = OpTypeInt 32 0",
378            "%scope = OpConstant %u32 {scope}",
379            "%semantics = OpConstant %u32 {semantics}",
380            "%value = OpLoad _ {value}",
381            "%old = OpAtomicSMax _ {ptr} %scope %semantics %value",
382            "OpStore {old} %old",
383            scope = const SCOPE,
384            semantics = const SEMANTICS,
385            ptr = in(reg) ptr,
386            old = in(reg) &mut old,
387            value = in(reg) &value
388        }
389
390        old
391    }
392}
393
394/// Perform the following steps atomically with respect to any other atomic
395/// accesses within Scope to the same location:
396///
397/// 1. Load through `ptr` to get an original value,
398/// 2. Get a new value by finding the largest unsigned integer of original
399///    value and `value`, and
400/// 3. Store the new value back through `ptr`.
401///
402/// The result is the original value.
403#[spirv_std_macros::gpu_only]
404#[doc(alias = "OpAtomicUMax")]
405#[inline]
406pub unsafe fn atomic_u_max<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
407    ptr: &mut U,
408    value: U,
409) -> U {
410    unsafe {
411        let mut old = U::default();
412
413        asm! {
414            "%u32 = OpTypeInt 32 0",
415            "%scope = OpConstant %u32 {scope}",
416            "%semantics = OpConstant %u32 {semantics}",
417            "%value = OpLoad _ {value}",
418            "%old = OpAtomicUMax _ {ptr} %scope %semantics %value",
419            "OpStore {old} %old",
420            scope = const SCOPE,
421            semantics = const SEMANTICS,
422            ptr = in(reg) ptr,
423            old = in(reg) &mut old,
424            value = in(reg) &value
425        }
426
427        old
428    }
429}
430
431/// Perform the following steps atomically with respect to any other atomic
432/// accesses within Scope to the same location:
433///
434/// 1. Load through `ptr` to get an original value,
435/// 2. Get a new value by the bitwise AND of the original value and `value`, and
436/// 3. Store the new value back through `ptr`.
437///
438/// The result is the original value.
439#[spirv_std_macros::gpu_only]
440#[doc(alias = "OpAtomicAnd")]
441#[inline]
442pub unsafe fn atomic_and<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
443    ptr: &mut I,
444    value: I,
445) -> I {
446    unsafe {
447        let mut old = I::default();
448
449        asm! {
450            "%u32 = OpTypeInt 32 0",
451            "%scope = OpConstant %u32 {scope}",
452            "%semantics = OpConstant %u32 {semantics}",
453            "%value = OpLoad _ {value}",
454            "%old = OpAtomicAnd _ {ptr} %scope %semantics %value",
455            "OpStore {old} %old",
456            scope = const SCOPE,
457            semantics = const SEMANTICS,
458            ptr = in(reg) ptr,
459            old = in(reg) &mut old,
460            value = in(reg) &value
461        }
462
463        old
464    }
465}
466
467/// Perform the following steps atomically with respect to any other atomic
468/// accesses within Scope to the same location:
469///
470/// 1. Load through `ptr` to get an original value,
471/// 2. Get a new value by the bitwise OR of the original value and `value`, and
472/// 3. Store the new value back through `ptr`.
473///
474/// The result is the original value.
475#[spirv_std_macros::gpu_only]
476#[doc(alias = "OpAtomicOr")]
477#[inline]
478pub unsafe fn atomic_or<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
479    ptr: &mut I,
480    value: I,
481) -> I {
482    unsafe {
483        let mut old = I::default();
484
485        asm! {
486            "%u32 = OpTypeInt 32 0",
487            "%scope = OpConstant %u32 {scope}",
488            "%semantics = OpConstant %u32 {semantics}",
489            "%value = OpLoad _ {value}",
490            "%old = OpAtomicOr _ {ptr} %scope %semantics %value",
491            "OpStore {old} %old",
492            scope = const SCOPE,
493            semantics = const SEMANTICS,
494            ptr = in(reg) ptr,
495            old = in(reg) &mut old,
496            value = in(reg) &value
497        }
498
499        old
500    }
501}
502
503/// Perform the following steps atomically with respect to any other atomic
504/// accesses within Scope to the same location:
505///
506/// 1. Load through `ptr` to get an original value,
507/// 2. Get a new value by the bitwise XOR of the original value and `value`, and
508/// 3. Store the new value back through `ptr`.
509///
510/// The result is the original value.
511#[spirv_std_macros::gpu_only]
512#[doc(alias = "OpAtomicXor")]
513#[inline]
514pub unsafe fn atomic_xor<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
515    ptr: &mut I,
516    value: I,
517) -> I {
518    unsafe {
519        let mut old = I::default();
520
521        asm! {
522            "%u32 = OpTypeInt 32 0",
523            "%scope = OpConstant %u32 {scope}",
524            "%semantics = OpConstant %u32 {semantics}",
525            "%value = OpLoad _ {value}",
526            "%old = OpAtomicXor _ {ptr} %scope %semantics %value",
527            "OpStore {old} %old",
528            scope = const SCOPE,
529            semantics = const SEMANTICS,
530            ptr = in(reg) ptr,
531            old = in(reg) &mut old,
532            value = in(reg) &value
533        }
534
535        old
536    }
537}
538
539/// Perform the following steps atomically with respect to any other atomic
540/// accesses within Scope to the same location:
541///
542/// 1. Load through `ptr` to get an original value,
543/// 2. Get a new value by finding the smallest signed integer of original value
544///    and `value`, and
545/// 3. Store the new value back through `ptr`.
546///
547/// The result is the original value.
548#[spirv_std_macros::gpu_only]
549#[doc(alias = "OpAtomicFMinEXT")]
550#[inline]
551pub unsafe fn atomic_f_min<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
552    ptr: &mut F,
553    value: F,
554) -> F {
555    unsafe {
556        let mut old = F::default();
557
558        asm! {
559            "%u32 = OpTypeInt 32 0",
560            "%scope = OpConstant %u32 {scope}",
561            "%semantics = OpConstant %u32 {semantics}",
562            "%value = OpLoad _ {value}",
563            "%old = OpAtomicFMinEXT _ {ptr} %scope %semantics %value",
564            "OpStore {old} %old",
565            scope = const SCOPE,
566            semantics = const SEMANTICS,
567            ptr = in(reg) ptr,
568            old = in(reg) &mut old,
569            value = in(reg) &value
570        }
571
572        old
573    }
574}
575
576/// Perform the following steps atomically with respect to any other atomic
577/// accesses within Scope to the same location:
578///
579/// 1. Load through `ptr` to get an original value,
580/// 2. Get a new value by finding the largest signed integer of original value
581///    and `value`, and
582/// 3. Store the new value back through `ptr`.
583///
584/// The result is the original value.
585#[spirv_std_macros::gpu_only]
586#[doc(alias = "OpAtomicFMaxEXT")]
587#[inline]
588pub unsafe fn atomic_f_max<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
589    ptr: &mut F,
590    value: F,
591) -> F {
592    unsafe {
593        let mut old = F::default();
594
595        asm! {
596            "%u32 = OpTypeInt 32 0",
597            "%scope = OpConstant %u32 {scope}",
598            "%semantics = OpConstant %u32 {semantics}",
599            "%value = OpLoad _ {value}",
600            "%old = OpAtomicFMaxEXT _ {ptr} %scope %semantics %value",
601            "OpStore {old} %old",
602            scope = const SCOPE,
603            semantics = const SEMANTICS,
604            ptr = in(reg) ptr,
605            old = in(reg) &mut old,
606            value = in(reg) &value
607        }
608
609        old
610    }
611}
612
613/// Perform the following steps atomically with respect to any other atomic
614/// accesses within `SCOPE` to the same location:
615///
616/// 1) load through `ptr` to get an original value,
617/// 2) get a new value by integer addition of original value and `value`, and
618/// 3) store the new value back through `ptr`.
619///
620/// The result is the Original Value.
621#[spirv_std_macros::gpu_only]
622#[doc(alias = "OpAtomicFAddEXT")]
623#[inline]
624pub unsafe fn atomic_f_add<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
625    ptr: &mut F,
626    value: F,
627) -> F {
628    unsafe {
629        let mut old = F::default();
630
631        asm! {
632            "%u32 = OpTypeInt 32 0",
633            "%scope = OpConstant %u32 {scope}",
634            "%semantics = OpConstant %u32 {semantics}",
635            "%value = OpLoad _ {value}",
636            "%old = OpAtomicFAddEXT _ {ptr} %scope %semantics %value",
637            "OpStore {old} %old",
638            scope = const SCOPE,
639            semantics = const SEMANTICS,
640            ptr = in(reg) ptr,
641            old = in(reg) &mut old,
642            value = in(reg) &value
643        }
644
645        old
646    }
647}
spirv_std/arch/atomics.rs

spirv_std/arch/
atomics.rs
