#![allow(clippy::missing_safety_doc)]
#[cfg(target_arch = "spirv")]
use crate::arch::barrier;
use crate::float::Float;
use crate::integer::{Integer, SignedInteger, UnsignedInteger};
#[cfg(target_arch = "spirv")]
use crate::memory::{Scope, Semantics};
use crate::vector::VectorOrScalar;
#[cfg(target_arch = "spirv")]
use core::arch::asm;
#[cfg(target_arch = "spirv")]
const SUBGROUP: u32 = Scope::Subgroup as u32;
#[repr(transparent)]
#[derive(Copy, Clone, Default, Eq, PartialEq)]
#[cfg_attr(feature = "bytemuck", derive(bytemuck::Zeroable, bytemuck::Pod))]
pub struct SubgroupMask(pub glam::UVec4);
#[non_exhaustive]
#[derive(Debug, PartialEq, Eq)]
pub enum GroupOperation {
Reduce = 0,
InclusiveScan = 1,
ExclusiveScan = 2,
ClusteredReduce = 3,
PartitionedReduceNV = 6,
PartitionedInclusiveScanNV = 7,
PartitionedExclusiveScanNV = 8,
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "subgroupBarrier")]
#[inline]
pub unsafe fn subgroup_barrier() {
unsafe {
barrier::control_barrier::<
SUBGROUP,
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "subgroupMemoryBarrier")]
#[inline]
pub unsafe fn subgroup_memory_barrier() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{
Semantics::ACQUIRE_RELEASE.bits()
| Semantics::UNIFORM_MEMORY.bits()
| Semantics::WORKGROUP_MEMORY.bits()
| Semantics::IMAGE_MEMORY.bits()
},
>();
}
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "subgroupMemoryBarrierBuffer")]
#[inline]
pub unsafe fn subgroup_memory_barrier_buffer() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::UNIFORM_MEMORY.bits() },
>();
}
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "subgroupMemoryBarrierShared")]
#[inline]
pub unsafe fn subgroup_memory_barrier_shared() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::WORKGROUP_MEMORY.bits() },
>();
}
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "subgroupMemoryBarrierImage")]
#[inline]
pub unsafe fn subgroup_memory_barrier_image() {
unsafe {
barrier::memory_barrier::<
SUBGROUP,
{ Semantics::ACQUIRE_RELEASE.bits() | Semantics::IMAGE_MEMORY.bits() },
>();
}
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformElect")]
#[inline]
pub unsafe fn subgroup_elect() -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%result = OpGroupNonUniformElect %bool %subgroup",
"OpStore {result} %result",
subgroup = const SUBGROUP,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformAll")]
#[inline]
pub unsafe fn subgroup_all(predicate: bool) -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%predicate = OpLoad _ {predicate}",
"%result = OpGroupNonUniformAll %bool %subgroup %predicate",
"OpStore {result} %result",
subgroup = const SUBGROUP,
predicate = in(reg) &predicate,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformAny")]
#[inline]
pub unsafe fn subgroup_any(predicate: bool) -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%predicate = OpLoad _ {predicate}",
"%result = OpGroupNonUniformAny %bool %subgroup %predicate",
"OpStore {result} %result",
subgroup = const SUBGROUP,
predicate = in(reg) &predicate,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformAllEqual")]
#[inline]
pub unsafe fn subgroup_all_equal<T: VectorOrScalar>(value: T) -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%result = OpGroupNonUniformAllEqual %bool %subgroup %value",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBroadcast")]
#[inline]
pub unsafe fn subgroup_broadcast<T: VectorOrScalar>(value: T, id: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%id = OpLoad _ {id}",
"%result = OpGroupNonUniformBroadcast _ %subgroup %value %id",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
id = in(reg) &id,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBroadcastFirst")]
#[inline]
pub unsafe fn subgroup_broadcast_first<T: VectorOrScalar>(value: T) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%result = OpGroupNonUniformBroadcastFirst _ %subgroup %value",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBallot")]
#[inline]
pub unsafe fn subgroup_ballot(predicate: bool) -> SubgroupMask {
let mut result = SubgroupMask::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%groupmask = OpTypeVector %u32 4",
"%subgroup = OpConstant %u32 {subgroup}",
"%predicate = OpLoad _ {predicate}",
"%result = OpGroupNonUniformBallot %groupmask %subgroup %predicate",
"OpStore {result} %result",
subgroup = const SUBGROUP,
predicate = in(reg) &predicate,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformInverseBallot")]
#[inline]
pub unsafe fn subgroup_inverse_ballot(subgroup_mask: SubgroupMask) -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%subgroup_mask = OpLoad _ {subgroup_mask}",
"%result = OpGroupNonUniformInverseBallot %bool %subgroup %subgroup_mask",
"OpStore {result} %result",
subgroup = const SUBGROUP,
subgroup_mask = in(reg) &subgroup_mask,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBallotBitExtract")]
#[inline]
pub unsafe fn subgroup_ballot_bit_extract(subgroup_mask: SubgroupMask, id: u32) -> bool {
let mut result = false;
unsafe {
asm! {
"%bool = OpTypeBool",
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%subgroup_mask = OpLoad _ {subgroup_mask}",
"%id = OpLoad _ {id}",
"%result = OpGroupNonUniformBallotBitExtract %bool %subgroup %subgroup_mask %id",
"OpStore {result} %result",
subgroup = const SUBGROUP,
subgroup_mask = in(reg) &subgroup_mask,
id = in(reg) &id,
result = in(reg) &mut result,
}
}
result
}
macro_rules! macro_subgroup_ballot_bit_count {
($name:ident, $group_op:expr) => {
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBallotBitCount")]
#[inline]
pub unsafe fn $name(subgroup_mask: SubgroupMask) -> u32 {
let mut result = 0;
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%subgroup_mask = OpLoad _ {subgroup_mask}",
"%result = OpGroupNonUniformBallotBitCount %u32 %subgroup {groupop} %subgroup_mask",
"OpStore {result} %result",
subgroup = const SUBGROUP,
groupop = const ($group_op as u32),
subgroup_mask = in(reg) &subgroup_mask,
result = in(reg) &mut result,
}
}
result
}
};
}
macro_subgroup_ballot_bit_count!(subgroup_ballot_bit_count, GroupOperation::Reduce);
macro_subgroup_ballot_bit_count!(
subgroup_ballot_inclusive_bit_count,
GroupOperation::InclusiveScan
);
macro_subgroup_ballot_bit_count!(
subgroup_ballot_exclusive_bit_count,
GroupOperation::ExclusiveScan
);
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBallotFindLSB")]
#[inline]
pub unsafe fn subgroup_ballot_find_lsb(subgroup_mask: SubgroupMask) -> u32 {
let mut result = 0;
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%subgroup_mask = OpLoad _ {subgroup_mask}",
"%result = OpGroupNonUniformBallotFindLSB %u32 %subgroup %subgroup_mask",
"OpStore {result} %result",
subgroup = const SUBGROUP,
subgroup_mask = in(reg) &subgroup_mask,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformBallotFindMSB")]
#[inline]
pub unsafe fn subgroup_ballot_find_msb(subgroup_mask: SubgroupMask) -> u32 {
let mut result = 0;
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%subgroup_mask = OpLoad _ {subgroup_mask}",
"%result = OpGroupNonUniformBallotFindMSB %u32 %subgroup %subgroup_mask",
"OpStore {result} %result",
subgroup = const SUBGROUP,
subgroup_mask = in(reg) &subgroup_mask,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformShuffle")]
#[inline]
pub unsafe fn subgroup_shuffle<T: VectorOrScalar>(value: T, id: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%id = OpLoad _ {id}",
"%result = OpGroupNonUniformShuffle _ %subgroup %value %id",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
id = in(reg) &id,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformShuffleXor")]
#[inline]
pub unsafe fn subgroup_shuffle_xor<T: VectorOrScalar>(value: T, mask: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%mask = OpLoad _ {mask}",
"%result = OpGroupNonUniformShuffleXor _ %subgroup %value %mask",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
mask = in(reg) &mask,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformShuffleUp")]
#[inline]
pub unsafe fn subgroup_shuffle_up<T: VectorOrScalar>(value: T, delta: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%delta = OpLoad _ {delta}",
"%result = OpGroupNonUniformShuffleUp _ %subgroup %value %delta",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
delta = in(reg) &delta,
result = in(reg) &mut result,
}
}
result
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformShuffleDown")]
#[inline]
pub unsafe fn subgroup_shuffle_down<T: VectorOrScalar>(value: T, delta: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%delta = OpLoad _ {delta}",
"%result = OpGroupNonUniformShuffleDown _ %subgroup %value %delta",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
delta = in(reg) &delta,
result = in(reg) &mut result,
}
}
result
}
macro_rules! macro_subgroup_op {
($scalar:ty, $asm_op:literal, $($name:ident, $group_op:expr),+; $docs:literal) => { $(
#[doc = $docs]
#[spirv_std_macros::gpu_only]
#[doc(alias = $asm_op)]
#[inline]
pub unsafe fn $name<I: VectorOrScalar<Scalar = $scalar>>(
value: I,
) -> I {
let mut result = I::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
concat!("%result = ", $asm_op, " _ %subgroup {groupop} %value"),
"OpStore {result} %result",
subgroup = const SUBGROUP,
groupop = const ($group_op as u32),
value = in(reg) &value,
result = in(reg) &mut result,
}
}
result
}
)+ };
}
macro_rules! macro_subgroup_op_clustered {
($scalar:ty, $asm_op:literal, $name:ident; $docs:literal) => {
#[doc = $docs]
#[spirv_std_macros::gpu_only]
#[doc(alias = $asm_op)]
#[inline]
pub unsafe fn $name<const CLUSTER_SIZE: u32, I: VectorOrScalar<Scalar = $scalar>>(
value: I,
) -> I {
let mut result = I::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%clustersize = OpConstant %u32 {clustersize}",
concat!("%result = ", $asm_op, " _ %subgroup {groupop} %value %clustersize"),
"OpStore {result} %result",
subgroup = const SUBGROUP,
groupop = const (GroupOperation::ClusteredReduce as u32),
clustersize = const CLUSTER_SIZE,
value = in(reg) &value,
result = in(reg) &mut result,
}
}
result
}
};
}
macro_subgroup_op!(impl Integer, "OpGroupNonUniformIAdd", subgroup_i_add, GroupOperation::Reduce, subgroup_inclusive_i_add, GroupOperation::InclusiveScan, subgroup_exclusive_i_add, GroupOperation::ExclusiveScan; r"
An integer add group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Integer, "OpGroupNonUniformIAdd", subgroup_clustered_i_add; r"
An integer add group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Float, "OpGroupNonUniformFAdd", subgroup_f_add, GroupOperation::Reduce, subgroup_inclusive_f_add, GroupOperation::InclusiveScan, subgroup_exclusive_f_add, GroupOperation::ExclusiveScan; r"
A floating point add group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Float, "OpGroupNonUniformFAdd", subgroup_clustered_f_add; r"
A floating point add group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Integer, "OpGroupNonUniformIMul", subgroup_i_mul, GroupOperation::Reduce, subgroup_inclusive_i_mul, GroupOperation::InclusiveScan, subgroup_exclusive_i_mul, GroupOperation::ExclusiveScan; r"
An integer multiply group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 1.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Integer, "OpGroupNonUniformIMul", subgroup_clustered_i_mul; r"
An integer multiply group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 1. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Float, "OpGroupNonUniformFMul", subgroup_f_mul, GroupOperation::Reduce, subgroup_inclusive_f_mul, GroupOperation::InclusiveScan, subgroup_exclusive_f_mul, GroupOperation::ExclusiveScan; r"
A floating point multiply group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 1.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Float, "OpGroupNonUniformFMul", subgroup_clustered_f_mul; r"
A floating point multiply group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 1. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl SignedInteger, "OpGroupNonUniformSMin", subgroup_s_min, GroupOperation::Reduce, subgroup_inclusive_s_min, GroupOperation::InclusiveScan, subgroup_exclusive_s_min, GroupOperation::ExclusiveScan; r"
A signed integer minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is INT_MAX.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl SignedInteger, "OpGroupNonUniformSMin", subgroup_clustered_s_min; r"
A signed integer minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is INT_MAX. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl UnsignedInteger, "OpGroupNonUniformUMin", subgroup_u_min, GroupOperation::Reduce, subgroup_inclusive_u_min, GroupOperation::InclusiveScan, subgroup_exclusive_u_min, GroupOperation::ExclusiveScan; r"
An unsigned integer minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type, whose Signedness operand is 0.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is UINT_MAX.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl UnsignedInteger, "OpGroupNonUniformUMin", subgroup_clustered_u_min; r"
An unsigned integer minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type, whose Signedness operand is 0.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is UINT_MAX. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Float, "OpGroupNonUniformFMin", subgroup_f_min, GroupOperation::Reduce, subgroup_inclusive_f_min, GroupOperation::InclusiveScan, subgroup_exclusive_f_min, GroupOperation::ExclusiveScan; r"
A floating point minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is +INF.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined. From the set of Value(s) provided by active invocations within a subgroup, if for any two Values one of them is a NaN, the other is chosen. If all Value(s) that are used by the current invocation are NaN, then the result is an undefined value.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Float, "OpGroupNonUniformFMin", subgroup_clustered_f_min; r"
A floating point minimum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is +INF. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined. From the set of Value(s) provided by active invocations within a subgroup, if for any two Values one of them is a NaN, the other is chosen. If all Value(s) that are used by the current invocation are NaN, then the result is an undefined value.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl SignedInteger, "OpGroupNonUniformSMax", subgroup_s_max, GroupOperation::Reduce, subgroup_inclusive_s_max, GroupOperation::InclusiveScan, subgroup_exclusive_s_max, GroupOperation::ExclusiveScan; r"
A signed integer maximum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is INT_MIN.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl SignedInteger, "OpGroupNonUniformSMax", subgroup_clustered_s_max; r"
A signed integer maximum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is INT_MIN. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl UnsignedInteger, "OpGroupNonUniformUMax", subgroup_u_max, GroupOperation::Reduce, subgroup_inclusive_u_max, GroupOperation::InclusiveScan, subgroup_exclusive_u_max, GroupOperation::ExclusiveScan; r"
An unsigned integer maximum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type, whose Signedness operand is 0.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl UnsignedInteger, "OpGroupNonUniformUMax", subgroup_clustered_u_max; r"
An unsigned integer maximum group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type, whose Signedness operand is 0.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Float, "OpGroupNonUniformFMax", subgroup_f_max, GroupOperation::Reduce, subgroup_inclusive_f_max, GroupOperation::InclusiveScan, subgroup_exclusive_f_max, GroupOperation::ExclusiveScan; r"
A floating point maximum group operation of all Value operands contributed by active invocations in by group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is -INF.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined. From the set of Value(s) provided by active invocations within a subgroup, if for any two Values one of them is a NaN, the other is chosen. If all Value(s) that are used by the current invocation are NaN, then the result is an undefined value.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Float, "OpGroupNonUniformFMax", subgroup_clustered_f_max; r"
A floating point maximum group operation of all Value operands contributed by active invocations in by group.
Result Type must be a scalar or vector of floating-point type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is -INF.
The type of Value must be the same as Result Type. The method used to perform the group operation on the contributed Value(s) from active invocations is implementation defined. From the set of Value(s) provided by active invocations within a subgroup, if for any two Values one of them is a NaN, the other is chosen. If all Value(s) that are used by the current invocation are NaN, then the result is an undefined value.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Integer, "OpGroupNonUniformBitwiseAnd", subgroup_and, GroupOperation::Reduce, subgroup_inclusive_and, GroupOperation::InclusiveScan, subgroup_exclusive_and, GroupOperation::ExclusiveScan; r"
A bitwise and group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is ~0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Integer, "OpGroupNonUniformBitwiseAnd", subgroup_clustered_and; r"
A bitwise and group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is ~0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Integer, "OpGroupNonUniformBitwiseOr", subgroup_or, GroupOperation::Reduce, subgroup_inclusive_or, GroupOperation::InclusiveScan, subgroup_exclusive_or, GroupOperation::ExclusiveScan; r"
A bitwise or group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Integer, "OpGroupNonUniformBitwiseOr", subgroup_clustered_or; r"
A bitwise or group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(impl Integer, "OpGroupNonUniformBitwiseXor", subgroup_xor, GroupOperation::Reduce, subgroup_inclusive_xor, GroupOperation::InclusiveScan, subgroup_exclusive_xor, GroupOperation::ExclusiveScan; r"
A bitwise xor group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(impl Integer, "OpGroupNonUniformBitwiseXor", subgroup_clustered_xor; r"
A bitwise xor group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of integer type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(bool, "OpGroupNonUniformLogicalAnd", subgroup_logical_and, GroupOperation::Reduce, subgroup_inclusive_logical_and, GroupOperation::InclusiveScan, subgroup_exclusive_logical_and, GroupOperation::ExclusiveScan; r"
A logical and group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is ~0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(bool, "OpGroupNonUniformLogicalAnd", subgroup_clustered_logical_and; r"
A logical and group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is ~0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(bool, "OpGroupNonUniformLogicalOr", subgroup_logical_or, GroupOperation::Reduce, subgroup_inclusive_logical_or, GroupOperation::InclusiveScan, subgroup_exclusive_logical_or, GroupOperation::ExclusiveScan; r"
A logical or group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(bool, "OpGroupNonUniformLogicalOr", subgroup_clustered_logical_or; r"
A logical or group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
macro_subgroup_op!(bool, "OpGroupNonUniformLogicalXor", subgroup_logical_xor, GroupOperation::Reduce, subgroup_inclusive_logical_xor, GroupOperation::InclusiveScan, subgroup_exclusive_logical_xor, GroupOperation::ExclusiveScan; r"
A logical xor group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0.
The type of Value must be the same as Result Type.
Requires Capability `GroupNonUniformArithmetic`.
");
macro_subgroup_op_clustered!(bool, "OpGroupNonUniformLogicalXor", subgroup_clustered_logical_xor; r"
A logical xor group operation of all Value operands contributed by active invocations in the group.
Result Type must be a scalar or vector of Boolean type.
Execution is a Scope that identifies the group of invocations affected by this command. It must be Subgroup.
The identity I for Operation is 0. If Operation is ClusteredReduce, ClusterSize must be present.
The type of Value must be the same as Result Type.
ClusterSize is the size of cluster to use. ClusterSize must be a scalar of integer type, whose Signedness operand is 0. ClusterSize must come from a constant instruction. Behavior is undefined unless ClusterSize is at least 1 and a power of 2. If ClusterSize is greater than the size of the group, executing this instruction results in undefined behavior.
Requires Capability `GroupNonUniformArithmetic` and `GroupNonUniformClustered`.
");
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformQuadBroadcast")]
#[inline]
pub unsafe fn subgroup_quad_broadcast<T: VectorOrScalar>(value: T, id: u32) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%value = OpLoad _ {value}",
"%id = OpLoad _ {id}",
"%result = OpGroupNonUniformQuadBroadcast _ %subgroup %value %id",
"OpStore {result} %result",
subgroup = const SUBGROUP,
value = in(reg) &value,
id = in(reg) &id,
result = in(reg) &mut result,
}
}
result
}
pub enum QuadDirection {
Horizontal = 0,
Vertical = 1,
Diagonal = 2,
}
#[spirv_std_macros::gpu_only]
#[doc(alias = "OpGroupNonUniformQuadSwap")]
#[inline]
pub unsafe fn subgroup_quad_swap<const DIRECTION: u32, T: VectorOrScalar>(value: T) -> T {
let mut result = T::default();
unsafe {
asm! {
"%u32 = OpTypeInt 32 0",
"%subgroup = OpConstant %u32 {subgroup}",
"%direction = OpConstant %u32 {direction}",
"%value = OpLoad _ {value}",
"%result = OpGroupNonUniformQuadSwap _ %subgroup %value %direction",
"OpStore {result} %result",
subgroup = const SUBGROUP,
direction = const DIRECTION,
value = in(reg) &value,
result = in(reg) &mut result,
}
}
result
}