rustc_codegen_spirv/builder/
intrinsics.rs

1// HACK(eddyb) avoids rewriting all of the imports (see `lib.rs` and `build.rs`).
2use crate::maybe_pqp_cg_ssa as rustc_codegen_ssa;
3
4use super::Builder;
5use crate::abi::ConvSpirvType;
6use crate::builder_spirv::{SpirvValue, SpirvValueExt};
7use crate::codegen_cx::CodegenCx;
8use crate::custom_insts::CustomInst;
9use crate::spirv_type::SpirvType;
10use rspirv::dr::Operand;
11use rspirv::spirv::GlslStd450Op as GLOp;
12use rustc_abi::Align;
13use rustc_codegen_ssa::RetagInfo;
14use rustc_codegen_ssa::mir::IntrinsicResult;
15use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
16use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
17use rustc_codegen_ssa::traits::{BuilderMethods, IntrinsicCallBuilderMethods};
18use rustc_middle::ty::layout::LayoutOf;
19use rustc_middle::ty::{FnDef, Instance, Ty, TyKind, TypingEnv};
20use rustc_middle::{bug, ty};
21use rustc_span::sym;
22use rustc_span::{Span, Symbol};
23
24fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_>) -> Option<(u64, bool)> {
25    match ty.kind() {
26        TyKind::Int(t) => Some((
27            t.bit_width()
28                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
29            true,
30        )),
31        TyKind::Uint(t) => Some((
32            t.bit_width()
33                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
34            false,
35        )),
36        _ => None,
37    }
38}
39
40impl Builder<'_, '_> {
41    pub fn copysign(&mut self, val: SpirvValue, sign: SpirvValue) -> SpirvValue {
42        let width = match self.lookup_type(val.ty) {
43            SpirvType::Float(width) => width,
44            other => bug!(
45                "copysign must have float argument, not {}",
46                other.debug(val.ty, self)
47            ),
48        };
49        let int_ty = SpirvType::Integer(width, false).def(self.span(), self);
50        let [mask_sign, mask_value] = {
51            let sign_bit = 1u128.checked_shl(width - 1).unwrap();
52            let value_mask = sign_bit - 1;
53            [sign_bit, value_mask].map(|v| self.constant_int(int_ty, v))
54        };
55        let val_bits = self.bitcast(val, int_ty);
56        let sign_bits = self.bitcast(sign, int_ty);
57        let val_masked = self.and(val_bits, mask_value);
58        let sign_masked = self.and(sign_bits, mask_sign);
59        let result_bits = self.or(val_masked, sign_masked);
60        self.bitcast(result_bits, val.ty)
61    }
62}
63
64impl<'a, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'tcx> {
65    fn codegen_intrinsic_call(
66        &mut self,
67        instance: Instance<'tcx>,
68        args: &[OperandRef<'tcx, Self::Value>],
69        result_layout: ty::layout::TyAndLayout<'tcx>,
70        result_place: Option<PlaceValue<Self::Value>>,
71        _span: Span,
72    ) -> IntrinsicResult<'tcx, Self::Value> {
73        let result = PlaceRef {
74            val: result_place.unwrap(),
75            layout: result_layout,
76        };
77        let callee_ty = instance.ty(self.tcx, TypingEnv::fully_monomorphized());
78
79        let (def_id, fn_args) = match *callee_ty.kind() {
80            FnDef(def_id, fn_args) => (def_id, fn_args.skip_binder()),
81            _ => bug!("expected fn item type, found {}", callee_ty),
82        };
83
84        let sig = callee_ty.fn_sig(self.tcx);
85        let sig = self
86            .tcx
87            .normalize_erasing_late_bound_regions(TypingEnv::fully_monomorphized(), sig);
88        let arg_tys = sig.inputs();
89        let name = self.tcx.item_name(def_id);
90
91        let ret_ty = self.layout_of(sig.output()).spirv_type(self.span(), self);
92
93        let value = match name {
94            // `sym::likely` no longer exists on newer rustc nightlies, but the
95            // intrinsic item name remains `likely`; ignore hint intrinsics.
96            _ if self.tcx.is_intrinsic(def_id, sym::unlikely)
97                || self.tcx.is_intrinsic(def_id, Symbol::intern("likely")) =>
98            {
99                args[0].immediate()
100            }
101
102            sym::breakpoint => {
103                self.abort();
104                assert!(result.layout.ty.is_unit());
105                return IntrinsicResult::WroteIntoPlace;
106            }
107
108            sym::volatile_load | sym::unaligned_volatile_load => {
109                let ptr = args[0].immediate();
110                let layout = self.layout_of(fn_args.type_at(0));
111                let load =
112                    self.volatile_load(layout.spirv_type(self.span(), self), ptr, Align::ONE);
113                if !result.layout.is_zst() {
114                    self.store(load, result.val.llval, result.val.align);
115                }
116                return IntrinsicResult::WroteIntoPlace;
117            }
118
119            sym::prefetch_read_data
120            | sym::prefetch_write_data
121            | sym::prefetch_read_instruction
122            | sym::prefetch_write_instruction => {
123                // ignore
124                assert!(result.layout.ty.is_unit());
125                return IntrinsicResult::WroteIntoPlace;
126            }
127
128            sym::saturating_add => {
129                assert_eq!(arg_tys[0], arg_tys[1]);
130                let result = match arg_tys[0].kind() {
131                    TyKind::Int(_) | TyKind::Uint(_) => {
132                        self.add(args[0].immediate(), args[1].immediate())
133                    }
134                    TyKind::Float(_) => self.fadd(args[0].immediate(), args[1].immediate()),
135                    other => self.fatal(format!(
136                        "Unimplemented saturating_add intrinsic type: {other:#?}"
137                    )),
138                };
139                // TODO: Implement this
140                self.zombie(result.def(self), "saturating_add is not implemented yet");
141                result
142            }
143            sym::saturating_sub => {
144                assert_eq!(arg_tys[0], arg_tys[1]);
145                let result = match &arg_tys[0].kind() {
146                    TyKind::Int(_) | TyKind::Uint(_) => {
147                        self.sub(args[0].immediate(), args[1].immediate())
148                    }
149                    TyKind::Float(_) => self.fsub(args[0].immediate(), args[1].immediate()),
150                    other => self.fatal(format!(
151                        "Unimplemented saturating_sub intrinsic type: {other:#?}"
152                    )),
153                };
154                // TODO: Implement this
155                self.zombie(result.def(self), "saturating_sub is not implemented yet");
156                result
157            }
158
159            sym::sqrtf32 | sym::sqrtf64 | sym::sqrtf128 => {
160                self.gl_op(GLOp::Sqrt, ret_ty, [args[0].immediate()])
161            }
162            sym::powif32 | sym::powif64 | sym::powif128 => {
163                let float = self.sitofp(args[1].immediate(), args[0].immediate().ty);
164                self.gl_op(GLOp::Pow, ret_ty, [args[0].immediate(), float])
165            }
166            sym::sinf32 | sym::sinf64 | sym::sinf128 => {
167                self.gl_op(GLOp::Sin, ret_ty, [args[0].immediate()])
168            }
169            sym::cosf32 | sym::cosf64 | sym::cosf128 => {
170                self.gl_op(GLOp::Cos, ret_ty, [args[0].immediate()])
171            }
172            sym::powf32 | sym::powf64 | sym::powf128 => self.gl_op(
173                GLOp::Pow,
174                ret_ty,
175                [args[0].immediate(), args[1].immediate()],
176            ),
177            sym::expf32 | sym::expf64 | sym::expf128 => {
178                self.gl_op(GLOp::Exp, ret_ty, [args[0].immediate()])
179            }
180            sym::exp2f32 | sym::exp2f64 | sym::exp2f128 => {
181                self.gl_op(GLOp::Exp2, ret_ty, [args[0].immediate()])
182            }
183            sym::logf32 | sym::logf64 | sym::logf128 => {
184                self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()])
185            }
186            sym::log2f32 | sym::log2f64 | sym::log2f128 => {
187                self.gl_op(GLOp::Log2, ret_ty, [args[0].immediate()])
188            }
189            sym::log10f32 | sym::log10f64 | sym::log10f128 => {
190                // spir-v glsl doesn't have log10, so,
191                // log10(x) == (1 / ln(10)) * ln(x)
192                let mul = self.constant_float(args[0].immediate().ty, 1.0 / 10.0f64.ln());
193                let ln = self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()]);
194                self.fmul(mul, ln)
195            }
196            sym::fmaf32 | sym::fmaf64 | sym::fmaf128 => self.gl_op(
197                GLOp::Fma,
198                ret_ty,
199                [
200                    args[0].immediate(),
201                    args[1].immediate(),
202                    args[2].immediate(),
203                ],
204            ),
205            sym::fabs => self.gl_op(GLOp::FAbs, ret_ty, [args[0].immediate()]),
206            sym::minimumf32 | sym::minimumf64 | sym::minimumf128 => self.gl_op(
207                GLOp::FMin,
208                ret_ty,
209                [args[0].immediate(), args[1].immediate()],
210            ),
211            sym::maximumf32 | sym::maximumf64 | sym::maximumf128 => self.gl_op(
212                GLOp::FMax,
213                ret_ty,
214                [args[0].immediate(), args[1].immediate()],
215            ),
216            sym::copysignf32 | sym::copysignf64 | sym::copysignf128 => {
217                let val = args[0].immediate();
218                let sign = args[1].immediate();
219                self.copysign(val, sign)
220            }
221            sym::floorf32 | sym::floorf64 | sym::floorf128 => {
222                self.gl_op(GLOp::Floor, ret_ty, [args[0].immediate()])
223            }
224            sym::ceilf32 | sym::ceilf64 | sym::ceilf128 => {
225                self.gl_op(GLOp::Ceil, ret_ty, [args[0].immediate()])
226            }
227            sym::truncf32 | sym::truncf64 | sym::truncf128 => {
228                self.gl_op(GLOp::Trunc, ret_ty, [args[0].immediate()])
229            }
230            sym::round_ties_even_f32 | sym::round_ties_even_f64 | sym::round_ties_even_f128 => {
231                self.gl_op(GLOp::RoundEven, ret_ty, [args[0].immediate()])
232            }
233            sym::roundf32 | sym::roundf64 | sym::roundf128 => {
234                self.gl_op(GLOp::Round, ret_ty, [args[0].immediate()])
235            }
236
237            sym::rotate_left | sym::rotate_right => {
238                let is_left = name == sym::rotate_left;
239                let val = args[0].immediate();
240                let shift = args[1].immediate();
241                self.rotate(val, shift, is_left)
242            }
243
244            sym::ctlz => self.count_leading_trailing_zeros(args[0].immediate(), false, false),
245            sym::ctlz_nonzero => {
246                self.count_leading_trailing_zeros(args[0].immediate(), false, true)
247            }
248            sym::cttz => self.count_leading_trailing_zeros(args[0].immediate(), true, false),
249            sym::cttz_nonzero => self.count_leading_trailing_zeros(args[0].immediate(), true, true),
250
251            sym::ctpop => self.count_ones(args[0].immediate()),
252            sym::bitreverse => self.bit_reverse(args[0].immediate()),
253            sym::black_box => {
254                // TODO(LegNeato): do something more sophisticated that prevents DCE
255                self.tcx
256                    .dcx()
257                    .warn("black_box intrinsic does not prevent optimization in Rust GPU");
258
259                let layout = self.layout_of(arg_tys[0]);
260                let llty = layout.spirv_type(self.span(), self);
261
262                match args[0].val {
263                    // Scalars pass through unchanged
264                    OperandValue::Immediate(v) => v,
265                    // Pack scalar pairs to a single SSA aggregate
266                    OperandValue::Pair(..) => args[0].immediate_or_packed_pair(self),
267                    // Lvalues get loaded
268                    OperandValue::Ref(place) => self.load(llty, place.llval, place.align),
269                    // ZSTs become undef of the right type
270                    OperandValue::ZeroSized => self.undef(llty),
271                }
272            }
273            sym::bswap => {
274                // https://github.com/KhronosGroup/SPIRV-LLVM/pull/221/files
275                // TODO: Definitely add tests to make sure this impl is right.
276                let arg = args[0].immediate();
277                let (width, is_signed) = int_type_width_signed(arg_tys[0], self)
278                    .expect("bswap must have an integer argument");
279
280                // Cast to unsigned type for byte-swapping
281                let unsigned_ty: u32 =
282                    SpirvType::Integer(width.try_into().unwrap(), false).def(self.span(), self);
283                let unsigned_arg = if is_signed {
284                    self.bitcast(arg, unsigned_ty)
285                } else {
286                    arg
287                };
288
289                let swapped = match width {
290                    8 => unsigned_arg,
291                    16 => {
292                        let offset8 = self.constant_u16(self.span(), 8);
293                        let tmp1 = self.shl(unsigned_arg, offset8);
294                        let tmp2 = self.lshr(unsigned_arg, offset8);
295                        self.or(tmp1, tmp2)
296                    }
297                    32 => {
298                        let offset8 = self.constant_u32(self.span(), 8);
299                        let offset24 = self.constant_u32(self.span(), 24);
300                        let mask16 = self.constant_u32(self.span(), 0xFF00);
301                        let mask24 = self.constant_u32(self.span(), 0xFF0000);
302                        let tmp4 = self.shl(unsigned_arg, offset24);
303                        let tmp3 = self.shl(unsigned_arg, offset8);
304                        let tmp2 = self.lshr(unsigned_arg, offset8);
305                        let tmp1 = self.lshr(unsigned_arg, offset24);
306                        let tmp3 = self.and(tmp3, mask24);
307                        let tmp2 = self.and(tmp2, mask16);
308                        let res1 = self.or(tmp1, tmp2);
309                        let res2 = self.or(tmp3, tmp4);
310                        self.or(res1, res2)
311                    }
312                    64 => {
313                        let offset8 = self.constant_u64(self.span(), 8);
314                        let offset24 = self.constant_u64(self.span(), 24);
315                        let offset40 = self.constant_u64(self.span(), 40);
316                        let offset56 = self.constant_u64(self.span(), 56);
317                        let mask16 = self.constant_u64(self.span(), 0xff00);
318                        let mask24 = self.constant_u64(self.span(), 0xff0000);
319                        let mask32 = self.constant_u64(self.span(), 0xff000000);
320                        let mask40 = self.constant_u64(self.span(), 0xff00000000);
321                        let mask48 = self.constant_u64(self.span(), 0xff0000000000);
322                        let mask56 = self.constant_u64(self.span(), 0xff000000000000);
323                        let tmp8 = self.shl(unsigned_arg, offset56);
324                        let tmp7 = self.shl(unsigned_arg, offset40);
325                        let tmp6 = self.shl(unsigned_arg, offset24);
326                        let tmp5 = self.shl(unsigned_arg, offset8);
327                        let tmp4 = self.lshr(unsigned_arg, offset8);
328                        let tmp3 = self.lshr(unsigned_arg, offset24);
329                        let tmp2 = self.lshr(unsigned_arg, offset40);
330                        let tmp1 = self.lshr(unsigned_arg, offset56);
331                        let tmp7 = self.and(tmp7, mask56);
332                        let tmp6 = self.and(tmp6, mask48);
333                        let tmp5 = self.and(tmp5, mask40);
334                        let tmp4 = self.and(tmp4, mask32);
335                        let tmp3 = self.and(tmp3, mask24);
336                        let tmp2 = self.and(tmp2, mask16);
337                        let res1 = self.or(tmp8, tmp7);
338                        let res2 = self.or(tmp6, tmp5);
339                        let res3 = self.or(tmp4, tmp3);
340                        let res4 = self.or(tmp2, tmp1);
341                        let res1 = self.or(res1, res2);
342                        let res3 = self.or(res3, res4);
343                        self.or(res1, res3)
344                    }
345                    other => self.undef_zombie(
346                        ret_ty,
347                        &format!("bswap not implemented for int width {other}"),
348                    ),
349                };
350
351                // Cast back to the original signed type if necessary
352                if is_signed {
353                    self.bitcast(swapped, arg.ty)
354                } else {
355                    swapped
356                }
357            }
358
359            sym::compare_bytes => self.undef_zombie(ret_ty, "memcmp not implemented"),
360
361            _ => {
362                // Call the fallback body instead of generating the intrinsic code
363                return IntrinsicResult::Fallback(Instance::new_raw(def_id, instance.args));
364            }
365        };
366
367        if result.layout.ty.is_bool() {
368            let val = self.from_immediate(value);
369            self.store_to_place(val, result.val);
370        } else if !result.layout.ty.is_unit() {
371            // FIXME(eddyb) upstream uses `self.store_to_place(value, result.val);`,
372            // which AFAICT does not handle packed pairs explicitly, meaning it
373            // can/will store e.g. LLVM `{A, B}` values, which is legal (in LLVM),
374            // but seems suboptimal (or even risky with e.g. layout randomization).
375            OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
376                .val
377                .store(self, result);
378        }
379        IntrinsicResult::WroteIntoPlace
380    }
381
382    fn codegen_llvm_intrinsic_call(
383        &mut self,
384        instance: ty::Instance<'tcx>,
385        _args: &[OperandRef<'tcx, Self::Value>],
386        _is_cleanup: bool,
387    ) -> Self::Value {
388        bug!("LLVM intrinsic call not supported in SPIR-V backend: {instance:?}")
389    }
390
391    fn abort(&mut self) {
392        self.abort_with_kind_and_message_debug_printf("abort", "intrinsics::abort() called", []);
393    }
394
395    // FIXME(eddyb) `assume` is not implemented atm, so all of its forms should
396    // avoid computing its (potentially illegal) bool input in the first place.
397    fn assume(&mut self, _val: Self::Value) {}
398
399    fn expect(&mut self, cond: Self::Value, _expected: bool) -> Self::Value {
400        // TODO: llvm.expect
401        cond
402    }
403
404    fn type_checked_load(
405        &mut self,
406        _llvtable: Self::Value,
407        _vtable_byte_offset: u64,
408        _typeid: &[u8],
409    ) -> Self::Value {
410        todo!()
411    }
412
413    fn va_start(&mut self, _val: Self::Value) {}
414
415    fn retag_mem(&mut self, _place: Self::Value, _info: &RetagInfo<Self::Value>) {
416        bug!("retag not supported")
417    }
418
419    fn retag_reg(&mut self, _ptr: Self::Value, _info: &RetagInfo<Self::Value>) -> Self::Value {
420        bug!("retag not supported")
421    }
422}
423
424impl Builder<'_, '_> {
425    pub fn count_ones(&mut self, arg: SpirvValue) -> SpirvValue {
426        let ty = arg.ty;
427        match self.cx.lookup_type(ty) {
428            SpirvType::Integer(bits, false) => {
429                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
430
431                match bits {
432                    8 | 16 => {
433                        let arg = arg.def(self);
434                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
435                        self.emit().bit_count(u32, None, arg).unwrap()
436                    }
437                    32 => self.emit().bit_count(u32, None, arg.def(self)).unwrap(),
438                    64 => {
439                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
440                        let arg = arg.def(self);
441                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
442                        let higher = self
443                            .emit()
444                            .shift_right_logical(ty, None, arg, u32_32)
445                            .unwrap();
446                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
447
448                        let lower_bits = self.emit().bit_count(u32, None, lower).unwrap();
449                        let higher_bits = self.emit().bit_count(u32, None, higher).unwrap();
450                        self.emit()
451                            .i_add(u32, None, lower_bits, higher_bits)
452                            .unwrap()
453                    }
454                    _ => {
455                        return self.undef_zombie(
456                            ty,
457                            &format!("count_ones() on unsupported {ty:?} bit integer type"),
458                        );
459                    }
460                }
461                .with_type(u32)
462            }
463            _ => self.fatal(format!(
464                "count_ones() expected an unsigned integer type, got {:?}",
465                self.cx.lookup_type(ty)
466            )),
467        }
468    }
469
470    pub fn bit_reverse(&mut self, arg: SpirvValue) -> SpirvValue {
471        let ty = arg.ty;
472        match self.cx.lookup_type(ty) {
473            SpirvType::Integer(bits, false) => {
474                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
475                let uint = SpirvType::Integer(bits, false).def(self.span(), self);
476
477                match bits {
478                    8 | 16 => {
479                        let arg = arg.def(self);
480                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
481
482                        let reverse = self.emit().bit_reverse(u32, None, arg).unwrap();
483                        let shift = self.constant_u32(self.span(), 32 - bits).def(self);
484                        let reverse = self
485                            .emit()
486                            .shift_right_logical(u32, None, reverse, shift)
487                            .unwrap();
488                        self.emit().u_convert(uint, None, reverse).unwrap()
489                    }
490                    32 => self.emit().bit_reverse(u32, None, arg.def(self)).unwrap(),
491                    64 => {
492                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
493                        let arg = arg.def(self);
494                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
495                        let higher = self
496                            .emit()
497                            .shift_right_logical(ty, None, arg, u32_32)
498                            .unwrap();
499                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
500
501                        // note that higher and lower have swapped
502                        let higher_bits = self.emit().bit_reverse(u32, None, lower).unwrap();
503                        let lower_bits = self.emit().bit_reverse(u32, None, higher).unwrap();
504
505                        let higher_bits = self.emit().u_convert(uint, None, higher_bits).unwrap();
506                        let higher_bits = self
507                            .emit()
508                            .shift_left_logical(uint, None, higher_bits, u32_32)
509                            .unwrap();
510                        let lower_bits = self.emit().u_convert(uint, None, lower_bits).unwrap();
511
512                        self.emit()
513                            .bitwise_or(ty, None, lower_bits, higher_bits)
514                            .unwrap()
515                    }
516                    _ => {
517                        return self.undef_zombie(
518                            ty,
519                            &format!("bit_reverse() on unsupported {ty:?} bit integer type"),
520                        );
521                    }
522                }
523                .with_type(ty)
524            }
525            _ => self.fatal(format!(
526                "bit_reverse() expected an unsigned integer type, got {:?}",
527                self.cx.lookup_type(ty)
528            )),
529        }
530    }
531
532    pub fn count_leading_trailing_zeros(
533        &mut self,
534        arg: SpirvValue,
535        trailing: bool,
536        non_zero: bool,
537    ) -> SpirvValue {
538        let ty = arg.ty;
539        match self.cx.lookup_type(ty) {
540            SpirvType::Integer(bits, false) => {
541                let bool = SpirvType::Bool.def(self.span(), self);
542                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
543
544                let glsl = self.ext_inst.borrow_mut().import_glsl(self);
545                let find_xsb = |this: &mut Self, arg, offset: i32| {
546                    if trailing {
547                        let lsb = this
548                            .emit()
549                            .ext_inst(
550                                u32,
551                                None,
552                                glsl,
553                                GLOp::FindILsb as u32,
554                                [Operand::IdRef(arg)],
555                            )
556                            .unwrap();
557                        if offset == 0 {
558                            lsb
559                        } else {
560                            let const_offset = this.constant_i32(this.span(), offset).def(this);
561                            this.emit().i_add(u32, None, const_offset, lsb).unwrap()
562                        }
563                    } else {
564                        // rust is always unsigned, so FindUMsb
565                        let msb_bit = this
566                            .emit()
567                            .ext_inst(
568                                u32,
569                                None,
570                                glsl,
571                                GLOp::FindUMsb as u32,
572                                [Operand::IdRef(arg)],
573                            )
574                            .unwrap();
575                        // the glsl op returns the Msb bit, not the amount of leading zeros of this u32
576                        // leading zeros = 31 - Msb bit
577                        let const_offset = this.constant_i32(this.span(), 31 - offset).def(this);
578                        this.emit().i_sub(u32, None, const_offset, msb_bit).unwrap()
579                    }
580                };
581
582                let converted = match bits {
583                    8 | 16 => {
584                        let arg = self.emit().u_convert(u32, None, arg.def(self)).unwrap();
585                        if trailing {
586                            find_xsb(self, arg, 0)
587                        } else {
588                            find_xsb(self, arg, bits as i32 - 32)
589                        }
590                    }
591                    32 => find_xsb(self, arg.def(self), 0),
592                    64 => {
593                        let u32_0 = self.constant_int(u32, 0).def(self);
594                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
595
596                        let arg = arg.def(self);
597                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
598                        let higher = self
599                            .emit()
600                            .shift_right_logical(ty, None, arg, u32_32)
601                            .unwrap();
602                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
603
604                        if trailing {
605                            let use_lower = self.emit().i_equal(bool, None, lower, u32_0).unwrap();
606                            let lower_bits = find_xsb(self, lower, 0);
607                            let higher_bits = find_xsb(self, higher, 32);
608                            self.emit()
609                                .select(u32, None, use_lower, higher_bits, lower_bits)
610                                .unwrap()
611                        } else {
612                            let use_higher =
613                                self.emit().i_equal(bool, None, higher, u32_0).unwrap();
614                            let lower_bits = find_xsb(self, lower, 0);
615                            let higher_bits = find_xsb(self, higher, 32);
616                            self.emit()
617                                .select(u32, None, use_higher, lower_bits, higher_bits)
618                                .unwrap()
619                        }
620                    }
621                    _ => {
622                        return self.undef_zombie(ty, &format!(
623                            "count_leading_trailing_zeros() on unsupported {ty:?} bit integer type"
624                        ));
625                    }
626                };
627
628                if non_zero {
629                    converted
630                } else {
631                    let int_0 = self.constant_int(ty, 0).def(self);
632                    let int_bits = self.constant_int(u32, bits as u128).def(self);
633                    let is_0 = self
634                        .emit()
635                        .i_equal(bool, None, arg.def(self), int_0)
636                        .unwrap();
637                    self.emit()
638                        .select(u32, None, is_0, int_bits, converted)
639                        .unwrap()
640                }
641                .with_type(u32)
642            }
643            SpirvType::Integer(bits, true) => {
644                // rustc wants `[i8,i16,i32,i64]::leading_zeros()` with `non_zero: true` for some reason. I do not know
645                // how these are reachable, marking them as zombies makes none of our compiletests fail.
646                let unsigned = SpirvType::Integer(bits, false).def(self.span(), self);
647                let arg = self
648                    .emit()
649                    .bitcast(unsigned, None, arg.def(self))
650                    .unwrap()
651                    .with_type(unsigned);
652                let result = self.count_leading_trailing_zeros(arg, trailing, non_zero);
653                self.emit()
654                    .bitcast(ty, None, result.def(self))
655                    .unwrap()
656                    .with_type(ty)
657            }
658            e => {
659                self.fatal(format!(
660                    "count_leading_trailing_zeros(trailing: {trailing}, non_zero: {non_zero}) expected an integer type, got {e:?}",
661                ));
662            }
663        }
664    }
665
666    pub fn abort_with_kind_and_message_debug_printf(
667        &mut self,
668        kind: &str,
669        message_debug_printf_fmt_str: impl Into<String>,
670        message_debug_printf_args: impl IntoIterator<Item = SpirvValue>,
671    ) {
672        // FIXME(eddyb) this should be cached more efficiently.
673        let void_ty = SpirvType::Void.def(rustc_span::DUMMY_SP, self);
674
675        // HACK(eddyb) there is no `abort` or `trap` instruction in SPIR-V,
676        // so the best thing we can do is use our own custom instruction.
677        let kind_id = self.emit().string(kind);
678        let message_debug_printf_fmt_str_id = self.emit().string(message_debug_printf_fmt_str);
679        self.custom_inst(
680            void_ty,
681            CustomInst::Abort {
682                kind: Operand::IdRef(kind_id),
683                message_debug_printf: [message_debug_printf_fmt_str_id]
684                    .into_iter()
685                    .chain(
686                        message_debug_printf_args
687                            .into_iter()
688                            .map(|arg| arg.def(self)),
689                    )
690                    .map(Operand::IdRef)
691                    .collect(),
692            },
693        );
694        self.unreachable();
695
696        // HACK(eddyb) we still need an active block in case the user of this
697        // `Builder` will continue to emit instructions after the `.abort()`.
698        let post_abort_dead_bb = self.append_sibling_block("post_abort_dead");
699        self.switch_to_block(post_abort_dead_bb);
700    }
701}
rustc_codegen_spirv/builder/intrinsics.rs

rustc_codegen_spirv/builder/
intrinsics.rs
