rustc_codegen_spirv/builder/
intrinsics.rs

1// HACK(eddyb) avoids rewriting all of the imports (see `lib.rs` and `build.rs`).
2use crate::maybe_pqp_cg_ssa as rustc_codegen_ssa;
3
4use super::Builder;
5use crate::abi::ConvSpirvType;
6use crate::builder_spirv::{SpirvValue, SpirvValueExt};
7use crate::codegen_cx::CodegenCx;
8use crate::custom_insts::CustomInst;
9use crate::spirv_type::SpirvType;
10use rspirv::dr::Operand;
11use rspirv::spirv::GlslStd450Op as GLOp;
12use rustc_codegen_ssa::RetagInfo;
13use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
14use rustc_codegen_ssa::mir::place::PlaceRef;
15use rustc_codegen_ssa::traits::{BuilderMethods, IntrinsicCallBuilderMethods};
16use rustc_middle::ty::layout::LayoutOf;
17use rustc_middle::ty::{FnDef, Instance, Ty, TyKind, TypingEnv};
18use rustc_middle::{bug, ty};
19use rustc_span::sym;
20use rustc_span::{Span, Symbol};
21
22fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_>) -> Option<(u64, bool)> {
23    match ty.kind() {
24        TyKind::Int(t) => Some((
25            t.bit_width()
26                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
27            true,
28        )),
29        TyKind::Uint(t) => Some((
30            t.bit_width()
31                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
32            false,
33        )),
34        _ => None,
35    }
36}
37
38impl Builder<'_, '_> {
39    pub fn copysign(&mut self, val: SpirvValue, sign: SpirvValue) -> SpirvValue {
40        let width = match self.lookup_type(val.ty) {
41            SpirvType::Float(width) => width,
42            other => bug!(
43                "copysign must have float argument, not {}",
44                other.debug(val.ty, self)
45            ),
46        };
47        let int_ty = SpirvType::Integer(width, false).def(self.span(), self);
48        let [mask_sign, mask_value] = {
49            let sign_bit = 1u128.checked_shl(width - 1).unwrap();
50            let value_mask = sign_bit - 1;
51            [sign_bit, value_mask].map(|v| self.constant_int(int_ty, v))
52        };
53        let val_bits = self.bitcast(val, int_ty);
54        let sign_bits = self.bitcast(sign, int_ty);
55        let val_masked = self.and(val_bits, mask_value);
56        let sign_masked = self.and(sign_bits, mask_sign);
57        let result_bits = self.or(val_masked, sign_masked);
58        self.bitcast(result_bits, val.ty)
59    }
60}
61
62impl<'a, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'tcx> {
63    fn codegen_intrinsic_call(
64        &mut self,
65        instance: Instance<'tcx>,
66        args: &[OperandRef<'tcx, Self::Value>],
67        result: PlaceRef<'tcx, Self::Value>,
68        _span: Span,
69    ) -> Result<(), ty::Instance<'tcx>> {
70        let callee_ty = instance.ty(self.tcx, TypingEnv::fully_monomorphized());
71
72        let (def_id, fn_args) = match *callee_ty.kind() {
73            FnDef(def_id, fn_args) => (def_id, fn_args),
74            _ => bug!("expected fn item type, found {}", callee_ty),
75        };
76
77        let sig = callee_ty.fn_sig(self.tcx);
78        let sig = self
79            .tcx
80            .normalize_erasing_late_bound_regions(TypingEnv::fully_monomorphized(), sig);
81        let arg_tys = sig.inputs();
82        let name = self.tcx.item_name(def_id);
83
84        let ret_ty = self.layout_of(sig.output()).spirv_type(self.span(), self);
85
86        let value = match name {
87            // `sym::likely` no longer exists on newer rustc nightlies, but the
88            // intrinsic item name remains `likely`; ignore hint intrinsics.
89            _ if self.tcx.is_intrinsic(def_id, sym::unlikely)
90                || self.tcx.is_intrinsic(def_id, Symbol::intern("likely")) =>
91            {
92                args[0].immediate()
93            }
94
95            sym::breakpoint => {
96                self.abort();
97                assert!(result.layout.ty.is_unit());
98                return Ok(());
99            }
100
101            sym::volatile_load | sym::unaligned_volatile_load => {
102                let ptr = args[0].immediate();
103                let layout = self.layout_of(fn_args.type_at(0));
104                let load = self.volatile_load(layout.spirv_type(self.span(), self), ptr);
105                if !result.layout.is_zst() {
106                    self.store(load, result.val.llval, result.val.align);
107                }
108                return Ok(());
109            }
110
111            sym::prefetch_read_data
112            | sym::prefetch_write_data
113            | sym::prefetch_read_instruction
114            | sym::prefetch_write_instruction => {
115                // ignore
116                assert!(result.layout.ty.is_unit());
117                return Ok(());
118            }
119
120            sym::saturating_add => {
121                assert_eq!(arg_tys[0], arg_tys[1]);
122                let result = match arg_tys[0].kind() {
123                    TyKind::Int(_) | TyKind::Uint(_) => {
124                        self.add(args[0].immediate(), args[1].immediate())
125                    }
126                    TyKind::Float(_) => self.fadd(args[0].immediate(), args[1].immediate()),
127                    other => self.fatal(format!(
128                        "Unimplemented saturating_add intrinsic type: {other:#?}"
129                    )),
130                };
131                // TODO: Implement this
132                self.zombie(result.def(self), "saturating_add is not implemented yet");
133                result
134            }
135            sym::saturating_sub => {
136                assert_eq!(arg_tys[0], arg_tys[1]);
137                let result = match &arg_tys[0].kind() {
138                    TyKind::Int(_) | TyKind::Uint(_) => {
139                        self.sub(args[0].immediate(), args[1].immediate())
140                    }
141                    TyKind::Float(_) => self.fsub(args[0].immediate(), args[1].immediate()),
142                    other => self.fatal(format!(
143                        "Unimplemented saturating_sub intrinsic type: {other:#?}"
144                    )),
145                };
146                // TODO: Implement this
147                self.zombie(result.def(self), "saturating_sub is not implemented yet");
148                result
149            }
150
151            sym::sqrtf32 | sym::sqrtf64 | sym::sqrtf128 => {
152                self.gl_op(GLOp::Sqrt, ret_ty, [args[0].immediate()])
153            }
154            sym::powif32 | sym::powif64 | sym::powif128 => {
155                let float = self.sitofp(args[1].immediate(), args[0].immediate().ty);
156                self.gl_op(GLOp::Pow, ret_ty, [args[0].immediate(), float])
157            }
158            sym::sinf32 | sym::sinf64 | sym::sinf128 => {
159                self.gl_op(GLOp::Sin, ret_ty, [args[0].immediate()])
160            }
161            sym::cosf32 | sym::cosf64 | sym::cosf128 => {
162                self.gl_op(GLOp::Cos, ret_ty, [args[0].immediate()])
163            }
164            sym::powf32 | sym::powf64 | sym::powf128 => self.gl_op(
165                GLOp::Pow,
166                ret_ty,
167                [args[0].immediate(), args[1].immediate()],
168            ),
169            sym::expf32 | sym::expf64 | sym::expf128 => {
170                self.gl_op(GLOp::Exp, ret_ty, [args[0].immediate()])
171            }
172            sym::exp2f32 | sym::exp2f64 | sym::exp2f128 => {
173                self.gl_op(GLOp::Exp2, ret_ty, [args[0].immediate()])
174            }
175            sym::logf32 | sym::logf64 | sym::logf128 => {
176                self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()])
177            }
178            sym::log2f32 | sym::log2f64 | sym::log2f128 => {
179                self.gl_op(GLOp::Log2, ret_ty, [args[0].immediate()])
180            }
181            sym::log10f32 | sym::log10f64 | sym::log10f128 => {
182                // spir-v glsl doesn't have log10, so,
183                // log10(x) == (1 / ln(10)) * ln(x)
184                let mul = self.constant_float(args[0].immediate().ty, 1.0 / 10.0f64.ln());
185                let ln = self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()]);
186                self.fmul(mul, ln)
187            }
188            sym::fmaf32 | sym::fmaf64 | sym::fmaf128 => self.gl_op(
189                GLOp::Fma,
190                ret_ty,
191                [
192                    args[0].immediate(),
193                    args[1].immediate(),
194                    args[2].immediate(),
195                ],
196            ),
197            sym::fabs => self.gl_op(GLOp::FAbs, ret_ty, [args[0].immediate()]),
198            sym::minimumf32 | sym::minimumf64 | sym::minimumf128 => self.gl_op(
199                GLOp::FMin,
200                ret_ty,
201                [args[0].immediate(), args[1].immediate()],
202            ),
203            sym::maximumf32 | sym::maximumf64 | sym::maximumf128 => self.gl_op(
204                GLOp::FMax,
205                ret_ty,
206                [args[0].immediate(), args[1].immediate()],
207            ),
208            sym::copysignf32 | sym::copysignf64 | sym::copysignf128 => {
209                let val = args[0].immediate();
210                let sign = args[1].immediate();
211                self.copysign(val, sign)
212            }
213            sym::floorf32 | sym::floorf64 | sym::floorf128 => {
214                self.gl_op(GLOp::Floor, ret_ty, [args[0].immediate()])
215            }
216            sym::ceilf32 | sym::ceilf64 | sym::ceilf128 => {
217                self.gl_op(GLOp::Ceil, ret_ty, [args[0].immediate()])
218            }
219            sym::truncf32 | sym::truncf64 | sym::truncf128 => {
220                self.gl_op(GLOp::Trunc, ret_ty, [args[0].immediate()])
221            }
222            sym::round_ties_even_f32 | sym::round_ties_even_f64 | sym::round_ties_even_f128 => {
223                self.gl_op(GLOp::RoundEven, ret_ty, [args[0].immediate()])
224            }
225            sym::roundf32 | sym::roundf64 | sym::roundf128 => {
226                self.gl_op(GLOp::Round, ret_ty, [args[0].immediate()])
227            }
228
229            sym::rotate_left | sym::rotate_right => {
230                let is_left = name == sym::rotate_left;
231                let val = args[0].immediate();
232                let shift = args[1].immediate();
233                self.rotate(val, shift, is_left)
234            }
235
236            sym::ctlz => self.count_leading_trailing_zeros(args[0].immediate(), false, false),
237            sym::ctlz_nonzero => {
238                self.count_leading_trailing_zeros(args[0].immediate(), false, true)
239            }
240            sym::cttz => self.count_leading_trailing_zeros(args[0].immediate(), true, false),
241            sym::cttz_nonzero => self.count_leading_trailing_zeros(args[0].immediate(), true, true),
242
243            sym::ctpop => self.count_ones(args[0].immediate()),
244            sym::bitreverse => self.bit_reverse(args[0].immediate()),
245            sym::black_box => {
246                // TODO(LegNeato): do something more sophisticated that prevents DCE
247                self.tcx
248                    .dcx()
249                    .warn("black_box intrinsic does not prevent optimization in Rust GPU");
250
251                let layout = self.layout_of(arg_tys[0]);
252                let llty = layout.spirv_type(self.span(), self);
253
254                match args[0].val {
255                    // Scalars pass through unchanged
256                    OperandValue::Immediate(v) => v,
257                    // Pack scalar pairs to a single SSA aggregate
258                    OperandValue::Pair(..) => args[0].immediate_or_packed_pair(self),
259                    // Lvalues get loaded
260                    OperandValue::Ref(place) => self.load(llty, place.llval, place.align),
261                    // ZSTs become undef of the right type
262                    OperandValue::ZeroSized => self.undef(llty),
263                }
264            }
265            sym::bswap => {
266                // https://github.com/KhronosGroup/SPIRV-LLVM/pull/221/files
267                // TODO: Definitely add tests to make sure this impl is right.
268                let arg = args[0].immediate();
269                let (width, is_signed) = int_type_width_signed(arg_tys[0], self)
270                    .expect("bswap must have an integer argument");
271
272                // Cast to unsigned type for byte-swapping
273                let unsigned_ty: u32 =
274                    SpirvType::Integer(width.try_into().unwrap(), false).def(self.span(), self);
275                let unsigned_arg = if is_signed {
276                    self.bitcast(arg, unsigned_ty)
277                } else {
278                    arg
279                };
280
281                let swapped = match width {
282                    8 => unsigned_arg,
283                    16 => {
284                        let offset8 = self.constant_u16(self.span(), 8);
285                        let tmp1 = self.shl(unsigned_arg, offset8);
286                        let tmp2 = self.lshr(unsigned_arg, offset8);
287                        self.or(tmp1, tmp2)
288                    }
289                    32 => {
290                        let offset8 = self.constant_u32(self.span(), 8);
291                        let offset24 = self.constant_u32(self.span(), 24);
292                        let mask16 = self.constant_u32(self.span(), 0xFF00);
293                        let mask24 = self.constant_u32(self.span(), 0xFF0000);
294                        let tmp4 = self.shl(unsigned_arg, offset24);
295                        let tmp3 = self.shl(unsigned_arg, offset8);
296                        let tmp2 = self.lshr(unsigned_arg, offset8);
297                        let tmp1 = self.lshr(unsigned_arg, offset24);
298                        let tmp3 = self.and(tmp3, mask24);
299                        let tmp2 = self.and(tmp2, mask16);
300                        let res1 = self.or(tmp1, tmp2);
301                        let res2 = self.or(tmp3, tmp4);
302                        self.or(res1, res2)
303                    }
304                    64 => {
305                        let offset8 = self.constant_u64(self.span(), 8);
306                        let offset24 = self.constant_u64(self.span(), 24);
307                        let offset40 = self.constant_u64(self.span(), 40);
308                        let offset56 = self.constant_u64(self.span(), 56);
309                        let mask16 = self.constant_u64(self.span(), 0xff00);
310                        let mask24 = self.constant_u64(self.span(), 0xff0000);
311                        let mask32 = self.constant_u64(self.span(), 0xff000000);
312                        let mask40 = self.constant_u64(self.span(), 0xff00000000);
313                        let mask48 = self.constant_u64(self.span(), 0xff0000000000);
314                        let mask56 = self.constant_u64(self.span(), 0xff000000000000);
315                        let tmp8 = self.shl(unsigned_arg, offset56);
316                        let tmp7 = self.shl(unsigned_arg, offset40);
317                        let tmp6 = self.shl(unsigned_arg, offset24);
318                        let tmp5 = self.shl(unsigned_arg, offset8);
319                        let tmp4 = self.lshr(unsigned_arg, offset8);
320                        let tmp3 = self.lshr(unsigned_arg, offset24);
321                        let tmp2 = self.lshr(unsigned_arg, offset40);
322                        let tmp1 = self.lshr(unsigned_arg, offset56);
323                        let tmp7 = self.and(tmp7, mask56);
324                        let tmp6 = self.and(tmp6, mask48);
325                        let tmp5 = self.and(tmp5, mask40);
326                        let tmp4 = self.and(tmp4, mask32);
327                        let tmp3 = self.and(tmp3, mask24);
328                        let tmp2 = self.and(tmp2, mask16);
329                        let res1 = self.or(tmp8, tmp7);
330                        let res2 = self.or(tmp6, tmp5);
331                        let res3 = self.or(tmp4, tmp3);
332                        let res4 = self.or(tmp2, tmp1);
333                        let res1 = self.or(res1, res2);
334                        let res3 = self.or(res3, res4);
335                        self.or(res1, res3)
336                    }
337                    other => self.undef_zombie(
338                        ret_ty,
339                        &format!("bswap not implemented for int width {other}"),
340                    ),
341                };
342
343                // Cast back to the original signed type if necessary
344                if is_signed {
345                    self.bitcast(swapped, arg.ty)
346                } else {
347                    swapped
348                }
349            }
350
351            sym::compare_bytes => self.undef_zombie(ret_ty, "memcmp not implemented"),
352
353            _ => {
354                // Call the fallback body instead of generating the intrinsic code
355                return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
356            }
357        };
358
359        if result.layout.ty.is_bool() {
360            let val = self.from_immediate(value);
361            self.store_to_place(val, result.val);
362        } else if !result.layout.ty.is_unit() {
363            // FIXME(eddyb) upstream uses `self.store_to_place(value, result.val);`,
364            // which AFAICT does not handle packed pairs explicitly, meaning it
365            // can/will store e.g. LLVM `{A, B}` values, which is legal (in LLVM),
366            // but seems suboptimal (or even risky with e.g. layout randomization).
367            OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
368                .val
369                .store(self, result);
370        }
371        Ok(())
372    }
373
374    fn codegen_llvm_intrinsic_call(
375        &mut self,
376        instance: ty::Instance<'tcx>,
377        _args: &[OperandRef<'tcx, Self::Value>],
378        _is_cleanup: bool,
379    ) -> Self::Value {
380        bug!("LLVM intrinsic call not supported in SPIR-V backend: {instance:?}")
381    }
382
383    fn abort(&mut self) {
384        self.abort_with_kind_and_message_debug_printf("abort", "intrinsics::abort() called", []);
385    }
386
387    // FIXME(eddyb) `assume` is not implemented atm, so all of its forms should
388    // avoid computing its (potentially illegal) bool input in the first place.
389    fn assume(&mut self, _val: Self::Value) {}
390
391    fn expect(&mut self, cond: Self::Value, _expected: bool) -> Self::Value {
392        // TODO: llvm.expect
393        cond
394    }
395
396    fn type_checked_load(
397        &mut self,
398        _llvtable: Self::Value,
399        _vtable_byte_offset: u64,
400        _typeid: &[u8],
401    ) -> Self::Value {
402        todo!()
403    }
404
405    fn va_start(&mut self, val: Self::Value) -> Self::Value {
406        // SPIR-V backend has no variadic ABI support; keep the placeholder
407        // operand unchanged so MIR lowering can proceed without crashing.
408        val
409    }
410
411    fn va_end(&mut self, val: Self::Value) -> Self::Value {
412        // See `va_start` above.
413        val
414    }
415
416    fn retag_mem(&mut self, _place: Self::Value, _info: &RetagInfo<Self::Value>) {
417        bug!("retag not supported")
418    }
419
420    fn retag_reg(&mut self, _ptr: Self::Value, _info: &RetagInfo<Self::Value>) -> Self::Value {
421        bug!("retag not supported")
422    }
423}
424
425impl Builder<'_, '_> {
426    pub fn count_ones(&mut self, arg: SpirvValue) -> SpirvValue {
427        let ty = arg.ty;
428        match self.cx.lookup_type(ty) {
429            SpirvType::Integer(bits, false) => {
430                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
431
432                match bits {
433                    8 | 16 => {
434                        let arg = arg.def(self);
435                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
436                        self.emit().bit_count(u32, None, arg).unwrap()
437                    }
438                    32 => self.emit().bit_count(u32, None, arg.def(self)).unwrap(),
439                    64 => {
440                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
441                        let arg = arg.def(self);
442                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
443                        let higher = self
444                            .emit()
445                            .shift_right_logical(ty, None, arg, u32_32)
446                            .unwrap();
447                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
448
449                        let lower_bits = self.emit().bit_count(u32, None, lower).unwrap();
450                        let higher_bits = self.emit().bit_count(u32, None, higher).unwrap();
451                        self.emit()
452                            .i_add(u32, None, lower_bits, higher_bits)
453                            .unwrap()
454                    }
455                    _ => {
456                        return self.undef_zombie(
457                            ty,
458                            &format!("count_ones() on unsupported {ty:?} bit integer type"),
459                        );
460                    }
461                }
462                .with_type(u32)
463            }
464            _ => self.fatal(format!(
465                "count_ones() expected an unsigned integer type, got {:?}",
466                self.cx.lookup_type(ty)
467            )),
468        }
469    }
470
471    pub fn bit_reverse(&mut self, arg: SpirvValue) -> SpirvValue {
472        let ty = arg.ty;
473        match self.cx.lookup_type(ty) {
474            SpirvType::Integer(bits, false) => {
475                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
476                let uint = SpirvType::Integer(bits, false).def(self.span(), self);
477
478                match bits {
479                    8 | 16 => {
480                        let arg = arg.def(self);
481                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
482
483                        let reverse = self.emit().bit_reverse(u32, None, arg).unwrap();
484                        let shift = self.constant_u32(self.span(), 32 - bits).def(self);
485                        let reverse = self
486                            .emit()
487                            .shift_right_logical(u32, None, reverse, shift)
488                            .unwrap();
489                        self.emit().u_convert(uint, None, reverse).unwrap()
490                    }
491                    32 => self.emit().bit_reverse(u32, None, arg.def(self)).unwrap(),
492                    64 => {
493                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
494                        let arg = arg.def(self);
495                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
496                        let higher = self
497                            .emit()
498                            .shift_right_logical(ty, None, arg, u32_32)
499                            .unwrap();
500                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
501
502                        // note that higher and lower have swapped
503                        let higher_bits = self.emit().bit_reverse(u32, None, lower).unwrap();
504                        let lower_bits = self.emit().bit_reverse(u32, None, higher).unwrap();
505
506                        let higher_bits = self.emit().u_convert(uint, None, higher_bits).unwrap();
507                        let higher_bits = self
508                            .emit()
509                            .shift_left_logical(uint, None, higher_bits, u32_32)
510                            .unwrap();
511                        let lower_bits = self.emit().u_convert(uint, None, lower_bits).unwrap();
512
513                        self.emit()
514                            .bitwise_or(ty, None, lower_bits, higher_bits)
515                            .unwrap()
516                    }
517                    _ => {
518                        return self.undef_zombie(
519                            ty,
520                            &format!("bit_reverse() on unsupported {ty:?} bit integer type"),
521                        );
522                    }
523                }
524                .with_type(ty)
525            }
526            _ => self.fatal(format!(
527                "bit_reverse() expected an unsigned integer type, got {:?}",
528                self.cx.lookup_type(ty)
529            )),
530        }
531    }
532
533    pub fn count_leading_trailing_zeros(
534        &mut self,
535        arg: SpirvValue,
536        trailing: bool,
537        non_zero: bool,
538    ) -> SpirvValue {
539        let ty = arg.ty;
540        match self.cx.lookup_type(ty) {
541            SpirvType::Integer(bits, false) => {
542                let bool = SpirvType::Bool.def(self.span(), self);
543                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
544
545                let glsl = self.ext_inst.borrow_mut().import_glsl(self);
546                let find_xsb = |this: &mut Self, arg, offset: i32| {
547                    if trailing {
548                        let lsb = this
549                            .emit()
550                            .ext_inst(
551                                u32,
552                                None,
553                                glsl,
554                                GLOp::FindILsb as u32,
555                                [Operand::IdRef(arg)],
556                            )
557                            .unwrap();
558                        if offset == 0 {
559                            lsb
560                        } else {
561                            let const_offset = this.constant_i32(this.span(), offset).def(this);
562                            this.emit().i_add(u32, None, const_offset, lsb).unwrap()
563                        }
564                    } else {
565                        // rust is always unsigned, so FindUMsb
566                        let msb_bit = this
567                            .emit()
568                            .ext_inst(
569                                u32,
570                                None,
571                                glsl,
572                                GLOp::FindUMsb as u32,
573                                [Operand::IdRef(arg)],
574                            )
575                            .unwrap();
576                        // the glsl op returns the Msb bit, not the amount of leading zeros of this u32
577                        // leading zeros = 31 - Msb bit
578                        let const_offset = this.constant_i32(this.span(), 31 - offset).def(this);
579                        this.emit().i_sub(u32, None, const_offset, msb_bit).unwrap()
580                    }
581                };
582
583                let converted = match bits {
584                    8 | 16 => {
585                        let arg = self.emit().u_convert(u32, None, arg.def(self)).unwrap();
586                        if trailing {
587                            find_xsb(self, arg, 0)
588                        } else {
589                            find_xsb(self, arg, bits as i32 - 32)
590                        }
591                    }
592                    32 => find_xsb(self, arg.def(self), 0),
593                    64 => {
594                        let u32_0 = self.constant_int(u32, 0).def(self);
595                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
596
597                        let arg = arg.def(self);
598                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
599                        let higher = self
600                            .emit()
601                            .shift_right_logical(ty, None, arg, u32_32)
602                            .unwrap();
603                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
604
605                        if trailing {
606                            let use_lower = self.emit().i_equal(bool, None, lower, u32_0).unwrap();
607                            let lower_bits = find_xsb(self, lower, 0);
608                            let higher_bits = find_xsb(self, higher, 32);
609                            self.emit()
610                                .select(u32, None, use_lower, higher_bits, lower_bits)
611                                .unwrap()
612                        } else {
613                            let use_higher =
614                                self.emit().i_equal(bool, None, higher, u32_0).unwrap();
615                            let lower_bits = find_xsb(self, lower, 0);
616                            let higher_bits = find_xsb(self, higher, 32);
617                            self.emit()
618                                .select(u32, None, use_higher, lower_bits, higher_bits)
619                                .unwrap()
620                        }
621                    }
622                    _ => {
623                        return self.undef_zombie(ty, &format!(
624                            "count_leading_trailing_zeros() on unsupported {ty:?} bit integer type"
625                        ));
626                    }
627                };
628
629                if non_zero {
630                    converted
631                } else {
632                    let int_0 = self.constant_int(ty, 0).def(self);
633                    let int_bits = self.constant_int(u32, bits as u128).def(self);
634                    let is_0 = self
635                        .emit()
636                        .i_equal(bool, None, arg.def(self), int_0)
637                        .unwrap();
638                    self.emit()
639                        .select(u32, None, is_0, int_bits, converted)
640                        .unwrap()
641                }
642                .with_type(u32)
643            }
644            SpirvType::Integer(bits, true) => {
645                // rustc wants `[i8,i16,i32,i64]::leading_zeros()` with `non_zero: true` for some reason. I do not know
646                // how these are reachable, marking them as zombies makes none of our compiletests fail.
647                let unsigned = SpirvType::Integer(bits, false).def(self.span(), self);
648                let arg = self
649                    .emit()
650                    .bitcast(unsigned, None, arg.def(self))
651                    .unwrap()
652                    .with_type(unsigned);
653                let result = self.count_leading_trailing_zeros(arg, trailing, non_zero);
654                self.emit()
655                    .bitcast(ty, None, result.def(self))
656                    .unwrap()
657                    .with_type(ty)
658            }
659            e => {
660                self.fatal(format!(
661                    "count_leading_trailing_zeros(trailing: {trailing}, non_zero: {non_zero}) expected an integer type, got {e:?}",
662                ));
663            }
664        }
665    }
666
667    pub fn abort_with_kind_and_message_debug_printf(
668        &mut self,
669        kind: &str,
670        message_debug_printf_fmt_str: impl Into<String>,
671        message_debug_printf_args: impl IntoIterator<Item = SpirvValue>,
672    ) {
673        // FIXME(eddyb) this should be cached more efficiently.
674        let void_ty = SpirvType::Void.def(rustc_span::DUMMY_SP, self);
675
676        // HACK(eddyb) there is no `abort` or `trap` instruction in SPIR-V,
677        // so the best thing we can do is use our own custom instruction.
678        let kind_id = self.emit().string(kind);
679        let message_debug_printf_fmt_str_id = self.emit().string(message_debug_printf_fmt_str);
680        self.custom_inst(
681            void_ty,
682            CustomInst::Abort {
683                kind: Operand::IdRef(kind_id),
684                message_debug_printf: [message_debug_printf_fmt_str_id]
685                    .into_iter()
686                    .chain(
687                        message_debug_printf_args
688                            .into_iter()
689                            .map(|arg| arg.def(self)),
690                    )
691                    .map(Operand::IdRef)
692                    .collect(),
693            },
694        );
695        self.unreachable();
696
697        // HACK(eddyb) we still need an active block in case the user of this
698        // `Builder` will continue to emit instructions after the `.abort()`.
699        let post_abort_dead_bb = self.append_sibling_block("post_abort_dead");
700        self.switch_to_block(post_abort_dead_bb);
701    }
702}
rustc_codegen_spirv/builder/intrinsics.rs

rustc_codegen_spirv/builder/
intrinsics.rs
