rustc_codegen_spirv/builder/
intrinsics.rs

1// HACK(eddyb) avoids rewriting all of the imports (see `lib.rs` and `build.rs`).
2use crate::maybe_pqp_cg_ssa as rustc_codegen_ssa;
3
4use super::Builder;
5use crate::abi::ConvSpirvType;
6use crate::builder_spirv::{SpirvValue, SpirvValueExt};
7use crate::codegen_cx::CodegenCx;
8use crate::custom_insts::CustomInst;
9use crate::spirv_type::SpirvType;
10use rspirv::dr::Operand;
11use rspirv::spirv::GLOp;
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::{BuilderMethods, IntrinsicCallBuilderMethods};
15use rustc_middle::ty::layout::LayoutOf;
16use rustc_middle::ty::{FnDef, Instance, Ty, TyKind, TypingEnv};
17use rustc_middle::{bug, ty};
18use rustc_span::sym;
19use rustc_span::{Span, Symbol};
20
21fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_>) -> Option<(u64, bool)> {
22    match ty.kind() {
23        TyKind::Int(t) => Some((
24            t.bit_width()
25                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
26            true,
27        )),
28        TyKind::Uint(t) => Some((
29            t.bit_width()
30                .unwrap_or(cx.tcx.sess.target.pointer_width as u64),
31            false,
32        )),
33        _ => None,
34    }
35}
36
37impl Builder<'_, '_> {
38    pub fn copysign(&mut self, val: SpirvValue, sign: SpirvValue) -> SpirvValue {
39        let width = match self.lookup_type(val.ty) {
40            SpirvType::Float(width) => width,
41            other => bug!(
42                "copysign must have float argument, not {}",
43                other.debug(val.ty, self)
44            ),
45        };
46        let int_ty = SpirvType::Integer(width, false).def(self.span(), self);
47        let [mask_sign, mask_value] = {
48            let sign_bit = 1u128.checked_shl(width - 1).unwrap();
49            let value_mask = sign_bit - 1;
50            [sign_bit, value_mask].map(|v| self.constant_int(int_ty, v))
51        };
52        let val_bits = self.bitcast(val, int_ty);
53        let sign_bits = self.bitcast(sign, int_ty);
54        let val_masked = self.and(val_bits, mask_value);
55        let sign_masked = self.and(sign_bits, mask_sign);
56        let result_bits = self.or(val_masked, sign_masked);
57        self.bitcast(result_bits, val.ty)
58    }
59}
60
61impl<'a, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'tcx> {
62    fn codegen_intrinsic_call(
63        &mut self,
64        instance: Instance<'tcx>,
65        args: &[OperandRef<'tcx, Self::Value>],
66        result: PlaceRef<'tcx, Self::Value>,
67        _span: Span,
68    ) -> Result<(), ty::Instance<'tcx>> {
69        let callee_ty = instance.ty(self.tcx, TypingEnv::fully_monomorphized());
70
71        let (def_id, fn_args) = match *callee_ty.kind() {
72            FnDef(def_id, fn_args) => (def_id, fn_args),
73            _ => bug!("expected fn item type, found {}", callee_ty),
74        };
75
76        let sig = callee_ty.fn_sig(self.tcx);
77        let sig = self
78            .tcx
79            .normalize_erasing_late_bound_regions(TypingEnv::fully_monomorphized(), sig);
80        let arg_tys = sig.inputs();
81        let name = self.tcx.item_name(def_id);
82
83        let ret_ty = self.layout_of(sig.output()).spirv_type(self.span(), self);
84
85        let value = match name {
86            // `sym::likely` no longer exists on newer rustc nightlies, but the
87            // intrinsic item name remains `likely`; ignore hint intrinsics.
88            _ if self.tcx.is_intrinsic(def_id, sym::unlikely)
89                || self.tcx.is_intrinsic(def_id, Symbol::intern("likely")) =>
90            {
91                args[0].immediate()
92            }
93
94            sym::breakpoint => {
95                self.abort();
96                assert!(result.layout.ty.is_unit());
97                return Ok(());
98            }
99
100            sym::volatile_load | sym::unaligned_volatile_load => {
101                let ptr = args[0].immediate();
102                let layout = self.layout_of(fn_args.type_at(0));
103                let load = self.volatile_load(layout.spirv_type(self.span(), self), ptr);
104                if !result.layout.is_zst() {
105                    self.store(load, result.val.llval, result.val.align);
106                }
107                return Ok(());
108            }
109
110            sym::prefetch_read_data
111            | sym::prefetch_write_data
112            | sym::prefetch_read_instruction
113            | sym::prefetch_write_instruction => {
114                // ignore
115                assert!(result.layout.ty.is_unit());
116                return Ok(());
117            }
118
119            sym::saturating_add => {
120                assert_eq!(arg_tys[0], arg_tys[1]);
121                let result = match arg_tys[0].kind() {
122                    TyKind::Int(_) | TyKind::Uint(_) => {
123                        self.add(args[0].immediate(), args[1].immediate())
124                    }
125                    TyKind::Float(_) => self.fadd(args[0].immediate(), args[1].immediate()),
126                    other => self.fatal(format!(
127                        "Unimplemented saturating_add intrinsic type: {other:#?}"
128                    )),
129                };
130                // TODO: Implement this
131                self.zombie(result.def(self), "saturating_add is not implemented yet");
132                result
133            }
134            sym::saturating_sub => {
135                assert_eq!(arg_tys[0], arg_tys[1]);
136                let result = match &arg_tys[0].kind() {
137                    TyKind::Int(_) | TyKind::Uint(_) => {
138                        self.sub(args[0].immediate(), args[1].immediate())
139                    }
140                    TyKind::Float(_) => self.fsub(args[0].immediate(), args[1].immediate()),
141                    other => self.fatal(format!(
142                        "Unimplemented saturating_sub intrinsic type: {other:#?}"
143                    )),
144                };
145                // TODO: Implement this
146                self.zombie(result.def(self), "saturating_sub is not implemented yet");
147                result
148            }
149
150            sym::sqrtf32 | sym::sqrtf64 | sym::sqrtf128 => {
151                self.gl_op(GLOp::Sqrt, ret_ty, [args[0].immediate()])
152            }
153            sym::powif32 | sym::powif64 | sym::powif128 => {
154                let float = self.sitofp(args[1].immediate(), args[0].immediate().ty);
155                self.gl_op(GLOp::Pow, ret_ty, [args[0].immediate(), float])
156            }
157            sym::sinf32 | sym::sinf64 | sym::sinf128 => {
158                self.gl_op(GLOp::Sin, ret_ty, [args[0].immediate()])
159            }
160            sym::cosf32 | sym::cosf64 | sym::cosf128 => {
161                self.gl_op(GLOp::Cos, ret_ty, [args[0].immediate()])
162            }
163            sym::powf32 | sym::powf64 | sym::powf128 => self.gl_op(
164                GLOp::Pow,
165                ret_ty,
166                [args[0].immediate(), args[1].immediate()],
167            ),
168            sym::expf32 | sym::expf64 | sym::expf128 => {
169                self.gl_op(GLOp::Exp, ret_ty, [args[0].immediate()])
170            }
171            sym::exp2f32 | sym::exp2f64 | sym::exp2f128 => {
172                self.gl_op(GLOp::Exp2, ret_ty, [args[0].immediate()])
173            }
174            sym::logf32 | sym::logf64 | sym::logf128 => {
175                self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()])
176            }
177            sym::log2f32 | sym::log2f64 | sym::log2f128 => {
178                self.gl_op(GLOp::Log2, ret_ty, [args[0].immediate()])
179            }
180            sym::log10f32 | sym::log10f64 | sym::log10f128 => {
181                // spir-v glsl doesn't have log10, so,
182                // log10(x) == (1 / ln(10)) * ln(x)
183                let mul = self.constant_float(args[0].immediate().ty, 1.0 / 10.0f64.ln());
184                let ln = self.gl_op(GLOp::Log, ret_ty, [args[0].immediate()]);
185                self.fmul(mul, ln)
186            }
187            sym::fmaf32 | sym::fmaf64 | sym::fmaf128 => self.gl_op(
188                GLOp::Fma,
189                ret_ty,
190                [
191                    args[0].immediate(),
192                    args[1].immediate(),
193                    args[2].immediate(),
194                ],
195            ),
196            sym::fabs => self.gl_op(GLOp::FAbs, ret_ty, [args[0].immediate()]),
197            sym::minimumf32 | sym::minimumf64 | sym::minimumf128 => self.gl_op(
198                GLOp::FMin,
199                ret_ty,
200                [args[0].immediate(), args[1].immediate()],
201            ),
202            sym::maximumf32 | sym::maximumf64 | sym::maximumf128 => self.gl_op(
203                GLOp::FMax,
204                ret_ty,
205                [args[0].immediate(), args[1].immediate()],
206            ),
207            sym::copysignf32 | sym::copysignf64 | sym::copysignf128 => {
208                let val = args[0].immediate();
209                let sign = args[1].immediate();
210                self.copysign(val, sign)
211            }
212            sym::floorf32 | sym::floorf64 | sym::floorf128 => {
213                self.gl_op(GLOp::Floor, ret_ty, [args[0].immediate()])
214            }
215            sym::ceilf32 | sym::ceilf64 | sym::ceilf128 => {
216                self.gl_op(GLOp::Ceil, ret_ty, [args[0].immediate()])
217            }
218            sym::truncf32 | sym::truncf64 | sym::truncf128 => {
219                self.gl_op(GLOp::Trunc, ret_ty, [args[0].immediate()])
220            }
221            sym::round_ties_even_f32 | sym::round_ties_even_f64 | sym::round_ties_even_f128 => {
222                self.gl_op(GLOp::RoundEven, ret_ty, [args[0].immediate()])
223            }
224            sym::roundf32 | sym::roundf64 | sym::roundf128 => {
225                self.gl_op(GLOp::Round, ret_ty, [args[0].immediate()])
226            }
227
228            sym::rotate_left | sym::rotate_right => {
229                let is_left = name == sym::rotate_left;
230                let val = args[0].immediate();
231                let shift = args[1].immediate();
232                self.rotate(val, shift, is_left)
233            }
234
235            sym::ctlz => self.count_leading_trailing_zeros(args[0].immediate(), false, false),
236            sym::ctlz_nonzero => {
237                self.count_leading_trailing_zeros(args[0].immediate(), false, true)
238            }
239            sym::cttz => self.count_leading_trailing_zeros(args[0].immediate(), true, false),
240            sym::cttz_nonzero => self.count_leading_trailing_zeros(args[0].immediate(), true, true),
241
242            sym::ctpop => self.count_ones(args[0].immediate()),
243            sym::bitreverse => self.bit_reverse(args[0].immediate()),
244            sym::black_box => {
245                // TODO(LegNeato): do something more sophisticated that prevents DCE
246                self.tcx
247                    .dcx()
248                    .warn("black_box intrinsic does not prevent optimization in Rust GPU");
249
250                let layout = self.layout_of(arg_tys[0]);
251                let llty = layout.spirv_type(self.span(), self);
252
253                match args[0].val {
254                    // Scalars pass through unchanged
255                    OperandValue::Immediate(v) => v,
256                    // Pack scalar pairs to a single SSA aggregate
257                    OperandValue::Pair(..) => args[0].immediate_or_packed_pair(self),
258                    // Lvalues get loaded
259                    OperandValue::Ref(place) => self.load(llty, place.llval, place.align),
260                    // ZSTs become undef of the right type
261                    OperandValue::ZeroSized => self.undef(llty),
262                }
263            }
264            sym::bswap => {
265                // https://github.com/KhronosGroup/SPIRV-LLVM/pull/221/files
266                // TODO: Definitely add tests to make sure this impl is right.
267                let arg = args[0].immediate();
268                let (width, is_signed) = int_type_width_signed(arg_tys[0], self)
269                    .expect("bswap must have an integer argument");
270
271                // Cast to unsigned type for byte-swapping
272                let unsigned_ty: u32 =
273                    SpirvType::Integer(width.try_into().unwrap(), false).def(self.span(), self);
274                let unsigned_arg = if is_signed {
275                    self.bitcast(arg, unsigned_ty)
276                } else {
277                    arg
278                };
279
280                let swapped = match width {
281                    8 => unsigned_arg,
282                    16 => {
283                        let offset8 = self.constant_u16(self.span(), 8);
284                        let tmp1 = self.shl(unsigned_arg, offset8);
285                        let tmp2 = self.lshr(unsigned_arg, offset8);
286                        self.or(tmp1, tmp2)
287                    }
288                    32 => {
289                        let offset8 = self.constant_u32(self.span(), 8);
290                        let offset24 = self.constant_u32(self.span(), 24);
291                        let mask16 = self.constant_u32(self.span(), 0xFF00);
292                        let mask24 = self.constant_u32(self.span(), 0xFF0000);
293                        let tmp4 = self.shl(unsigned_arg, offset24);
294                        let tmp3 = self.shl(unsigned_arg, offset8);
295                        let tmp2 = self.lshr(unsigned_arg, offset8);
296                        let tmp1 = self.lshr(unsigned_arg, offset24);
297                        let tmp3 = self.and(tmp3, mask24);
298                        let tmp2 = self.and(tmp2, mask16);
299                        let res1 = self.or(tmp1, tmp2);
300                        let res2 = self.or(tmp3, tmp4);
301                        self.or(res1, res2)
302                    }
303                    64 => {
304                        let offset8 = self.constant_u64(self.span(), 8);
305                        let offset24 = self.constant_u64(self.span(), 24);
306                        let offset40 = self.constant_u64(self.span(), 40);
307                        let offset56 = self.constant_u64(self.span(), 56);
308                        let mask16 = self.constant_u64(self.span(), 0xff00);
309                        let mask24 = self.constant_u64(self.span(), 0xff0000);
310                        let mask32 = self.constant_u64(self.span(), 0xff000000);
311                        let mask40 = self.constant_u64(self.span(), 0xff00000000);
312                        let mask48 = self.constant_u64(self.span(), 0xff0000000000);
313                        let mask56 = self.constant_u64(self.span(), 0xff000000000000);
314                        let tmp8 = self.shl(unsigned_arg, offset56);
315                        let tmp7 = self.shl(unsigned_arg, offset40);
316                        let tmp6 = self.shl(unsigned_arg, offset24);
317                        let tmp5 = self.shl(unsigned_arg, offset8);
318                        let tmp4 = self.lshr(unsigned_arg, offset8);
319                        let tmp3 = self.lshr(unsigned_arg, offset24);
320                        let tmp2 = self.lshr(unsigned_arg, offset40);
321                        let tmp1 = self.lshr(unsigned_arg, offset56);
322                        let tmp7 = self.and(tmp7, mask56);
323                        let tmp6 = self.and(tmp6, mask48);
324                        let tmp5 = self.and(tmp5, mask40);
325                        let tmp4 = self.and(tmp4, mask32);
326                        let tmp3 = self.and(tmp3, mask24);
327                        let tmp2 = self.and(tmp2, mask16);
328                        let res1 = self.or(tmp8, tmp7);
329                        let res2 = self.or(tmp6, tmp5);
330                        let res3 = self.or(tmp4, tmp3);
331                        let res4 = self.or(tmp2, tmp1);
332                        let res1 = self.or(res1, res2);
333                        let res3 = self.or(res3, res4);
334                        self.or(res1, res3)
335                    }
336                    other => self.undef_zombie(
337                        ret_ty,
338                        &format!("bswap not implemented for int width {other}"),
339                    ),
340                };
341
342                // Cast back to the original signed type if necessary
343                if is_signed {
344                    self.bitcast(swapped, arg.ty)
345                } else {
346                    swapped
347                }
348            }
349
350            sym::compare_bytes => self.undef_zombie(ret_ty, "memcmp not implemented"),
351
352            _ => {
353                // Call the fallback body instead of generating the intrinsic code
354                return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
355            }
356        };
357
358        if result.layout.ty.is_bool() {
359            let val = self.from_immediate(value);
360            self.store_to_place(val, result.val);
361        } else if !result.layout.ty.is_unit() {
362            // FIXME(eddyb) upstream uses `self.store_to_place(value, result.val);`,
363            // which AFAICT does not handle packed pairs explicitly, meaning it
364            // can/will store e.g. LLVM `{A, B}` values, which is legal (in LLVM),
365            // but seems suboptimal (or even risky with e.g. layout randomization).
366            OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
367                .val
368                .store(self, result);
369        }
370        Ok(())
371    }
372
373    fn codegen_llvm_intrinsic_call(
374        &mut self,
375        instance: ty::Instance<'tcx>,
376        _args: &[OperandRef<'tcx, Self::Value>],
377        _is_cleanup: bool,
378    ) -> Self::Value {
379        bug!("LLVM intrinsic call not supported in SPIR-V backend: {instance:?}")
380    }
381
382    fn abort(&mut self) {
383        self.abort_with_kind_and_message_debug_printf("abort", "intrinsics::abort() called", []);
384    }
385
386    // FIXME(eddyb) `assume` is not implemented atm, so all of its forms should
387    // avoid computing its (potentially illegal) bool input in the first place.
388    fn assume(&mut self, _val: Self::Value) {}
389
390    fn expect(&mut self, cond: Self::Value, _expected: bool) -> Self::Value {
391        // TODO: llvm.expect
392        cond
393    }
394
395    fn type_checked_load(
396        &mut self,
397        _llvtable: Self::Value,
398        _vtable_byte_offset: u64,
399        _typeid: &[u8],
400    ) -> Self::Value {
401        todo!()
402    }
403
404    fn va_start(&mut self, val: Self::Value) -> Self::Value {
405        // SPIR-V backend has no variadic ABI support; keep the placeholder
406        // operand unchanged so MIR lowering can proceed without crashing.
407        val
408    }
409
410    fn va_end(&mut self, val: Self::Value) -> Self::Value {
411        // See `va_start` above.
412        val
413    }
414}
415
416impl Builder<'_, '_> {
417    pub fn count_ones(&mut self, arg: SpirvValue) -> SpirvValue {
418        let ty = arg.ty;
419        match self.cx.lookup_type(ty) {
420            SpirvType::Integer(bits, false) => {
421                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
422
423                match bits {
424                    8 | 16 => {
425                        let arg = arg.def(self);
426                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
427                        self.emit().bit_count(u32, None, arg).unwrap()
428                    }
429                    32 => self.emit().bit_count(u32, None, arg.def(self)).unwrap(),
430                    64 => {
431                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
432                        let arg = arg.def(self);
433                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
434                        let higher = self
435                            .emit()
436                            .shift_right_logical(ty, None, arg, u32_32)
437                            .unwrap();
438                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
439
440                        let lower_bits = self.emit().bit_count(u32, None, lower).unwrap();
441                        let higher_bits = self.emit().bit_count(u32, None, higher).unwrap();
442                        self.emit()
443                            .i_add(u32, None, lower_bits, higher_bits)
444                            .unwrap()
445                    }
446                    _ => {
447                        return self.undef_zombie(
448                            ty,
449                            &format!("count_ones() on unsupported {ty:?} bit integer type"),
450                        );
451                    }
452                }
453                .with_type(u32)
454            }
455            _ => self.fatal(format!(
456                "count_ones() expected an unsigned integer type, got {:?}",
457                self.cx.lookup_type(ty)
458            )),
459        }
460    }
461
462    pub fn bit_reverse(&mut self, arg: SpirvValue) -> SpirvValue {
463        let ty = arg.ty;
464        match self.cx.lookup_type(ty) {
465            SpirvType::Integer(bits, false) => {
466                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
467                let uint = SpirvType::Integer(bits, false).def(self.span(), self);
468
469                match bits {
470                    8 | 16 => {
471                        let arg = arg.def(self);
472                        let arg = self.emit().u_convert(u32, None, arg).unwrap();
473
474                        let reverse = self.emit().bit_reverse(u32, None, arg).unwrap();
475                        let shift = self.constant_u32(self.span(), 32 - bits).def(self);
476                        let reverse = self
477                            .emit()
478                            .shift_right_logical(u32, None, reverse, shift)
479                            .unwrap();
480                        self.emit().u_convert(uint, None, reverse).unwrap()
481                    }
482                    32 => self.emit().bit_reverse(u32, None, arg.def(self)).unwrap(),
483                    64 => {
484                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
485                        let arg = arg.def(self);
486                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
487                        let higher = self
488                            .emit()
489                            .shift_right_logical(ty, None, arg, u32_32)
490                            .unwrap();
491                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
492
493                        // note that higher and lower have swapped
494                        let higher_bits = self.emit().bit_reverse(u32, None, lower).unwrap();
495                        let lower_bits = self.emit().bit_reverse(u32, None, higher).unwrap();
496
497                        let higher_bits = self.emit().u_convert(uint, None, higher_bits).unwrap();
498                        let higher_bits = self
499                            .emit()
500                            .shift_left_logical(uint, None, higher_bits, u32_32)
501                            .unwrap();
502                        let lower_bits = self.emit().u_convert(uint, None, lower_bits).unwrap();
503
504                        self.emit()
505                            .bitwise_or(ty, None, lower_bits, higher_bits)
506                            .unwrap()
507                    }
508                    _ => {
509                        return self.undef_zombie(
510                            ty,
511                            &format!("bit_reverse() on unsupported {ty:?} bit integer type"),
512                        );
513                    }
514                }
515                .with_type(ty)
516            }
517            _ => self.fatal(format!(
518                "bit_reverse() expected an unsigned integer type, got {:?}",
519                self.cx.lookup_type(ty)
520            )),
521        }
522    }
523
524    pub fn count_leading_trailing_zeros(
525        &mut self,
526        arg: SpirvValue,
527        trailing: bool,
528        non_zero: bool,
529    ) -> SpirvValue {
530        let ty = arg.ty;
531        match self.cx.lookup_type(ty) {
532            SpirvType::Integer(bits, false) => {
533                let bool = SpirvType::Bool.def(self.span(), self);
534                let u32 = SpirvType::Integer(32, false).def(self.span(), self);
535
536                let glsl = self.ext_inst.borrow_mut().import_glsl(self);
537                let find_xsb = |this: &mut Self, arg, offset: i32| {
538                    if trailing {
539                        let lsb = this
540                            .emit()
541                            .ext_inst(
542                                u32,
543                                None,
544                                glsl,
545                                GLOp::FindILsb as u32,
546                                [Operand::IdRef(arg)],
547                            )
548                            .unwrap();
549                        if offset == 0 {
550                            lsb
551                        } else {
552                            let const_offset = this.constant_i32(this.span(), offset).def(this);
553                            this.emit().i_add(u32, None, const_offset, lsb).unwrap()
554                        }
555                    } else {
556                        // rust is always unsigned, so FindUMsb
557                        let msb_bit = this
558                            .emit()
559                            .ext_inst(
560                                u32,
561                                None,
562                                glsl,
563                                GLOp::FindUMsb as u32,
564                                [Operand::IdRef(arg)],
565                            )
566                            .unwrap();
567                        // the glsl op returns the Msb bit, not the amount of leading zeros of this u32
568                        // leading zeros = 31 - Msb bit
569                        let const_offset = this.constant_i32(this.span(), 31 - offset).def(this);
570                        this.emit().i_sub(u32, None, const_offset, msb_bit).unwrap()
571                    }
572                };
573
574                let converted = match bits {
575                    8 | 16 => {
576                        let arg = self.emit().u_convert(u32, None, arg.def(self)).unwrap();
577                        if trailing {
578                            find_xsb(self, arg, 0)
579                        } else {
580                            find_xsb(self, arg, bits as i32 - 32)
581                        }
582                    }
583                    32 => find_xsb(self, arg.def(self), 0),
584                    64 => {
585                        let u32_0 = self.constant_int(u32, 0).def(self);
586                        let u32_32 = self.constant_u32(self.span(), 32).def(self);
587
588                        let arg = arg.def(self);
589                        let lower = self.emit().u_convert(u32, None, arg).unwrap();
590                        let higher = self
591                            .emit()
592                            .shift_right_logical(ty, None, arg, u32_32)
593                            .unwrap();
594                        let higher = self.emit().u_convert(u32, None, higher).unwrap();
595
596                        if trailing {
597                            let use_lower = self.emit().i_equal(bool, None, lower, u32_0).unwrap();
598                            let lower_bits = find_xsb(self, lower, 0);
599                            let higher_bits = find_xsb(self, higher, 32);
600                            self.emit()
601                                .select(u32, None, use_lower, higher_bits, lower_bits)
602                                .unwrap()
603                        } else {
604                            let use_higher =
605                                self.emit().i_equal(bool, None, higher, u32_0).unwrap();
606                            let lower_bits = find_xsb(self, lower, 0);
607                            let higher_bits = find_xsb(self, higher, 32);
608                            self.emit()
609                                .select(u32, None, use_higher, lower_bits, higher_bits)
610                                .unwrap()
611                        }
612                    }
613                    _ => {
614                        return self.undef_zombie(ty, &format!(
615                            "count_leading_trailing_zeros() on unsupported {ty:?} bit integer type"
616                        ));
617                    }
618                };
619
620                if non_zero {
621                    converted
622                } else {
623                    let int_0 = self.constant_int(ty, 0).def(self);
624                    let int_bits = self.constant_int(u32, bits as u128).def(self);
625                    let is_0 = self
626                        .emit()
627                        .i_equal(bool, None, arg.def(self), int_0)
628                        .unwrap();
629                    self.emit()
630                        .select(u32, None, is_0, int_bits, converted)
631                        .unwrap()
632                }
633                .with_type(u32)
634            }
635            SpirvType::Integer(bits, true) => {
636                // rustc wants `[i8,i16,i32,i64]::leading_zeros()` with `non_zero: true` for some reason. I do not know
637                // how these are reachable, marking them as zombies makes none of our compiletests fail.
638                let unsigned = SpirvType::Integer(bits, false).def(self.span(), self);
639                let arg = self
640                    .emit()
641                    .bitcast(unsigned, None, arg.def(self))
642                    .unwrap()
643                    .with_type(unsigned);
644                let result = self.count_leading_trailing_zeros(arg, trailing, non_zero);
645                self.emit()
646                    .bitcast(ty, None, result.def(self))
647                    .unwrap()
648                    .with_type(ty)
649            }
650            e => {
651                self.fatal(format!(
652                    "count_leading_trailing_zeros(trailing: {trailing}, non_zero: {non_zero}) expected an integer type, got {e:?}",
653                ));
654            }
655        }
656    }
657
658    pub fn abort_with_kind_and_message_debug_printf(
659        &mut self,
660        kind: &str,
661        message_debug_printf_fmt_str: impl Into<String>,
662        message_debug_printf_args: impl IntoIterator<Item = SpirvValue>,
663    ) {
664        // FIXME(eddyb) this should be cached more efficiently.
665        let void_ty = SpirvType::Void.def(rustc_span::DUMMY_SP, self);
666
667        // HACK(eddyb) there is no `abort` or `trap` instruction in SPIR-V,
668        // so the best thing we can do is use our own custom instruction.
669        let kind_id = self.emit().string(kind);
670        let message_debug_printf_fmt_str_id = self.emit().string(message_debug_printf_fmt_str);
671        self.custom_inst(
672            void_ty,
673            CustomInst::Abort {
674                kind: Operand::IdRef(kind_id),
675                message_debug_printf: [message_debug_printf_fmt_str_id]
676                    .into_iter()
677                    .chain(
678                        message_debug_printf_args
679                            .into_iter()
680                            .map(|arg| arg.def(self)),
681                    )
682                    .map(Operand::IdRef)
683                    .collect(),
684            },
685        );
686        self.unreachable();
687
688        // HACK(eddyb) we still need an active block in case the user of this
689        // `Builder` will continue to emit instructions after the `.abort()`.
690        let post_abort_dead_bb = self.append_sibling_block("post_abort_dead");
691        self.switch_to_block(post_abort_dead_bb);
692    }
693}
rustc_codegen_spirv/builder/intrinsics.rs

rustc_codegen_spirv/builder/
intrinsics.rs
