spirt/qptr/
lift.rs

1//! [`QPtr`](crate::TypeKind::QPtr) lifting (e.g. to SPIR-V).
2
3// HACK(eddyb) sharing layout code with other modules.
4use super::layout::*;
5
6use crate::func_at::FuncAtMut;
7use crate::qptr::{QPtrAttr, QPtrMemUsage, QPtrMemUsageKind, QPtrOp, QPtrUsage, shapes};
8use crate::transform::{InnerInPlaceTransform, InnerTransform, Transformed, Transformer};
9use crate::{
10    AddrSpace, Attr, AttrSet, AttrSetDef, Const, ConstDef, ConstKind, Context, ControlNode,
11    ControlNodeKind, DataInst, DataInstDef, DataInstFormDef, DataInstKind, DeclDef, Diag,
12    DiagLevel, EntityDefs, EntityOrientedDenseMap, Func, FuncDecl, FxIndexMap, GlobalVar,
13    GlobalVarDecl, Module, Type, TypeDef, TypeKind, TypeOrConst, Value, spv,
14};
15use smallvec::SmallVec;
16use std::cell::Cell;
17use std::mem;
18use std::num::NonZeroU32;
19use std::rc::Rc;
20
21struct LiftError(Diag);
22
23/// Context for lifting `QPtr`s to SPIR-V `OpTypePointer`s.
24///
25/// See also `passes::qptr::lift_to_spv_ptrs` (which drives this).
26pub struct LiftToSpvPtrs<'a> {
27    cx: Rc<Context>,
28    wk: &'static spv::spec::WellKnown,
29    layout_cache: LayoutCache<'a>,
30
31    cached_u32_type: Cell<Option<Type>>,
32}
33
34impl<'a> LiftToSpvPtrs<'a> {
35    pub fn new(cx: Rc<Context>, layout_config: &'a LayoutConfig) -> Self {
36        Self {
37            cx: cx.clone(),
38            wk: &spv::spec::Spec::get().well_known,
39            layout_cache: LayoutCache::new(cx, layout_config),
40            cached_u32_type: Default::default(),
41        }
42    }
43
44    pub fn lift_global_var(&self, global_var_decl: &mut GlobalVarDecl) {
45        match self.spv_ptr_type_and_addr_space_for_global_var(global_var_decl) {
46            Ok((spv_ptr_type, addr_space)) => {
47                global_var_decl.attrs = self.strip_qptr_usage_attr(global_var_decl.attrs);
48                global_var_decl.type_of_ptr_to = spv_ptr_type;
49                global_var_decl.addr_space = addr_space;
50                global_var_decl.shape = None;
51            }
52            Err(LiftError(e)) => {
53                global_var_decl.attrs.push_diag(&self.cx, e);
54            }
55        }
56        // FIXME(eddyb) if globals have initializers pointing at other globals,
57        // here is where they might get fixed up, but that usage is illegal so
58        // likely needs to get legalized on `qptr`s, before here.
59    }
60
61    pub fn lift_all_funcs(&self, module: &mut Module, funcs: impl IntoIterator<Item = Func>) {
62        for func in funcs {
63            LiftToSpvPtrInstsInFunc {
64                lifter: self,
65                global_vars: &module.global_vars,
66
67                deferred_ptr_noops: Default::default(),
68                data_inst_use_counts: Default::default(),
69
70                func_has_qptr_analysis_bug_diags: false,
71            }
72            .in_place_transform_func_decl(&mut module.funcs[func]);
73        }
74    }
75
76    fn find_qptr_usage_attr(&self, attrs: AttrSet) -> Result<&QPtrUsage, LiftError> {
77        self.cx[attrs]
78            .attrs
79            .iter()
80            .find_map(|attr| match attr {
81                Attr::QPtr(QPtrAttr::Usage(usage)) => Some(&usage.0),
82                _ => None,
83            })
84            .ok_or_else(|| LiftError(Diag::bug(["missing `qptr.usage` attribute".into()])))
85    }
86
87    fn strip_qptr_usage_attr(&self, attrs: AttrSet) -> AttrSet {
88        self.cx.intern(AttrSetDef {
89            attrs: self.cx[attrs]
90                .attrs
91                .iter()
92                .filter(|attr| !matches!(attr, Attr::QPtr(QPtrAttr::Usage(_))))
93                .cloned()
94                .collect(),
95        })
96    }
97
98    fn spv_ptr_type_and_addr_space_for_global_var(
99        &self,
100        global_var_decl: &GlobalVarDecl,
101    ) -> Result<(Type, AddrSpace), LiftError> {
102        let wk = self.wk;
103
104        let qptr_usage = self.find_qptr_usage_attr(global_var_decl.attrs)?;
105
106        let shape =
107            global_var_decl.shape.ok_or_else(|| LiftError(Diag::bug(["missing shape".into()])))?;
108        let (storage_class, pointee_type) = match (global_var_decl.addr_space, shape) {
109            (AddrSpace::Handles, shapes::GlobalVarShape::Handles { handle, fixed_count }) => {
110                let (storage_class, handle_type) = match handle {
111                    shapes::Handle::Opaque(ty) => {
112                        if self.pointee_type_for_usage(qptr_usage)? != ty {
113                            return Err(LiftError(Diag::bug([
114                                "mismatched opaque handle types in `qptr.usage` vs `shape`".into(),
115                            ])));
116                        }
117                        (wk.UniformConstant, ty)
118                    }
119                    // FIXME(eddyb) validate usage against `buf` and/or expand
120                    // the type to make sure it has the right size.
121                    shapes::Handle::Buffer(AddrSpace::SpvStorageClass(storage_class), _buf) => {
122                        (storage_class, self.pointee_type_for_usage(qptr_usage)?)
123                    }
124                    shapes::Handle::Buffer(AddrSpace::Handles, _) => {
125                        return Err(LiftError(Diag::bug([
126                            "invalid `AddrSpace::Handles` in `Handle::Buffer`".into(),
127                        ])));
128                    }
129                };
130                (
131                    storage_class,
132                    if fixed_count == Some(NonZeroU32::new(1).unwrap()) {
133                        handle_type
134                    } else {
135                        self.spv_op_type_array(handle_type, fixed_count.map(|c| c.get()), None)?
136                    },
137                )
138            }
139            // FIXME(eddyb) validate usage against `layout` and/or expand
140            // the type to make sure it has the right size.
141            (
142                AddrSpace::SpvStorageClass(storage_class),
143                shapes::GlobalVarShape::UntypedData(_layout),
144            ) => (storage_class, self.pointee_type_for_usage(qptr_usage)?),
145            (
146                AddrSpace::SpvStorageClass(storage_class),
147                shapes::GlobalVarShape::TypedInterface(ty),
148            ) => (storage_class, ty),
149
150            (
151                AddrSpace::Handles,
152                shapes::GlobalVarShape::UntypedData(_) | shapes::GlobalVarShape::TypedInterface(_),
153            )
154            | (AddrSpace::SpvStorageClass(_), shapes::GlobalVarShape::Handles { .. }) => {
155                return Err(LiftError(Diag::bug(["mismatched `addr_space` and `shape`".into()])));
156            }
157        };
158        let addr_space = AddrSpace::SpvStorageClass(storage_class);
159        Ok((self.spv_ptr_type(addr_space, pointee_type), addr_space))
160    }
161
162    /// Returns `Some` iff `ty` is a SPIR-V `OpTypePointer`.
163    //
164    // FIXME(eddyb) deduplicate with `qptr::lower`.
165    fn as_spv_ptr_type(&self, ty: Type) -> Option<(AddrSpace, Type)> {
166        match &self.cx[ty].kind {
167            TypeKind::SpvInst { spv_inst, type_and_const_inputs }
168                if spv_inst.opcode == self.wk.OpTypePointer =>
169            {
170                let sc = match spv_inst.imms[..] {
171                    [spv::Imm::Short(_, sc)] => sc,
172                    _ => unreachable!(),
173                };
174                let pointee = match type_and_const_inputs[..] {
175                    [TypeOrConst::Type(elem_type)] => elem_type,
176                    _ => unreachable!(),
177                };
178                Some((AddrSpace::SpvStorageClass(sc), pointee))
179            }
180            _ => None,
181        }
182    }
183
184    fn spv_ptr_type(&self, addr_space: AddrSpace, pointee_type: Type) -> Type {
185        let wk = self.wk;
186
187        let storage_class = match addr_space {
188            AddrSpace::Handles => unreachable!(),
189            AddrSpace::SpvStorageClass(storage_class) => storage_class,
190        };
191        self.cx.intern(TypeKind::SpvInst {
192            spv_inst: spv::Inst {
193                opcode: wk.OpTypePointer,
194                imms: [spv::Imm::Short(wk.StorageClass, storage_class)].into_iter().collect(),
195            },
196            type_and_const_inputs: [TypeOrConst::Type(pointee_type)].into_iter().collect(),
197        })
198    }
199
200    fn pointee_type_for_usage(&self, usage: &QPtrUsage) -> Result<Type, LiftError> {
201        let wk = self.wk;
202
203        match usage {
204            &QPtrUsage::Handles(shapes::Handle::Opaque(ty)) => Ok(ty),
205            QPtrUsage::Handles(shapes::Handle::Buffer(_, data_usage)) => {
206                let attr_spv_decorate_block = Attr::SpvAnnotation(spv::Inst {
207                    opcode: wk.OpDecorate,
208                    imms: [spv::Imm::Short(wk.Decoration, wk.Block)].into_iter().collect(),
209                });
210                match &data_usage.kind {
211                    QPtrMemUsageKind::Unused => {
212                        self.spv_op_type_struct([], [attr_spv_decorate_block])
213                    }
214                    QPtrMemUsageKind::OffsetBase(fields) => self.spv_op_type_struct(
215                        fields.iter().map(|(&field_offset, field_usage)| {
216                            Ok((field_offset, self.pointee_type_for_mem_usage(field_usage)?))
217                        }),
218                        [attr_spv_decorate_block],
219                    ),
220                    QPtrMemUsageKind::StrictlyTyped(_)
221                    | QPtrMemUsageKind::DirectAccess(_)
222                    | QPtrMemUsageKind::DynOffsetBase { .. } => self.spv_op_type_struct(
223                        [Ok((0, self.pointee_type_for_mem_usage(data_usage)?))],
224                        [attr_spv_decorate_block],
225                    ),
226                }
227            }
228            QPtrUsage::Memory(usage) => self.pointee_type_for_mem_usage(usage),
229        }
230    }
231
232    fn pointee_type_for_mem_usage(&self, usage: &QPtrMemUsage) -> Result<Type, LiftError> {
233        match &usage.kind {
234            QPtrMemUsageKind::Unused => self.spv_op_type_struct([], []),
235            &QPtrMemUsageKind::StrictlyTyped(ty) | &QPtrMemUsageKind::DirectAccess(ty) => Ok(ty),
236            QPtrMemUsageKind::OffsetBase(fields) => self.spv_op_type_struct(
237                fields.iter().map(|(&field_offset, field_usage)| {
238                    Ok((field_offset, self.pointee_type_for_mem_usage(field_usage)?))
239                }),
240                [],
241            ),
242            QPtrMemUsageKind::DynOffsetBase { element, stride } => {
243                let element_type = self.pointee_type_for_mem_usage(element)?;
244
245                let fixed_len = usage
246                    .max_size
247                    .map(|size| {
248                        if size % stride.get() != 0 {
249                            return Err(LiftError(Diag::bug([format!(
250                                "DynOffsetBase: size ({size}) not a multiple of stride ({stride})"
251                            )
252                            .into()])));
253                        }
254                        Ok(size / stride.get())
255                    })
256                    .transpose()?;
257
258                self.spv_op_type_array(element_type, fixed_len, Some(*stride))
259            }
260        }
261    }
262
263    fn spv_op_type_array(
264        &self,
265        element_type: Type,
266        fixed_len: Option<u32>,
267        stride: Option<NonZeroU32>,
268    ) -> Result<Type, LiftError> {
269        let wk = self.wk;
270
271        let stride_attrs = stride.map(|stride| {
272            self.cx.intern(AttrSetDef {
273                attrs: [Attr::SpvAnnotation(spv::Inst {
274                    opcode: wk.OpDecorate,
275                    imms: [
276                        spv::Imm::Short(wk.Decoration, wk.ArrayStride),
277                        spv::Imm::Short(wk.LiteralInteger, stride.get()),
278                    ]
279                    .into_iter()
280                    .collect(),
281                })]
282                .into(),
283            })
284        });
285
286        let spv_opcode = if fixed_len.is_some() { wk.OpTypeArray } else { wk.OpTypeRuntimeArray };
287
288        Ok(self.cx.intern(TypeDef {
289            attrs: stride_attrs.unwrap_or_default(),
290            kind: TypeKind::SpvInst {
291                spv_inst: spv_opcode.into(),
292                type_and_const_inputs: [TypeOrConst::Type(element_type)]
293                    .into_iter()
294                    .chain(fixed_len.map(|len| TypeOrConst::Const(self.const_u32(len))))
295                    .collect(),
296            },
297        }))
298    }
299
300    fn spv_op_type_struct(
301        &self,
302        field_offsets_and_types: impl IntoIterator<Item = Result<(u32, Type), LiftError>>,
303        extra_attrs: impl IntoIterator<Item = Attr>,
304    ) -> Result<Type, LiftError> {
305        let wk = self.wk;
306
307        let field_offsets_and_types = field_offsets_and_types.into_iter();
308        let mut attrs = AttrSetDef::default();
309        let mut type_and_const_inputs =
310            SmallVec::with_capacity(field_offsets_and_types.size_hint().0);
311        for (i, field_offset_and_type) in field_offsets_and_types.enumerate() {
312            let (offset, field_type) = field_offset_and_type?;
313            attrs.attrs.insert(Attr::SpvAnnotation(spv::Inst {
314                opcode: wk.OpMemberDecorate,
315                imms: [
316                    spv::Imm::Short(wk.LiteralInteger, i.try_into().unwrap()),
317                    spv::Imm::Short(wk.Decoration, wk.Offset),
318                    spv::Imm::Short(wk.LiteralInteger, offset),
319                ]
320                .into_iter()
321                .collect(),
322            }));
323            type_and_const_inputs.push(TypeOrConst::Type(field_type));
324        }
325        attrs.attrs.extend(extra_attrs);
326        Ok(self.cx.intern(TypeDef {
327            attrs: self.cx.intern(attrs),
328            kind: TypeKind::SpvInst { spv_inst: wk.OpTypeStruct.into(), type_and_const_inputs },
329        }))
330    }
331
332    /// Get the (likely cached) `u32` type.
333    fn u32_type(&self) -> Type {
334        if let Some(cached) = self.cached_u32_type.get() {
335            return cached;
336        }
337        let wk = self.wk;
338        let ty = self.cx.intern(TypeKind::SpvInst {
339            spv_inst: spv::Inst {
340                opcode: wk.OpTypeInt,
341                imms: [
342                    spv::Imm::Short(wk.LiteralInteger, 32),
343                    spv::Imm::Short(wk.LiteralInteger, 0),
344                ]
345                .into_iter()
346                .collect(),
347            },
348            type_and_const_inputs: [].into_iter().collect(),
349        });
350        self.cached_u32_type.set(Some(ty));
351        ty
352    }
353
354    fn const_u32(&self, x: u32) -> Const {
355        let wk = self.wk;
356
357        self.cx.intern(ConstDef {
358            attrs: AttrSet::default(),
359            ty: self.u32_type(),
360            kind: ConstKind::SpvInst {
361                spv_inst_and_const_inputs: Rc::new((
362                    spv::Inst {
363                        opcode: wk.OpConstant,
364                        imms: [spv::Imm::Short(wk.LiteralContextDependentNumber, x)]
365                            .into_iter()
366                            .collect(),
367                    },
368                    [].into_iter().collect(),
369                )),
370            },
371        })
372    }
373
374    /// Attempt to compute a `TypeLayout` for a given (SPIR-V) `Type`.
375    fn layout_of(&self, ty: Type) -> Result<TypeLayout, LiftError> {
376        self.layout_cache.layout_of(ty).map_err(|LayoutError(err)| LiftError(err))
377    }
378}
379
380struct LiftToSpvPtrInstsInFunc<'a> {
381    lifter: &'a LiftToSpvPtrs<'a>,
382    global_vars: &'a EntityDefs<GlobalVar>,
383
384    /// Some `QPtr`->`QPtr` `QPtrOp`s must be noops in SPIR-V, but because some
385    /// of them have meaningful semantic differences in SPIR-T, replacement of
386    /// their uses must be deferred until after `try_lift_data_inst_def` has had
387    /// a chance to observe the distinction.
388    ///
389    /// E.g. `QPtrOp::BufferData`s cannot adjust the SPIR-V pointer type, due to
390    /// interactions between the `Block` annotation and any potential trailing
391    /// `OpTypeRuntimeArray`s (which cannot be nested in non-`Block` structs).
392    ///
393    /// The `QPtrOp` itself is only removed after the entire function is lifted,
394    /// (using `data_inst_use_counts` to determine whether they're truly unused).
395    deferred_ptr_noops: FxIndexMap<DataInst, DeferredPtrNoop>,
396
397    // FIXME(eddyb) consider removing this and just do a full second traversal.
398    data_inst_use_counts: EntityOrientedDenseMap<DataInst, NonZeroU32>,
399
400    // HACK(eddyb) this is used to avoid noise when `qptr::analyze` failed.
401    func_has_qptr_analysis_bug_diags: bool,
402}
403
404struct DeferredPtrNoop {
405    output_pointer: Value,
406
407    output_pointer_addr_space: AddrSpace,
408
409    /// Should be equivalent to `layout_of` on `output_pointer`'s pointee type,
410    /// except in the case of `QPtrOp::BufferData`.
411    output_pointee_layout: TypeLayout,
412
413    parent_block: ControlNode,
414}
415
416impl LiftToSpvPtrInstsInFunc<'_> {
417    fn try_lift_data_inst_def(
418        &mut self,
419        mut func_at_data_inst: FuncAtMut<'_, DataInst>,
420        parent_block: ControlNode,
421    ) -> Result<Transformed<DataInstDef>, LiftError> {
422        let wk = self.lifter.wk;
423        let cx = &self.lifter.cx;
424
425        let func_at_data_inst_frozen = func_at_data_inst.reborrow().freeze();
426        let data_inst = func_at_data_inst_frozen.position;
427        let data_inst_def = func_at_data_inst_frozen.def();
428        let data_inst_form_def = &cx[data_inst_def.form];
429        let func = func_at_data_inst_frozen.at(());
430        let type_of_val = |v: Value| func.at(v).type_of(cx);
431        // FIXME(eddyb) maybe all this data should be packaged up together in a
432        // type with fields like those of `DeferredPtrNoop` (or even more).
433        let type_of_val_as_spv_ptr_with_layout = |v: Value| {
434            if let Value::DataInstOutput(v_data_inst) = v {
435                if let Some(ptr_noop) = self.deferred_ptr_noops.get(&v_data_inst) {
436                    return Ok((
437                        ptr_noop.output_pointer_addr_space,
438                        ptr_noop.output_pointee_layout.clone(),
439                    ));
440                }
441            }
442
443            let (addr_space, pointee_type) =
444                self.lifter.as_spv_ptr_type(type_of_val(v)).ok_or_else(|| {
445                    LiftError(Diag::bug(["pointer input not an `OpTypePointer`".into()]))
446                })?;
447
448            Ok((addr_space, self.lifter.layout_of(pointee_type)?))
449        };
450        let replacement_data_inst_def = match &data_inst_form_def.kind {
451            &DataInstKind::FuncCall(_callee) => {
452                for &v in &data_inst_def.inputs {
453                    if self.lifter.as_spv_ptr_type(type_of_val(v)).is_some() {
454                        return Err(LiftError(Diag::bug([
455                            "unimplemented calls with pointer args".into(),
456                        ])));
457                    }
458                }
459                return Ok(Transformed::Unchanged);
460            }
461
462            DataInstKind::QPtr(QPtrOp::FuncLocalVar(_mem_layout)) => {
463                let qptr_usage = self.lifter.find_qptr_usage_attr(data_inst_def.attrs)?;
464
465                // FIXME(eddyb) validate against `mem_layout`!
466                let pointee_type = self.lifter.pointee_type_for_usage(qptr_usage)?;
467                DataInstDef {
468                    attrs: self.lifter.strip_qptr_usage_attr(data_inst_def.attrs),
469                    form: cx.intern(DataInstFormDef {
470                        kind: DataInstKind::SpvInst(spv::Inst {
471                            opcode: wk.OpVariable,
472                            imms: [spv::Imm::Short(wk.StorageClass, wk.Function)]
473                                .into_iter()
474                                .collect(),
475                        }),
476                        output_type: Some(
477                            self.lifter.spv_ptr_type(
478                                AddrSpace::SpvStorageClass(wk.Function),
479                                pointee_type,
480                            ),
481                        ),
482                    }),
483                    inputs: data_inst_def.inputs.clone(),
484                }
485            }
486            DataInstKind::QPtr(QPtrOp::HandleArrayIndex) => {
487                let (addr_space, layout) =
488                    type_of_val_as_spv_ptr_with_layout(data_inst_def.inputs[0])?;
489                let handle = match layout {
490                    // FIXME(eddyb) standardize variant order in enum/match.
491                    TypeLayout::HandleArray(handle, _) => handle,
492                    TypeLayout::Handle(_) => {
493                        return Err(LiftError(Diag::bug(["cannot index single Handle".into()])));
494                    }
495                    TypeLayout::Concrete(_) => {
496                        return Err(LiftError(Diag::bug(
497                            ["cannot index memory as handles".into()],
498                        )));
499                    }
500                };
501                let handle_type = match handle {
502                    shapes::Handle::Opaque(ty) => ty,
503                    shapes::Handle::Buffer(_, buf) => buf.original_type,
504                };
505                DataInstDef {
506                    attrs: data_inst_def.attrs,
507                    form: cx.intern(DataInstFormDef {
508                        kind: DataInstKind::SpvInst(wk.OpAccessChain.into()),
509                        output_type: Some(self.lifter.spv_ptr_type(addr_space, handle_type)),
510                    }),
511                    inputs: data_inst_def.inputs.clone(),
512                }
513            }
514            DataInstKind::QPtr(QPtrOp::BufferData) => {
515                let buf_ptr = data_inst_def.inputs[0];
516                let (addr_space, buf_layout) = type_of_val_as_spv_ptr_with_layout(buf_ptr)?;
517
518                let buf_data_layout = match buf_layout {
519                    TypeLayout::Handle(shapes::Handle::Buffer(_, buf)) => TypeLayout::Concrete(buf),
520                    _ => return Err(LiftError(Diag::bug(["non-Buffer pointee".into()]))),
521                };
522
523                self.deferred_ptr_noops.insert(data_inst, DeferredPtrNoop {
524                    output_pointer: buf_ptr,
525                    output_pointer_addr_space: addr_space,
526                    output_pointee_layout: buf_data_layout,
527                    parent_block,
528                });
529
530                DataInstDef {
531                    // FIXME(eddyb) avoid the repeated call to `type_of_val`
532                    // (and the interning of a temporary `DataInstFormDef`),
533                    // maybe don't even replace the `QPtrOp::Buffer` instruction?
534                    form: cx.intern(DataInstFormDef {
535                        kind: QPtrOp::BufferData.into(),
536                        output_type: Some(type_of_val(buf_ptr)),
537                    }),
538                    ..data_inst_def.clone()
539                }
540            }
541            &DataInstKind::QPtr(QPtrOp::BufferDynLen { fixed_base_size, dyn_unit_stride }) => {
542                let buf_ptr = data_inst_def.inputs[0];
543                let (_, buf_layout) = type_of_val_as_spv_ptr_with_layout(buf_ptr)?;
544
545                let buf_data_layout = match buf_layout {
546                    TypeLayout::Handle(shapes::Handle::Buffer(_, buf)) => buf,
547                    _ => return Err(LiftError(Diag::bug(["non-Buffer pointee".into()]))),
548                };
549
550                let field_idx = match &buf_data_layout.components {
551                    Components::Fields { offsets, layouts }
552                        if offsets.last() == Some(&fixed_base_size)
553                            && layouts.last().map_or(false, |last_field| {
554                                last_field.mem_layout.fixed_base.size == 0
555                                    && last_field.mem_layout.dyn_unit_stride
556                                        == Some(dyn_unit_stride)
557                                    && matches!(last_field.components, Components::Elements {
558                                        fixed_len: None,
559                                        ..
560                                    })
561                            }) =>
562                    {
563                        u32::try_from(offsets.len() - 1).unwrap()
564                    }
565                    // FIXME(eddyb) support/diagnose more cases.
566                    _ => {
567                        return Err(LiftError(Diag::bug([
568                            "buffer data type shape mismatch".into()
569                        ])));
570                    }
571                };
572
573                DataInstDef {
574                    form: cx.intern(DataInstFormDef {
575                        kind: DataInstKind::SpvInst(spv::Inst {
576                            opcode: wk.OpArrayLength,
577                            imms: [spv::Imm::Short(wk.LiteralInteger, field_idx)]
578                                .into_iter()
579                                .collect(),
580                        }),
581                        output_type: data_inst_form_def.output_type,
582                    }),
583                    ..data_inst_def.clone()
584                }
585            }
586            &DataInstKind::QPtr(QPtrOp::Offset(offset)) => {
587                let base_ptr = data_inst_def.inputs[0];
588                let (addr_space, layout) = type_of_val_as_spv_ptr_with_layout(base_ptr)?;
589                let mut layout = match layout {
590                    TypeLayout::Handle(_) | TypeLayout::HandleArray(..) => {
591                        return Err(LiftError(Diag::bug(["cannot offset Handles".into()])));
592                    }
593                    TypeLayout::Concrete(mem_layout) => mem_layout,
594                };
595                let mut offset = u32::try_from(offset)
596                    .ok()
597                    .ok_or_else(|| LiftError(Diag::bug(["negative offset".into()])))?;
598
599                let mut access_chain_inputs: SmallVec<_> = [base_ptr].into_iter().collect();
600                // FIXME(eddyb) deduplicate with access chain loop for Load/Store.
601                while offset > 0 {
602                    let idx = {
603                        // HACK(eddyb) supporting ZSTs would be a pain because
604                        // they can "fit" in weird ways, e.g. given 3 offsets
605                        // A, B, C (before/between/after a pair of fields),
606                        // `B..B` is included in both `A..B` and `B..C`.
607                        let allow_zst = false;
608                        let offset_range = if allow_zst {
609                            offset..offset
610                        } else {
611                            offset..offset.saturating_add(1)
612                        };
613                        let mut component_indices =
614                            layout.components.find_components_containing(offset_range);
615                        match (component_indices.next(), component_indices.next()) {
616                            (None, _) => {
617                                // FIXME(eddyb) this could include the chosen indices,
618                                // and maybe the current type and/or layout.
619                                return Err(LiftError(Diag::bug([format!(
620                                    "offset {offset} not found in type layout, after {} access chain indices",
621                                    access_chain_inputs.len() - 1
622                                ).into()])));
623                            }
624                            (Some(idx), Some(_)) => {
625                                // FIXME(eddyb) !!! this can also be illegal overlap
626                                if allow_zst {
627                                    return Err(LiftError(Diag::bug([
628                                        "ambiguity due to ZSTs in type layout".into(),
629                                    ])));
630                                }
631                                // HACK(eddyb) letting illegal overlap through
632                                idx
633                            }
634                            (Some(idx), None) => idx,
635                        }
636                    };
637
638                    let idx_as_i32 = i32::try_from(idx).ok().ok_or_else(|| {
639                        LiftError(Diag::bug([
640                            format!("{idx} not representable as a positive s32").into()
641                        ]))
642                    })?;
643                    access_chain_inputs
644                        .push(Value::Const(self.lifter.const_u32(idx_as_i32 as u32)));
645
646                    match &layout.components {
647                        Components::Scalar => unreachable!(),
648                        Components::Elements { stride, elem, .. } => {
649                            offset %= stride.get();
650                            layout = elem.clone();
651                        }
652                        Components::Fields { offsets, layouts } => {
653                            offset -= offsets[idx];
654                            layout = layouts[idx].clone();
655                        }
656                    }
657                }
658
659                if access_chain_inputs.len() == 1 {
660                    self.deferred_ptr_noops.insert(data_inst, DeferredPtrNoop {
661                        output_pointer: base_ptr,
662                        output_pointer_addr_space: addr_space,
663                        output_pointee_layout: TypeLayout::Concrete(layout),
664                        parent_block,
665                    });
666                    DataInstDef {
667                        // FIXME(eddyb) avoid the repeated call to `type_of_val`
668                        // (and the interning of a temporary `DataInstFormDef`),
669                        // maybe don't even replace the `QPtrOp::Offset` instruction?
670                        form: cx.intern(DataInstFormDef {
671                            kind: QPtrOp::Offset(0).into(),
672                            output_type: Some(type_of_val(base_ptr)),
673                        }),
674                        ..data_inst_def.clone()
675                    }
676                } else {
677                    DataInstDef {
678                        attrs: data_inst_def.attrs,
679                        form: cx.intern(DataInstFormDef {
680                            kind: DataInstKind::SpvInst(wk.OpAccessChain.into()),
681                            output_type: Some(
682                                self.lifter.spv_ptr_type(addr_space, layout.original_type),
683                            ),
684                        }),
685                        inputs: access_chain_inputs,
686                    }
687                }
688            }
689            DataInstKind::QPtr(QPtrOp::DynOffset { stride, index_bounds }) => {
690                let base_ptr = data_inst_def.inputs[0];
691                let (addr_space, layout) = type_of_val_as_spv_ptr_with_layout(base_ptr)?;
692                let mut layout = match layout {
693                    TypeLayout::Handle(_) | TypeLayout::HandleArray(..) => {
694                        return Err(LiftError(Diag::bug(["cannot offset Handles".into()])));
695                    }
696                    TypeLayout::Concrete(mem_layout) => mem_layout,
697                };
698
699                let mut access_chain_inputs: SmallVec<_> = [base_ptr].into_iter().collect();
700                loop {
701                    if let Components::Elements { stride: layout_stride, elem, fixed_len } =
702                        &layout.components
703                    {
704                        if layout_stride == stride
705                            && Ok(index_bounds.clone())
706                                == fixed_len
707                                    .map(|len| i32::try_from(len.get()).map(|len| 0..len))
708                                    .transpose()
709                        {
710                            access_chain_inputs.push(data_inst_def.inputs[1]);
711                            layout = elem.clone();
712                            break;
713                        }
714                    }
715
716                    // FIXME(eddyb) deduplicate with `maybe_adjust_pointer_for_access`.
717                    let idx = {
718                        // FIXME(eddyb) there might be a better way to
719                        // estimate a relevant offset range for the array,
720                        // maybe assume length >= 1 so the minimum range
721                        // is always `0..stride`?
722                        let min_expected_len = index_bounds
723                            .clone()
724                            .and_then(|index_bounds| u32::try_from(index_bounds.end).ok())
725                            .unwrap_or(0);
726                        let offset_range =
727                            0..min_expected_len.checked_add(stride.get()).unwrap_or(0);
728                        let mut component_indices =
729                            layout.components.find_components_containing(offset_range);
730                        match (component_indices.next(), component_indices.next()) {
731                            (None, _) => {
732                                return Err(LiftError(Diag::bug([
733                                    "matching array not found in pointee type layout".into(),
734                                ])));
735                            }
736                            // FIXME(eddyb) obsolete this case entirely,
737                            // by removing stores of ZSTs, and replacing
738                            // loads of ZSTs with `OpUndef` constants.
739                            (Some(_), Some(_)) => {
740                                return Err(LiftError(Diag::bug([
741                                    "ambiguity due to ZSTs in pointee type layout".into(),
742                                ])));
743                            }
744                            (Some(idx), None) => idx,
745                        }
746                    };
747
748                    let idx_as_i32 = i32::try_from(idx).ok().ok_or_else(|| {
749                        LiftError(Diag::bug([
750                            format!("{idx} not representable as a positive s32").into()
751                        ]))
752                    })?;
753                    access_chain_inputs
754                        .push(Value::Const(self.lifter.const_u32(idx_as_i32 as u32)));
755
756                    layout = match &layout.components {
757                        Components::Scalar => unreachable!(),
758                        Components::Elements { elem, .. } => elem.clone(),
759                        Components::Fields { layouts, .. } => layouts[idx].clone(),
760                    };
761                }
762                DataInstDef {
763                    attrs: data_inst_def.attrs,
764                    form: cx.intern(DataInstFormDef {
765                        kind: DataInstKind::SpvInst(wk.OpAccessChain.into()),
766                        output_type: Some(
767                            self.lifter.spv_ptr_type(addr_space, layout.original_type),
768                        ),
769                    }),
770                    inputs: access_chain_inputs,
771                }
772            }
773            DataInstKind::QPtr(op @ (QPtrOp::Load | QPtrOp::Store)) => {
774                let (spv_opcode, access_type) = match op {
775                    QPtrOp::Load => (wk.OpLoad, data_inst_form_def.output_type.unwrap()),
776                    QPtrOp::Store => (wk.OpStore, type_of_val(data_inst_def.inputs[1])),
777                    _ => unreachable!(),
778                };
779
780                // FIXME(eddyb) written in a more general style for future deduplication.
781                let maybe_ajustment = {
782                    let input_idx = 0;
783                    let ptr = data_inst_def.inputs[input_idx];
784                    let (addr_space, pointee_layout) = type_of_val_as_spv_ptr_with_layout(ptr)?;
785                    self.maybe_adjust_pointer_for_access(
786                        ptr,
787                        addr_space,
788                        pointee_layout,
789                        access_type,
790                    )?
791                    .map(|access_chain_data_inst_def| (input_idx, access_chain_data_inst_def))
792                    .into_iter()
793                };
794
795                let mut new_data_inst_def = DataInstDef {
796                    form: cx.intern(DataInstFormDef {
797                        kind: DataInstKind::SpvInst(spv_opcode.into()),
798                        output_type: data_inst_form_def.output_type,
799                    }),
800                    ..data_inst_def.clone()
801                };
802
803                // FIXME(eddyb) written in a more general style for future deduplication.
804                for (input_idx, mut access_chain_data_inst_def) in maybe_ajustment {
805                    // HACK(eddyb) account for `deferred_ptr_noops` interactions.
806                    self.resolve_deferred_ptr_noop_uses(&mut access_chain_data_inst_def.inputs);
807                    self.add_value_uses(&access_chain_data_inst_def.inputs);
808
809                    let access_chain_data_inst = func_at_data_inst
810                        .reborrow()
811                        .data_insts
812                        .define(cx, access_chain_data_inst_def.into());
813
814                    // HACK(eddyb) can't really use helpers like `FuncAtMut::def`,
815                    // due to the need to borrow `control_nodes` and `data_insts`
816                    // at the same time - perhaps some kind of `FuncAtMut` position
817                    // types for "where a list is in a parent entity" could be used
818                    // to make this more ergonomic, although the potential need for
819                    // an actual list entity of its own, should be considered.
820                    let data_inst = func_at_data_inst.position;
821                    let func = func_at_data_inst.reborrow().at(());
822                    match &mut func.control_nodes[parent_block].kind {
823                        ControlNodeKind::Block { insts } => {
824                            insts.insert_before(access_chain_data_inst, data_inst, func.data_insts);
825                        }
826                        _ => unreachable!(),
827                    }
828
829                    new_data_inst_def.inputs[input_idx] =
830                        Value::DataInstOutput(access_chain_data_inst);
831                }
832
833                new_data_inst_def
834            }
835
836            DataInstKind::SpvInst(_) | DataInstKind::SpvExtInst { .. } => {
837                let mut to_spv_ptr_input_adjustments = vec![];
838                let mut from_spv_ptr_output = None;
839                for attr in &cx[data_inst_def.attrs].attrs {
840                    match *attr {
841                        Attr::QPtr(QPtrAttr::ToSpvPtrInput {
842                            input_idx,
843                            pointee: expected_pointee_type,
844                        }) => {
845                            let input_idx = usize::try_from(input_idx).unwrap();
846                            let expected_pointee_type = expected_pointee_type.0;
847
848                            let input_ptr = data_inst_def.inputs[input_idx];
849                            let (input_ptr_addr_space, input_pointee_layout) =
850                                type_of_val_as_spv_ptr_with_layout(input_ptr)?;
851
852                            if let Some(access_chain_data_inst_def) = self
853                                .maybe_adjust_pointer_for_access(
854                                    input_ptr,
855                                    input_ptr_addr_space,
856                                    input_pointee_layout,
857                                    expected_pointee_type,
858                                )?
859                            {
860                                to_spv_ptr_input_adjustments
861                                    .push((input_idx, access_chain_data_inst_def));
862                            }
863                        }
864                        Attr::QPtr(QPtrAttr::FromSpvPtrOutput { addr_space, pointee }) => {
865                            assert!(from_spv_ptr_output.is_none());
866                            from_spv_ptr_output = Some((addr_space.0, pointee.0));
867                        }
868                        _ => {}
869                    }
870                }
871
872                if to_spv_ptr_input_adjustments.is_empty() && from_spv_ptr_output.is_none() {
873                    return Ok(Transformed::Unchanged);
874                }
875
876                let mut new_data_inst_def = data_inst_def.clone();
877
878                // FIXME(eddyb) deduplicate with `Load`/`Store`.
879                for (input_idx, mut access_chain_data_inst_def) in to_spv_ptr_input_adjustments {
880                    // HACK(eddyb) account for `deferred_ptr_noops` interactions.
881                    self.resolve_deferred_ptr_noop_uses(&mut access_chain_data_inst_def.inputs);
882                    self.add_value_uses(&access_chain_data_inst_def.inputs);
883
884                    let access_chain_data_inst = func_at_data_inst
885                        .reborrow()
886                        .data_insts
887                        .define(cx, access_chain_data_inst_def.into());
888
889                    // HACK(eddyb) can't really use helpers like `FuncAtMut::def`,
890                    // due to the need to borrow `control_nodes` and `data_insts`
891                    // at the same time - perhaps some kind of `FuncAtMut` position
892                    // types for "where a list is in a parent entity" could be used
893                    // to make this more ergonomic, although the potential need for
894                    // an actual list entity of its own, should be considered.
895                    let data_inst = func_at_data_inst.position;
896                    let func = func_at_data_inst.reborrow().at(());
897                    match &mut func.control_nodes[parent_block].kind {
898                        ControlNodeKind::Block { insts } => {
899                            insts.insert_before(access_chain_data_inst, data_inst, func.data_insts);
900                        }
901                        _ => unreachable!(),
902                    }
903
904                    new_data_inst_def.inputs[input_idx] =
905                        Value::DataInstOutput(access_chain_data_inst);
906                }
907
908                if let Some((addr_space, pointee_type)) = from_spv_ptr_output {
909                    new_data_inst_def.form = cx.intern(DataInstFormDef {
910                        output_type: Some(self.lifter.spv_ptr_type(addr_space, pointee_type)),
911                        ..cx[new_data_inst_def.form].clone()
912                    });
913                }
914
915                new_data_inst_def
916            }
917        };
918        Ok(Transformed::Changed(replacement_data_inst_def))
919    }
920
921    /// If necessary, construct an `OpAccessChain` instruction to turn `ptr`
922    /// (pointing to a type with `pointee_layout`) into a pointer to `access_type`
923    /// (which can then be used with e.g. `OpLoad`/`OpStore`).
924    //
925    // FIXME(eddyb) customize errors, to tell apart Load/Store/ToSpvPtrInput.
926    fn maybe_adjust_pointer_for_access(
927        &self,
928        ptr: Value,
929        addr_space: AddrSpace,
930        mut pointee_layout: TypeLayout,
931        access_type: Type,
932    ) -> Result<Option<DataInstDef>, LiftError> {
933        let wk = self.lifter.wk;
934
935        let access_layout = self.lifter.layout_of(access_type)?;
936
937        // The access type might be merely a prefix of the pointee type,
938        // requiring injecting an extra `OpAccessChain` to "dig in".
939        let mut access_chain_inputs: SmallVec<_> = [ptr].into_iter().collect();
940
941        if let TypeLayout::HandleArray(handle, _) = pointee_layout {
942            access_chain_inputs.push(Value::Const(self.lifter.const_u32(0)));
943            pointee_layout = TypeLayout::Handle(handle);
944        }
945        match (pointee_layout, access_layout) {
946            (TypeLayout::HandleArray(..), _) => unreachable!(),
947
948            // All the illegal cases are here to keep the rest tidier.
949            (_, TypeLayout::Handle(shapes::Handle::Buffer(..))) => {
950                return Err(LiftError(Diag::bug(["cannot access whole Buffer".into()])));
951            }
952            (_, TypeLayout::HandleArray(..)) => {
953                return Err(LiftError(Diag::bug(["cannot access whole HandleArray".into()])));
954            }
955            (_, TypeLayout::Concrete(access_layout))
956                if access_layout.mem_layout.dyn_unit_stride.is_some() =>
957            {
958                return Err(LiftError(Diag::bug(["cannot access unsized type".into()])));
959            }
960            (TypeLayout::Handle(shapes::Handle::Buffer(..)), _) => {
961                return Err(LiftError(Diag::bug(["cannot access into Buffer".into()])));
962            }
963            (TypeLayout::Handle(_), TypeLayout::Concrete(_)) => {
964                return Err(LiftError(Diag::bug(["cannot access Handle as memory".into()])));
965            }
966            (TypeLayout::Concrete(_), TypeLayout::Handle(_)) => {
967                return Err(LiftError(Diag::bug(["cannot access memory as Handle".into()])));
968            }
969
970            (
971                TypeLayout::Handle(shapes::Handle::Opaque(pointee_handle_type)),
972                TypeLayout::Handle(shapes::Handle::Opaque(access_handle_type)),
973            ) => {
974                if pointee_handle_type != access_handle_type {
975                    return Err(LiftError(Diag::bug([
976                        "(opaque handle) pointer vs access type mismatch".into(),
977                    ])));
978                }
979            }
980
981            (TypeLayout::Concrete(mut pointee_layout), TypeLayout::Concrete(access_layout)) => {
982                // FIXME(eddyb) deduplicate with access chain loop for Offset.
983                while pointee_layout.original_type != access_layout.original_type {
984                    let idx = {
985                        let offset_range = 0..access_layout.mem_layout.fixed_base.size;
986                        let mut component_indices =
987                            pointee_layout.components.find_components_containing(offset_range);
988                        match (component_indices.next(), component_indices.next()) {
989                            (None, _) => {
990                                return Err(LiftError(Diag::bug([
991                                    "accessed type not found in pointee type layout".into(),
992                                ])));
993                            }
994                            // FIXME(eddyb) obsolete this case entirely,
995                            // by removing stores of ZSTs, and replacing
996                            // loads of ZSTs with `OpUndef` constants.
997                            (Some(_), Some(_)) => {
998                                return Err(LiftError(Diag::bug([
999                                    "ambiguity due to ZSTs in pointee type layout".into(),
1000                                ])));
1001                            }
1002                            (Some(idx), None) => idx,
1003                        }
1004                    };
1005
1006                    let idx_as_i32 = i32::try_from(idx).ok().ok_or_else(|| {
1007                        LiftError(Diag::bug([
1008                            format!("{idx} not representable as a positive s32").into()
1009                        ]))
1010                    })?;
1011                    access_chain_inputs
1012                        .push(Value::Const(self.lifter.const_u32(idx_as_i32 as u32)));
1013
1014                    pointee_layout = match &pointee_layout.components {
1015                        Components::Scalar => unreachable!(),
1016                        Components::Elements { elem, .. } => elem.clone(),
1017                        Components::Fields { layouts, .. } => layouts[idx].clone(),
1018                    };
1019                }
1020            }
1021        }
1022
1023        Ok(if access_chain_inputs.len() > 1 {
1024            Some(DataInstDef {
1025                attrs: Default::default(),
1026                form: self.lifter.cx.intern(DataInstFormDef {
1027                    kind: DataInstKind::SpvInst(wk.OpAccessChain.into()),
1028                    output_type: Some(self.lifter.spv_ptr_type(addr_space, access_type)),
1029                }),
1030                inputs: access_chain_inputs,
1031            })
1032        } else {
1033            None
1034        })
1035    }
1036
1037    /// Apply rewrites implied by `deferred_ptr_noops` to `values`.
1038    ///
1039    /// This **does not** update `data_inst_use_counts` - in order to do that,
1040    /// you must call `self.remove_value_uses(values)` beforehand, and then also
1041    /// call `self.after_value_uses(values)` afterwards.
1042    fn resolve_deferred_ptr_noop_uses(&self, values: &mut [Value]) {
1043        for v in values {
1044            // FIXME(eddyb) the loop could theoretically be avoided, but that'd
1045            // make tracking use counts harder.
1046            while let Value::DataInstOutput(data_inst) = *v {
1047                match self.deferred_ptr_noops.get(&data_inst) {
1048                    Some(ptr_noop) => {
1049                        *v = ptr_noop.output_pointer;
1050                    }
1051                    None => break,
1052                }
1053            }
1054        }
1055    }
1056
1057    // FIXME(eddyb) these are only this whacky because an `u32` is being
1058    // encoded as `Option<NonZeroU32>` for (dense) map entry reasons.
1059    fn add_value_uses(&mut self, values: &[Value]) {
1060        for &v in values {
1061            if let Value::DataInstOutput(data_inst) = v {
1062                let count = self.data_inst_use_counts.entry(data_inst);
1063                *count = Some(
1064                    NonZeroU32::new(count.map_or(0, |c| c.get()).checked_add(1).unwrap()).unwrap(),
1065                );
1066            }
1067        }
1068    }
1069    fn remove_value_uses(&mut self, values: &[Value]) {
1070        for &v in values {
1071            if let Value::DataInstOutput(data_inst) = v {
1072                let count = self.data_inst_use_counts.entry(data_inst);
1073                *count = NonZeroU32::new(count.unwrap().get() - 1);
1074            }
1075        }
1076    }
1077}
1078
1079impl Transformer for LiftToSpvPtrInstsInFunc<'_> {
1080    // FIXME(eddyb) this is intentionally *shallow* and will not handle pointers
1081    // "hidden" in composites (which should be handled in SPIR-T explicitly).
1082    fn transform_const_use(&mut self, ct: Const) -> Transformed<Const> {
1083        // FIXME(eddyb) maybe cache this remap (in `LiftToSpvPtrs`, globally).
1084        let ct_def = &self.lifter.cx[ct];
1085        if let ConstKind::PtrToGlobalVar(gv) = ct_def.kind {
1086            Transformed::Changed(self.lifter.cx.intern(ConstDef {
1087                attrs: ct_def.attrs,
1088                ty: self.global_vars[gv].type_of_ptr_to,
1089                kind: ct_def.kind.clone(),
1090            }))
1091        } else {
1092            Transformed::Unchanged
1093        }
1094    }
1095
1096    fn transform_value_use(&mut self, v: &Value) -> Transformed<Value> {
1097        self.add_value_uses(&[*v]);
1098
1099        v.inner_transform_with(self)
1100    }
1101
1102    // HACK(eddyb) while we want to transform `DataInstDef`s, we can't inject
1103    // adjacent instructions without access to the parent `ControlNodeKind::Block`,
1104    // and to fix this would likely require list nodes to carry some handle to
1105    // the list they're part of, either the whole semantic parent, or something
1106    // more contrived, where lists are actually allocated entities of their own,
1107    // perhaps something where an `EntityListDefs<DataInstDef>` contains both:
1108    // - an `EntityDefs<EntityListNode<DataInstDef>>` (keyed by `DataInst`)
1109    // - an `EntityDefs<EntityListDef<DataInst>>` (keyed by `EntityList<DataInst>`)
1110    fn in_place_transform_control_node_def(
1111        &mut self,
1112        mut func_at_control_node: FuncAtMut<'_, ControlNode>,
1113    ) {
1114        func_at_control_node.reborrow().inner_in_place_transform_with(self);
1115
1116        let control_node = func_at_control_node.position;
1117        if let ControlNodeKind::Block { insts } = func_at_control_node.reborrow().def().kind {
1118            let mut func_at_inst_iter = func_at_control_node.reborrow().at(insts).into_iter();
1119            while let Some(mut func_at_inst) = func_at_inst_iter.next() {
1120                let mut lifted = self.try_lift_data_inst_def(func_at_inst.reborrow(), control_node);
1121                if let Ok(Transformed::Unchanged) = lifted {
1122                    let data_inst_def = func_at_inst.reborrow().def();
1123                    let data_inst_form_def = &self.lifter.cx[data_inst_def.form];
1124                    if let DataInstKind::QPtr(_) = data_inst_form_def.kind {
1125                        lifted =
1126                            Err(LiftError(Diag::bug(["unimplemented qptr instruction".into()])));
1127                    } else if let Some(ty) = data_inst_form_def.output_type {
1128                        if matches!(self.lifter.cx[ty].kind, TypeKind::QPtr) {
1129                            lifted = Err(LiftError(Diag::bug([
1130                                "unimplemented qptr-producing instruction".into(),
1131                            ])));
1132                        }
1133                    }
1134                }
1135                match lifted {
1136                    Ok(Transformed::Unchanged) => {}
1137                    Ok(Transformed::Changed(new_def)) => {
1138                        // HACK(eddyb) this whole dance ensures that use counts
1139                        // remain accurate, no matter what rewrites occur.
1140                        let data_inst_def = func_at_inst.def();
1141                        self.remove_value_uses(&data_inst_def.inputs);
1142                        *data_inst_def = new_def;
1143                        self.resolve_deferred_ptr_noop_uses(&mut data_inst_def.inputs);
1144                        self.add_value_uses(&data_inst_def.inputs);
1145                    }
1146                    Err(LiftError(e)) => {
1147                        let data_inst_def = func_at_inst.def();
1148
1149                        // HACK(eddyb) do not add redundant errors to `qptr::analyze` bugs.
1150                        self.func_has_qptr_analysis_bug_diags = self
1151                            .func_has_qptr_analysis_bug_diags
1152                            || self.lifter.cx[data_inst_def.attrs].attrs.iter().any(|attr| {
1153                                match attr {
1154                                    Attr::Diagnostics(diags) => {
1155                                        diags.0.iter().any(|diag| match diag.level {
1156                                            DiagLevel::Bug(loc) => {
1157                                                loc.file().ends_with("qptr/analyze.rs")
1158                                                    || loc.file().ends_with("qptr\\analyze.rs")
1159                                            }
1160                                            _ => false,
1161                                        })
1162                                    }
1163                                    _ => false,
1164                                }
1165                            });
1166
1167                        if !self.func_has_qptr_analysis_bug_diags {
1168                            data_inst_def.attrs.push_diag(&self.lifter.cx, e);
1169                        }
1170                    }
1171                }
1172            }
1173        }
1174    }
1175
1176    fn in_place_transform_func_decl(&mut self, func_decl: &mut FuncDecl) {
1177        func_decl.inner_in_place_transform_with(self);
1178
1179        // Remove all `deferred_ptr_noops` instructions that are truly unused.
1180        if let DeclDef::Present(func_def_body) = &mut func_decl.def {
1181            let deferred_ptr_noops = mem::take(&mut self.deferred_ptr_noops);
1182            // NOTE(eddyb) reverse order is important, as each removal can reduce
1183            // use counts of an earlier definition, allowing further removal.
1184            for (inst, ptr_noop) in deferred_ptr_noops.into_iter().rev() {
1185                if self.data_inst_use_counts.get(inst).is_none() {
1186                    // HACK(eddyb) can't really use helpers like `FuncAtMut::def`,
1187                    // due to the need to borrow `control_nodes` and `data_insts`
1188                    // at the same time - perhaps some kind of `FuncAtMut` position
1189                    // types for "where a list is in a parent entity" could be used
1190                    // to make this more ergonomic, although the potential need for
1191                    // an actual list entity of its own, should be considered.
1192                    match &mut func_def_body.control_nodes[ptr_noop.parent_block].kind {
1193                        ControlNodeKind::Block { insts } => {
1194                            insts.remove(inst, &mut func_def_body.data_insts);
1195                        }
1196                        _ => unreachable!(),
1197                    }
1198
1199                    self.remove_value_uses(&func_def_body.at(inst).def().inputs);
1200                }
1201            }
1202        }
1203    }
1204}
spirt/qptr/lift.rs

spirt/qptr/
lift.rs
