Skip to content

Commit 93eb02e

Browse files
adwinwhiteadwinwhite
committed
make enum size not depend on the order of variants
1 parent 9322d18 commit 93eb02e

File tree

8 files changed

+315
-140
lines changed

8 files changed

+315
-140
lines changed

‎compiler/rustc_abi/src/layout.rs

+12-4
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
196196
pub fn layout_of_struct_or_enum<
197197
'a,
198198
FieldIdx: Idx,
199-
VariantIdx: Idx,
199+
VariantIdx: Idx + PartialOrd,
200200
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
201201
>(
202202
&self,
@@ -468,7 +468,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
468468
fn layout_of_enum<
469469
'a,
470470
FieldIdx: Idx,
471-
VariantIdx: Idx,
471+
VariantIdx: Idx + PartialOrd,
472472
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
473473
>(
474474
&self,
@@ -528,8 +528,16 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
528528
let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
529529
..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
530530

531-
let count =
532-
(niche_variants.end().index() as u128 - niche_variants.start().index() as u128) + 1;
531+
let count = {
532+
let niche_variants_len = (niche_variants.end().index() as u128
533+
- niche_variants.start().index() as u128)
534+
+ 1;
535+
if niche_variants.contains(&largest_variant_index) {
536+
niche_variants_len - 1
537+
} else {
538+
niche_variants_len
539+
}
540+
};
533541

534542
// Use the largest niche in the largest variant.
535543
let niche = variant_layouts[largest_variant_index].largest_niche?;

‎compiler/rustc_abi/src/lib.rs

+50-6
Original file line numberDiff line numberDiff line change
@@ -1498,15 +1498,59 @@ pub enum TagEncoding<VariantIdx: Idx> {
14981498
Direct,
14991499

15001500
/// Niche (values invalid for a type) encoding the discriminant:
1501-
/// Discriminant and variant index coincide.
1501+
/// Discriminant and variant index doesn't always coincide.
1502+
///
15021503
/// The variant `untagged_variant` contains a niche at an arbitrary
15031504
/// offset (field `tag_field` of the enum), which for a variant with
1504-
/// discriminant `d` is set to
1505-
/// `(d - niche_variants.start).wrapping_add(niche_start)`.
1505+
/// discriminant `d` is set to `d.wrapping_add(niche_start)`.
15061506
///
1507-
/// For example, `Option<(usize, &T)>` is represented such that
1508-
/// `None` has a null pointer for the second tuple field, and
1509-
/// `Some` is the identity function (with a non-null reference).
1507+
/// As for how to compute the discriminant, we have an optimization here that we allocate discriminant
1508+
/// value starting from the variant after the `untagged_variant` when the `untagged_variant` is
1509+
/// contained in `niche_variants`' range. Thus the `untagged_variant` won't be allocated with a
1510+
/// unneeded discriminant. Motivation for this is issue #117238.
1511+
/// For example,
1512+
/// ```rust
1513+
/// enum {
1514+
/// A, // 1
1515+
/// B, // 2
1516+
/// C(bool), // untagged_variant, no discriminant
1517+
/// D, // has a discriminant of 0
1518+
/// }
1519+
/// ```
1520+
/// The algorithm is as follows:
1521+
/// ```rust
1522+
/// // We ignore leading and trailing variants that don't need discriminants.
1523+
/// adjusted_len = niche_variants.end - niche_variants.start + 1
1524+
/// adjusted_index = variant_index - niche_variants.start
1525+
/// d = if niche_variants.contains(untagged_variant) {
1526+
/// adjusted_untagged_index = untagged_variant - niche_variants.start
1527+
/// (adjusted_index + adjusted_len - adjusted_untagged_index) % adjusted_len - 1
1528+
/// } else {
1529+
/// adjusted_index
1530+
/// }
1531+
/// tag_value = d.wrapping_add(niche_start)
1532+
/// ```
1533+
/// To load variant index from tag value:
1534+
/// ```rust
1535+
/// adjusted_len = niche_variants.end - niche_variants.start + 1
1536+
/// d = tag_value.wrapping_sub(niche_start)
1537+
/// variant_index = if niche_variants.contains(untagged_variant) {
1538+
/// if d < adjusted_len - 1 {
1539+
/// adjusted_untagged_index = untagged_variant - niche_variants.start
1540+
/// (d + 1 + adjusted_untagged_index) % adjusted_len + niche_variants.start
1541+
/// } else {
1542+
/// // When the discriminant is larger than the number of variants having
1543+
/// // discriminant, we know it represents the untagged_variant.
1544+
/// untagged_variant
1545+
/// }
1546+
/// } else {
1547+
/// if d < adjusted_len {
1548+
/// d + niche_variants.start
1549+
/// } else {
1550+
/// untagged_variant
1551+
/// }
1552+
/// }
1553+
/// ```
15101554
Niche {
15111555
untagged_variant: VariantIdx,
15121556
niche_variants: RangeInclusive<VariantIdx>,

‎compiler/rustc_codegen_cranelift/src/discriminant.rs

+92-63
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,20 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
5252
variants: _,
5353
} => {
5454
if variant_index != untagged_variant {
55+
let discr_len = niche_variants.end().index() - niche_variants.start().index() + 1;
56+
let adj_idx = variant_index.index() - niche_variants.start().index();
57+
5558
let niche = place.place_field(fx, FieldIdx::new(tag_field));
5659
let niche_type = fx.clif_type(niche.layout().ty).unwrap();
57-
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
58-
let niche_value = (niche_value as u128).wrapping_add(niche_start);
60+
61+
let discr = if niche_variants.contains(&untagged_variant) {
62+
let adj_untagged_idx =
63+
untagged_variant.index() - niche_variants.start().index();
64+
(adj_idx + discr_len - adj_untagged_idx) % discr_len - 1
65+
} else {
66+
adj_idx
67+
};
68+
let niche_value = (discr as u128).wrapping_add(niche_start);
5969
let niche_value = match niche_type {
6070
types::I128 => {
6171
let lsb = fx.bcx.ins().iconst(types::I64, niche_value as u64 as i64);
@@ -131,72 +141,91 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
131141
dest.write_cvalue(fx, res);
132142
}
133143
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
134-
let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
135-
136-
// We have a subrange `niche_start..=niche_end` inside `range`.
137-
// If the value of the tag is inside this subrange, it's a
138-
// "niche value", an increment of the discriminant. Otherwise it
139-
// indicates the untagged variant.
140-
// A general algorithm to extract the discriminant from the tag
141-
// is:
142-
// relative_tag = tag - niche_start
143-
// is_niche = relative_tag <= (ule) relative_max
144-
// discr = if is_niche {
145-
// cast(relative_tag) + niche_variants.start()
146-
// } else {
147-
// untagged_variant
148-
// }
149-
// However, we will likely be able to emit simpler code.
150-
151-
let (is_niche, tagged_discr, delta) = if relative_max == 0 {
152-
// Best case scenario: only one tagged variant. This will
153-
// likely become just a comparison and a jump.
154-
// The algorithm is:
155-
// is_niche = tag == niche_start
156-
// discr = if is_niche {
157-
// niche_start
158-
// } else {
159-
// untagged_variant
160-
// }
144+
// See the algorithm explanation in the definition of `TagEncoding::Niche`.
145+
let discr_len = niche_variants.end().index() - niche_variants.start().index() + 1;
146+
147+
let niche_start_value = match fx.bcx.func.dfg.value_type(tag) {
148+
types::I128 => {
149+
let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64);
150+
let msb = fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64);
151+
fx.bcx.ins().iconcat(lsb, msb)
152+
}
153+
ty => fx.bcx.ins().iconst(ty, niche_start as i64),
154+
};
155+
156+
let (is_niche, tagged_discr) = if discr_len == 1 {
157+
// Special case where we only have a single tagged variant.
158+
// The untagged variant can't be contained in niche_variant's range in this case.
159+
// Thus the discriminant of the only tagged variant is 0 and its variant index
160+
// is the start of niche_variants.
161161
let is_niche = codegen_icmp_imm(fx, IntCC::Equal, tag, niche_start as i128);
162162
let tagged_discr =
163163
fx.bcx.ins().iconst(cast_to, niche_variants.start().as_u32() as i64);
164-
(is_niche, tagged_discr, 0)
164+
(is_niche, tagged_discr)
165165
} else {
166-
// The special cases don't apply, so we'll have to go with
167-
// the general algorithm.
168-
let niche_start = match fx.bcx.func.dfg.value_type(tag) {
169-
types::I128 => {
170-
let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64);
171-
let msb =
172-
fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64);
173-
fx.bcx.ins().iconcat(lsb, msb)
174-
}
175-
ty => fx.bcx.ins().iconst(ty, niche_start as i64),
176-
};
177-
let relative_discr = fx.bcx.ins().isub(tag, niche_start);
178-
let cast_tag = clif_intcast(fx, relative_discr, cast_to, false);
179-
let is_niche = crate::common::codegen_icmp_imm(
180-
fx,
181-
IntCC::UnsignedLessThanOrEqual,
182-
relative_discr,
183-
i128::from(relative_max),
184-
);
185-
(is_niche, cast_tag, niche_variants.start().as_u32() as u128)
186-
};
166+
// General case.
167+
let discr = fx.bcx.ins().isub(tag, niche_start_value);
168+
let tagged_discr = clif_intcast(fx, discr, cast_to, false);
169+
if niche_variants.contains(&untagged_variant) {
170+
let is_niche = crate::common::codegen_icmp_imm(
171+
fx,
172+
IntCC::UnsignedLessThan,
173+
discr,
174+
(discr_len - 1) as i128,
175+
);
176+
let adj_untagged_idx =
177+
untagged_variant.index() - niche_variants.start().index();
178+
let untagged_delta = 1 + adj_untagged_idx;
179+
let untagged_delta = match cast_to {
180+
types::I128 => {
181+
let lsb = fx.bcx.ins().iconst(types::I64, untagged_delta as i64);
182+
let msb = fx.bcx.ins().iconst(types::I64, 0);
183+
fx.bcx.ins().iconcat(lsb, msb)
184+
}
185+
ty => fx.bcx.ins().iconst(ty, untagged_delta as i64),
186+
};
187+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, untagged_delta);
187188

188-
let tagged_discr = if delta == 0 {
189-
tagged_discr
190-
} else {
191-
let delta = match cast_to {
192-
types::I128 => {
193-
let lsb = fx.bcx.ins().iconst(types::I64, delta as u64 as i64);
194-
let msb = fx.bcx.ins().iconst(types::I64, (delta >> 64) as u64 as i64);
195-
fx.bcx.ins().iconcat(lsb, msb)
196-
}
197-
ty => fx.bcx.ins().iconst(ty, delta as i64),
198-
};
199-
fx.bcx.ins().iadd(tagged_discr, delta)
189+
let discr_len = match cast_to {
190+
types::I128 => {
191+
let lsb = fx.bcx.ins().iconst(types::I64, discr_len as i64);
192+
let msb = fx.bcx.ins().iconst(types::I64, 0);
193+
fx.bcx.ins().iconcat(lsb, msb)
194+
}
195+
ty => fx.bcx.ins().iconst(ty, discr_len as i64),
196+
};
197+
let tagged_discr = fx.bcx.ins().urem(tagged_discr, discr_len);
198+
199+
let niche_variants_start = niche_variants.start().index();
200+
let niche_variants_start = match cast_to {
201+
types::I128 => {
202+
let lsb = fx.bcx.ins().iconst(types::I64, niche_variants_start as i64);
203+
let msb = fx.bcx.ins().iconst(types::I64, 0);
204+
fx.bcx.ins().iconcat(lsb, msb)
205+
}
206+
ty => fx.bcx.ins().iconst(ty, niche_variants_start as i64),
207+
};
208+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, niche_variants_start);
209+
(is_niche, tagged_discr)
210+
} else {
211+
let is_niche = crate::common::codegen_icmp_imm(
212+
fx,
213+
IntCC::UnsignedLessThan,
214+
discr,
215+
(discr_len - 1) as i128,
216+
);
217+
let niche_variants_start = niche_variants.start().index();
218+
let niche_variants_start = match cast_to {
219+
types::I128 => {
220+
let lsb = fx.bcx.ins().iconst(types::I64, niche_variants_start as i64);
221+
let msb = fx.bcx.ins().iconst(types::I64, 0);
222+
fx.bcx.ins().iconcat(lsb, msb)
223+
}
224+
ty => fx.bcx.ins().iconst(ty, niche_variants_start as i64),
225+
};
226+
let tagged_discr = fx.bcx.ins().iadd(tagged_discr, niche_variants_start);
227+
(is_niche, tagged_discr)
228+
}
200229
};
201230

202231
let untagged_variant = if cast_to == types::I128 {

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs

+15-3
Original file line numberDiff line numberDiff line change
@@ -391,9 +391,21 @@ fn compute_discriminant_value<'ll, 'tcx>(
391391

392392
DiscrResult::Range(min, max)
393393
} else {
394-
let value = (variant_index.as_u32() as u128)
395-
.wrapping_sub(niche_variants.start().as_u32() as u128)
396-
.wrapping_add(niche_start);
394+
let discr_len = niche_variants.end().as_u32() as u128
395+
- niche_variants.start().as_u32() as u128
396+
+ 1;
397+
// FIXME: Why do we even return discriminant for absent variants?
398+
let adj_idx = (variant_index.as_u32() as u128)
399+
.wrapping_sub(niche_variants.start().as_u32() as u128);
400+
401+
let discr = if niche_variants.contains(&untagged_variant) {
402+
let adj_untagged_idx =
403+
(untagged_variant.as_u32() - niche_variants.start().as_u32()) as u128;
404+
(adj_idx + discr_len - adj_untagged_idx) % discr_len - 1
405+
} else {
406+
adj_idx
407+
};
408+
let value = discr.wrapping_add(niche_start);
397409
let value = tag.size(cx).truncate(value);
398410
DiscrResult::Value(value)
399411
}

0 commit comments

Comments
 (0)