Merge pull request 'llvm: only load/store ABI-sized integers to/from memory' (#35711) from mlugg/llvm-weird-int-in-memory into master

Resolves: https://github.com/ziglang/zig/issues/17768
Resolves: https://github.com/ziglang/zig/issues/18936
Resolves: https://github.com/ziglang/zig/issues/19755
Resolves: https://github.com/ziglang/zig/issues/24282
Resolves: https://github.com/ziglang/zig/issues/24883
Resolves: https://github.com/ziglang/zig/issues/25042
Resolves: https://github.com/ziglang/zig/issues/25555
Resolves: https://codeberg.org/ziglang/zig/issues/32036
Resolves: https://codeberg.org/ziglang/zig/issues/35560
Reviewed-on: https://codeberg.org/ziglang/zig/pulls/35711
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

CMakeLists.txt

@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES
     src/Package/Module.zig
     src/RangeSet.zig
     src/Sema.zig
-    src/Sema/bitcast.zig
+    src/Sema/reinterpret.zig
     src/Sema/comptime_ptr_access.zig
     src/Sema/type_resolution.zig
     src/Type.zig

doc/langref/test_packed_structs.zig

@@ -26,16 +26,8 @@ fn doTheTest() !void {
     try expectEqual(0x1, divided.quarter4);
 
     const ordered: [2]u8 = @bitCast(full);
-    switch (native_endian) {
-        .big => {
-            try expectEqual(0x12, ordered[0]);
-            try expectEqual(0x34, ordered[1]);
-        },
-        .little => {
-            try expectEqual(0x34, ordered[0]);
-            try expectEqual(0x12, ordered[1]);
-        },
-    }
+    try expectEqual(0x34, ordered[0]);
+    try expectEqual(0x12, ordered[1]);
 }
 
 // test

doc/langref/test_pointer_casting.zig

@@ -1,18 +1,22 @@
 const std = @import("std");
+const native_endian = @import("builtin").target.cpu.arch.endian();
 const expectEqual = std.testing.expectEqual;
 
 test "pointer casting" {
-    const bytes align(@alignOf(u32)) = [_]u8{ 0x12, 0x12, 0x12, 0x12 };
+    const bytes: [4]u8 align(@alignOf(u32)) = .{ 0x10, 0x20, 0x30, 0x40 };
     const u32_ptr: *const u32 = @ptrCast(&bytes);
-    try expectEqual(0x12121212, u32_ptr.*);
 
-    // Even this example is contrived - there are better ways to do the above than
-    // pointer casting. For example, using a slice narrowing cast:
-    const u32_value = std.mem.bytesAsSlice(u32, bytes[0..])[0];
-    try expectEqual(0x12121212, u32_value);
+    // Because we directly reinterpreted bytes of memory, the `u32` value we
+    // load from `u32_ptr` depends on the target endian:
+    switch (native_endian) {
+        .little => try expectEqual(0x40302010, u32_ptr.*),
+        .big => try expectEqual(0x10203040, u32_ptr.*),
+    }
 
-    // And even another way, the most straightforward way to do it:
-    try expectEqual(0x12121212, @as(u32, @bitCast(bytes)));
+    // To instead reinterpret the logical bit representation of `bytes` with no
+    // dependency on the target endian, use `@bitCast`, which always places
+    // earlier array elements into less-significant bits:
+    try expectEqual(0x40302010, @as(u32, @bitCast(bytes)));
 }
 
 test "pointer child type" {

lib/compiler_rt/float_from_int.zig

@@ -64,10 +64,21 @@ inline fn limb(limbs: []const u32, index: usize) u32 {
 pub inline fn floatFromBigInt(comptime T: type, comptime signedness: std.builtin.Signedness, x: []const u32) T {
     switch (x.len) {
         0 => return 0,
-        inline 1...4 => |limbs_len| return @floatFromInt(@as(
-            @Int(signedness, 32 * limbs_len),
-            @bitCast(x[0..limbs_len].*),
-        )),
+        inline 1...4 => |limbs_len| {
+            const low_to_high: [limbs_len]u32 = switch (@import("builtin").cpu.arch.endian()) {
+                .little => x[0..limbs_len].*,
+                .big => switch (limbs_len) {
+                    1 => .{x[0]},
+                    2 => .{ x[1], x[0] },
+                    3 => .{ x[2], x[1], x[0] },
+                    4 => .{ x[3], x[2], x[1], x[0] },
+                    else => comptime unreachable,
+                },
+            };
+            const I = @Int(signedness, 32 * limbs_len);
+            const int: I = @bitCast(low_to_high);
+            return @floatFromInt(int);
+        },
         else => {},
     }
 

lib/compiler_rt/int_from_float.zig

@@ -80,10 +80,18 @@ pub inline fn bigIntFromFloat(comptime signedness: std.builtin.Signedness, resul
     switch (result.len) {
         0 => return,
         inline 1...4 => |limbs_len| {
-            result[0..limbs_len].* = @bitCast(@as(
-                @Int(signedness, 32 * limbs_len),
-                @intFromFloat(a),
-            ));
+            const I = @Int(signedness, 32 * limbs_len);
+            const low_to_high: [limbs_len]u32 = @bitCast(@as(I, @intFromFloat(a)));
+            result[0..limbs_len].* = switch (@import("builtin").cpu.arch.endian()) {
+                .little => low_to_high,
+                .big => switch (limbs_len) {
+                    1 => .{low_to_high[0]},
+                    2 => .{ low_to_high[1], low_to_high[0] },
+                    3 => .{ low_to_high[2], low_to_high[1], low_to_high[0] },
+                    4 => .{ low_to_high[3], low_to_high[2], low_to_high[1], low_to_high[0] },
+                    else => comptime unreachable,
+                },
+            };
             return;
         },
         else => {},

lib/compiler_rt/limb64.zig

@@ -75,7 +75,17 @@ fn asLimbs(v: anytype) Limbs(@TypeOf(v)) {
     const int_info = @typeInfo(T).int;
     const limb_cnt = comptime limbCount(int_info.bits);
     const ET = @Int(int_info.signedness, limb_cnt * 64);
-    return @bitCast(@as(ET, v));
+    const low_to_high: Limbs(T) = @bitCast(@as(ET, v));
+    switch (endian) {
+        .little => return low_to_high,
+        .big => {
+            var swapped: Limbs(T) = undefined;
+            for (low_to_high, 0..) |x, i| {
+                swapped[limb_cnt - i - 1] = x;
+            }
+            return swapped;
+        },
+    }
 }
 
 fn limbWrap(limb: u64, is_signed: bool, bits: u16) u64 {
@@ -944,11 +954,7 @@ inline fn add3(x: *[3]u64, start: usize, v0: u64) void {
 
 fn mulwide(a: u64, b: u64) [2]u64 {
     const muldXi = @import("mulXi3.zig").muldXi;
-    const limbs: [2]u64 = @bitCast(muldXi(u64, a, b));
-    return switch (endian) {
-        .little => limbs,
-        .big => .{ limbs[1], limbs[0] },
-    };
+    return @bitCast(muldXi(u64, a, b));
 }
 
 fn __mulo_limb64(out_ptr: [*]u64, a_ptr: [*]const u64, b_ptr: [*]const u64, is_signed: bool, bits: u16) callconv(.c) bool {

lib/compiler_rt/udivmod.zig

@@ -61,12 +61,6 @@ pub fn __umodti3(a: u128, b: u128) callconv(.c) u128 {
     return r;
 }
 
-const lo = switch (builtin.cpu.arch.endian()) {
-    .big => 1,
-    .little => 0,
-};
-const hi = 1 - lo;
-
 // Let _u1 and _u0 be the high and low limbs of U respectively.
 // Returns U / v_ and sets r = U % v_.
 fn divwide_generic(comptime T: type, _u1: T, _u0: T, v_: T, r: *T) T {
@@ -158,22 +152,22 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
         return 0;
     }
 
-    const a: [2]HalfT = @bitCast(a_);
-    const b: [2]HalfT = @bitCast(b_);
+    const a: [2]HalfT = @bitCast(a_); // [0] is low bits, [1] is high bits
+    const b: [2]HalfT = @bitCast(b_); // [0] is low bits, [1] is high bits
     var q: [2]HalfT = undefined;
     var r: [2]HalfT = undefined;
 
     // When the divisor fits in 64 bits, we can use an optimized path
-    if (b[hi] == 0) {
-        r[hi] = 0;
-        if (a[hi] < b[lo]) {
+    if (b[1] == 0) {
+        r[1] = 0;
+        if (a[1] < b[0]) {
             // The result fits in 64 bits
-            q[hi] = 0;
-            q[lo] = divwide(HalfT, a[hi], a[lo], b[lo], &r[lo]);
+            q[1] = 0;
+            q[0] = divwide(HalfT, a[1], a[0], b[0], &r[0]);
         } else {
             // First, divide with the high part to get the remainder. After that a_hi < b_lo.
-            q[hi] = a[hi] / b[lo];
-            q[lo] = divwide(HalfT, a[hi] % b[lo], a[lo], b[lo], &r[lo]);
+            q[1] = a[1] / b[0];
+            q[0] = divwide(HalfT, a[1] % b[0], a[0], b[0], &r[0]);
         }
         if (maybe_rem) |rem| {
             rem.* = @bitCast(r);
@@ -181,21 +175,21 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
         return @bitCast(q);
     }
 
-    // Large-divisor case: b[hi] != 0, so the quotient fits in one HalfT word.
+    // Large-divisor case: b[1] != 0, so the quotient fits in one HalfT word.
     //
     // Trial quotient via divwide (Knuth Vol 2, Section 4.3.1):
     // Normalize the divisor so its high half has the MSB set, then use divwide
     // on the top bits to get a trial quotient that is at most 1 too large.
     // This replaces the O(shift) bit-by-bit loop with O(1) operations.
-    const s: Log2Int(HalfT) = @intCast(@clz(b[hi]));
+    const s: Log2Int(HalfT) = @intCast(@clz(b[1]));
 
     if (s == 0) {
-        // b[hi] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b
+        // b[1] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b
         // (we passed the b_ > a_ check), a >= 2^(T_bits - 1) too, meaning
-        // a[hi] also has its MSB set. Therefore a / b < 2, and the quotient
+        // a[1] also has its MSB set. Therefore a / b < 2, and the quotient
         // is exactly 1.
         q = @bitCast(@as(T, 0));
-        q[lo] = 1;
+        q[0] = 1;
         if (maybe_rem) |rem| {
             rem.* = a_ - b_;
         }
@@ -207,12 +201,12 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
         std.math.IntFittingRange(0, half_bits),
         @intCast(s),
     ));
-    const bn_hi: HalfT = (b[hi] << s) | (b[lo] >> sr);
+    const bn_hi: HalfT = (b[1] << s) | (b[0] >> sr);
 
     // Trial numerator: the top (half_bits + s) bits of (a << s), as [a2:a1].
     // a2 < bn_hi is guaranteed since a2 < 2^s and bn_hi >= 2^(half_bits - 1).
-    const a2: HalfT = a[hi] >> sr;
-    const a1: HalfT = (a[hi] << s) | (a[lo] >> sr);
+    const a2: HalfT = a[1] >> sr;
+    const a1: HalfT = (a[1] << s) | (a[0] >> sr);
 
     // Trial quotient via divwide: q_hat = floor([a2:a1] / bn_hi).
     // By Knuth's theorem (normalized divisor), q <= q_hat <= q + 1.
@@ -223,42 +217,42 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
     // Compute the product using HalfT * HalfT -> T widening multiplications,
     // which are native single-instruction ops when HalfT fits in a register
     // (e.g. u64 * u64 -> u128 via mulq on x86_64, mul on aarch64).
-    // product = q_hat * [b[hi]:b[lo]] = [p_top : p_mid : p_lo] (3 half-words)
-    const prod_lo: T = @as(T, q_hat) * @as(T, b[lo]);
-    const prod_hi: T = @as(T, q_hat) * @as(T, b[hi]);
+    // product = q_hat * [b[1]:b[0]] = [p_top : p_mid : p_lo] (3 half-words)
+    const prod_lo: T = @as(T, q_hat) * @as(T, b[0]);
+    const prod_hi: T = @as(T, q_hat) * @as(T, b[1]);
 
     const prod_lo_parts: [2]HalfT = @bitCast(prod_lo);
     const prod_hi_parts: [2]HalfT = @bitCast(prod_hi);
 
-    const mid_add = @addWithOverflow(prod_hi_parts[lo], prod_lo_parts[hi]);
+    const mid_add = @addWithOverflow(prod_hi_parts[0], prod_lo_parts[1]);
     var p_mid: HalfT = mid_add[0];
-    const p_top: HalfT = prod_hi_parts[hi] +% @as(HalfT, mid_add[1]);
-    var p_lo: HalfT = prod_lo_parts[lo];
+    const p_top: HalfT = prod_hi_parts[1] +% @as(HalfT, mid_add[1]);
+    var p_lo: HalfT = prod_lo_parts[0];
 
     // If product > a, decrement q_hat (at most once, guaranteed by Knuth).
-    if (p_top > 0 or p_mid > a[hi] or (p_mid == a[hi] and p_lo > a[lo])) {
+    if (p_top > 0 or p_mid > a[1] or (p_mid == a[1] and p_lo > a[0])) {
         q_hat -= 1;
         // Subtract b from the product for correct remainder computation.
         // After correction, (q_hat * b) fits in T bits, so borrows into
         // p_top cancel it to zero -- we only need [p_mid:p_lo].
-        const sub_lo = @subWithOverflow(p_lo, b[lo]);
+        const sub_lo = @subWithOverflow(p_lo, b[0]);
         p_lo = sub_lo[0];
-        const sub_mid = @subWithOverflow(p_mid, b[hi]);
+        const sub_mid = @subWithOverflow(p_mid, b[1]);
         const sub_mid2 = @subWithOverflow(sub_mid[0], @as(HalfT, sub_lo[1]));
         p_mid = sub_mid2[0];
     }
 
     q = @bitCast(@as(T, 0));
-    q[lo] = q_hat;
+    q[0] = q_hat;
 
     if (maybe_rem) |rem| {
-        // remainder = a - q_hat * b = [a[hi]:a[lo]] - [p_mid:p_lo]
+        // remainder = a - q_hat * b = [a[1]:a[0]] - [p_mid:p_lo]
         // This subtraction is non-negative since q_hat <= true quotient.
-        const rem_lo = @subWithOverflow(a[lo], p_lo);
-        r[lo] = rem_lo[0];
-        const rem_hi = @subWithOverflow(a[hi], p_mid);
+        const rem_lo = @subWithOverflow(a[0], p_lo);
+        r[0] = rem_lo[0];
+        const rem_hi = @subWithOverflow(a[1], p_mid);
         const rem_hi2 = @subWithOverflow(rem_hi[0], @as(HalfT, rem_lo[1]));
-        r[hi] = rem_hi2[0];
+        r[1] = rem_hi2[0];
         rem.* = @bitCast(r);
     }
     return @bitCast(q);

lib/std/Build/Configuration.zig

@@ -3215,7 +3215,8 @@ pub const Storage = enum {
                 .@"extern" => {
                     const n = @divExact(@sizeOf(Field), @sizeOf(u32));
                     defer i.* += n;
-                    return @bitCast(buffer[i.*..][0..n].*);
+                    const ptr: *align(@alignOf(u32)) const Field = @ptrCast(buffer[i.*..][0..n]);
+                    return ptr.*;
                 },
             },
             else => comptime unreachable,
@@ -3381,7 +3382,8 @@ pub const Storage = enum {
                 },
                 .@"extern" => {
                     const n = @divExact(@sizeOf(Field), @sizeOf(u32));
-                    buffer[i..][0..n].* = @bitCast(value);
+                    const ptr: *align(@alignOf(Field)) const [n]u32 = @ptrCast(&value);
+                    buffer[i..][0..n].* = ptr.*;
                     return n;
                 },
             },

lib/std/Io/Threaded.zig

@@ -14188,9 +14188,11 @@ fn addressUnixToPosix(a: *const net.UnixAddress, storage: *UnixAddress) posix.so
 }
 
 fn address4FromPosix(in: *const posix.sockaddr.in) net.Ip4Address {
+    // The network byte order address in `in.addr` is already the byte order we want.
+    const addr_bytes: *const [4]u8 = @ptrCast(&in.addr);
     return .{
         .port = std.mem.bigToNative(u16, in.port),
-        .bytes = @bitCast(in.addr),
+        .bytes = addr_bytes.*,
     };
 }
 
@@ -14204,9 +14206,11 @@ fn address6FromPosix(in6: *const posix.sockaddr.in6) net.Ip6Address {
 }
 
 fn address4ToPosix(a: net.Ip4Address) posix.sockaddr.in {
+    // The byte order of `a.bytes` is already equivalent to a network byte order address.
+    const addr_raw: *align(1) const u32 = @ptrCast(&a.bytes);
     return .{
         .port = std.mem.nativeToBig(u16, a.port),
-        .addr = @bitCast(a.bytes),
+        .addr = addr_raw.*,
     };
 }
 

lib/std/Io/net.zig

@@ -592,11 +592,7 @@ pub const Ip6Address = struct {
                         if (remaining != 0) return .incomplete;
                     }
 
-                    // Workaround that can be removed when this proposal is
-                    // implemented https://github.com/ziglang/zig/issues/19755
-                    if ((comptime @import("builtin").cpu.arch.endian()) != .big) {
-                        for (&parts) |*part| part.* = @byteSwap(part.*);
-                    }
+                    for (&parts) |*part| part.* = @byteSwap(part.*);
 
                     return .{ .success = .{
                         .bytes = @bitCast(parts),

lib/std/Random/RomuTrio.zig

@@ -5,6 +5,7 @@
 const std = @import("std");
 const math = std.math;
 const RomuTrio = @This();
+const builtin = @import("builtin");
 
 x_state: u64,
 y_state: u64,
@@ -33,7 +34,7 @@ fn next(self: *RomuTrio) u64 {
 }
 
 pub fn seedWithBuf(self: *RomuTrio, buf: [24]u8) void {
-    const seed_buf = @as([3]u64, @bitCast(buf));
+    const seed_buf: [3]u64 = @bitCast(buf);
     self.x_state = seed_buf[0];
     self.y_state = seed_buf[1];
     self.z_state = seed_buf[2];
@@ -121,7 +122,8 @@ test fill {
 }
 
 test "buf seeding test" {
-    const buf0 = @as([24]u8, @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 }));
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
+    const buf0: [24]u8 = @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 });
     const resulting_state = .{ .x = 16294208416658607535, .y = 13964609475759908645, .z = 4703697494102998476 };
     var r = RomuTrio.init(0);
     r.seedWithBuf(buf0);

lib/std/Random/Xoshiro256.zig

@@ -46,12 +46,12 @@ pub fn jump(self: *Xoshiro256) void {
 
     while (table != 0) : (table >>= 1) {
         if (@as(u1, @truncate(table)) != 0) {
-            s ^= @as(u256, @bitCast(self.s));
+            s ^= @bitCast(self.s);
         }
         _ = self.next();
     }
 
-    self.s = @as([4]u64, @bitCast(s));
+    self.s = @bitCast(s);
 }
 
 pub fn seed(self: *Xoshiro256, init_s: u64) void {

lib/std/crypto/tls/Client.zig

@@ -376,7 +376,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
                         const nonce = nonce: {
                             const V = @Vector(P.AEAD.nonce_length, u8);
                             const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
-                            const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
+                            const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(read_seq)));
                             break :nonce @as(V, pv.server_handshake_iv) ^ operand;
                         };
                         P.AEAD.decrypt(cleartext, ciphertext, auth_tag, record_header, nonce, pv.server_handshake_key) catch
@@ -416,7 +416,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
                         const nonce: [P.AEAD.nonce_length]u8 = nonce: {
                             const V = @Vector(P.AEAD.nonce_length, u8);
                             const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
-                            const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
+                            const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
                             break :nonce @as(V, pv.app_cipher.server_write_IV ++ record_iv) ^ operand;
                         };
                         const ciphertext = record_decoder.slice(message_len);
@@ -792,7 +792,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
                                 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
                                     const V = @Vector(P.AEAD.nonce_length, u8);
                                     const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
-                                    const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
+                                    const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(write_seq)));
                                     break :nonce @as(V, pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt) ^ operand;
                                 };
                                 var client_verify_msg = .{@intFromEnum(tls.ContentType.handshake)} ++
@@ -1105,7 +1105,7 @@ fn prepareCiphertextRecord(
                     const nonce: [P.AEAD.nonce_length]u8 = nonce: {
                         const V = @Vector(P.AEAD.nonce_length, u8);
                         const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
-                        const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
+                        const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(c.write_seq)));
                         break :nonce @as(V, pv.client_write_IV ++ pv.client_salt) ^ operand;
                     };
                     record_iv.* = nonce[P.fixed_iv_length..].*;
@@ -1214,7 +1214,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {
                 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
                     const V = @Vector(P.AEAD.nonce_length, u8);
                     const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
-                    const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
+                    const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
                     break :nonce @as(V, pv.server_write_IV ++ record_iv) ^ operand;
                 };
                 const ciphertext = input.take(message_len) catch unreachable; // already peeked

lib/std/hash/xxhash.zig

@@ -2,7 +2,6 @@ const std = @import("std");
 const builtin = @import("builtin");
 const mem = std.mem;
 const expectEqual = std.testing.expectEqual;
-const native_endian = builtin.cpu.arch.endian();
 
 const rotl = std.math.rotl;
 
@@ -421,7 +420,18 @@ pub const XxHash32 = struct {
 };
 
 pub const XxHash3 = struct {
+    const block_bytes = 64;
     const Block = @Vector(8, u64);
+    const InputBlock = extern struct {
+        raw: [block_bytes]u8,
+        inline fn load(ptr: *const InputBlock) Block {
+            return @bitCast(ptr.raw);
+        }
+        inline fn store(ptr: *InputBlock, val: Block) void {
+            ptr.raw = @bitCast(val);
+        }
+    };
+
     const default_secret: [192]u8 = .{
         0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
         0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
@@ -464,10 +474,6 @@ pub const XxHash3 = struct {
         return wide[0] ^ wide[1];
     }
 
-    inline fn swap(x: anytype) @TypeOf(x) {
-        return if (native_endian == .big) @byteSwap(x) else x;
-    }
-
     inline fn disableAutoVectorization(x: anytype) void {
         if (!@inComptime()) asm volatile (""
             :
@@ -476,20 +482,20 @@ pub const XxHash3 = struct {
     }
 
     inline fn mix16(seed: u64, input: []const u8, secret: []const u8) u64 {
-        const blk: [4]u64 = @bitCast([_][16]u8{ input[0..16].*, secret[0..16].* });
+        const blk: [4]u64 = @bitCast([2][16]u8{ input[0..16].*, secret[0..16].* });
         disableAutoVectorization(seed);
 
         return fold(
-            swap(blk[0]) ^ (swap(blk[2]) +% seed),
-            swap(blk[1]) ^ (swap(blk[3]) -% seed),
+            blk[0] ^ (blk[2] +% seed),
+            blk[1] ^ (blk[3] -% seed),
         );
     }
 
-    const Accumulator = extern struct {
+    const Accumulator = struct {
         consumed: usize = 0,
         seed: u64,
-        secret: [192]u8 = undefined,
-        state: Block = Block{
+        secret: [192]u8,
+        state: Block = .{
             XxHash32.prime_3,
             XxHash64.prime_1,
             XxHash64.prime_2,
@@ -501,33 +507,35 @@ pub const XxHash3 = struct {
         },
 
         inline fn init(seed: u64) Accumulator {
-            var self = Accumulator{ .seed = seed };
-            for (
-                std.mem.bytesAsSlice(Block, &self.secret),
-                std.mem.bytesAsSlice(Block, &default_secret),
-            ) |*dst, src| {
-                dst.* = swap(swap(src) +% Block{
-                    seed, @as(u64, 0) -% seed,
-                    seed, @as(u64, 0) -% seed,
-                    seed, @as(u64, 0) -% seed,
-                    seed, @as(u64, 0) -% seed,
-                });
+            const seed_block: Block = .{
+                seed, @as(u64, 0) -% seed,
+                seed, @as(u64, 0) -% seed,
+                seed, @as(u64, 0) -% seed,
+                seed, @as(u64, 0) -% seed,
+            };
+
+            var secret: [192]u8 = undefined;
+            const secret_blocks: []InputBlock = @ptrCast(&secret);
+            const default_secret_blocks: []const InputBlock = @ptrCast(&default_secret);
+            for (secret_blocks, default_secret_blocks) |*dst, *src| {
+                dst.store(src.load() +% seed_block);
             }
-            return self;
+
+            return .{ .seed = seed, .secret = secret };
         }
 
         inline fn round(
             noalias state: *Block,
-            noalias input_block: *align(1) const Block,
-            noalias secret_block: *align(1) const Block,
+            noalias input_block: *const InputBlock,
+            noalias secret_block: *const InputBlock,
         ) void {
-            const data = swap(input_block.*);
-            const mixed = data ^ swap(secret_block.*);
+            const data = input_block.load();
+            const mixed = data ^ secret_block.load();
             state.* +%= (mixed & @as(Block, @splat(0xffffffff))) *% (mixed >> @splat(32));
             state.* +%= @shuffle(u64, data, undefined, [_]i32{ 1, 0, 3, 2, 5, 4, 7, 6 });
         }
 
-        fn accumulate(noalias self: *Accumulator, blocks: []align(1) const Block) void {
+        fn accumulate(noalias self: *Accumulator, blocks: []const InputBlock) void {
             const secret = std.mem.bytesAsSlice(u64, self.secret[self.consumed * 8 ..]);
             for (blocks, secret[0..blocks.len]) |*input_block, *secret_block| {
                 @prefetch(@as([*]const u8, @ptrCast(input_block)) + 320, .{});
@@ -536,14 +544,14 @@ pub const XxHash3 = struct {
         }
 
         fn scramble(self: *Accumulator) void {
-            const secret_block: Block = @bitCast(self.secret[192 - @sizeOf(Block) .. 192].*);
+            const secret_block: Block = @bitCast(self.secret[192 - block_bytes .. 192].*);
             self.state ^= self.state >> @splat(47);
-            self.state ^= swap(secret_block);
+            self.state ^= secret_block;
             self.state *%= @as(Block, @splat(XxHash32.prime_1));
         }
 
-        fn consume(noalias self: *Accumulator, input_blocks: []align(1) const Block) void {
-            const blocks_per_scramble = 1024 / @sizeOf(Block);
+        fn consume(noalias self: *Accumulator, input_blocks: []const InputBlock) void {
+            const blocks_per_scramble = 1024 / block_bytes;
             std.debug.assert(self.consumed <= blocks_per_scramble);
 
             var blocks = input_blocks;
@@ -561,12 +569,12 @@ pub const XxHash3 = struct {
             self.consumed += blocks.len;
         }
 
-        fn digest(noalias self: *Accumulator, total_len: u64, noalias last_block: *align(1) const Block) u64 {
-            const secret_block = self.secret[192 - @sizeOf(Block) - 7 ..][0..@sizeOf(Block)];
+        fn digest(noalias self: *Accumulator, total_len: u64, noalias last_block: *const InputBlock) u64 {
+            const secret_block = self.secret[192 - block_bytes - 7 ..][0..block_bytes];
             round(&self.state, last_block, @ptrCast(secret_block));
 
-            const merge_block: Block = @bitCast(self.secret[11 .. 11 + @sizeOf(Block)].*);
-            self.state ^= swap(merge_block);
+            const merge_block: Block = @bitCast(self.secret[11 .. 11 + block_bytes].*);
+            self.state ^= merge_block;
 
             var result = XxHash64.prime_1 *% total_len;
             inline for (0..4) |i| {
@@ -588,7 +596,7 @@ pub const XxHash3 = struct {
         if (input.len > 0) return hash3(seed, input, secret);
 
         const flip: [2]u64 = @bitCast(secret[56..72].*);
-        const key = swap(flip[0]) ^ swap(flip[1]);
+        const key = flip[0] ^ flip[1];
         return avalanche(.h64, seed ^ key);
     }
 
@@ -604,8 +612,8 @@ pub const XxHash3 = struct {
             input[input.len / 2],
         });
 
-        const key = @as(u64, swap(flip[0]) ^ swap(flip[1])) +% seed;
-        return avalanche(.h64, key ^ swap(blk));
+        const key = @as(u64, flip[0] ^ flip[1]) +% seed;
+        return avalanche(.h64, key ^ blk);
     }
 
     fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
@@ -619,8 +627,8 @@ pub const XxHash3 = struct {
         });
 
         const mixed = seed ^ (@as(u64, @byteSwap(@as(u32, @truncate(seed)))) << 32);
-        const key = (swap(flip[0]) ^ swap(flip[1])) -% mixed;
-        const combined = (@as(u64, swap(blk[0])) << 32) +% swap(blk[1]);
+        const key = (flip[0] ^ flip[1]) -% mixed;
+        const combined = (@as(u64, blk[0]) << 32) +% blk[1];
         return avalanche(.{ .rrmxmx = input.len }, key ^ combined);
     }
 
@@ -634,8 +642,8 @@ pub const XxHash3 = struct {
             input[input.len - 8 ..][0..8].*,
         });
 
-        const lo = swap(blk[0]) ^ ((swap(flip[0]) ^ swap(flip[1])) +% seed);
-        const hi = swap(blk[1]) ^ ((swap(flip[2]) ^ swap(flip[3])) -% seed);
+        const lo = blk[0] ^ ((flip[0] ^ flip[1]) +% seed);
+        const hi = blk[1] ^ ((flip[2] ^ flip[3]) -% seed);
         const combined = @as(u64, input.len) +% @byteSwap(lo) +% hi +% fold(lo, hi);
         return avalanche(.h3, combined);
     }
@@ -679,11 +687,11 @@ pub const XxHash3 = struct {
         @branchHint(.unlikely);
         std.debug.assert(input.len >= 240);
 
-        const block_count = ((input.len - 1) / @sizeOf(Block)) * @sizeOf(Block);
-        const last_block = input[input.len - @sizeOf(Block) ..][0..@sizeOf(Block)];
+        const block_count = ((input.len - 1) / block_bytes) * block_bytes;
+        const last_block = input[input.len - block_bytes ..][0..block_bytes];
 
         var acc = Accumulator.init(seed);
-        acc.consume(std.mem.bytesAsSlice(Block, input[0..block_count]));
+        acc.consume(std.mem.bytesAsSlice(InputBlock, input[0..block_count]));
         return acc.digest(input.len, @ptrCast(last_block));
     }
 
@@ -716,21 +724,21 @@ pub const XxHash3 = struct {
             @memcpy(self.buffer[self.buffered..], consumable[0..remaining]);
             consumable = consumable[remaining..];
 
-            self.accumulator.consume(std.mem.bytesAsSlice(Block, &self.buffer));
+            self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, &self.buffer));
             self.buffered = 0;
         }
 
         // The input isn't small enough to fit in the buffer. Consume it directly.
         if (consumable.len > self.buffer.len) {
-            const block_count = ((consumable.len - 1) / @sizeOf(Block)) * @sizeOf(Block);
-            self.accumulator.consume(std.mem.bytesAsSlice(Block, consumable[0..block_count]));
+            const block_count = ((consumable.len - 1) / block_bytes) * block_bytes;
+            self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, consumable[0..block_count]));
             consumable = consumable[block_count..];
 
             // In case we consume all remaining input, write the last block to end of the buffer
             // to populate the last_block_copy in final() similar to hashLong()'s last_block.
             @memcpy(
-                self.buffer[self.buffer.len - @sizeOf(Block) .. self.buffer.len],
-                (consumable.ptr - @sizeOf(Block))[0..@sizeOf(Block)],
+                self.buffer[self.buffer.len - block_bytes .. self.buffer.len],
+                (consumable.ptr - block_bytes)[0..block_bytes],
             );
         }
 
@@ -751,16 +759,16 @@ pub const XxHash3 = struct {
 
         // Make a copy of the Accumulator state in case `self` needs to update() / be used later.
         var accumulator_copy = self.accumulator;
-        var last_block_copy: [@sizeOf(Block)]u8 = undefined;
+        var last_block_copy: [block_bytes]u8 = undefined;
 
         // Digest the last block onthe Accumulator copy.
         return accumulator_copy.digest(self.total_len, last_block: {
-            if (self.buffered >= @sizeOf(Block)) {
-                const block_count = ((self.buffered - 1) / @sizeOf(Block)) * @sizeOf(Block);
-                accumulator_copy.consume(std.mem.bytesAsSlice(Block, self.buffer[0..block_count]));
-                break :last_block @ptrCast(self.buffer[self.buffered - @sizeOf(Block) ..][0..@sizeOf(Block)]);
+            if (self.buffered >= block_bytes) {
+                const block_count = ((self.buffered - 1) / block_bytes) * block_bytes;
+                accumulator_copy.consume(std.mem.bytesAsSlice(InputBlock, self.buffer[0..block_count]));
+                break :last_block @ptrCast(self.buffer[self.buffered - block_bytes ..][0..block_bytes]);
             } else {
-                const remaining = @sizeOf(Block) - self.buffered;
+                const remaining = block_bytes - self.buffered;
                 @memcpy(last_block_copy[0..remaining], self.buffer[self.buffer.len - remaining ..][0..remaining]);
                 @memcpy(last_block_copy[remaining..][0..self.buffered], self.buffer[0..self.buffered]);
                 break :last_block @ptrCast(&last_block_copy);
@@ -780,6 +788,7 @@ fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64)
 }
 
 test "xxhash3" {
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     const H = XxHash3;
     // Non-Seeded Tests
     try testExpect(H, 0, "", 0x2d06800538d394c2);
@@ -811,6 +820,7 @@ test "xxhash3" {
 }
 
 test "xxhash3 smhasher" {
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     const Test = struct {
         fn do() !void {
             try expectEqual(verify.smhasher(XxHash3.hash), 0x9a636405);
@@ -822,6 +832,7 @@ test "xxhash3 smhasher" {
 }
 
 test "xxhash3 iterative api" {
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     const Test = struct {
         fn do() !void {
             try verify.iterativeApi(XxHash3);

lib/std/hash_map.zig

@@ -593,8 +593,8 @@ fn Custom(
             fingerprint: FingerPrint = free,
             used: u1 = 0,
 
-            const slot_free = @as(u8, @bitCast(Metadata{ .fingerprint = free }));
-            const slot_tombstone = @as(u8, @bitCast(Metadata{ .fingerprint = tombstone }));
+            const slot_free: u8 = @bitCast(Metadata{ .fingerprint = free });
+            const slot_tombstone: u8 = @bitCast(Metadata{ .fingerprint = tombstone });
 
             pub fn isUsed(self: Metadata) bool {
                 return self.used == 1;

lib/std/http/HeadParser.zig

@@ -116,11 +116,8 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
 
                     const chunk = bytes[index..][0..vector_len];
                     const v: Vector = chunk.*;
-                    // depends on https://github.com/ziglang/zig/issues/19755
-                    // const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
-                    // const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
-                    const matches_r: BitVector = @select(u1, v == @as(Vector, @splat('\r')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
-                    const matches_n: BitVector = @select(u1, v == @as(Vector, @splat('\n')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
+                    const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
+                    const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
                     const matches_or: SizeVector = matches_r | matches_n;
 
                     const matches = @reduce(.Add, matches_or);
@@ -331,15 +328,15 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
 }
 
 inline fn int16(array: *const [2]u8) u16 {
-    return @bitCast(array.*);
+    return std.mem.toNative(u16, @bitCast(array.*), .little);
 }
 
 inline fn int24(array: *const [3]u8) u24 {
-    return @bitCast(array.*);
+    return std.mem.toNative(u24, @bitCast(array.*), .little);
 }
 
 inline fn int32(array: *const [4]u8) u32 {
-    return @bitCast(array.*);
+    return std.mem.toNative(u32, @bitCast(array.*), .little);
 }
 
 inline fn intShift(comptime T: type, x: anytype) T {

lib/std/http/Server.zig

@@ -726,7 +726,7 @@ pub const WebSocket = struct {
                 else => @intFromEnum(h1.payload_len),
             };
             if (len > in.buffer.len) return error.MessageOversize;
-            const mask: u32 = @bitCast((try in.takeArray(4)).*);
+            const mask: [4]u8 = (try in.takeArray(4)).*;
             const payload = try in.take(len);
 
             // Skip pongs.
@@ -734,11 +734,16 @@ pub const WebSocket = struct {
 
             // The last item may contain a partial word of unused data.
             const floored_len = (payload.len / 4) * 4;
-            const u32_payload: []align(1) u32 = @ptrCast(payload[0..floored_len]);
-            for (u32_payload) |*elem| elem.* ^= mask;
-            const mask_bytes: []const u8 = @ptrCast(&mask);
-            for (payload[floored_len..], mask_bytes[0 .. payload.len - floored_len]) |*leftover, m|
+
+            const payload_chunks: [][4]u8 = @ptrCast(payload[0..floored_len]);
+            for (payload_chunks) |*chunk| {
+                const mask_i: u32 = @bitCast(mask);
+                const chunk_i: u32 = @bitCast(chunk.*);
+                chunk.* = @bitCast(chunk_i ^ mask_i);
+            }
+            for (payload[floored_len..], mask[0 .. payload.len - floored_len]) |*leftover, m| {
                 leftover.* ^= m;
+            }
 
             return .{
                 .opcode = h0.opcode,

lib/std/mem.zig

@@ -1848,18 +1848,18 @@ pub fn readVarPackedInt(
     if (@bitSizeOf(T) <= 8) {
         // These are the same shifts/masks we perform below, but adds `@truncate`/`@intCast`
         // where needed since int is smaller than a byte.
-        const value = if (read_size == 1) b: {
-            break :b @as(uN, @truncate(read_bytes[0] >> bit_shift));
+        const value: uN = if (read_size == 1) b: {
+            break :b @truncate(read_bytes[0] >> bit_shift);
         } else b: {
             const i: u1 = @intFromBool(endian == .big);
-            const head = @as(uN, @truncate(read_bytes[i] >> bit_shift));
-            const tail_shift = @as(Log2N, @intCast(@as(u4, 8) - bit_shift));
-            const tail = @as(uN, @truncate(read_bytes[1 - i]));
+            const head: uN = @truncate(read_bytes[i] >> bit_shift);
+            const tail_shift: Log2N = @intCast(@as(u4, 8) - bit_shift);
+            const tail: uN = @truncate(read_bytes[1 - i]);
             break :b (tail << tail_shift) | head;
         };
         switch (signedness) {
-            .signed => return @as(T, @intCast((@as(iN, @bitCast(value)) << pad) >> pad)),
-            .unsigned => return @as(T, @intCast((@as(uN, @bitCast(value)) << pad) >> pad)),
+            .signed => return @intCast((@as(iN, @bitCast(value)) << pad) >> pad),
+            .unsigned => return @intCast((value << pad) >> pad),
         }
     }
 
@@ -1880,8 +1880,8 @@ pub fn readVarPackedInt(
         },
     }
     switch (signedness) {
-        .signed => return @as(T, @intCast((@as(iN, @bitCast(int)) << pad) >> pad)),
-        .unsigned => return @as(T, @intCast((@as(uN, @bitCast(int)) << pad) >> pad)),
+        .signed => return @intCast((@as(iN, @bitCast(int)) << pad) >> pad),
+        .unsigned => return @intCast((int << pad) >> pad),
     }
 }
 
@@ -1896,8 +1896,13 @@ test readVarPackedInt {
 /// The bit count of T must be evenly divisible by 8.
 /// This function cannot fail and cannot cause undefined behavior.
 pub inline fn readInt(comptime T: type, buffer: *const [@divExact(@typeInfo(T).int.bits, 8)]u8, endian: Endian) T {
-    const value: T = @bitCast(buffer.*);
-    return if (endian == native_endian) value else @byteSwap(value);
+    // Zig's logical bit order aligns with a little-endian byte array, so when reading in big-endian
+    // we must `@byteSwap` the int after we `@bitCast` to it.
+    const little_val: T = @bitCast(buffer.*);
+    return switch (endian) {
+        .little => little_val,
+        .big => @byteSwap(little_val),
+    };
 }
 
 test readInt {
@@ -1940,13 +1945,15 @@ fn readPackedIntLittle(comptime T: type, bytes: []const u8, bit_offset: usize) T
     // Read by loading a LoadInt, and then follow it up with a 1-byte read
     // of the tail if bit_offset pushed us over a byte boundary.
     const read_bytes = bytes[bit_offset / 8 ..];
-    const val = @as(uN, @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift));
+    const val: uN = @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift);
     if (bit_shift > load_tail_bits) {
         const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
         const tail_byte = read_bytes[load_size];
         const tail_truncated = if (bit_count < 8) @as(uN, @truncate(tail_byte)) else @as(uN, tail_byte);
-        return @as(T, @bitCast(val | (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));
-    } else return @as(T, @bitCast(val));
+        return @bitCast(val | (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
+    } else {
+        return @bitCast(val);
+    }
 }
 
 fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
@@ -1972,8 +1979,10 @@ fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
     if (bit_shift > load_tail_bits) {
         const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
         const tail_byte = if (bit_count < 8) @as(uN, @truncate(read_bytes[0])) else @as(uN, read_bytes[0]);
-        return @as(T, @bitCast(val | (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));
-    } else return @as(T, @bitCast(val));
+        return @bitCast(val | (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
+    } else {
+        return @bitCast(val);
+    }
 }
 
 /// Loads an integer from packed memory.
@@ -2011,7 +2020,12 @@ test "comptime read/write int" {
 /// This function always succeeds, has defined behavior for all inputs, but
 /// the integer bit width must be divisible by 8.
 pub inline fn writeInt(comptime T: type, buffer: *[@divExact(@typeInfo(T).int.bits, 8)]u8, value: T, endian: Endian) void {
-    buffer.* = @bitCast(if (endian == native_endian) value else @byteSwap(value));
+    // Zig's logical bit order aligns with a little-endian byte array, so when writing in big-endian
+    // we must `@byteSwap` the int before we `@bitCast` to an array.
+    buffer.* = switch (endian) {
+        .little => @bitCast(value),
+        .big => @bitCast(@byteSwap(value)),
+    };
 }
 
 test writeInt {
@@ -2209,10 +2223,10 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
 /// (Changing their endianness)
 pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *align(a.toByteUnits()) S) void {
     switch (@typeInfo(S)) {
-        .@"struct" => |struct_info| {
-            if (struct_info.backing_integer) |Int| {
+        .@"struct" => |@"struct"| {
+            if (@"struct".backing_integer) |Int| {
                 ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
-            } else inline for (struct_info.field_types, struct_info.field_names, struct_info.field_attrs) |f_type, f_name, f_attr| {
+            } else inline for (@"struct".field_types, @"struct".field_names, @"struct".field_attrs) |f_type, f_name, f_attr| {
                 switch (@typeInfo(f_type)) {
                     .@"struct" => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
                     .@"union", .array => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
@@ -2220,8 +2234,8 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
                         @field(ptr, f_name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f_name))));
                     },
                     .bool => {},
-                    .float => |float_info| {
-                        @field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(@field(ptr, f_name)))));
+                    .float => |float| {
+                        @field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float.bits), @bitCast(@field(ptr, f_name)))));
                     },
                     else => {
                         @field(ptr, f_name) = @byteSwap(@field(ptr, f_name));
@@ -2229,23 +2243,26 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
                 }
             }
         },
-        .@"union" => |union_info| {
-            if (union_info.tag_type != null) {
-                @compileError("byteSwapAllFields expects an untagged union");
+        .@"union" => |@"union"| if (@"union".backing_integer) |Int| {
+            ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
+        } else {
+            if (@"union".layout != .@"extern") {
+                @compileError("byteSwapAllFields expects a packed or extern union");
             }
 
-            const first_size = @bitSizeOf(union_info.field_types[0]);
-            inline for (union_info.field_types) |field_type| {
+            const first_size = @bitSizeOf(@"union".field_types[0]);
+            inline for (@"union".field_types) |field_type| {
                 if (@bitSizeOf(field_type) != first_size) {
                     @compileError("Unable to byte-swap unions with varying field sizes");
                 }
             }
 
-            const BackingInt = @Int(.unsigned, @bitSizeOf(S));
-            ptr.* = @bitCast(@byteSwap(@as(BackingInt, @bitCast(ptr.*))));
+            const FieldInt = @Int(.unsigned, first_size);
+            const field_ptr = &@field(ptr, @"union".field_names[0]);
+            field_ptr.* = @bitCast(@byteSwap(@as(FieldInt, @bitCast(field_ptr.*))));
         },
-        .array => |info| {
-            byteSwapAllElements(info.child, ptr);
+        .array => |array| {
+            byteSwapAllElements(array.child, ptr);
         },
         else => {
             ptr.* = @byteSwap(ptr.*);
@@ -2291,7 +2308,7 @@ test byteSwapAllFields {
         .f2 = 0x12345678,
         .f3 = .{0x12},
         .f4 = true,
-        .f5 = @as(f32, @bitCast(@as(u32, 0x4640e400))),
+        .f5 = @bitCast(@as(u32, 0x4640e400)),
         .f6 = .{ .f0 = 0x1234 },
     };
     var k = K{
@@ -2300,7 +2317,7 @@ test byteSwapAllFields {
         .f2 = 0x1234,
         .f3 = .{0x12},
         .f4 = false,
-        .f5 = @as(f32, @bitCast(@as(u32, 0x45d42800))),
+        .f5 = @bitCast(@as(u32, 0x45d42800)),
     };
     var p: P = @bitCast(@as(u32, 0x01234567));
     var a: A = A{
@@ -2318,7 +2335,7 @@ test byteSwapAllFields {
         .f2 = 0x78563412,
         .f3 = .{0x12},
         .f4 = true,
-        .f5 = @as(f32, @bitCast(@as(u32, 0x00e44046))),
+        .f5 = @bitCast(@as(u32, 0x00e44046)),
         .f6 = .{ .f0 = 0x3412 },
     }, s);
     try std.testing.expectEqual(K{
@@ -2327,7 +2344,7 @@ test byteSwapAllFields {
         .f2 = 0x3412,
         .f3 = .{0x12},
         .f4 = false,
-        .f5 = @as(f32, @bitCast(@as(u32, 0x0028d445))),
+        .f5 = @bitCast(@as(u32, 0x0028d445)),
     }, k);
     try std.testing.expectEqual(@as(P, @bitCast(@as(u32, 0x67452301))), p);
     try std.testing.expectEqual(A{
@@ -2348,8 +2365,9 @@ pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void {
                 elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*)));
             },
             .bool => {},
-            .float => |float_info| {
-                elem.* = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(elem.*))));
+            .float => |float| {
+                const int_repr: @Int(.unsigned, float.bits) = @bitCast(elem.*);
+                elem.* = @bitCast(@byteSwap(int_repr));
             },
             else => {
                 elem.* = @byteSwap(elem.*);
@@ -3870,25 +3888,29 @@ inline fn reverseVector(comptime N: usize, comptime T: type, a: []T) [N]T {
 pub fn reverse(comptime T: type, items: []T) void {
     var i: usize = 0;
     const end = items.len / 2;
-    if (use_vectors and
-        !@inComptime() and
-        @bitSizeOf(T) > 0 and
-        std.math.isPowerOfTwo(@bitSizeOf(T)))
-    {
-        if (std.simd.suggestVectorLength(T)) |simd_size| {
-            if (simd_size <= end) {
-                const simd_end = end - (simd_size - 1);
-                while (i < simd_end) : (i += simd_size) {
-                    const left_slice = items[i .. i + simd_size];
-                    const right_slice = items[items.len - i - simd_size .. items.len - i];
 
-                    const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);
-                    const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);
+    vec: {
+        if (!use_vectors) break :vec;
+        if (@inComptime()) break :vec;
+        switch (@typeInfo(T)) {
+            .int, .float => {},
+            .pointer => |pointer| if (pointer.size == .slice) break :vec,
+            else => break :vec,
+        }
+        if (@bitSizeOf(T) == 0 or !comptime std.math.isPowerOfTwo(@bitSizeOf(T))) break :vec;
+        const simd_size = std.simd.suggestVectorLength(T) orelse break :vec;
+        if (simd_size > end) break :vec;
 
-                    @memcpy(right_slice, &left_shuffled);
-                    @memcpy(left_slice, &right_shuffled);
-                }
-            }
+        const simd_end = end - (simd_size - 1);
+        while (i < simd_end) : (i += simd_size) {
+            const left_slice = items[i .. i + simd_size];
+            const right_slice = items[items.len - i - simd_size .. items.len - i];
+
+            const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);
+            const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);
+
+            @memcpy(right_slice, &left_shuffled);
+            @memcpy(left_slice, &right_shuffled);
         }
     }
 
@@ -5009,8 +5031,8 @@ test "read/write(Var)PackedInt" {
                     for ([_]PackedType{
                         ~@as(PackedType, 0), // all ones: -1 iN / maxInt uN
                         @as(PackedType, 0), // all zeros: 0 iN / 0 uN
-                        @as(PackedType, @bitCast(@as(iPackedType, math.maxInt(iPackedType)))), // maxInt iN
-                        @as(PackedType, @bitCast(@as(iPackedType, math.minInt(iPackedType)))), // maxInt iN
+                        @bitCast(@as(iPackedType, math.maxInt(iPackedType))), // maxInt iN
+                        @bitCast(@as(iPackedType, math.minInt(iPackedType))), // maxInt iN
                         random.int(PackedType), // random
                         random.int(PackedType), // random
                     }) |write_value| {

lib/std/os/linux/IoUring/test.zig

@@ -526,7 +526,9 @@ test "sendmsg/recvmsg" {
 
     var address_server: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
 
     const server = try socket(address_server.family, posix.SOCK.DGRAM, 0);
@@ -1028,7 +1030,9 @@ test "shutdown" {
 
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
 
     // Socket bound, expect shutdown to work
@@ -1740,7 +1744,9 @@ test "accept multishot" {
 
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
     const listener_socket = try createListenerSocket(&address);
     defer _ = linux.close(listener_socket);
@@ -1842,7 +1848,9 @@ test "accept_direct" {
     defer ring.deinit();
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
 
     // register direct file descriptors
@@ -1931,7 +1939,9 @@ test "accept_multishot_direct" {
 
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
 
     var registered_fds: [2]linux.fd_t = @splat(-1);
@@ -2041,7 +2051,9 @@ test "socket_direct/socket_direct_alloc/close_direct" {
     // use sockets from registered_fds in connect operation
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
     const listener_socket = try createListenerSocket(&address);
     defer _ = linux.close(listener_socket);
@@ -2426,7 +2438,9 @@ test "bind/listen/connect" {
 
     var addr: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
     const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;
 
@@ -2614,7 +2628,9 @@ pub fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
     // Create a TCP server socket
     var address: linux.sockaddr.in = .{
         .port = 0,
-        .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
+        .addr = @as(*align(1) const u32, @ptrCast(
+            &@as([4]u8, .{ 127, 0, 0, 1 }),
+        )).*,
     };
     const listener_socket = try createListenerSocket(&address);
     errdefer _ = linux.close(listener_socket);

lib/std/os/uefi.zig

@@ -218,7 +218,7 @@ pub const TimeCapabilities = extern struct {
 pub const FileHandle = *opaque {};
 
 test "GUID formatting" {
-    const bytes = [_]u8{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };
+    const bytes: [16]u8 = .{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };
     const guid: Guid = @bitCast(bytes);
 
     const str = try std.fmt.allocPrint(std.testing.allocator, "{f}", .{guid});

lib/std/os/windows.zig

@@ -4189,19 +4189,11 @@ pub const GUID = extern struct {
     Data3: u16,
     Data4: [8]u8,
 
-    const hex_offsets = switch (builtin.target.cpu.arch.endian()) {
-        .big => [16]u6{
-            0,  2,  4,  6,
-            9,  11, 14, 16,
-            19, 21, 24, 26,
-            28, 30, 32, 34,
-        },
-        .little => [16]u6{
-            6,  4,  2,  0,
-            11, 9,  16, 14,
-            19, 21, 24, 26,
-            28, 30, 32, 34,
-        },
+    const hex_offsets: [16]u6 = .{
+        6,  4,  2,  0,
+        11, 9,  16, 14,
+        19, 21, 24, 26,
+        28, 30, 32, 34,
     };
 
     pub fn parse(s: []const u8) GUID {
@@ -4216,12 +4208,21 @@ pub const GUID = extern struct {
         assert(s[13] == '-');
         assert(s[18] == '-');
         assert(s[23] == '-');
-        var bytes: [16]u8 = undefined;
-        for (hex_offsets, 0..) |hex_offset, i| {
-            bytes[i] = (try std.fmt.charToDigit(s[hex_offset], 16)) << 4 |
-                try std.fmt.charToDigit(s[hex_offset + 1], 16);
-        }
-        return @as(GUID, @bitCast(bytes));
+        var raw1: [4]u8 = undefined;
+        var raw2: [2]u8 = undefined;
+        var raw3: [2]u8 = undefined;
+        var raw4: [8]u8 = undefined;
+        assert((try std.fmt.hexToBytes(&raw1, s[0..8])).len == raw1.len);
+        assert((try std.fmt.hexToBytes(&raw2, s[9..13])).len == raw2.len);
+        assert((try std.fmt.hexToBytes(&raw3, s[14..18])).len == raw3.len);
+        assert((try std.fmt.hexToBytes(raw4[0..2], s[19..23])).len == 2);
+        assert((try std.fmt.hexToBytes(raw4[2..8], s[24..36])).len == 6);
+        return .{
+            .Data1 = @byteSwap(@as(u32, @bitCast(raw1))),
+            .Data2 = @byteSwap(@as(u16, @bitCast(raw2))),
+            .Data3 = @byteSwap(@as(u16, @bitCast(raw3))),
+            .Data4 = raw4,
+        };
     }
 
     pub fn format(self: GUID, w: *std.Io.Writer) std.Io.Writer.Error!void {
@@ -4233,28 +4234,28 @@ pub const GUID = extern struct {
             self.Data4[2..8],
         });
     }
-};
 
-test GUID {
-    try std.testing.expectEqual(
-        GUID{
+    test parse {
+        const expected: GUID = .{
             .Data1 = 0x01234567,
             .Data2 = 0x89ab,
             .Data3 = 0xef10,
             .Data4 = "\x32\x54\x76\x98\xba\xdc\xfe\x91".*,
-        },
-        GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"),
-    );
-    try std.testing.expectFmt(
-        "{01234567-89ab-ef10-3254-7698badcfe91}",
-        "{f}",
-        .{GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}")},
-    );
-    try std.testing.expectFmt(
-        "{00000001-0001-0001-0001-000000000001}",
-        "{f}",
-        .{GUID{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = [_]u8{ 0, 1, 0, 0, 0, 0, 0, 1 } }},
-    );
+        };
+        try std.testing.expectEqual(expected, GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"));
+    }
+
+    test format {
+        const guid0: GUID = .{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = .{ 0, 1, 0, 0, 0, 0, 0, 1 } };
+        try std.testing.expectFmt("{00000001-0001-0001-0001-000000000001}", "{f}", .{guid0});
+
+        const guid1: GUID = .parse("{01234567-89AB-EF10-3254-7698badcfe91}");
+        try std.testing.expectFmt("{01234567-89ab-ef10-3254-7698badcfe91}", "{f}", .{guid1});
+    }
+};
+
+test {
+    _ = GUID;
 }
 
 pub const COORD = extern struct {

lib/std/testing.zig

@@ -141,39 +141,45 @@ fn expectEqualInner(comptime T: type, expected: T, actual: T) !void {
             try expectEqualSlices(info.child, &expect_array, &actual_array);
         },
 
-        .@"struct" => |structType| {
-            inline for (structType.field_names) |field_name| {
+        .@"struct" => |@"struct"| {
+            inline for (@"struct".field_names) |field_name| {
                 try expectEqual(@field(expected, field_name), @field(actual, field_name));
             }
         },
 
-        .@"union" => |union_info| {
-            if (union_info.tag_type == null) {
-                const first_size = @bitSizeOf(union_info.field_types[0]);
-                inline for (union_info.field_types) |field_type| {
+        .@"union" => |@"union"| if (@"union".backing_integer) |Int| {
+            try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
+        } else switch (@"union".layout) {
+            .@"packed" => {
+                const Int = @Int(.unsigned, @bitSizeOf(T));
+                try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
+            },
+            .@"extern" => {
+                const first_size = @bitSizeOf(@"union".field_types[0]);
+                inline for (@"union".field_types) |field_type| {
                     if (@bitSizeOf(field_type) != first_size) {
-                        @compileError("Unable to compare untagged unions with varying field sizes for type " ++ @typeName(@TypeOf(actual)));
+                        @compileError("Unable to compare extern unions with varying field sizes for type " ++ @typeName(T));
                     }
                 }
-
-                const BackingInt = @Int(.unsigned, @bitSizeOf(T));
+                const FieldInt = @Int(.unsigned, first_size);
+                const expected_field = @field(expected, @"union".field_names[0]);
+                const actual_field = @field(actual, @"union".field_names[0]);
                 return expectEqual(
-                    @as(BackingInt, @bitCast(expected)),
-                    @as(BackingInt, @bitCast(actual)),
+                    @as(FieldInt, @bitCast(expected_field)),
+                    @as(FieldInt, @bitCast(actual_field)),
                 );
-            }
+            },
+            .auto => {
+                const Tag = @"union".tag_type orelse @compileError("byteSwapAllFields expects packed, extern, or tagged union");
 
-            const Tag = std.meta.Tag(@TypeOf(expected));
-
-            const expectedTag = @as(Tag, expected);
-            const actualTag = @as(Tag, actual);
-
-            try expectEqual(expectedTag, actualTag);
-
-            // we only reach this switch if the tags are equal
-            switch (expected) {
-                inline else => |val, tag| try expectEqual(val, @field(actual, @tagName(tag))),
-            }
+                try expectEqual(@as(Tag, expected), @as(Tag, actual));
+                switch (expected) {
+                    inline else => |expected_payload, tag| {
+                        const actual_payload = @field(actual, @tagName(tag));
+                        try expectEqual(expected_payload, actual_payload);
+                    },
+                }
+            },
         },
 
         .optional => {

lib/std/zig/llvm/Builder.zig

@@ -1816,7 +1816,7 @@ pub const Linkage = enum(u4) {
     }
 };
 
-pub const Preemption = enum {
+pub const Preemption = enum(u2) {
     dso_preemptable,
     dso_local,
     implicit_dso_local,
@@ -2011,7 +2011,7 @@ pub const AddrSpace = enum(u24) {
     }
 };
 
-pub const ExternallyInitialized = enum {
+pub const ExternallyInitialized = enum(u1) {
     default,
     externally_initialized,
 
@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {
         };
     }
 
+    /// Asserts that neither `a` nor `b` is `.default`.
+    pub fn max(a: Alignment, b: Alignment) Alignment {
+        assert(a != .default);
+        assert(b != .default);
+        return @enumFromInt(@max(@intFromEnum(a), @intFromEnum(b)));
+    }
+
     pub fn toLlvm(self: Alignment) u6 {
         return switch (self) {
             .default => 0,
@@ -4314,6 +4321,9 @@ pub const Function = struct {
             @"tail call",
             @"tail call fast",
             trunc,
+            @"trunc nuw",
+            @"trunc nsw",
+            @"trunc nuw nsw",
             udiv,
             @"udiv exact",
             urem,
@@ -4377,7 +4387,10 @@ pub const Function = struct {
                 };
             }
 
-            pub fn toCastOpcode(self: Tag) CastOpcode {
+            /// Does not accept `.@"trunc nuw"`, `.@"trunc nsw"`, or `.@"trunc nuw nsw"`, because
+            /// they do not have distinct `CastOpcode` values, and are instead encoded in bitcode
+            /// using flags on a normal `trunc` operation.
+            fn toCastOpcode(self: Tag) CastOpcode {
                 return switch (self) {
                     .trunc => .trunc,
                     .zext => .zext,
@@ -4572,6 +4585,9 @@ pub const Function = struct {
                     .sext,
                     .sitofp,
                     .trunc,
+                    .@"trunc nuw",
+                    .@"trunc nsw",
+                    .@"trunc nuw nsw",
                     .uitofp,
                     .zext,
                     => wip.extraData(Cast, instruction.data).type,
@@ -4758,6 +4774,9 @@ pub const Function = struct {
                     .sext,
                     .sitofp,
                     .trunc,
+                    .@"trunc nuw",
+                    .@"trunc nsw",
+                    .@"trunc nuw nsw",
                     .uitofp,
                     .zext,
                     => function.extraData(Cast, instruction.data).type,
@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {
             .sext,
             .sitofp,
             .trunc,
+            .@"trunc nuw",
+            .@"trunc nsw",
+            .@"trunc nuw nsw",
             .uitofp,
             .zext,
             => {},
@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {
                     .sext,
                     .sitofp,
                     .trunc,
+                    .@"trunc nuw",
+                    .@"trunc nsw",
+                    .@"trunc nuw nsw",
                     .uitofp,
                     .zext,
                     => {
@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void
                     .sext,
                     .sitofp,
                     .trunc,
+                    .@"trunc nuw",
+                    .@"trunc nsw",
+                    .@"trunc nuw nsw",
                     .uitofp,
                     .zext,
                     => |tag| {
@@ -11649,7 +11677,11 @@ fn convTag(
                     .unneeded => unreachable,
                 },
                 .eq => unreachable,
-                .gt => .trunc,
+                .gt => switch (signedness) {
+                    .unsigned => .@"trunc nuw",
+                    .signed => .@"trunc nsw",
+                    .unneeded => .trunc,
+                },
             },
             .pointer => .inttoptr,
             else => unreachable,
@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
                                 .opcode = kind.toCastOpcode(),
                             });
                         },
+                        .@"trunc nuw",
+                        .@"trunc nsw",
+                        .@"trunc nuw nsw",
+                        => |kind| {
+                            const extra = func.extraData(Function.Instruction.Cast, data);
+                            try function_block.writeAbbrev(FunctionBlock.TruncNoWrap{
+                                .val = adapter.getOffsetValueIndex(extra.val),
+                                .type_index = extra.type,
+                                .flags = .{
+                                    .no_unsigned_wrap = kind == .@"trunc nuw" or kind == .@"trunc nuw nsw",
+                                    .no_signed_wrap = kind == .@"trunc nsw" or kind == .@"trunc nuw nsw",
+                                },
+                            });
+                        },
                         .@"fcmp false",
                         .@"fcmp oeq",
                         .@"fcmp oge",

lib/std/zig/llvm/ir.zig

@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {
             ModuleBlock.FunctionBlock.Select,
             ModuleBlock.FunctionBlock.SelectFast,
             ModuleBlock.FunctionBlock.Cast,
+            ModuleBlock.FunctionBlock.TruncNoWrap,
             ModuleBlock.FunctionBlock.Alloca,
             ModuleBlock.FunctionBlock.GetElementPtr,
             ModuleBlock.FunctionBlock.ExtractValue,
@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {
             opcode: CastOpcode,
         };
 
+        pub const TruncNoWrap = struct {
+            pub const Flags = packed struct(u2) {
+                no_unsigned_wrap: bool,
+                no_signed_wrap: bool,
+            };
+            pub const ops = [_]AbbrevOp{
+                .{ .literal = @intFromEnum(ModuleBlock.FunctionBlock.Code.INST_CAST) },
+                ValueAbbrev,
+                .{ .fixed_runtime = Builder.Type },
+                .{ .literal = @intFromEnum(Builder.CastOpcode.trunc) },
+                .{ .fixed = @bitSizeOf(Flags) },
+            };
+
+            val: u32,
+            type_index: Builder.Type,
+            flags: Flags,
+        };
+
         pub const Alloca = struct {
             pub const Flags = packed struct(u11) {
                 align_lower: u5,

src/Air.zig

@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");
 
 pub const Legalize = @import("Air/Legalize.zig");
 pub const Liveness = @import("Air/Liveness.zig");
+pub const Verify = @import("Air/Verify.zig");
 
 instructions: std.MultiArrayList(Inst).Slice,
 /// The meaning of this data is determined by `Inst.Tag` value.
@@ -276,10 +277,50 @@ pub const Inst = struct {
         /// Boolean or binary NOT.
         /// Uses the `ty_op` field.
         not,
-        /// Reinterpret the bits of a value as a different type.  This is like `@bitCast` but
-        /// also supports enums and pointers.
+        /// Implements `@bitCast`.
+        ///
         /// Uses the `ty_op` field.
-        bitcast,
+        bit_cast,
+        /// Cast a pointer to a different pointer type. The result type is a slice iff the operand
+        /// type is a slice (the length of the slice does not change). All other pointer attributes
+        /// except for the address space may change.
+        ///
+        /// Supports vectors of pointers.
+        ///
+        /// Uses the `ty_op` field.
+        ptr_cast,
+        /// Cast an integer to a pointer (not a slice). Operand type is always `usize`.
+        ///
+        /// Supports vectors of integers.
+        ///
+        /// Uses the `ty_op` field.
+        ptr_from_int,
+        /// Cast a pointer (not a slice) to an integer. Result type is always `usize`.
+        ///
+        /// Supports vectors of pointers.
+        ///
+        /// Uses the `ty_op` field.
+        int_from_ptr,
+        /// Cast an error set `E1` to a different error set `E2`, or cast an error union `E1!T` to
+        /// an error union `E2!T` with the same payload type but a different error set type.
+        ///
+        /// Uses the `ty_op` field.
+        error_cast,
+        /// Cast an integer to an error set type. The integer operand type is unsigned and has bit
+        /// width equal to `zcu.errorSetBits()`.
+        ///
+        /// Uses the `ty_op` field.
+        error_from_int,
+        /// Cast an error set to an integer type. The integer destination type is unsigned and has
+        /// bit width equal to `zcu.errorSetBits()`.
+        ///
+        /// Uses the `ty_op` field.
+        int_from_error,
+        /// Cast an enum value to a tagged union, whose tag type is that enum, and which has no
+        /// payload bits (i.e. all payloads are equivalent to `void`).
+        ///
+        /// Uses the `ty_op` field.
+        union_from_enum,
         /// A block runs its body which always ends with a `noreturn` instruction,
         /// so the only way to proceed to the code after the `block` is to encounter a `br`
         /// that targets this `block`.  If the `block` type is `noreturn`,
@@ -589,13 +630,13 @@ pub const Inst = struct {
         /// the integer tag type of the enum.
         /// See `trunc` for integer truncation.
         /// Uses the `ty_op` field.
-        intcast,
-        /// Like `intcast`, but includes two safety checks:
+        int_cast,
+        /// Like `int_cast`, but includes two safety checks:
         /// * triggers a safety panic if the cast truncates bits
         /// * triggers a safety panic if the destination type is an exhaustive enum
         ///   and the operand is not a valid value of this type; i.e. equivalent to
         ///   a safety check based on `.is_named_enum_value`
-        intcast_safe,
+        int_cast_safe,
         /// Truncate higher bits from an integer, resulting in an integer type with the same
         /// sign but an equal or smaller number of bits.
         /// Uses the `ty_op` field.
@@ -955,7 +996,7 @@ pub const Inst = struct {
         /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
         /// this instruction when scalarizing vector operations.
         ///
-        /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
+        /// Uses the `bin_op` field. `lhs` is the vector value. `rhs` is the element index. Result
         /// type is the vector element type.
         legalize_vec_elem_val,
 
@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
         => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
 
         .not,
-        .bitcast,
+        .bit_cast,
+        .ptr_cast,
+        .ptr_from_int,
+        .int_from_ptr,
+        .error_cast,
+        .error_from_int,
+        .int_from_error,
+        .union_from_enum,
         .load,
         .fpext,
         .fptrunc,
-        .intcast,
-        .intcast_safe,
+        .int_cast,
+        .int_cast_safe,
         .trunc,
         .optional_payload,
         .optional_payload_ptr,
@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .add_safe,
         .sub_safe,
         .mul_safe,
-        .intcast_safe,
+        .int_cast_safe,
         .int_from_float_safe,
         .int_from_float_optimized_safe,
         .legalize_vec_store_elem,
@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .shl_sat,
         .xor,
         .not,
-        .bitcast,
+        .bit_cast,
+        .ptr_cast,
+        .ptr_from_int,
+        .int_from_ptr,
+        .error_cast,
+        .error_from_int,
+        .int_from_error,
+        .union_from_enum,
         .ret_addr,
         .frame_addr,
         .clz,
@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .is_non_err,
         .fptrunc,
         .fpext,
-        .intcast,
+        .int_cast,
         .trunc,
         .optional_payload,
         .optional_payload_ptr,

src/Air/Legalize.zig

@@ -75,13 +75,9 @@ pub const Feature = enum {
     scalarize_shl_sat,
     scalarize_xor,
     scalarize_not,
-    /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements.
-    /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`.
-    /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast
-    /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and
-    /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be
-    /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics.
-    scalarize_bitcast,
+    scalarize_ptr_cast,
+    scalarize_ptr_from_int,
+    scalarize_int_from_ptr,
     scalarize_clz,
     scalarize_ctz,
     scalarize_popcount,
@@ -107,8 +103,8 @@ pub const Feature = enum {
     scalarize_cmp_vector_optimized,
     scalarize_fptrunc,
     scalarize_fpext,
-    scalarize_intcast,
-    scalarize_intcast_safe,
+    scalarize_int_cast,
+    scalarize_int_cast_safe,
     scalarize_trunc,
     scalarize_int_from_float,
     scalarize_int_from_float_optimized,
@@ -122,16 +118,45 @@ pub const Feature = enum {
     scalarize_select,
     scalarize_mul_add,
 
+    // Below are several different features for scalarizing `bit_cast` in different scenarios. It is
+    // valid to enable any combination of these features.
+
+    /// Scalarize `bit_cast` where the operand or result type is an array.
+    scalarize_bit_cast_array,
+    /// Scalarize `bit_cast` where either:
+    ///
+    /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
+    /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
+    ///
+    /// This effectively scalarizes any `bit_cast` to/from a vector, *unless* the operation can be
+    /// performed by bitcasting each vector element and returning a vector of the results.
+    ///
+    /// If this feature is enabled, the following AIR instruction tags may be emitted:
+    /// * `.legalize_vec_elem_val`
+    /// * `.legalize_vec_store_elem`
+    scalarize_bit_cast_vector_non_elementwise,
+    /// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
+    /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
+    /// need to sign- or zero-extend multiple elements to populate "padding" bits.
+    ///
+    /// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig`
+    /// to conform to the new bitcast semantics.
+    ///
+    /// If this feature is enabled, the following AIR instruction tags may be emitted:
+    /// * `.legalize_vec_elem_val`
+    /// * `.legalize_vec_store_elem`
+    scalarize_bit_cast_padded_elems,
+
     /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
     unsplat_shift_rhs,
     /// Legalize reduce of a one element vector to a bitcast.
-    reduce_one_elem_to_bitcast,
+    reduce_one_elem_to_bit_cast,
     /// Legalize splat to a one element vector to a bitcast.
-    splat_one_elem_to_bitcast,
+    splat_one_elem_to_bit_cast,
 
-    /// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.
-    /// Not compatible with `scalarize_intcast_safe`.
-    expand_intcast_safe,
+    /// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
+    /// Not compatible with `scalarize_int_cast_safe`.
+    expand_int_cast_safe,
     /// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
     /// Not compatible with `scalarize_int_from_float_safe`.
     expand_int_from_float_safe,
@@ -156,9 +181,9 @@ pub const Feature = enum {
     /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
     /// first byte of memory until bit pointers know their backing type.
     expand_packed_store,
-    /// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
+    /// Replace `struct_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
     expand_packed_struct_field_val,
-    /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
+    /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
     expand_packed_aggregate_init,
 
     /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
@@ -227,7 +252,6 @@ pub const Feature = enum {
             .shl_sat => .scalarize_shl_sat,
             .xor => .scalarize_xor,
             .not => .scalarize_not,
-            .bitcast => .scalarize_bitcast,
             .clz => .scalarize_clz,
             .ctz => .scalarize_ctz,
             .popcount => .scalarize_popcount,
@@ -253,8 +277,11 @@ pub const Feature = enum {
             .cmp_vector_optimized => .scalarize_cmp_vector_optimized,
             .fptrunc => .scalarize_fptrunc,
             .fpext => .scalarize_fpext,
-            .intcast => .scalarize_intcast,
-            .intcast_safe => .scalarize_intcast_safe,
+            .int_cast => .scalarize_int_cast,
+            .int_cast_safe => .scalarize_int_cast_safe,
+            .ptr_cast => .scalarize_ptr_cast,
+            .ptr_from_int => .scalarize_ptr_from_int,
+            .int_from_ptr => .scalarize_int_from_ptr,
             .trunc => .scalarize_trunc,
             .int_from_float => .scalarize_int_from_float,
             .int_from_float_optimized => .scalarize_int_from_float_optimized,
@@ -474,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
             .popcount,
             .byte_swap,
             .bit_reverse,
-            .intcast,
+            .int_cast,
+            .ptr_cast,
+            .ptr_from_int,
+            .int_from_ptr,
             .trunc,
             => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
                 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
@@ -548,15 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
                     },
                 }
             },
-            .bitcast => if (l.features.has(.scalarize_bitcast)) {
+            .bit_cast => if (l.features.hasAny(&.{
+                .scalarize_bit_cast_array,
+                .scalarize_bit_cast_vector_non_elementwise,
+                .scalarize_bit_cast_padded_elems,
+            })) {
                 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
                     continue :inst l.replaceInst(inst, .block, payload);
                 }
             },
-            .intcast_safe => if (l.features.has(.expand_intcast_safe)) {
-                assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
+            .int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
+                assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
                 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
-            } else if (l.features.has(.scalarize_intcast_safe)) {
+            } else if (l.features.has(.scalarize_int_cast_safe)) {
                 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
                 if (ty_op.ty.toType().isVector(zcu)) {
                     continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
@@ -772,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
             inline .reduce, .reduce_optimized => |air_tag| {
                 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
                 const vector_ty = l.typeOf(reduce.operand);
-                if (l.features.has(.reduce_one_elem_to_bitcast)) {
+                if (l.features.has(.reduce_one_elem_to_bit_cast)) {
                     switch (vector_ty.vectorLen(zcu)) {
                         0 => unreachable,
-                        1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
+                        1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
                             .ty = .fromType(vector_ty.childType(zcu)),
                             .operand = reduce.operand,
                         } }),
@@ -792,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
                     .soft_float => unreachable, // the operand is not a scalar
                 }
             },
-            .splat => if (l.features.has(.splat_one_elem_to_bitcast)) {
+            .splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
                 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
                 switch (ty_op.ty.toType().vectorLen(zcu)) {
                     0 => unreachable,
-                    1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
+                    1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
                         .ty = ty_op.ty,
                         .operand = ty_op.operand,
                     } }),
@@ -862,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
                                 const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
                                 if (field_bits == struct_bits) {
                                     // Just bitcast this field.
-                                    continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
+                                    continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
                                         .ty = .fromType(agg_ty),
                                         .operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),
                                     } });
@@ -909,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
             .legalize_vec_store_elem,
             .legalize_compiler_rt_call,
             .spirv_runtime_array_len,
+            .error_cast,
+            .error_from_int,
+            .int_from_error,
+            .union_from_enum,
             => {},
         }
     }
@@ -931,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz
 
     if (result_is_array) {
         // This is only allowed when legalizing an elementwise bitcast.
-        assert(orig.tag == .bitcast);
+        assert(orig.tag == .bit_cast);
         assert(form == .ty_op);
     }
 
@@ -1423,35 +1461,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
     const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
 
     const dest_ty = ty_op.ty.toType();
-    const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
-        else => true,
-        .array, .vector => legal: {
-            if (dest_ty.arrayLen(zcu) == 1) break :legal true;
-            const dest_elem_ty = dest_ty.childType(zcu);
-            break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
-        },
-    };
-
     const operand_ty = l.typeOf(ty_op.operand);
-    const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
-        else => true,
-        .array, .vector => legal: {
-            if (operand_ty.arrayLen(zcu) == 1) break :legal true;
-            const operand_elem_ty = operand_ty.childType(zcu);
-            break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
-        },
+
+    // We exit this block only if the scalarization is actually necessary. Otherwise we will return
+    // `null` from within the block.
+    const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: {
+        const operand_tag = operand_ty.zigTypeTag(zcu);
+        const dest_tag = dest_ty.zigTypeTag(zcu);
+
+        if (operand_tag != .array and
+            operand_tag != .vector and
+            dest_tag != .array and
+            dest_tag != .vector)
+        {
+            return null;
+        }
+
+        // We track the validity of 3 different bitcast operations:
+        // * operand -> dest
+        // * operand -> uint
+        // * uint -> dest
+        // If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing
+        // the validity of the other operations helps us lower the scalarization efficiently.
+        var operand_to_dest: bool = true;
+        var operand_to_int: bool = true;
+        var int_to_dest: bool = true;
+
+        if (l.features.has(.scalarize_bit_cast_array)) {
+            if (operand_tag == .array) {
+                operand_to_dest = false;
+                operand_to_int = false;
+            }
+            if (dest_tag == .array) {
+                operand_to_dest = false;
+                int_to_dest = false;
+            }
+        }
+
+        if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
+            if (operand_tag == .vector) operand_to_int = false;
+            if (dest_tag == .vector) int_to_dest = false;
+
+            if (operand_tag == .vector or dest_tag == .vector) {
+                if (operand_tag != .vector or
+                    dest_tag != .vector or
+                    operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu))
+                {
+                    operand_to_dest = false;
+                }
+            }
+        }
+
+        if (l.features.has(.scalarize_bit_cast_padded_elems)) {
+            if (operand_tag == .array or operand_tag == .vector) {
+                const elem_ty = operand_ty.childType(zcu);
+                if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
+                    operand_to_int = false;
+                    operand_to_dest = false;
+                }
+            }
+            if (dest_tag == .array or dest_tag == .vector) {
+                const elem_ty = dest_ty.childType(zcu);
+                if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
+                    int_to_dest = false;
+                    operand_to_dest = false;
+                }
+            }
+        }
+
+        if (operand_to_dest) {
+            return null; // no scalarization needed!
+        }
+
+        // We need a scalarization, but before breaking from the block, check if we can do it
+        // elementwise---if we can, that's preferable to the generic lowering.
+        if ((operand_tag == .array or operand_tag == .vector) and
+            (dest_tag == .array or dest_tag == .vector) and
+            operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu))
+        {
+            // Operand and result types are both arrays/vectors whose element types have the same
+            // bit size, so we can do an elementwise bitcast.
+            return try l.scalarizeBlockPayload(orig_inst, .ty_op);
+        }
+
+        break :int_ok .{ operand_to_int, int_to_dest };
     };
 
-    if (dest_legal and operand_legal) return null;
-
-    if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
-        // from_ty and to_ty are both arrays or vectors of types with the same bit size,
-        // so we can do an elementwise bitcast.
-        return try l.scalarizeBlockPayload(orig_inst, .ty_op);
-    }
-
-    // Fallback path. Our strategy is to use an unsigned integer type as an intermediate
-    // "bag of bits" representation which can be manipulated by bitwise operations.
+    // Generic scalarization implementation. Our strategy is to use an unsigned integer type as an
+    // intermediate "bag of bits" representation which can be manipulated by bitwise operations.
 
     const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
     assert(operand_ty.bitSize(zcu) == num_bits);
@@ -1465,11 +1562,17 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
     // First, convert `operand_ty` to `uint_ty` (`uN`).
 
     const uint_val: Air.Inst.Ref = uint_val: {
-        if (operand_legal) {
+        if (operand_to_int_ok) {
             _ = main_block.stealCapacity(19);
             break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
         }
 
+        if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
+            _ = main_block.stealCapacity(18);
+            const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
+            break :uint_val main_block.addBitCast(l, uint_ty, elem);
+        }
+
         // %1 = block({
         //   %2 = alloc(*usize)
         //   %3 = alloc(*uN)
@@ -1478,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
         //   %6 = loop({
         //     %7 = load(%2)
         //     %8 = array_elem_val(orig_operand, %7)
-        //     %9 = bitcast(uE, %8)
-        //     %10 = intcast(uN, %9)
+        //     %9 = bit_cast(uE, %8)
+        //     %10 = int_cast(uN, %9)
         //     %11 = load(%3)
         //     %12 = shl_exact(%11, <uS, E>)
         //     %13 = bit_or(%12, %10)
@@ -1529,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
             index_val,
         ).toRef();
         const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
-        const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
+        const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
         const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
         const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
         const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
@@ -1560,10 +1663,23 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
 
     // Now convert `uint_ty` (`uN`) to `dest_ty`.
 
-    if (dest_legal) {
+    if (int_to_dest_ok) {
         _ = main_block.stealCapacity(17);
         const result = main_block.addBitCast(l, dest_ty, uint_val);
         main_block.addBr(l, orig_inst, result);
+    } else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
+        _ = main_block.stealCapacity(16);
+        const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
+        const aggregate_init_payload_start = l.air_extra.items.len;
+        try l.air_extra.append(zcu.gpa, @intFromEnum(elem));
+        const result = main_block.add(l, .{
+            .tag = .aggregate_init,
+            .data = .{ .ty_pl = .{
+                .ty = .fromType(dest_ty),
+                .payload = @intCast(aggregate_init_payload_start),
+            } },
+        }).toRef();
+        main_block.addBr(l, orig_inst, result);
     } else {
         // %1 = alloc(*usize)
         // %2 = alloc(*@Vector(N, Result))
@@ -1571,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
         // %4 = loop({
         //   %5 = load(%1)
         //   %6 = mul(%5, <usize, E>)
-        //   %7 = intcast(uS, %6)
+        //   %7 = int_cast(uS, %6)
         //   %8 = shr(uint_val, %7)
         //   %9 = trunc(uE, %8)
-        //   %10 = bitcast(Result, %9)
+        //   %10 = bit_cast(Result, %9)
         //   %11 = legalize_vec_store_elem(%2, %5, %10)
         //   %12 = cmp_eq(%5, <usize, vec_len>)
         //   %13 = cond_br(%12, {
@@ -1603,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
 
         const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
         const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
-        const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
+        const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
         const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
         const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
         const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
@@ -1981,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
     //     %5 = call(@panic.invalidEnumValue, [])
     //     %6 = unreach()
     //   }, {
-    //     %7 = intcast(@res_ty, %y)
+    //     %7 = int_cast(@res_ty, %y)
     //     %8 = is_named_enum_value(%7)
     //     %9 = cond_br(%8, {
     //       %10 = br(%x, %7)
@@ -2003,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
     //     %6 = call(@panic.invalidEnumValue, [])
     //     %7 = unreach()
     //   }, {
-    //     %8 = intcast(@res_ty, %y)
+    //     %8 = int_cast(@res_ty, %y)
     //     %9 = br(%x, %8)
     //   })
     // })
@@ -2056,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
         cur_block = &condbr.else_block;
     }
 
-    // Now we know we're in-range, we can intcast:
+    // Now we know we're in-range, we can int_cast:
     const cast_inst = cur_block.add(l, .{
-        .tag = .intcast,
+        .tag = .int_cast,
         .data = .{ .ty_op = .{
             .ty = Air.internedToRef(dest_ty.toIntern()),
             .operand = operand_ref,
@@ -2229,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
     // %1 = add_with_overflow(%x, %y)
     // %2 = struct_field_val(%1, .@"1")
     // %3 = reduce(%2, .@"or")
-    // %4 = bitcast(%3, @bool_type)
+    // %4 = bit_cast(%3, @bool_type)
     // %5 = cond_br(%4, {
     //   %6 = call(@panic.integerOverflow, [])
     //   %7 = unreach()
@@ -2335,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins
                                 .tag = .load,
                                 .data = .{ .ty_op = .{
                                     .ty = Air.internedToRef(load_ty.toIntern()),
-                                    .operand = res_block.addBitCast(l, load_ptr_ty: {
+                                    .operand = res_block.addPtrCast(l, load_ptr_ty: {
                                         var load_ptr_info = ptr_info;
                                         load_ptr_info.child = load_ty.toIntern();
                                         load_ptr_info.flags.vector_index = .none;
@@ -2378,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
 
     var res_block: Block = .init(&inst_buf);
     {
-        const backing_ptr_inst = res_block.add(l, .{
-            .tag = .bitcast,
-            .data = .{ .ty_op = .{
-                .ty = Air.internedToRef((load_store_ptr_ty: {
-                    var load_ptr_info = ptr_info;
-                    load_ptr_info.child = load_store_ty.toIntern();
-                    load_ptr_info.flags.vector_index = .none;
-                    load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
-                    break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
-                }).toIntern()),
-                .operand = orig_bin_op.lhs,
-            } },
-        });
+        const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
+            var load_ptr_info = ptr_info;
+            load_ptr_info.child = load_store_ty.toIntern();
+            load_ptr_info.flags.vector_index = .none;
+            load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
+            break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
+        }, orig_bin_op.lhs);
         _ = res_block.add(l, .{
             .tag = .store,
             .data = .{ .bin_op = .{
-                .lhs = backing_ptr_inst.toRef(),
+                .lhs = backing_ptr,
                 .rhs = res_block.add(l, .{
                     .tag = .bit_or,
                     .data = .{ .bin_op = .{
@@ -2405,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
                                     .tag = .load,
                                     .data = .{ .ty_op = .{
                                         .ty = Air.internedToRef(load_store_ty.toIntern()),
-                                        .operand = backing_ptr_inst.toRef(),
+                                        .operand = backing_ptr,
                                     } },
                                 }).toRef(),
                                 .rhs = Air.internedToRef((keep_mask: {
@@ -2434,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
                             .tag = .shl_exact,
                             .data = .{ .bin_op = .{
                                 .lhs = res_block.add(l, .{
-                                    .tag = .intcast,
+                                    .tag = .int_cast,
                                     .data = .{ .ty_op = .{
                                         .ty = Air.internedToRef(load_store_ty.toIntern()),
                                         .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
@@ -2532,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro
 
         const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
         const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
-        const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
+        const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
         cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
     }
 
@@ -2721,18 +2831,51 @@ const Block = struct {
         });
     }
 
-    /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
+    /// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
     /// no-op casts.
     fn addBitCast(
         b: *Block,
         l: *Legalize,
-        ty: Type,
+        result_ty: Type,
         operand: Air.Inst.Ref,
     ) Air.Inst.Ref {
-        if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{
-            .tag = .bitcast,
+        const zcu = l.pt.zcu;
+        const operand_ty = l.typeOf(operand);
+        assert(!operand_ty.isPtrAtRuntime(zcu));
+        assert(!operand_ty.isSliceAtRuntime(zcu));
+        assert(!result_ty.isPtrAtRuntime(zcu));
+        assert(!result_ty.isSliceAtRuntime(zcu));
+        if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
+            .tag = .bit_cast,
             .data = .{ .ty_op = .{
-                .ty = Air.internedToRef(ty.toIntern()),
+                .ty = .fromType(result_ty),
+                .operand = operand,
+            } },
+        }).toRef();
+        _ = b.stealCapacity(1);
+        return operand;
+    }
+
+    /// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
+    /// no-op casts.
+    fn addPtrCast(
+        b: *Block,
+        l: *Legalize,
+        result_ty: Type,
+        operand: Air.Inst.Ref,
+    ) Air.Inst.Ref {
+        const zcu = l.pt.zcu;
+        const operand_ty = l.typeOf(operand);
+        if (operand_ty.isSliceAtRuntime(zcu)) {
+            assert(result_ty.isSliceAtRuntime(zcu));
+        } else {
+            assert(operand_ty.isPtrAtRuntime(zcu));
+            assert(result_ty.isPtrAtRuntime(zcu));
+        }
+        if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
+            .tag = .ptr_cast,
+            .data = .{ .ty_op = .{
+                .ty = .fromType(result_ty),
                 .operand = operand,
             } },
         }).toRef();
@@ -3073,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
         var main_block: Block = .init(&inst_buf);
         try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
 
-        const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
+        const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
         const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
         const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
         main_block.addBr(l, orig_inst, casted_result);
@@ -3100,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
     try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
 
     const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
-        break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
+        break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
     } else ext: {
         _ = main_block.stealCapacity(1);
         break :ext ty_op.operand;
@@ -3165,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
         try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
 
         const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
-        const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();
+        const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
         main_block.addBr(l, orig_inst, casted_val);
 
         return .{ .block_payload = .{ .ty_pl = .{
@@ -3189,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
     const bits_val = try pt.intValue(.usize, dest_info.bits);
     _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
     const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
-    const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();
+    const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
     main_block.addBr(l, orig_inst, result_val);
 
     return .{ .block_payload = .{ .ty_pl = .{

src/Air/Liveness.zig

@@ -488,12 +488,19 @@ fn analyzeInst(
         => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
 
         .not,
-        .bitcast,
+        .bit_cast,
+        .ptr_cast,
+        .ptr_from_int,
+        .int_from_ptr,
+        .error_cast,
+        .error_from_int,
+        .int_from_error,
+        .union_from_enum,
         .load,
         .fpext,
         .fptrunc,
-        .intcast,
-        .intcast_safe,
+        .int_cast,
+        .int_cast_safe,
         .trunc,
         .optional_payload,
         .optional_payload_ptr,

src/Air/Liveness/Verify.zig

@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
 
             // unary
             .not,
-            .bitcast,
+            .bit_cast,
+            .ptr_cast,
+            .ptr_from_int,
+            .int_from_ptr,
+            .error_cast,
+            .error_from_int,
+            .int_from_error,
+            .union_from_enum,
             .load,
             .fpext,
             .fptrunc,
-            .intcast,
-            .intcast_safe,
+            .int_cast,
+            .int_cast_safe,
             .trunc,
             .optional_payload,
             .optional_payload_ptr,

src/Air/Verify.zig

@@ -0,0 +1,465 @@
+/// Verifies that AIR is valid, in that every instruction has valid operands and types. In compiler
+/// builds with debug extensions, this is run on all AIR, both before `Air.Legalize` is run and (if
+/// it is run) after it.
+///
+/// This verification pass is currently highly incomplete---expand it as needed.
+const Verify = @This();
+
+zcu: *Zcu,
+func_index: InternPool.Index,
+ret_ty: Type,
+air: *const Air,
+cur_inst: Air.Inst.Index,
+
+pub fn run(pt: Zcu.PerThread, func_index: InternPool.Index, air: *const Air) void {
+    if (!@import("build_options").enable_debug_extensions) {
+        // `Air.Verify` is a debugging feature---it should not be used in release builds because it
+        // has little benefit and negatively affects compiler performance.
+        return;
+    }
+
+    const zcu = pt.zcu;
+
+    const func_ty: Type = Value.fromInterned(func_index).typeOf(zcu);
+    const ret_ty = func_ty.fnReturnType(zcu);
+
+    var verify: Verify = .{
+        .zcu = zcu,
+        .func_index = func_index,
+        .ret_ty = ret_ty,
+        .air = air,
+        .cur_inst = undefined, // populated by `body(...)`
+    };
+    verify.body(air.getMainBody()) catch |verify_err| switch (verify_err) {
+        error.VerifyFail => {
+            const ip = &zcu.intern_pool;
+            const func_nav = ip.indexToKey(func_index).func.owner_nav;
+            const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
+            log.info("AIR for '{s}':", .{func_fqn});
+            const io = zcu.comp.io;
+            const stderr = io.lockStderr(&.{}, null) catch |err| switch (err) {
+                error.Canceled => return io.recancel(),
+            };
+            defer io.unlockStderr();
+            air.write(&stderr.file_writer.interface, pt, null) catch |err| switch (err) {
+                error.WriteFailed => switch (stderr.file_writer.err.?) {
+                    error.Canceled => return io.recancel(),
+                    else => {},
+                },
+            };
+        },
+    };
+}
+
+const Error = error{VerifyFail};
+
+fn fail(verify: *Verify, msg: []const u8) Error {
+    const ip = &verify.zcu.intern_pool;
+    const func_nav = ip.indexToKey(verify.func_index).func.owner_nav;
+    const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
+    log.err("'{s}', %{d}: {s}", .{ func_fqn, verify.cur_inst, msg });
+    return error.VerifyFail;
+}
+
+fn body(verify: *Verify, body_insts: []const Air.Inst.Index) Error!void {
+    const zcu = verify.zcu;
+    const ip = &zcu.intern_pool;
+    const air = verify.air;
+    const tags = air.instructions.items(.tag);
+    const data = air.instructions.items(.data);
+    for (body_insts, 0..) |inst, body_index| {
+        verify.cur_inst = inst;
+        switch (tags[@intFromEnum(inst)]) {
+            .block => {
+                const block = air.unwrapBlock(inst);
+                try verify.body(block.body);
+            },
+            .dbg_inline_block => {
+                const block = air.unwrapDbgBlock(inst);
+                try verify.body(block.body);
+            },
+            .@"try", .try_cold => {
+                const @"try" = air.unwrapTry(inst);
+                try verify.body(@"try".else_body);
+            },
+            .try_ptr, .try_ptr_cold => {
+                const try_ptr = air.unwrapTryPtr(inst);
+                try verify.body(try_ptr.else_body);
+            },
+            .loop => {
+                const block = air.unwrapBlock(inst);
+                try verify.body(block.body);
+            },
+            .cond_br => {
+                const cond_br = air.unwrapCondBr(inst);
+                try verify.body(cond_br.then_body);
+                try verify.body(cond_br.else_body);
+            },
+            .switch_br, .loop_switch_br => {
+                const switch_br = air.unwrapSwitch(inst);
+                var it = switch_br.iterateCases();
+                while (it.next()) |case| {
+                    try verify.body(case.body);
+                }
+                const else_body = it.elseBody();
+                if (else_body.len > 0) {
+                    try verify.body(else_body);
+                }
+            },
+            .ret, .ret_safe => {
+                const operand = data[@intFromEnum(inst)].un_op;
+                if (air.typeOf(operand, ip).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
+            },
+            .ret_load => {
+                const operand = data[@intFromEnum(inst)].un_op;
+                const ptr_ty = air.typeOf(operand, ip);
+                if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("operand is not a pointer");
+                if (ptr_ty.ptrSize(zcu) != .one) return verify.fail("pointer size is not '.one'");
+                if (ptr_ty.childType(zcu).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
+            },
+
+            .bit_cast => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                // Enums are allowed here even if their backing type is implicit.
+                if (!operand_ty.hasBitRepresentation(zcu) and operand_ty.zigTypeTag(zcu) != .@"enum") {
+                    return verify.fail("bad operand type");
+                }
+                if (!result_ty.hasBitRepresentation(zcu) and result_ty.zigTypeTag(zcu) != .@"enum") {
+                    return verify.fail("bad result type");
+                }
+                if (operand_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type (pointer)");
+                if (result_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type (pointer)");
+                if (operand_ty.bitSize(zcu) != result_ty.bitSize(zcu)) return verify.fail("bit size mismatch");
+            },
+            .ptr_cast => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                const operand_scalar_ty = operand_ty.scalarType(zcu);
+                const result_scalar_ty = result_ty.scalarType(zcu);
+                if (operand_ty.isSliceAtRuntime(zcu)) {
+                    if (!result_ty.isSliceAtRuntime(zcu)) return verify.fail("operand is slice, but result is not");
+                } else {
+                    if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
+                    if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("operand is pointer, but result is not");
+                    if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
+                    if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
+                }
+                if (operand_scalar_ty.ptrAddressSpace(zcu) != result_scalar_ty.ptrAddressSpace(zcu)) {
+                    return verify.fail("illegal change to address space");
+                }
+            },
+            .ptr_from_int => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                const operand_scalar_ty = operand_ty.scalarType(zcu);
+                const result_scalar_ty = result_ty.scalarType(zcu);
+                if (operand_scalar_ty.toIntern() != .usize_type) return verify.fail("bad operand type");
+                if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type");
+                if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
+                if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
+            },
+            .int_from_ptr => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                const operand_scalar_ty = operand_ty.scalarType(zcu);
+                const result_scalar_ty = result_ty.scalarType(zcu);
+                if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
+                if (result_scalar_ty.toIntern() != .usize_type) return verify.fail("bad result type");
+                if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
+                if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
+            },
+            .error_cast => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                switch (operand_ty.zigTypeTag(zcu)) {
+                    else => return verify.fail("bad operand type"),
+                    .error_union => {
+                        if (result_ty.zigTypeTag(zcu) != .error_union) {
+                            return verify.fail("operand is error union, but result is not");
+                        }
+                        if (operand_ty.errorUnionPayload(zcu).toIntern() != result_ty.errorUnionPayload(zcu).toIntern()) {
+                            return verify.fail("error union payload type differs");
+                        }
+                    },
+                    .error_set => if (result_ty.zigTypeTag(zcu) != .error_set) {
+                        return verify.fail("operand is error set, but result is not");
+                    },
+                }
+            },
+            .error_from_int => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                if (!operand_ty.isUnsignedInt(zcu)) return verify.fail("bad operand type");
+                if (operand_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad operand bit size");
+                if (result_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad result type");
+            },
+            .int_from_error => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                if (operand_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad operand type");
+                if (!result_ty.isUnsignedInt(zcu)) return verify.fail("bad result type");
+                if (result_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad result bit size");
+            },
+            .union_from_enum => {
+                const ty_op = data[@intFromEnum(inst)].ty_op;
+                const operand_ty = air.typeOf(ty_op.operand, ip);
+                const result_ty = ty_op.ty.toType();
+                if (operand_ty.zigTypeTag(zcu) != .@"enum") return verify.fail("bad operand type");
+                if (result_ty.zigTypeTag(zcu) != .@"union") return verify.fail("bad result type");
+                const union_tag_ty = result_ty.unionTagType(zcu) orelse return verify.fail("union type is not tagged");
+                if (union_tag_ty.toIntern() != operand_ty.toIntern()) return verify.fail("union tag type does not match operand type");
+            },
+
+            .ptr_elem_ptr => {
+                const ty_pl = data[@intFromEnum(inst)].ty_pl;
+                const bin_op = air.extraData(Air.Bin, ty_pl.payload).data;
+                const ptr_ty = air.typeOf(bin_op.lhs, ip);
+                const result_ty = ty_pl.ty.toType();
+                if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad pointer type");
+                if (result_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad result type");
+                const ptr_info = ptr_ty.ptrInfo(zcu);
+                const result_ptr_info = result_ty.ptrInfo(zcu);
+                if (ptr_info.packed_offset.host_size != 0) return verify.fail("pointer type is bitpacked pointer");
+                if (result_ptr_info.packed_offset.host_size != 0) return verify.fail("result type is bitpacked pointer");
+            },
+
+            .arg,
+            .add,
+            .add_safe,
+            .add_optimized,
+            .add_wrap,
+            .add_sat,
+            .sub,
+            .sub_safe,
+            .sub_optimized,
+            .sub_wrap,
+            .sub_sat,
+            .mul,
+            .mul_safe,
+            .mul_optimized,
+            .mul_wrap,
+            .mul_sat,
+            .div_float,
+            .div_float_optimized,
+            .div_trunc,
+            .div_trunc_optimized,
+            .div_floor,
+            .div_floor_optimized,
+            .div_exact,
+            .div_exact_optimized,
+            .rem,
+            .rem_optimized,
+            .mod,
+            .mod_optimized,
+            .ptr_add,
+            .ptr_sub,
+            .max,
+            .min,
+            .add_with_overflow,
+            .sub_with_overflow,
+            .mul_with_overflow,
+            .shl_with_overflow,
+            .alloc,
+            .inferred_alloc,
+            .inferred_alloc_comptime,
+            .ret_ptr,
+            .assembly,
+            .bit_and,
+            .bit_or,
+            .shr,
+            .shr_exact,
+            .shl,
+            .shl_exact,
+            .shl_sat,
+            .xor,
+            .not,
+            .repeat,
+            .br,
+            .trap,
+            .breakpoint,
+            .ret_addr,
+            .frame_addr,
+            .call,
+            .call_always_tail,
+            .call_never_tail,
+            .call_never_inline,
+            .clz,
+            .ctz,
+            .popcount,
+            .byte_swap,
+            .bit_reverse,
+            .sqrt,
+            .sin,
+            .cos,
+            .tan,
+            .exp,
+            .exp2,
+            .log,
+            .log2,
+            .log10,
+            .abs,
+            .floor,
+            .ceil,
+            .round,
+            .trunc_float,
+            .neg,
+            .neg_optimized,
+            .cmp_lt,
+            .cmp_lt_optimized,
+            .cmp_lte,
+            .cmp_lte_optimized,
+            .cmp_eq,
+            .cmp_eq_optimized,
+            .cmp_gte,
+            .cmp_gte_optimized,
+            .cmp_gt,
+            .cmp_gt_optimized,
+            .cmp_neq,
+            .cmp_neq_optimized,
+            .cmp_vector,
+            .cmp_vector_optimized,
+            .switch_dispatch,
+            .dbg_stmt,
+            .dbg_empty_stmt,
+            .dbg_var_ptr,
+            .dbg_var_val,
+            .dbg_arg_inline,
+            .is_null,
+            .is_non_null,
+            .is_null_ptr,
+            .is_non_null_ptr,
+            .is_err,
+            .is_non_err,
+            .is_err_ptr,
+            .is_non_err_ptr,
+            .load,
+            .store,
+            .store_safe,
+            .unreach,
+            .fptrunc,
+            .fpext,
+            .int_cast,
+            .int_cast_safe,
+            .trunc,
+            .optional_payload,
+            .optional_payload_ptr,
+            .optional_payload_ptr_set,
+            .wrap_optional,
+            .unwrap_errunion_payload,
+            .unwrap_errunion_err,
+            .unwrap_errunion_payload_ptr,
+            .unwrap_errunion_err_ptr,
+            .errunion_payload_ptr_set,
+            .wrap_errunion_payload,
+            .wrap_errunion_err,
+            .struct_field_ptr,
+            .struct_field_ptr_index_0,
+            .struct_field_ptr_index_1,
+            .struct_field_ptr_index_2,
+            .struct_field_ptr_index_3,
+            .struct_field_val,
+            .set_union_tag,
+            .get_union_tag,
+            .slice,
+            .slice_len,
+            .slice_ptr,
+            .ptr_slice_len_ptr,
+            .ptr_slice_ptr_ptr,
+            .array_elem_val,
+            .slice_elem_val,
+            .slice_elem_ptr,
+            .ptr_elem_val,
+            .array_to_slice,
+            .int_from_float,
+            .int_from_float_optimized,
+            .int_from_float_safe,
+            .int_from_float_optimized_safe,
+            .float_from_int,
+            .reduce,
+            .reduce_optimized,
+            .splat,
+            .shuffle_one,
+            .shuffle_two,
+            .select,
+            .memset,
+            .memset_safe,
+            .memcpy,
+            .memmove,
+            .cmpxchg_weak,
+            .cmpxchg_strong,
+            .atomic_load,
+            .atomic_store_unordered,
+            .atomic_store_monotonic,
+            .atomic_store_release,
+            .atomic_store_seq_cst,
+            .atomic_rmw,
+            .is_named_enum_value,
+            .tag_name,
+            .error_name,
+            .error_set_has_value,
+            .aggregate_init,
+            .union_init,
+            .prefetch,
+            .mul_add,
+            .field_parent_ptr,
+            .wasm_memory_size,
+            .wasm_memory_grow,
+            .cmp_lte_errors_len,
+            .err_return_trace,
+            .set_err_return_trace,
+            .addrspace_cast,
+            .save_err_return_trace_index,
+            .runtime_nav_ptr,
+            .c_va_arg,
+            .c_va_copy,
+            .c_va_end,
+            .c_va_start,
+            .spirv_runtime_array_len,
+            .work_item_id,
+            .work_group_size,
+            .work_group_id,
+            .legalize_vec_store_elem,
+            .legalize_vec_elem_val,
+            .legalize_compiler_rt_call,
+            => {},
+        }
+        if (air.typeOfIndex(inst, ip).isNoReturn(zcu)) {
+            if (body_index == body_insts.len - 1) return;
+
+            // HACK: right now, we emit the safety check for noreturn functions returning in a weird
+            // way, where the `call` instruction is `noreturn` but there are still instructions
+            // following it. We need to figure out a better way to represent that! That safety check
+            // probably just needs to live exclusively in backends; putting AIR instructions after a
+            // call implies that we have e.g. a valid stack at that point, which we can't actually
+            // assume when the user has gotten a function's ABI wrong.
+            switch (tags[@intFromEnum(inst)]) {
+                .call,
+                .call_always_tail,
+                .call_never_tail,
+                .call_never_inline,
+                => continue,
+                else => {},
+            }
+
+            return verify.fail("body contains instructions after noreturn");
+        }
+    }
+    return verify.fail("body does not terminate noreturn");
+}
+
+const std = @import("std");
+const log = std.log.scoped(.air_verify);
+
+const Zcu = @import("../Zcu.zig");
+const InternPool = @import("../InternPool.zig");
+const Air = @import("../Air.zig");
+const Type = @import("../Type.zig");
+const Value = @import("../Value.zig");

src/Air/print.zig

@@ -232,12 +232,19 @@ const Writer = struct {
             .arg => try w.writeArg(s, inst),
 
             .not,
-            .bitcast,
+            .bit_cast,
+            .ptr_cast,
+            .ptr_from_int,
+            .int_from_ptr,
+            .error_cast,
+            .error_from_int,
+            .int_from_error,
+            .union_from_enum,
             .load,
             .fptrunc,
             .fpext,
-            .intcast,
-            .intcast_safe,
+            .int_cast,
+            .int_cast_safe,
             .trunc,
             .optional_payload,
             .optional_payload_ptr,

src/InternPool.zig

@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {
         return r;
     }
 
-    const LlvmBuilderAlignment = std.zig.llvm.Builder.Alignment;
-
-    pub fn toLlvm(a: Alignment) LlvmBuilderAlignment {
-        return @enumFromInt(@intFromEnum(a));
-    }
-
-    pub fn fromLlvm(a: LlvmBuilderAlignment) Alignment {
+    pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
         return @enumFromInt(@intFromEnum(a));
     }
 };

src/Sema/LowerZon.zig

@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool
         .@"packed" => result: {
             const arena = self.sema.arena;
             const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));
+            @memset(buf, 0);
             var bit_offset: u16 = 0;
             for (field_values) |field_ip| {
                 const field_val: Value = .fromInterned(field_ip);
-                field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) {
-                    error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
-                    error.OutOfMemory => |e| return e,
-                };
+                field_val.writeToPackedMemory(zcu, buf, bit_offset);
                 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
             }
             assert(bit_offset == res_ty.bitSize(zcu));
-            break :result Value.readFromPackedMemory(res_ty, pt, buf, 0, arena) catch |err| switch (err) {
-                error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
-                error.OutOfMemory => |e| return e,
-            };
+            break :result try .readFromPackedMemory(res_ty, pt, buf, 0);
         },
     };
     return result.toIntern();
@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool.
     };
     const result: Value = switch (union_info.layout) {
         .auto, .@"extern" => try pt.unionValue(res_ty, tag, val),
-        .@"packed" => try self.sema.bitCastVal(val, res_ty, 0, 0, 0) orelse {
-            unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
-        },
+        .@"packed" => try self.sema.bitCastVal(val, res_ty),
     };
     return result.toIntern();
 }

src/Sema/bitcast.zig

@@ -1,774 +0,0 @@
-//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
-//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
-//! a sequence of values in *packed* memory, and then unflatten through a combination of special
-//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
-//!
-//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
-//! have their fields reversed when represented as packed memory on such targets.
-
-/// If `host_bits` is `0`, attempts to convert the memory at offset
-/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
-/// ignoring `bit_offset`.
-///
-/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
-/// non-packed value consisting of `host_bits` bits. A value of type
-/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
-/// into this value.
-///
-/// Returns `null` if the operation must be performed at runtime.
-pub fn bitCast(
-    sema: *Sema,
-    val: Value,
-    dest_ty: Type,
-    byte_offset: u64,
-    host_bits: u64,
-    bit_offset: u64,
-) CompileError!?Value {
-    return bitCastInner(sema, val, dest_ty, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
-        error.ReinterpretDeclRef => return null,
-        error.IllDefinedMemoryLayout => unreachable,
-        error.Unimplemented => @panic("unimplemented bitcast"),
-        else => |e| return e,
-    };
-}
-
-/// Uses bitcasting to splice the value `splice_val` into `val`,
-/// replacing overlapping bits and returning the modified value.
-///
-/// If `host_bits` is `0`, splices `splice_val` at an offset
-/// `byte_offset` bytes into the virtual memory of `val`, ignoring
-/// `bit_offset`.
-///
-/// Otherwise, `byte_offset` is an offset into bytes into `val` to
-/// a non-packed value consisting of `host_bits` bits. The value
-/// `splice_val` will be placed at a packed offset of `bit_offset`
-/// into this value.
-pub fn bitCastSplice(
-    sema: *Sema,
-    val: Value,
-    splice_val: Value,
-    byte_offset: u64,
-    host_bits: u64,
-    bit_offset: u64,
-) CompileError!?Value {
-    return bitCastSpliceInner(sema, val, splice_val, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
-        error.ReinterpretDeclRef => return null,
-        error.IllDefinedMemoryLayout => unreachable,
-        error.Unimplemented => @panic("unimplemented bitcast"),
-        else => |e| return e,
-    };
-}
-
-const BitCastError = CompileError || error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented };
-
-fn bitCastInner(
-    sema: *Sema,
-    val: Value,
-    dest_ty: Type,
-    byte_offset: u64,
-    host_bits: u64,
-    bit_offset: u64,
-) BitCastError!Value {
-    const pt = sema.pt;
-    const zcu = pt.zcu;
-    const endian = zcu.getTarget().cpu.arch.endian();
-
-    if (dest_ty.toIntern() == val.typeOf(zcu).toIntern() and bit_offset == 0) {
-        return val;
-    }
-
-    const val_ty = val.typeOf(zcu);
-
-    val_ty.assertHasLayout(zcu);
-    dest_ty.assertHasLayout(zcu);
-
-    assert(val_ty.hasWellDefinedLayout(zcu));
-
-    const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
-        .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
-    else
-        .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
-
-    const skip_bits = switch (endian) {
-        .little => bit_offset + byte_offset * 8,
-        .big => if (host_bits > 0)
-            val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
-        else
-            val_ty.abiSize(zcu) * 8 - byte_offset * 8 - dest_ty.bitSize(zcu),
-    };
-
-    var unpack: UnpackValueBits = .{
-        .pt = sema.pt,
-        .arena = sema.arena,
-        .skip_bits = skip_bits,
-        .remaining_bits = dest_ty.bitSize(zcu),
-        .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
-    };
-    switch (endian) {
-        .little => {
-            try unpack.add(val);
-            try unpack.padding(abi_pad_bits);
-        },
-        .big => {
-            try unpack.padding(abi_pad_bits);
-            try unpack.add(val);
-        },
-    }
-    try unpack.padding(host_pad_bits);
-
-    var pack: PackValueBits = .{
-        .pt = sema.pt,
-        .arena = sema.arena,
-        .unpacked = unpack.unpacked.items,
-    };
-    return pack.get(dest_ty);
-}
-
-fn bitCastSpliceInner(
-    sema: *Sema,
-    val: Value,
-    splice_val: Value,
-    byte_offset: u64,
-    host_bits: u64,
-    bit_offset: u64,
-) BitCastError!Value {
-    const pt = sema.pt;
-    const zcu = pt.zcu;
-    const endian = zcu.getTarget().cpu.arch.endian();
-    const val_ty = val.typeOf(zcu);
-    const splice_val_ty = splice_val.typeOf(zcu);
-
-    val_ty.assertHasLayout(zcu);
-    splice_val_ty.assertHasLayout(zcu);
-
-    const splice_bits = splice_val_ty.bitSize(zcu);
-
-    const splice_offset = switch (endian) {
-        .little => bit_offset + byte_offset * 8,
-        .big => if (host_bits > 0)
-            val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
-        else
-            val_ty.abiSize(zcu) * 8 - byte_offset * 8 - splice_bits,
-    };
-
-    assert(splice_offset + splice_bits <= val_ty.abiSize(zcu) * 8);
-
-    const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
-        .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
-    else
-        .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
-
-    var unpack: UnpackValueBits = .{
-        .pt = pt,
-        .arena = sema.arena,
-        .skip_bits = 0,
-        .remaining_bits = splice_offset,
-        .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
-    };
-    switch (endian) {
-        .little => {
-            try unpack.add(val);
-            try unpack.padding(abi_pad_bits);
-        },
-        .big => {
-            try unpack.padding(abi_pad_bits);
-            try unpack.add(val);
-        },
-    }
-    try unpack.padding(host_pad_bits);
-
-    unpack.remaining_bits = splice_bits;
-    try unpack.add(splice_val);
-
-    unpack.skip_bits = splice_offset + splice_bits;
-    unpack.remaining_bits = val_ty.abiSize(zcu) * 8 - splice_offset - splice_bits;
-    switch (endian) {
-        .little => {
-            try unpack.add(val);
-            try unpack.padding(abi_pad_bits);
-        },
-        .big => {
-            try unpack.padding(abi_pad_bits);
-            try unpack.add(val);
-        },
-    }
-    try unpack.padding(host_pad_bits);
-
-    var pack: PackValueBits = .{
-        .pt = pt,
-        .arena = sema.arena,
-        .unpacked = unpack.unpacked.items,
-    };
-    switch (endian) {
-        .little => {},
-        .big => try pack.padding(abi_pad_bits),
-    }
-    return pack.get(val_ty);
-}
-
-/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
-/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
-/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
-/// list contains bits from the *final* byte of the value.
-const UnpackValueBits = struct {
-    pt: Zcu.PerThread,
-    arena: Allocator,
-    skip_bits: u64,
-    remaining_bits: u64,
-    extra_bits: u64 = undefined,
-    unpacked: std.array_list.Managed(InternPool.Index),
-
-    fn add(unpack: *UnpackValueBits, val: Value) BitCastError!void {
-        const pt = unpack.pt;
-        const zcu = pt.zcu;
-        const endian = zcu.getTarget().cpu.arch.endian();
-        const ip = &zcu.intern_pool;
-
-        if (unpack.remaining_bits == 0) {
-            return;
-        }
-
-        const ty = val.typeOf(zcu);
-        const bit_size = ty.bitSize(zcu);
-
-        if (unpack.skip_bits >= bit_size) {
-            unpack.skip_bits -= bit_size;
-            return;
-        }
-
-        switch (ip.indexToKey(val.toIntern())) {
-            .int_type,
-            .ptr_type,
-            .array_type,
-            .vector_type,
-            .opt_type,
-            .anyframe_type,
-            .error_union_type,
-            .simple_type,
-            .struct_type,
-            .tuple_type,
-            .union_type,
-            .opaque_type,
-            .spirv_type,
-            .enum_type,
-            .func_type,
-            .error_set_type,
-            .inferred_error_set_type,
-            .@"extern",
-            .func,
-            .err,
-            .error_union,
-            .enum_literal,
-            .slice,
-            .memoized_call,
-            => unreachable, // ill-defined layout or not real values
-
-            .undef,
-            .int,
-            .enum_tag,
-            .simple_value,
-            .float,
-            .ptr,
-            .opt,
-            => try unpack.primitive(val),
-
-            .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
-
-            .aggregate => switch (ty.zigTypeTag(zcu)) {
-                .vector => {
-                    const len: usize = @intCast(ty.arrayLen(zcu));
-                    for (0..len) |i| {
-                        // We reverse vector elements in packed memory on BE targets.
-                        const real_idx = switch (endian) {
-                            .little => i,
-                            .big => len - i - 1,
-                        };
-                        const elem_val = try val.elemValue(pt, real_idx);
-                        try unpack.add(elem_val);
-                    }
-                },
-                .array => {
-                    // Each element is padded up to its ABI size. Padding bits are undefined.
-                    // The final element does not have trailing padding.
-                    // Elements are reversed in packed memory on BE targets.
-                    const elem_ty = ty.childType(zcu);
-                    const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
-                    const len = ty.arrayLen(zcu);
-                    const maybe_sent = ty.sentinel(zcu);
-
-                    if (endian == .big) if (maybe_sent) |s| {
-                        try unpack.add(s);
-                        if (len != 0) try unpack.padding(pad_bits);
-                    };
-
-                    for (0..@intCast(len)) |i| {
-                        // We reverse array elements in packed memory on BE targets.
-                        const real_idx = switch (endian) {
-                            .little => i,
-                            .big => len - i - 1,
-                        };
-                        const elem_val = try val.elemValue(pt, @intCast(real_idx));
-                        try unpack.add(elem_val);
-                        if (i != len - 1) try unpack.padding(pad_bits);
-                    }
-
-                    if (endian == .little) if (maybe_sent) |s| {
-                        if (len != 0) try unpack.padding(pad_bits);
-                        try unpack.add(s);
-                    };
-                },
-                .@"struct" => switch (ty.containerLayout(zcu)) {
-                    .auto => unreachable, // ill-defined layout
-                    .@"extern" => switch (endian) {
-                        .little => {
-                            var cur_bit_off: u64 = 0;
-                            var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
-                            while (it.next()) |field_idx| {
-                                const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
-                                const pad_bits = want_bit_off - cur_bit_off;
-                                const field_val = try val.fieldValue(pt, field_idx);
-                                try unpack.padding(pad_bits);
-                                try unpack.add(field_val);
-                                cur_bit_off = want_bit_off + field_val.typeOf(zcu).bitSize(zcu);
-                            }
-                            // Add trailing padding bits.
-                            try unpack.padding(bit_size - cur_bit_off);
-                        },
-                        .big => {
-                            var cur_bit_off: u64 = bit_size;
-                            var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
-                            while (it.next()) |field_idx| {
-                                const field_val = try val.fieldValue(pt, field_idx);
-                                const field_ty = field_val.typeOf(zcu);
-                                const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
-                                const pad_bits = cur_bit_off - want_bit_off;
-                                try unpack.padding(pad_bits);
-                                try unpack.add(field_val);
-                                cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
-                            }
-                            assert(cur_bit_off == 0);
-                        },
-                    },
-                    .@"packed" => {
-                        // Just add all fields in order. There are no padding bits.
-                        // This is identical between LE and BE targets.
-                        for (0..ty.structFieldCount(zcu)) |i| {
-                            const field_val = try val.fieldValue(pt, i);
-                            try unpack.add(field_val);
-                        }
-                    },
-                },
-                else => unreachable,
-            },
-
-            .un => |un| {
-                // We actually don't care about the tag here!
-                // Instead, we just need to write the payload value, plus any necessary padding.
-                // This correctly handles the case where `tag == .none`, since the payload is then
-                // either an integer or a byte array, both of which we can unpack.
-                const payload_val = Value.fromInterned(un.val);
-                const pad_bits = bit_size - payload_val.typeOf(zcu).bitSize(zcu);
-                if (endian == .little or ty.containerLayout(zcu) == .@"packed") {
-                    try unpack.add(payload_val);
-                    try unpack.padding(pad_bits);
-                } else {
-                    try unpack.padding(pad_bits);
-                    try unpack.add(payload_val);
-                }
-            },
-        }
-    }
-
-    fn padding(unpack: *UnpackValueBits, pad_bits: u64) BitCastError!void {
-        if (pad_bits == 0) return;
-        const pt = unpack.pt;
-        // Figure out how many full bytes and leftover bits there are.
-        const bytes = pad_bits / 8;
-        const bits = pad_bits % 8;
-        // Add undef u8 values for the bytes...
-        const undef_u8 = try pt.undefValue(Type.u8);
-        for (0..@intCast(bytes)) |_| {
-            try unpack.primitive(undef_u8);
-        }
-        // ...and an undef int for the leftover bits.
-        if (bits == 0) return;
-        const bits_ty = try pt.intType(.unsigned, @intCast(bits));
-        const bits_val = try pt.undefValue(bits_ty);
-        try unpack.primitive(bits_val);
-    }
-
-    fn primitive(unpack: *UnpackValueBits, val: Value) BitCastError!void {
-        const pt = unpack.pt;
-        const zcu = pt.zcu;
-
-        if (unpack.remaining_bits == 0) {
-            return;
-        }
-
-        const ty = val.typeOf(pt.zcu);
-        const bit_size = ty.bitSize(zcu);
-
-        // Note that this skips all zero-bit types.
-        if (unpack.skip_bits >= bit_size) {
-            unpack.skip_bits -= bit_size;
-            return;
-        }
-
-        if (unpack.skip_bits > 0) {
-            const skip = unpack.skip_bits;
-            unpack.skip_bits = 0;
-            return unpack.splitPrimitive(val, skip, bit_size - skip);
-        }
-
-        if (unpack.remaining_bits < bit_size) {
-            return unpack.splitPrimitive(val, 0, unpack.remaining_bits);
-        }
-
-        unpack.remaining_bits -|= bit_size;
-
-        try unpack.unpacked.append(val.toIntern());
-    }
-
-    fn splitPrimitive(unpack: *UnpackValueBits, val: Value, bit_offset: u64, bit_count: u64) BitCastError!void {
-        const pt = unpack.pt;
-        const zcu = pt.zcu;
-        const ty = val.typeOf(pt.zcu);
-
-        const val_bits = ty.bitSize(zcu);
-        assert(bit_offset + bit_count <= val_bits);
-
-        switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
-            // In the `ptr` case, this will return `error.ReinterpretDeclRef`
-            // if we're trying to split a non-integer pointer value.
-            .int, .float, .enum_tag, .ptr, .opt => {
-                // This @intCast is okay because no primitive can exceed the size of a u16.
-                const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
-                const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
-                try val.writeToPackedMemory(zcu, buf, 0);
-                const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
-                try unpack.primitive(sub_val);
-            },
-            .undef => try unpack.padding(bit_count),
-            // The only values here with runtime bits are `true` and `false.
-            // These are both 1 bit, so will never need truncating.
-            .simple_value => unreachable,
-            else => unreachable, // zero-bit or not primitives
-        }
-    }
-};
-
-/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBits`),
-/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
-/// values and of pointers which align in virtual memory.
-const PackValueBits = struct {
-    pt: Zcu.PerThread,
-    arena: Allocator,
-    bit_offset: u64 = 0,
-    unpacked: []const InternPool.Index,
-
-    fn get(pack: *PackValueBits, ty: Type) BitCastError!Value {
-        const pt = pack.pt;
-        const zcu = pt.zcu;
-        const endian = zcu.getTarget().cpu.arch.endian();
-        const ip = &zcu.intern_pool;
-        const arena = pack.arena;
-        switch (ty.zigTypeTag(zcu)) {
-            .vector => {
-                // Elements are bit-packed.
-                const len = ty.arrayLen(zcu);
-                const elem_ty = ty.childType(zcu);
-                const elems = try arena.alloc(InternPool.Index, @intCast(len));
-                // We reverse vector elements in packed memory on BE targets.
-                switch (endian) {
-                    .little => for (elems) |*elem| {
-                        elem.* = (try pack.get(elem_ty)).toIntern();
-                    },
-                    .big => {
-                        var i = elems.len;
-                        while (i > 0) {
-                            i -= 1;
-                            elems[i] = (try pack.get(elem_ty)).toIntern();
-                        }
-                    },
-                }
-                return pt.aggregateValue(ty, elems);
-            },
-            .array => {
-                // Each element is padded up to its ABI size. The final element does not have trailing padding.
-                const len = ty.arrayLen(zcu);
-                const elem_ty = ty.childType(zcu);
-                const maybe_sent = ty.sentinel(zcu);
-                const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
-                const elems = try arena.alloc(InternPool.Index, @intCast(len));
-
-                if (endian == .big and maybe_sent != null) {
-                    // TODO: validate sentinel was preserved!
-                    try pack.padding(elem_ty.bitSize(zcu));
-                    if (len != 0) try pack.padding(pad_bits);
-                }
-
-                for (0..elems.len) |i| {
-                    const real_idx = switch (endian) {
-                        .little => i,
-                        .big => len - i - 1,
-                    };
-                    elems[@intCast(real_idx)] = (try pack.get(elem_ty)).toIntern();
-                    if (i != len - 1) try pack.padding(pad_bits);
-                }
-
-                if (endian == .little and maybe_sent != null) {
-                    // TODO: validate sentinel was preserved!
-                    if (len != 0) try pack.padding(pad_bits);
-                    try pack.padding(elem_ty.bitSize(zcu));
-                }
-
-                return pt.aggregateValue(ty, elems);
-            },
-            .@"struct" => switch (ty.containerLayout(zcu)) {
-                .auto => unreachable, // ill-defined layout
-                .@"extern" => {
-                    const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
-                    @memset(elems, .none);
-                    switch (endian) {
-                        .little => {
-                            var cur_bit_off: u64 = 0;
-                            var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
-                            while (it.next()) |field_idx| {
-                                const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
-                                try pack.padding(want_bit_off - cur_bit_off);
-                                const field_ty = ty.fieldType(field_idx, zcu);
-                                elems[field_idx] = (try pack.get(field_ty)).toIntern();
-                                cur_bit_off = want_bit_off + field_ty.bitSize(zcu);
-                            }
-                            try pack.padding(ty.bitSize(zcu) - cur_bit_off);
-                        },
-                        .big => {
-                            var cur_bit_off: u64 = ty.bitSize(zcu);
-                            var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
-                            while (it.next()) |field_idx| {
-                                const field_ty = ty.fieldType(field_idx, zcu);
-                                const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
-                                try pack.padding(cur_bit_off - want_bit_off);
-                                elems[field_idx] = (try pack.get(field_ty)).toIntern();
-                                cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
-                            }
-                            assert(cur_bit_off == 0);
-                        },
-                    }
-                    // Any fields which do not have runtime bits should be OPV or comptime fields.
-                    // Fill those values now.
-                    for (elems, 0..) |*elem, field_idx| {
-                        if (elem.* != .none) continue;
-                        const val = (try ty.structFieldValueComptime(pt, field_idx)).?;
-                        elem.* = val.toIntern();
-                    }
-                    return pt.aggregateValue(ty, elems);
-                },
-                .@"packed" => {
-                    const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
-                    if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
-                    return pt.bitpackValue(ty, backing_int_val);
-                },
-            },
-            .@"union" => switch (ty.containerLayout(zcu)) {
-                .auto => unreachable, // ill-defined layout
-                .@"extern" => {
-                    // We will attempt to read as the backing representation. If this emits
-                    // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
-                    // We will also attempt smaller fields when we get `undefined`, as if some bits are
-                    // defined we want to include them.
-                    // TODO: this is very very bad. We need a more sophisticated union representation.
-
-                    const prev_unpacked = pack.unpacked;
-                    const prev_bit_offset = pack.bit_offset;
-
-                    const backing_ty = try ty.externUnionBackingType(pt);
-
-                    backing: {
-                        const backing_val = pack.get(backing_ty) catch |err| switch (err) {
-                            error.ReinterpretDeclRef => {
-                                pack.unpacked = prev_unpacked;
-                                pack.bit_offset = prev_bit_offset;
-                                break :backing;
-                            },
-                            else => |e| return e,
-                        };
-                        if (backing_val.isUndef(zcu)) {
-                            pack.unpacked = prev_unpacked;
-                            pack.bit_offset = prev_bit_offset;
-                            break :backing;
-                        }
-                        return Value.fromInterned(try pt.internUnion(.{
-                            .ty = ty.toIntern(),
-                            .tag = .none,
-                            .val = backing_val.toIntern(),
-                        }));
-                    }
-
-                    const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
-                    for (field_order, 0..) |*f, i| f.* = @intCast(i);
-                    // Sort `field_order` to put the fields with the largest bit sizes first.
-                    const SizeSortCtx = struct {
-                        zcu: *Zcu,
-                        field_types: []const InternPool.Index,
-                        fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
-                            const a_ty = Type.fromInterned(ctx.field_types[a_idx]);
-                            const b_ty = Type.fromInterned(ctx.field_types[b_idx]);
-                            return a_ty.bitSize(ctx.zcu) > b_ty.bitSize(ctx.zcu);
-                        }
-                    };
-                    std.mem.sortUnstable(u32, field_order, SizeSortCtx{
-                        .zcu = zcu,
-                        .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
-                    }, SizeSortCtx.lessThan);
-
-                    const padding_after = endian == .little or ty.containerLayout(zcu) == .@"packed";
-
-                    for (field_order) |field_idx| {
-                        const field_ty = Type.fromInterned(zcu.typeToUnion(ty).?.field_types.get(ip)[field_idx]);
-                        const pad_bits = ty.bitSize(zcu) - field_ty.bitSize(zcu);
-                        if (!padding_after) try pack.padding(pad_bits);
-                        const field_val = pack.get(field_ty) catch |err| switch (err) {
-                            error.ReinterpretDeclRef => {
-                                pack.unpacked = prev_unpacked;
-                                pack.bit_offset = prev_bit_offset;
-                                continue;
-                            },
-                            else => |e| return e,
-                        };
-                        if (padding_after) try pack.padding(pad_bits);
-                        if (field_val.isUndef(zcu)) {
-                            pack.unpacked = prev_unpacked;
-                            pack.bit_offset = prev_bit_offset;
-                            continue;
-                        }
-                        const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_idx);
-                        return Value.fromInterned(try pt.internUnion(.{
-                            .ty = ty.toIntern(),
-                            .tag = tag_val.toIntern(),
-                            .val = field_val.toIntern(),
-                        }));
-                    }
-
-                    // No field could represent the value. Just do whatever happens when we try to read
-                    // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
-                    const backing_val = try pack.get(backing_ty);
-                    return Value.fromInterned(try pt.internUnion(.{
-                        .ty = ty.toIntern(),
-                        .tag = .none,
-                        .val = backing_val.toIntern(),
-                    }));
-                },
-                .@"packed" => {
-                    const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
-                    if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
-                    return pt.bitpackValue(ty, backing_int_val);
-                },
-            },
-            else => return pack.primitive(ty),
-        }
-    }
-
-    fn padding(pack: *PackValueBits, pad_bits: u64) BitCastError!void {
-        _ = pack.prepareBits(pad_bits);
-    }
-
-    fn primitive(pack: *PackValueBits, want_ty: Type) BitCastError!Value {
-        const pt = pack.pt;
-        const zcu = pt.zcu;
-
-        if (try want_ty.onePossibleValue(pt)) |opv| return opv;
-
-        const vals, const bit_offset = pack.prepareBits(want_ty.bitSize(zcu));
-
-        for (vals) |val| {
-            if (!Value.fromInterned(val).isUndef(zcu)) break;
-        } else {
-            // All bits of the value are `undefined`.
-            return pt.undefValue(want_ty);
-        }
-
-        // TODO: we need to decide how to handle partially-undef values here.
-        // Currently, a value with some undefined bits becomes `0xAA` so that we
-        // preserve the well-defined bits, because we can't currently represent
-        // a partially-undefined primitive (e.g. an int with some undef bits).
-        // In future, we probably want to take one of these two routes:
-        // * Define that if any bits are `undefined`, the entire value is `undefined`.
-        //   This is a major breaking change, and probably a footgun.
-        // * Introduce tracking for partially-undef values at comptime.
-        //   This would complicate a lot of operations in Sema, such as basic
-        //   arithmetic.
-        // This design complexity is tracked by #19634.
-
-        ptr_cast: {
-            if (vals.len != 1) break :ptr_cast;
-            const val = Value.fromInterned(vals[0]);
-            if (!val.typeOf(zcu).isPtrAtRuntime(zcu)) break :ptr_cast;
-            if (!want_ty.isPtrAtRuntime(zcu)) break :ptr_cast;
-            return pt.getCoerced(val, want_ty);
-        }
-
-        // Reinterpret via an in-memory buffer.
-
-        var buf_bits: u64 = 0;
-        for (vals) |ip_val| {
-            const val = Value.fromInterned(ip_val);
-            const ty = val.typeOf(pt.zcu);
-            buf_bits += ty.bitSize(zcu);
-        }
-
-        const buf = try pack.arena.alloc(u8, @intCast((buf_bits + 7) / 8));
-        // We will skip writing undefined values, so mark the buffer as `0xAA` so we get "undefined" bits.
-        @memset(buf, 0xAA);
-        var cur_bit_off: usize = 0;
-        for (vals) |ip_val| {
-            const val = Value.fromInterned(ip_val);
-            const ty = val.typeOf(zcu);
-            if (!val.isUndef(zcu)) {
-                try val.writeToPackedMemory(zcu, buf, cur_bit_off);
-            }
-            cur_bit_off += @intCast(ty.bitSize(zcu));
-        }
-
-        return Value.readFromPackedMemory(want_ty, pt, buf, @intCast(bit_offset), pack.arena);
-    }
-
-    fn prepareBits(pack: *PackValueBits, need_bits: u64) struct { []const InternPool.Index, u64 } {
-        if (need_bits == 0) return .{ &.{}, 0 };
-
-        const pt = pack.pt;
-        const zcu = pt.zcu;
-
-        var bits: u64 = 0;
-        var len: usize = 0;
-        while (bits < pack.bit_offset + need_bits) {
-            bits += Value.fromInterned(pack.unpacked[len]).typeOf(pt.zcu).bitSize(zcu);
-            len += 1;
-        }
-
-        const result_vals = pack.unpacked[0..len];
-        const result_offset = pack.bit_offset;
-
-        const extra_bits = bits - pack.bit_offset - need_bits;
-        if (extra_bits == 0) {
-            pack.unpacked = pack.unpacked[len..];
-            pack.bit_offset = 0;
-        } else {
-            pack.unpacked = pack.unpacked[len - 1 ..];
-            pack.bit_offset = Value.fromInterned(pack.unpacked[0]).typeOf(pt.zcu).bitSize(zcu) - extra_bits;
-        }
-
-        return .{ result_vals, result_offset };
-    }
-};
-
-const std = @import("std");
-const Allocator = std.mem.Allocator;
-const assert = std.debug.assert;
-
-const Sema = @import("../Sema.zig");
-const Zcu = @import("../Zcu.zig");
-const InternPool = @import("../InternPool.zig");
-const Type = @import("../Type.zig");
-const Value = @import("../Value.zig");
-const CompileError = Zcu.CompileError;

src/Sema/comptime_ptr_access.zig

@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {
 pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {
     const pt = sema.pt;
     const zcu = pt.zcu;
+
     const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
-    // TODO: host size for vectors is terrible
-    const host_bits = switch (ptr_info.flags.vector_index) {
-        .none => ptr_info.packed_offset.host_size * 8,
-        else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
-    };
-    const bit_offset = if (host_bits != 0) bit_offset: {
-        const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
-        const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
-            .none => 0,
-            else => |idx| switch (pt.zcu.getTarget().cpu.arch.endian()) {
-                .little => child_bits * @intFromEnum(idx),
-                .big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
-            },
-        };
-        if (child_bits + bit_offset > host_bits) {
+    const elem_ty: Type = .fromInterned(ptr_info.child);
+    const host_size = ptr_info.packed_offset.host_size;
+
+    if (host_size == 0) {
+        return loadComptimePtrInner(sema, block, src, ptr, elem_ty, 0);
+    }
+
+    assert(elem_ty.hasBitRepresentation(zcu));
+    if (ptr_info.flags.vector_index == .none) {
+        if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
             return .exceeds_host_size;
         }
-        break :bit_offset bit_offset;
-    } else 0;
-    return loadComptimePtrInner(sema, block, src, ptr, bit_offset, host_bits, Type.fromInterned(ptr_info.child), 0);
+        const load_ty: Type = try pt.intType(.unsigned, host_size * 8);
+        const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
+            else => |result| return result,
+            .success => |mv| mv,
+        };
+        const backing_int_val = try backing_int_mv.intern(pt, sema.arena);
+        const buf = try sema.arena.alloc(u8, host_size);
+        @memset(buf, 0);
+        backing_int_val.writeToPackedMemory(zcu, buf, 0);
+        const result_val: Value = try .readFromPackedMemory(elem_ty, pt, buf, ptr_info.packed_offset.bit_offset);
+        return .{ .success = .{ .interned = result_val.toIntern() } };
+    }
+    if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
+        return .exceeds_host_size;
+    }
+    const load_ty: Type = try pt.vectorType(.{
+        .len = host_size,
+        .child = elem_ty.toIntern(),
+    });
+    const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
+        else => |result| return result,
+        .success => |mv| mv,
+    };
+    const vector_val = try vector_mv.intern(pt, sema.arena);
+    const result_val = try vector_val.elemValue(pt, @intFromEnum(ptr_info.flags.vector_index));
+    return .{ .success = .{ .interned = result_val.toIntern() } };
 }
 
 pub const ComptimeStoreResult = union(enum) {
@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {
 };
 
 /// Perform a comptime load of value `store_val` to a pointer.
-/// The pointer's type is ignored.
+///
+/// Asserts that the type of `store_val` equals the element type of the pointer type.
 pub fn storeComptimePtr(
     sema: *Sema,
     block: *Block,
@@ -62,42 +82,84 @@ pub fn storeComptimePtr(
 ) !ComptimeStoreResult {
     const pt = sema.pt;
     const zcu = pt.zcu;
-    const ptr_info = ptr.typeOf(zcu).ptrInfo(zcu);
-    assert(store_val.typeOf(zcu).toIntern() == ptr_info.child);
 
-    {
-        const store_ty: Type = .fromInterned(ptr_info.child);
-        if (!store_ty.comptimeOnly(zcu) and !store_ty.hasRuntimeBits(zcu)) {
-            // zero-bit store; nothing to do
-            return .success;
-        }
+    const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
+    const elem_ty: Type = .fromInterned(ptr_info.child);
+    const host_size = ptr_info.packed_offset.host_size;
+    assert(store_val.typeOf(zcu).toIntern() == elem_ty.toIntern());
+
+    if (host_size == 0) {
+        return storeComptimePtrInner(sema, block, src, ptr, store_val);
     }
 
-    // TODO: host size for vectors is terrible
-    const host_bits = switch (ptr_info.flags.vector_index) {
-        .none => ptr_info.packed_offset.host_size * 8,
-        else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
-    };
-    const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
-        .none => 0,
-        else => |idx| switch (zcu.getTarget().cpu.arch.endian()) {
-            .little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
-            .big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian
-        },
-    };
-    const pseudo_store_ty = if (host_bits > 0) t: {
-        const need_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
-        if (need_bits + bit_offset > host_bits) {
+    assert(elem_ty.hasBitRepresentation(zcu));
+    if (ptr_info.flags.vector_index == .none) {
+        if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
             return .exceeds_host_size;
         }
-        break :t try sema.pt.intType(.unsigned, @intCast(host_bits));
-    } else Type.fromInterned(ptr_info.child);
+        const backing_ty: Type = try pt.intType(.unsigned, host_size * 8);
+        const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, backing_ty, 0)) {
+            .success => |mv| mv,
+            .runtime_load => return .runtime_store,
+            inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
+        };
+        const old_backing_int_val = try backing_int_mv.intern(pt, sema.arena);
+        const buf = try sema.arena.alloc(u8, host_size);
+        @memset(buf, 0);
+        old_backing_int_val.writeToPackedMemory(zcu, buf, 0);
+        // Write the new element...
+        store_val.writeToPackedMemory(zcu, buf, ptr_info.packed_offset.bit_offset);
+        // ...then read the resulting backing integer value...
+        const new_backing_int_val: Value = try .readFromPackedMemory(backing_ty, pt, buf, 0);
+        // ...and store that back into memory
+        return storeComptimePtrInner(sema, block, src, ptr, new_backing_int_val);
+    }
 
-    const strat = try prepareComptimePtrStore(sema, block, src, ptr, pseudo_store_ty, 0);
+    if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
+        return .exceeds_host_size;
+    }
+    const vec_ty: Type = try pt.vectorType(.{
+        .len = host_size,
+        .child = elem_ty.toIntern(),
+    });
+    const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, vec_ty, 0)) {
+        .success => |mv| mv,
+        .runtime_load => return .runtime_store,
+        inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
+    };
+    const old_vector_val = try vector_mv.intern(pt, sema.arena);
+    const elems_buf = try sema.arena.alloc(InternPool.Index, host_size);
+    for (elems_buf, 0..) |*elem, elem_index| {
+        const elem_val = try old_vector_val.elemValue(pt, elem_index);
+        elem.* = elem_val.toIntern();
+    }
+    elems_buf[@intFromEnum(ptr_info.flags.vector_index)] = store_val.toIntern();
+    const new_vector_val = try pt.aggregateValue(vec_ty, elems_buf);
+    return storeComptimePtrInner(sema, block, src, ptr, new_vector_val);
+}
+
+/// Like `storeComptimePtr`, except ignores the type of `ptr`, instead treating it as a single-item
+/// pointer to `store_val.typeOf(zcu)`.
+fn storeComptimePtrInner(
+    sema: *Sema,
+    block: *Block,
+    src: LazySrcLoc,
+    ptr: Value,
+    store_val: Value,
+) !ComptimeStoreResult {
+    const pt = sema.pt;
+    const zcu = pt.zcu;
+    const store_ty = store_val.typeOf(zcu);
+
+    if (store_ty.classify(zcu) == .one_possible_value) {
+        // zero-bit store; nothing to do
+        return .success;
+    }
+
+    const strat = try prepareComptimePtrStore(sema, block, src, ptr, store_ty, 0);
 
     // Propagate errors and handle comptime fields.
     switch (strat) {
-        .direct, .index, .flat_index, .reinterpret => {},
         .comptime_field => {
             // To "store" to a comptime field, just perform a load of the field
             // and see if the store value matches.
@@ -125,79 +187,60 @@ pub fn storeComptimePtr(
         .inactive_union_field => return .inactive_union_field,
         .needed_well_defined => |ty| return .{ .needed_well_defined = ty },
         .out_of_bounds => |ty| return .{ .out_of_bounds = ty },
-    }
 
-    // Check the store is not inside a runtime condition
-    try checkComptimeVarStore(sema, block, src, strat.alloc());
-
-    if (host_bits == 0) {
-        // We can attempt a direct store depending on the strategy.
-        switch (strat) {
-            .direct => |direct| {
-                const want_ty = direct.val.typeOf(zcu);
-                const coerced_store_val = try pt.getCoerced(store_val, want_ty);
-                direct.val.* = .{ .interned = coerced_store_val.toIntern() };
-                return .success;
-            },
-            .index => |index| {
-                const want_ty = index.val.typeOf(zcu).childType(zcu);
-                const coerced_store_val = try pt.getCoerced(store_val, want_ty);
-                try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
-                return .success;
-            },
-            .flat_index => |flat| {
-                const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
-                const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
-                {
-                    var next_idx: u64 = 0;
-                    var skip: u64 = 0;
-                    try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
-                }
-                for (flat_elems, 0..) |elem, idx| {
-                    // TODO: recursiveIndex in a loop does a lot of redundant work!
-                    // Better would be to gather all the store targets into an array.
-                    var index: u64 = flat.flat_elem_index + idx;
-                    const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
-                    try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
-                }
-                return .success;
-            },
-            .reinterpret => {},
-            else => unreachable,
-        }
-    }
-
-    // Either there is a bit offset, or the strategy required reinterpreting.
-    // Therefore, we must perform a bitcast.
-
-    const val_ptr: *MutableValue, const byte_offset: u64 = switch (strat) {
-        .direct => |direct| .{ direct.val, 0 },
-        .index => |index| .{
-            index.val,
-            index.elem_index * index.val.typeOf(zcu).childType(zcu).abiSize(zcu),
+        .direct => |direct| {
+            try checkComptimeVarStore(sema, block, src, direct.alloc);
+            const want_ty = direct.val.typeOf(zcu);
+            const coerced_store_val = try pt.getCoerced(store_val, want_ty);
+            direct.val.* = .{ .interned = coerced_store_val.toIntern() };
+            return .success;
         },
-        .flat_index => |flat| .{ flat.val, flat.flat_elem_index * flat.val.typeOf(zcu).arrayBase(zcu)[0].abiSize(zcu) },
-        .reinterpret => |reinterpret| .{ reinterpret.val, reinterpret.byte_offset },
-        else => unreachable,
-    };
 
-    if (!val_ptr.typeOf(zcu).hasWellDefinedLayout(zcu)) {
-        return .{ .needed_well_defined = val_ptr.typeOf(zcu) };
+        .index => |index| {
+            try checkComptimeVarStore(sema, block, src, index.alloc);
+            const want_ty = index.val.typeOf(zcu).childType(zcu);
+            const coerced_store_val = try pt.getCoerced(store_val, want_ty);
+            try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
+            return .success;
+        },
+
+        .flat_index => |flat| {
+            try checkComptimeVarStore(sema, block, src, flat.alloc);
+            const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
+            const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
+            {
+                var next_idx: u64 = 0;
+                var skip: u64 = 0;
+                try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
+            }
+            for (flat_elems, 0..) |elem, idx| {
+                // TODO: recursiveIndex in a loop does a lot of redundant work!
+                // Better would be to gather all the store targets into an array.
+                var index: u64 = flat.flat_elem_index + idx;
+                const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
+                try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
+            }
+            return .success;
+        },
+
+        .reinterpret => |reinterpret| {
+            try checkComptimeVarStore(sema, block, src, reinterpret.alloc);
+            if (!reinterpret.val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
+                return .{ .needed_well_defined = reinterpret.val.typeOf(zcu) };
+            }
+            if (!store_ty.hasWellDefinedLayout(zcu)) {
+                return .{ .needed_well_defined = store_ty };
+            }
+            const old_val = try reinterpret.val.intern(pt, sema.arena);
+            const new_val = try sema.spliceMemory(
+                old_val,
+                store_val,
+                reinterpret.byte_offset,
+            ) orelse return .runtime_store;
+            reinterpret.val.* = .{ .interned = new_val.toIntern() };
+            return .success;
+        },
     }
-
-    if (!store_val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
-        return .{ .needed_well_defined = store_val.typeOf(zcu) };
-    }
-
-    const new_val = try sema.bitCastSpliceVal(
-        try val_ptr.intern(pt, sema.arena),
-        store_val,
-        byte_offset,
-        host_bits,
-        bit_offset,
-    ) orelse return .runtime_store;
-    val_ptr.* = .{ .interned = new_val.toIntern() };
-    return .success;
 }
 
 /// Perform a comptime load of type `load_ty` from a pointer.
@@ -207,8 +250,6 @@ fn loadComptimePtrInner(
     block: *Block,
     src: LazySrcLoc,
     ptr_val: Value,
-    bit_offset: u64,
-    host_bits: u64,
     load_ty: Type,
     /// If `load_ty` is an array, this is the number of array elements to skip
     /// before `load_ty`. Otherwise, it is ignored and may be `undefined`.
@@ -244,7 +285,7 @@ fn loadComptimePtrInner(
         .eu_payload => |base_ptr_ip| val: {
             const base_ptr = Value.fromInterned(base_ptr_ip);
             const base_ty = base_ptr.typeOf(zcu).childType(zcu);
-            switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
+            switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
                 .success => |eu_val| switch (eu_val.unpackErrorUnion(zcu)) {
                     .undef => return .undef,
                     .err => |err| return .{ .err_payload = err },
@@ -256,7 +297,7 @@ fn loadComptimePtrInner(
         .opt_payload => |base_ptr_ip| val: {
             const base_ptr = Value.fromInterned(base_ptr_ip);
             const base_ty = base_ptr.typeOf(zcu).childType(zcu);
-            switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
+            switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
                 .success => |eu_val| switch (eu_val.unpackOptional(zcu)) {
                     .undef => return .undef,
                     .null => return .null_payload,
@@ -283,7 +324,7 @@ fn loadComptimePtrInner(
                 .child = base_ty.toIntern(),
             });
 
-            switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, want_ty, base_index.index)) {
+            switch (try loadComptimePtrInner(sema, block, src, base_ptr, want_ty, base_index.index)) {
                 .success => |arr_val| break :val arr_val,
                 else => |err| return err,
             }
@@ -293,7 +334,7 @@ fn loadComptimePtrInner(
             const base_ty = base_ptr.typeOf(zcu).childType(zcu);
 
             // Field of a slice, or of an auto-layout struct or union.
-            const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
+            const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
                 .success => |val| val,
                 else => |err| return err,
             };
@@ -324,7 +365,7 @@ fn loadComptimePtrInner(
         },
     };
 
-    if (ptr.byte_offset == 0 and host_bits == 0) {
+    if (ptr.byte_offset == 0) {
         if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {
             if (.ok == try sema.coerceInMemoryAllowed(
                 block,
@@ -343,8 +384,6 @@ fn loadComptimePtrInner(
     }
 
     restructure_array: {
-        if (host_bits != 0) break :restructure_array;
-
         // We might also be changing the length of an array, or restructuring it.
         // e.g. [1][2][3]T -> [3][2]T.
         // This case is important because it's permitted for types with ill-defined layouts.
@@ -402,7 +441,7 @@ fn loadComptimePtrInner(
         cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;
     }
 
-    const need_bytes = if (host_bits > 0) (host_bits + 7) / 8 else load_ty.abiSize(zcu);
+    const need_bytes = load_ty.abiSize(zcu);
 
     if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {
         return .{ .out_of_bounds = cur_val.typeOf(zcu) };
@@ -453,7 +492,7 @@ fn loadComptimePtrInner(
             },
             .@"struct" => switch (cur_ty.containerLayout(zcu)) {
                 .auto => unreachable, // ill-defined layout
-                .@"packed" => break, // let the bitcast logic handle this
+                .@"packed" => break, // let the memory reinterpret logic handle this
                 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
                     const start_off = cur_ty.structFieldOffset(field_idx, zcu);
                     const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
@@ -466,9 +505,9 @@ fn loadComptimePtrInner(
             },
             .@"union" => switch (cur_ty.containerLayout(zcu)) {
                 .auto => unreachable, // ill-defined layout
-                .@"packed" => break, // let the bitcast logic handle this
+                .@"packed" => break, // let the memory reinterpret logic handle this
                 .@"extern" => {
-                    // TODO: we have to let bitcast logic handle this for now.
+                    // TODO: we have to let the memory reinterpret logic handle this for now.
                     // Otherwise, we might traverse into a union field which doesn't allow pointers.
                     // Figure out a solution!
                     if (true) break;
@@ -495,27 +534,13 @@ fn loadComptimePtrInner(
 
     // Fast path: check again if we're now at the type we want to load.
     // If so, just return the loaded value.
-    if (cur_offset == 0 and host_bits == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
+    if (cur_offset == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
         return .{ .success = cur_val };
     }
 
-    var bitcast_src_val = try cur_val.intern(sema.pt, sema.arena);
-
-    if (host_bits != 0) {
-        const src_bit_size = bitcast_src_val.typeOf(zcu).bitSize(zcu);
-        if (src_bit_size > host_bits) {
-            const truncate_ty = try pt.intType(.unsigned, @intCast(host_bits));
-            bitcast_src_val = try pt.getCoerced(bitcast_src_val, truncate_ty);
-        }
-    }
-
-    const result_val = try sema.bitCastVal(
-        bitcast_src_val,
-        load_ty,
-        cur_offset,
-        host_bits,
-        bit_offset,
-    ) orelse return .runtime_load;
+    // Otherwise, use the memory reinterpretation logic to pull out the bytes we need.
+    const reinterpret_val = try cur_val.intern(pt, sema.arena);
+    const result_val = try sema.castMemory(reinterpret_val, load_ty, cur_offset) orelse return .runtime_load;
     return .{ .success = .{ .interned = result_val.toIntern() } };
 }
 
@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {
         val: *MutableValue,
         flat_elem_index: u64,
     },
-    /// This value should be reinterpreted using bitcast logic to perform the
+    /// This value should be reinterpreted using `Sema.spliceMemory` to perform
     /// store. Only returned if `store_ty` and the type of `val` both have
     /// well-defined layouts.
     reinterpret: struct {
@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(
             },
             .@"struct" => switch (cur_ty.containerLayout(zcu)) {
                 .auto => unreachable, // ill-defined layout
-                .@"packed" => break, // let the bitcast logic handle this
+                .@"packed" => break, // let the memory reinterp logic handle this
                 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
                     const start_off = cur_ty.structFieldOffset(field_idx, zcu);
                     const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(
             },
             .@"union" => switch (cur_ty.containerLayout(zcu)) {
                 .auto => unreachable, // ill-defined layout
-                .@"packed" => break, // let the bitcast logic handle this
+                .@"packed" => break, // let the memory reinterp logic handle this
                 .@"extern" => {
-                    // TODO: we have to let bitcast logic handle this for now.
+                    // TODO: we have to let the memory reinterp logic handle this for now.
                     // Otherwise, we might traverse into a union field which doesn't allow pointers.
                     // Figure out a solution!
                     if (true) break;

src/Sema/reinterpret.zig

@@ -0,0 +1,576 @@
+//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
+//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
+//! a sequence of values in *packed* memory, and then unflatten through a combination of special
+//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
+//!
+//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
+//! have their fields reversed when represented as packed memory on such targets.
+
+/// If `host_bits` is `0`, attempts to convert the memory at offset
+/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
+/// ignoring `bit_offset`.
+///
+/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
+/// non-packed value consisting of `host_bits` bits. A value of type
+/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
+/// into this value.
+///
+/// Returns `null` if the operation must be performed at runtime.
+pub fn castMemory(
+    sema: *Sema,
+    val: Value,
+    dest_ty: Type,
+    byte_offset: u64,
+) CompileError!?Value {
+    const pt = sema.pt;
+    const zcu = pt.zcu;
+
+    const val_ty = val.typeOf(zcu);
+
+    if (dest_ty.toIntern() == val_ty.toIntern()) {
+        assert(byte_offset == 0);
+        return val;
+    }
+
+    val_ty.assertHasLayout(zcu);
+    dest_ty.assertHasLayout(zcu);
+
+    var unpack: UnpackValueBytes = .{
+        .pt = pt,
+        .arena = sema.arena,
+        .skip_bytes = byte_offset,
+        .remaining_bytes = dest_ty.abiSize(zcu),
+        .unpacked = .init(sema.arena),
+    };
+    unpack.add(val) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+
+    var pack: PackValueBytes = .{
+        .pt = pt,
+        .arena = sema.arena,
+        .unpacked = unpack.unpacked.items,
+    };
+    return pack.get(dest_ty) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+}
+
+/// Splice the value `splice_val` into `val` at the given `byte_offset`, replacing overlapping bits
+/// and returning the modified value.
+pub fn spliceMemory(
+    sema: *Sema,
+    val: Value,
+    splice_val: Value,
+    byte_offset: u64,
+) CompileError!?Value {
+    const pt = sema.pt;
+    const zcu = pt.zcu;
+    const val_ty = val.typeOf(zcu);
+    const splice_val_ty = splice_val.typeOf(zcu);
+
+    val_ty.assertHasLayout(zcu);
+    splice_val_ty.assertHasLayout(zcu);
+
+    var unpack: UnpackValueBytes = .{
+        .pt = pt,
+        .arena = sema.arena,
+        .skip_bytes = 0,
+        .remaining_bytes = byte_offset,
+        .unpacked = .init(sema.arena),
+    };
+    unpack.add(val) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+
+    const splice_len = splice_val_ty.abiSize(zcu);
+
+    unpack.remaining_bytes = splice_len;
+    unpack.add(splice_val) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+
+    unpack.skip_bytes = byte_offset + splice_len;
+    unpack.remaining_bytes = val_ty.abiSize(zcu) * 8 - byte_offset - splice_len;
+    unpack.add(val) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+
+    var pack: PackValueBytes = .{
+        .pt = pt,
+        .arena = sema.arena,
+        .unpacked = unpack.unpacked.items,
+    };
+    return pack.get(val_ty) catch |err| switch (err) {
+        error.ReinterpretDeclRef => return null,
+        error.OutOfMemory => |e| return e,
+    };
+}
+
+/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
+/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
+/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
+/// list contains bits from the *final* byte of the value.
+const UnpackValueBytes = struct {
+    pt: Zcu.PerThread,
+    arena: Allocator,
+    skip_bytes: u64,
+    remaining_bytes: u64,
+    unpacked: std.array_list.Managed(InternPool.Index),
+
+    fn add(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
+        const pt = unpack.pt;
+        const zcu = pt.zcu;
+        const ip = &zcu.intern_pool;
+
+        if (unpack.remaining_bytes == 0) {
+            return;
+        }
+
+        const ty = val.typeOf(zcu);
+        const size = ty.abiSize(zcu);
+
+        if (unpack.skip_bytes >= size) {
+            unpack.skip_bytes -= size;
+            return;
+        }
+
+        switch (ip.indexToKey(val.toIntern())) {
+            .int_type,
+            .ptr_type,
+            .array_type,
+            .vector_type,
+            .opt_type,
+            .anyframe_type,
+            .error_union_type,
+            .simple_type,
+            .struct_type,
+            .tuple_type,
+            .union_type,
+            .opaque_type,
+            .spirv_type,
+            .enum_type,
+            .func_type,
+            .error_set_type,
+            .inferred_error_set_type,
+            .@"extern",
+            .func,
+            .err,
+            .error_union,
+            .enum_literal,
+            .slice,
+            .memoized_call,
+            => unreachable, // ill-defined layout or not real values
+
+            .undef,
+            .int,
+            .enum_tag,
+            .simple_value,
+            .float,
+            .ptr,
+            .opt,
+            => try unpack.primitive(val),
+
+            .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
+
+            .aggregate => switch (ty.zigTypeTag(zcu)) {
+                .vector => unreachable, // ill-defined layout
+                .array => {
+                    for (0..@intCast(ty.arrayLen(zcu))) |elem_index| {
+                        const elem_val = try val.elemValue(pt, @intCast(elem_index));
+                        try unpack.add(elem_val);
+                    }
+                    if (ty.sentinel(zcu)) |s| {
+                        try unpack.add(s);
+                    }
+                },
+                .@"struct" => switch (ty.containerLayout(zcu)) {
+                    .auto => unreachable, // ill-defined layout
+                    .@"packed" => unreachable, // uses `.bitpack`, not `.aggregate`
+                    .@"extern" => {
+                        var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
+                        var offset: u64 = 0;
+                        while (it.next()) |field_index| {
+                            const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
+                            const field_val = try val.fieldValue(pt, field_index);
+                            try unpack.padding(pad_bytes);
+                            try unpack.add(field_val);
+                            offset += pad_bytes + field_val.typeOf(zcu).abiSize(zcu);
+                        }
+                        try unpack.padding(size - offset);
+                    },
+                },
+                else => unreachable,
+            },
+
+            .un => |un| {
+                const payload_val = Value.fromInterned(un.val);
+                const pad_bytes = size - payload_val.typeOf(zcu).abiSize(zcu);
+                try unpack.add(payload_val);
+                try unpack.padding(pad_bytes);
+            },
+        }
+    }
+
+    fn padding(unpack: *UnpackValueBytes, num_bytes: u64) Allocator.Error!void {
+        if (num_bytes == 0) return;
+        const undef_u8 = try unpack.pt.undefValue(Type.u8);
+        for (0..@intCast(num_bytes)) |_| {
+            unpack.primitive(undef_u8) catch |err| switch (err) {
+                error.OutOfMemory => |e| return e,
+                error.ReinterpretDeclRef => unreachable,
+            };
+        }
+    }
+
+    fn primitive(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
+        const pt = unpack.pt;
+        const zcu = pt.zcu;
+
+        if (unpack.remaining_bytes == 0) {
+            return;
+        }
+
+        const ty = val.typeOf(pt.zcu);
+        const size = ty.abiSize(zcu);
+
+        if (unpack.skip_bytes >= size) {
+            unpack.skip_bytes -= size;
+            return;
+        }
+
+        if (unpack.skip_bytes > 0) {
+            const offset = unpack.skip_bytes;
+            unpack.skip_bytes = 0;
+            return unpack.splitPrimitive(val, offset, @min(size - offset, unpack.remaining_bytes));
+        }
+
+        if (unpack.remaining_bytes < size) {
+            return unpack.splitPrimitive(val, 0, unpack.remaining_bytes);
+        }
+
+        unpack.remaining_bytes -= size;
+        try unpack.unpacked.append(val.toIntern());
+    }
+
+    fn splitPrimitive(unpack: *UnpackValueBytes, val: Value, offset: u64, len: u64) (error{ReinterpretDeclRef} || Allocator.Error)!void {
+        const pt = unpack.pt;
+        const zcu = pt.zcu;
+        const ty = val.typeOf(pt.zcu);
+
+        assert(offset + len <= ty.abiSize(zcu));
+
+        try unpack.unpacked.ensureUnusedCapacity(@intCast(len));
+        unpack.remaining_bytes -= len;
+
+        switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
+            // In the `ptr` case, this will return `error.ReinterpretDeclRef`
+            // if we're trying to split a non-integer pointer value.
+            .int, .float, .enum_tag, .ptr, .opt => {
+                const buf = try unpack.arena.alloc(u8, @intCast(ty.abiSize(zcu)));
+                val.writeToMemory(zcu, buf) catch |err| switch (err) {
+                    error.IllDefinedMemoryLayout => unreachable,
+                    else => |e| return e,
+                };
+                for (buf[@intCast(offset)..][0..@intCast(len)]) |byte_raw| {
+                    const byte_val = try pt.intValue(.u8, byte_raw);
+                    unpack.unpacked.appendAssumeCapacity(byte_val.toIntern());
+                }
+            },
+            .undef => {
+                const undef_u8 = try pt.undefValue(.u8);
+                for (0..@intCast(len)) |_| {
+                    unpack.unpacked.appendAssumeCapacity(undef_u8.toIntern());
+                }
+            },
+            // The only values here with runtime bits are `true` and `false`.
+            // These are both 1 byte, so will never need splitting.
+            .simple_value => unreachable,
+            else => unreachable, // zero-bit or not primitives
+        }
+    }
+};
+
+/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBytes`),
+/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
+/// values and of pointers which align in virtual memory.
+const PackValueBytes = struct {
+    pt: Zcu.PerThread,
+    arena: Allocator,
+    byte_offset: u64 = 0,
+    unpacked: []const InternPool.Index,
+
+    fn get(pack: *PackValueBytes, ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
+        const pt = pack.pt;
+        const zcu = pt.zcu;
+        const ip = &zcu.intern_pool;
+        const arena = pack.arena;
+        switch (ty.zigTypeTag(zcu)) {
+            .vector => unreachable, // ill-defined layout
+            .array => {
+                // Each element is padded up to its ABI size. The final element does not have trailing padding.
+                const elem_ty = ty.childType(zcu);
+                const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
+
+                for (elems) |*elem| {
+                    elem.* = (try pack.get(elem_ty)).toIntern();
+                }
+
+                if (ty.sentinel(zcu)) |s| {
+                    _ = s; // TODO: validate sentinel was preserved!
+                    pack.padding(elem_ty.abiSize(zcu));
+                }
+
+                return pt.aggregateValue(ty, elems);
+            },
+            .@"struct" => switch (ty.containerLayout(zcu)) {
+                .auto => unreachable, // ill-defined layout
+                .@"extern" => {
+                    const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
+                    @memset(elems, .none);
+                    var offset: u64 = 0;
+                    var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
+                    while (it.next()) |field_index| {
+                        const field_ty = ty.fieldType(field_index, zcu);
+                        const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
+                        pack.padding(pad_bytes);
+                        elems[field_index] = (try pack.get(field_ty)).toIntern();
+                        offset += pad_bytes + field_ty.abiSize(zcu);
+                    }
+                    pack.padding(ty.abiSize(zcu) - offset);
+                    // Any fields which do not have runtime bits should be OPV or comptime fields.
+                    // Fill those values now.
+                    for (elems, 0..) |*elem, field_index| {
+                        if (elem.* != .none) continue;
+                        const val = (try ty.structFieldValueComptime(pt, field_index)).?;
+                        elem.* = val.toIntern();
+                    }
+                    return pt.aggregateValue(ty, elems);
+                },
+                .@"packed" => {
+                    const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
+                    if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
+                    return pt.bitpackValue(ty, backing_int_val);
+                },
+            },
+            .@"union" => switch (ty.containerLayout(zcu)) {
+                .auto => unreachable, // ill-defined layout
+                .@"extern" => {
+                    // We will attempt to read as the backing representation. If this emits
+                    // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
+                    // We will also attempt smaller fields when we get `undefined`, as if some bits are
+                    // defined we want to include them.
+                    // TODO: this is very very bad. We need a more sophisticated union representation.
+
+                    const prev_unpacked = pack.unpacked;
+                    const prev_byte_offset = pack.byte_offset;
+
+                    const backing_ty = try ty.externUnionBackingType(pt);
+
+                    const backing_result: enum { undef, reinterpret_decl_ref } = backing: {
+                        const backing_val = pack.get(backing_ty) catch |err| switch (err) {
+                            error.ReinterpretDeclRef => break :backing .reinterpret_decl_ref,
+                            else => |e| return e,
+                        };
+                        if (backing_val.isUndef(zcu)) break :backing .undef;
+                        return .fromInterned(try pt.internUnion(.{
+                            .ty = ty.toIntern(),
+                            .tag = .none,
+                            .val = backing_val.toIntern(),
+                        }));
+                    };
+
+                    const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
+                    for (field_order, 0..) |*f, i| f.* = @intCast(i);
+                    // Sort `field_order` to put the fields with the largest ABI sizes first.
+                    const SizeSortCtx = struct {
+                        zcu: *const Zcu,
+                        field_types: []const InternPool.Index,
+                        fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
+                            const a_ty: Type = .fromInterned(ctx.field_types[a_idx]);
+                            const b_ty: Type = .fromInterned(ctx.field_types[b_idx]);
+                            return a_ty.abiSize(ctx.zcu) > b_ty.abiSize(ctx.zcu);
+                        }
+                    };
+                    std.mem.sortUnstable(u32, field_order, SizeSortCtx{
+                        .zcu = zcu,
+                        .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
+                    }, SizeSortCtx.lessThan);
+
+                    for (field_order) |field_index| {
+                        pack.unpacked = prev_unpacked;
+                        pack.byte_offset = prev_byte_offset;
+                        const field_ty = ty.fieldType(field_index, zcu);
+                        const field_val = pack.get(field_ty) catch |err| switch (err) {
+                            error.ReinterpretDeclRef => continue,
+                            else => |e| return e,
+                        };
+                        if (field_val.isUndef(zcu)) continue;
+                        pack.padding(ty.abiSize(zcu) - field_ty.abiSize(zcu));
+                        const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_index);
+                        return pt.unionValue(ty, tag_val, field_val);
+                    }
+
+                    // No field could represent the value. Just do whatever happens when we try to read
+                    // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
+                    switch (backing_result) {
+                        .undef => return pt.undefValue(ty),
+                        .reinterpret_decl_ref => return error.ReinterpretDeclRef,
+                    }
+                },
+                .@"packed" => {
+                    const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
+                    if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
+                    return pt.bitpackValue(ty, backing_int_val);
+                },
+            },
+            .@"enum" => {
+                const tag_int_val = try pack.primitive(ty.intTagType(zcu));
+                if (tag_int_val.isUndef(zcu)) return pt.undefValue(ty);
+                return pt.enumValue(ty, tag_int_val.toIntern());
+            },
+            else => return pack.primitive(ty),
+        }
+    }
+
+    fn padding(pack: *PackValueBytes, num_bytes: u64) void {
+        _ = pack.prepareBytes(num_bytes);
+    }
+
+    fn primitive(pack: *PackValueBytes, want_ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
+        const pt = pack.pt;
+        const zcu = pt.zcu;
+
+        if (try want_ty.onePossibleValue(pt)) |opv| return opv;
+
+        const vals, const byte_offset = pack.prepareBytes(want_ty.abiSize(zcu));
+
+        for (vals) |val| {
+            if (!Value.fromInterned(val).isUndef(zcu)) break;
+        } else {
+            // All bits of the value are `undefined`.
+            return pt.undefValue(want_ty);
+        }
+
+        // TODO: we need to decide how to handle partially-undef values here.
+        // Currently, a value with some undefined bits becomes `0xAA` so that we
+        // preserve the well-defined bits, because we can't currently represent
+        // a partially-undefined primitive (e.g. an int with some undef bits).
+        // In future, we probably want to take one of these two routes:
+        // * Define that if any bits are `undefined`, the entire value is `undefined`.
+        //   This is a major breaking change, and probably a footgun.
+        // * Introduce tracking for partially-undef values at comptime.
+        //   This would complicate a lot of operations in Sema, such as basic
+        //   arithmetic.
+        // This design complexity is tracked by #19634.
+
+        if (vals.len == 1 and
+            want_ty.isPtrAtRuntime(zcu) and
+            Value.fromInterned(vals[0]).typeOf(zcu).isPtrAtRuntime(zcu))
+        {
+            return pt.getCoerced(.fromInterned(vals[0]), want_ty);
+        }
+
+        // Reinterpret via an in-memory buffer.
+
+        var buf_len: u64 = 0;
+        for (vals) |ip_val| {
+            const val: Value = .fromInterned(ip_val);
+            buf_len += val.typeOf(zcu).abiSize(zcu);
+        }
+
+        const buf = try pack.arena.alloc(u8, @intCast(buf_len));
+        {
+            var offset: usize = 0;
+            for (vals) |ip_val| {
+                const val: Value = .fromInterned(ip_val);
+                const ty = val.typeOf(zcu);
+                const size = ty.abiSize(zcu);
+                if (val.isUndef(zcu)) {
+                    @memset(buf[offset..][0..@intCast(size)], 0xAA);
+                } else {
+                    val.writeToMemory(zcu, buf[offset..][0..@intCast(size)]) catch |err| switch (err) {
+                        error.IllDefinedMemoryLayout => unreachable,
+                        else => |e| return e,
+                    };
+                }
+                offset += @intCast(size);
+            }
+        }
+        const bytes = buf[@intCast(byte_offset)..];
+
+        const target = zcu.getTarget();
+        const endian = target.cpu.arch.endian();
+        switch (want_ty.zigTypeTag(zcu)) {
+            .bool => return .makeBool(bytes[0] != 0),
+            .int => return .readIntFromMemory(want_ty, pt, bytes, pack.arena),
+            .float => switch (want_ty.floatBits(target)) {
+                16 => return pt.floatValue(want_ty, @as(f16, @bitCast(std.mem.readInt(u16, bytes[0..2], endian)))),
+                32 => return pt.floatValue(want_ty, @as(f32, @bitCast(std.mem.readInt(u32, bytes[0..4], endian)))),
+                64 => return pt.floatValue(want_ty, @as(f64, @bitCast(std.mem.readInt(u64, bytes[0..8], endian)))),
+                80 => return pt.floatValue(want_ty, @as(f80, @bitCast(std.mem.readInt(u80, bytes[0..10], endian)))),
+                128 => return pt.floatValue(want_ty, @as(f128, @bitCast(std.mem.readInt(u128, bytes[0..16], endian)))),
+                else => unreachable,
+            },
+            .pointer => {
+                assert(!want_ty.isSlice(zcu));
+                const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
+                return pt.ptrIntValue(want_ty, ptr_addr);
+            },
+            .optional => {
+                assert(want_ty.isPtrLikeOptional(zcu));
+                const ptr_ty = want_ty.optionalChild(zcu);
+                const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
+                return .fromInterned(try pt.intern(.{ .opt = .{
+                    .ty = want_ty.toIntern(),
+                    .val = if (ptr_addr == 0) .none else (try pt.ptrIntValue(ptr_ty, ptr_addr)).toIntern(),
+                } }));
+            },
+            else => unreachable,
+        }
+    }
+
+    fn prepareBytes(pack: *PackValueBytes, need_bytes: u64) struct { []const InternPool.Index, u64 } {
+        if (need_bytes == 0) return .{ &.{}, 0 };
+
+        const pt = pack.pt;
+        const zcu = pt.zcu;
+
+        var bytes: u64 = 0;
+        var len: usize = 0;
+        while (bytes < pack.byte_offset + need_bytes) {
+            bytes += Value.fromInterned(pack.unpacked[len]).typeOf(zcu).abiSize(zcu);
+            len += 1;
+        }
+
+        const result_vals = pack.unpacked[0..len];
+        const result_offset = pack.byte_offset;
+
+        const extra_bytes = bytes - pack.byte_offset - need_bytes;
+        if (extra_bytes == 0) {
+            pack.unpacked = pack.unpacked[len..];
+            pack.byte_offset = 0;
+        } else {
+            pack.unpacked = pack.unpacked[len - 1 ..];
+            pack.byte_offset = Value.fromInterned(pack.unpacked[0]).typeOf(zcu).abiSize(zcu) - extra_bytes;
+        }
+
+        return .{ result_vals, result_offset };
+    }
+};
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+
+const Sema = @import("../Sema.zig");
+const Zcu = @import("../Zcu.zig");
+const InternPool = @import("../InternPool.zig");
+const Type = @import("../Type.zig");
+const Value = @import("../Value.zig");
+const CompileError = Zcu.CompileError;

src/Type.zig

@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {
     const ip = &zcu.intern_pool;
     return switch (ip.indexToKey(ty.toIntern())) {
         .int_type,
-        .vector_type,
         => true,
 
+        .vector_type,
         .error_union_type,
         .error_set_type,
         .inferred_error_set_type,
@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {
 }
 
 /// Asserts that `ty` is not an opaque or comptime-only type.
-/// Once #19755 is implemented, this query will only work on types with a defined bit-level representation.
 pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {
-    const target = zcu.getTarget();
-    const ip = &zcu.intern_pool;
-    assertHasLayout(ty, zcu);
-    return switch (ip.indexToKey(ty.toIntern())) {
-        .int_type => |int_type| int_type.bits,
-        .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
-            .slice => target.ptrBitWidth() * 2,
-            else => target.ptrBitWidth(),
+    return switch (ty.zigTypeTag(zcu)) {
+        .void => 0,
+        .bool => 1,
+        .float => ty.floatBits(zcu.getTarget()),
+        .pointer, .optional => {
+            assert(ty.isPtrAtRuntime(zcu));
+            return zcu.getTarget().ptrBitWidth();
         },
-        .anyframe_type => target.ptrBitWidth(),
-        .array_type => |array_type| {
-            const elem_ty: Type = .fromInterned(array_type.child);
-            const len = array_type.lenIncludingSentinel();
-            return switch (zcu.comp.getZigBackend()) {
-                .stage2_x86_64 => len * elem_ty.bitSize(zcu),
-                // this case will be removed under #19755
-                else => switch (len) {
-                    0 => 0,
-                    else => (len - 1) * 8 * elem_ty.abiSize(zcu) + elem_ty.bitSize(zcu),
-                },
-            };
-        },
-        .vector_type => |vec| vec.len * Type.fromInterned(vec.child).bitSize(zcu),
-        .error_set_type, .inferred_error_set_type => zcu.errorSetBits(),
-        .func_type => unreachable,
-
-        .simple_type => |t| switch (t) {
-            .void => 0,
-            .bool => 1,
-            .anyerror, .adhoc_inferred_error_set => zcu.errorSetBits(),
-            .usize, .isize => target.ptrBitWidth(),
-
-            .c_char => target.cTypeBitSize(.char),
-            .c_short => target.cTypeBitSize(.short),
-            .c_ushort => target.cTypeBitSize(.ushort),
-            .c_int => target.cTypeBitSize(.int),
-            .c_uint => target.cTypeBitSize(.uint),
-            .c_long => target.cTypeBitSize(.long),
-            .c_ulong => target.cTypeBitSize(.ulong),
-            .c_longlong => target.cTypeBitSize(.longlong),
-            .c_ulonglong => target.cTypeBitSize(.ulonglong),
-            .c_longdouble => target.cTypeBitSize(.longdouble),
-
-            .f16 => 16,
-            .f32 => 32,
-            .f64 => 64,
-            .f80 => 80,
-            .f128 => 128,
-
-            .anyopaque => unreachable,
-            .type => unreachable,
-            .comptime_int => unreachable,
-            .comptime_float => unreachable,
-            .noreturn => unreachable,
-            .null => unreachable,
-            .undefined => unreachable,
-            .enum_literal => unreachable,
-            .generic_poison => unreachable,
-        },
-
-        .struct_type => {
-            const struct_obj = ip.loadStructType(ty.toIntern());
-            switch (struct_obj.layout) {
-                .@"packed" => return Type.fromInterned(struct_obj.packed_backing_int_type).bitSize(zcu),
-                .auto, .@"extern" => return struct_obj.size * 8, // will be `unreachable` under #19755
-            }
-        },
-        .union_type => {
-            const union_obj = ip.loadUnionType(ty.toIntern());
-            switch (union_obj.layout) {
-                .@"packed" => return Type.fromInterned(union_obj.packed_backing_int_type).bitSize(zcu),
-                .auto, .@"extern" => return union_obj.size * 8, // will be `unreachable` under #19755
-            }
-        },
-        .enum_type => Type.fromInterned(ip.loadEnumType(ty.toIntern()).int_tag_type).bitSize(zcu),
-
-        // will be `unreachable` under #19755
-        .opt_type,
-        .error_union_type,
-        .tuple_type,
-        => ty.abiSize(zcu) * 8,
-
-        .opaque_type, .spirv_type => unreachable,
-
-        // values, not types
-        .undef,
-        .simple_value,
-        .@"extern",
-        .func,
-        .int,
-        .err,
-        .error_union,
-        .enum_literal,
-        .enum_tag,
-        .float,
-        .ptr,
-        .slice,
-        .opt,
-        .aggregate,
-        .un,
-        .bitpack,
-        // memoization, not types
-        .memoized_call,
-        => unreachable,
+        .array, .vector => ty.arrayLenIncludingSentinel(zcu) * ty.childType(zcu).bitSize(zcu),
+        else => ty.intInfo(zcu).bits,
     };
 }
 
@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {
 /// * `[*]T`
 /// * `[*c]T`
 /// * `@SpirvType(.{ .runtime_array = T })`
+/// * `*@SpirvType(.{ .runtime_array = T })`
 pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {
     const ip = &zcu.intern_pool;
     return switch (ip.indexToKey(ty.toIntern())) {
@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
         .frame,
         => false,
 
+        .vector => position == .param_ty or position == .ret_ty,
+
         .void => switch (position) {
             .ret_ty,
             .union_field,
@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
             .other,
             => ty.childType(zcu).validateExtern(.element, zcu),
         },
-        .vector => ty.childType(zcu).validateExtern(.element, zcu),
         .optional => ty.isPtrLikeOptional(zcu),
     };
 }
@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {
     };
 }
 
+/// Returns whether `ty` is considered by Zig to have a bit-level representation, meaning it is
+/// allowed as the operand to `@bitSizeOf`. This is a superset of packable types.
+pub fn hasBitRepresentation(ty: Type, zcu: *const Zcu) bool {
+    return switch (ty.zigTypeTag(zcu)) {
+        .@"fn",
+        .noreturn,
+        .undefined,
+        .null,
+        .@"opaque",
+        .spirv,
+        .type,
+        .enum_literal,
+        .comptime_float,
+        .comptime_int,
+        .error_set,
+        .error_union,
+        .frame,
+        .@"anyframe",
+        => false,
+
+        .void,
+        .bool,
+        .int,
+        .float,
+        => true,
+
+        .@"enum" => zcu.intern_pool.loadEnumType(ty.toIntern()).int_tag_mode == .explicit,
+        .pointer, .optional => ty.isPtrAtRuntime(zcu),
+        .@"struct", .@"union" => ty.containerLayout(zcu) == .@"packed",
+
+        .array, .vector => ty.childType(zcu).hasBitRepresentation(zcu),
+    };
+}
+
 /// Asserts that `ty` has resolved layout.
 pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {
     if (!std.debug.runtime_safety) {

src/Value.zig

@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {
 pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
     ReinterpretDeclRef,
     IllDefinedMemoryLayout,
-    Unimplemented,
     OutOfMemory,
 }!void {
     const target = zcu.getTarget();
@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
     const ty = val.typeOf(zcu);
     if (val.isUndef(zcu)) {
         const size: usize = @intCast(ty.abiSize(zcu));
-        @memset(buffer[0..size], 0xaa);
+        @memset(buffer[0..size], 0xAA);
         return;
     }
-    switch (ty.zigTypeTag(zcu)) {
+    tag: switch (ty.zigTypeTag(zcu)) {
+        .type => return error.IllDefinedMemoryLayout,
+        .comptime_float => return error.IllDefinedMemoryLayout,
+        .comptime_int => return error.IllDefinedMemoryLayout,
+        .undefined => return error.IllDefinedMemoryLayout,
+        .null => return error.IllDefinedMemoryLayout,
+        .error_union => return error.IllDefinedMemoryLayout,
+        .enum_literal => return error.IllDefinedMemoryLayout,
+        .@"fn" => return error.IllDefinedMemoryLayout,
+        .spirv => return error.IllDefinedMemoryLayout,
+        .@"opaque" => unreachable,
+        .frame => unreachable,
+        .@"anyframe" => unreachable,
+        .noreturn => unreachable,
         .void => {},
         .bool => {
             buffer[0] = @intFromBool(val.toBool());
         },
-        .int, .@"enum", .error_set, .pointer => |tag| {
-            const int_ty = if (tag == .pointer) int_ty: {
-                if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
-                if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
-                break :int_ty Type.usize;
-            } else ty;
-            const int_info = int_ty.intInfo(zcu);
-            const bits = int_info.bits;
-            const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
-
+        .pointer => {
+            if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
+            if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
+            continue :tag .int;
+        },
+        .int, .@"enum", .error_set => {
             var bigint_buffer: BigIntSpace = undefined;
             const bigint = val.toBigInt(&bigint_buffer, zcu);
-            bigint.writeTwosComplement(buffer[0..byte_count], endian);
+            bigint.writeTwosComplement(buffer[0..@intCast(ty.abiSize(zcu))], endian);
         },
-        .float => switch (ty.floatBits(target)) {
-            16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
-            32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
-            64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
-            80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
-            128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
-            else => unreachable,
+        .float => {
+            const float_bits = ty.floatBits(target);
+            switch (float_bits) {
+                16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
+                32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
+                64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
+                80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
+                128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
+                else => unreachable,
+            }
+            const float_bytes = @divExact(float_bits, 8);
+            const total_bytes: usize = @intCast(ty.abiSize(zcu));
+            @memset(buffer[float_bytes..total_bytes], 0); // padding
         },
         .array => {
             const aggregate = ip.indexToKey(val.toIntern()).aggregate;
@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
                 }
                 buf_off += elem_size;
             }
+            if (ty.sentinel(zcu)) |sentinel_val| {
+                try sentinel_val.writeToMemory(zcu, buffer[buf_off..]);
+            }
         },
-        .vector => {
-            // We use byte_count instead of abi_size here, so that any padding bytes
-            // follow the data bytes, on both big- and little-endian systems.
-            const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
-            return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
-        },
+        .vector => return error.IllDefinedMemoryLayout,
         .@"struct" => {
             const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
             switch (struct_type.layout) {
                 .auto => return error.IllDefinedMemoryLayout,
-                .@"extern" => for (0..struct_type.field_types.len) |field_index| {
-                    const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
-                    const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
-                        .bytes => |bytes| {
-                            buffer[off] = bytes.at(field_index, ip);
-                            continue;
-                        },
-                        .elems => |elems| elems[field_index],
-                        .repeated_elem => |elem| elem,
-                    });
-                    try writeToMemory(field_val, zcu, buffer[off..]);
+                .@"extern" => {
+                    var last_off: usize = 0;
+                    for (struct_type.field_types.get(ip), 0..) |field_ty_ip, field_index| {
+                        const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
+                        @memset(buffer[last_off..off], 0xAA);
+                        const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
+                            .bytes => |bytes| {
+                                buffer[off] = bytes.at(field_index, ip);
+                                continue;
+                            },
+                            .elems => |elems| elems[field_index],
+                            .repeated_elem => |elem| elem,
+                        });
+                        try writeToMemory(field_val, zcu, buffer[off..]);
+                        last_off = @intCast(off + Type.fromInterned(field_ty_ip).abiSize(zcu));
+                    }
+                    const struct_size: usize = @intCast(ty.abiSize(zcu));
+                    @memset(buffer[last_off..struct_size], 0xAA);
                 },
                 .@"packed" => {
                     const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
             .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
             .@"extern" => {
                 const payload_val = val.unionPayload(zcu);
+                const payload_size: usize = @intCast(payload_val.typeOf(zcu).abiSize(zcu));
+                const union_size: usize = @intCast(ty.abiSize(zcu));
+                @memset(buffer[payload_size..union_size], 0xAA);
                 return writeToMemory(payload_val, zcu, buffer);
             },
             .@"packed" => {
@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
                 @memset(buffer[0..@intCast(byte_count)], 0); // null pointer
             }
         },
-        else => return error.Unimplemented,
     }
 }
 
@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
 ///
 /// Both the start and the end of the provided buffer must be tight, since
 /// big-endian packed memory layouts start at the end of the buffer.
+///
+/// Supports arrays and vectors, for which the value is written in logical bit
+/// order, i.e. with the first element at bit offset 0.
 pub fn writeToPackedMemory(
     val: Value,
     zcu: *const Zcu,
     buffer: []u8,
     bit_offset: usize,
-) error{ ReinterpretDeclRef, OutOfMemory }!void {
+) void {
     const ip = &zcu.intern_pool;
     const target = zcu.getTarget();
     const endian = target.cpu.arch.endian();
@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(
         },
         .@"enum" => {
             const int_val = val.intFromEnum(zcu);
-            return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
-        },
-        .pointer => {
-            assert(!ty.isSlice(zcu)); // No well defined layout.
-            if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
-            const addr = val.toUnsignedInt(zcu);
-            std.mem.writeVarPackedInt(buffer, bit_offset, zcu.getTarget().ptrBitWidth(), addr, endian);
+            int_val.writeToPackedMemory(zcu, buffer, bit_offset);
         },
         .int => {
             const bits = ty.intInfo(zcu).bits;
@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(
             128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),
             else => unreachable,
         },
-        .vector => {
-            const elem_ty = ty.childType(zcu);
-            const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
-            const len: usize = @intCast(ty.arrayLen(zcu));
-
-            var bits: u16 = 0;
-            var elem_i: usize = 0;
-            const aggregate = ip.indexToKey(val.toIntern()).aggregate;
-            while (elem_i < len) : (elem_i += 1) {
-                // On big-endian systems, LLVM reverses the element order of vectors by default
-                const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
-                switch (aggregate.storage) {
-                    .bytes => |bytes| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
-                    .elems => |elems| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
-                    .repeated_elem => |elem| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
-                }
-                bits += elem_bit_size;
-            }
-        },
         .@"struct", .@"union" => {
             assert(ty.containerLayout(zcu) == .@"packed");
             const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
-            return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
+            int_val.writeToPackedMemory(zcu, buffer, bit_offset);
         },
-        .optional => {
-            assert(ty.isPtrLikeOptional(zcu));
-            if (val.optionalValue(zcu)) |ptr_val| {
-                return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);
-            } else {
-                return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);
+        .array, .vector => {
+            const elem_bits: usize = @intCast(ty.childType(zcu).bitSize(zcu));
+            const len: usize = @intCast(ty.arrayLen(zcu));
+            var elem_bit_off: usize = bit_offset;
+            switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
+                .repeated_elem => |elem_val_ip| {
+                    const elem_val: Value = .fromInterned(elem_val_ip);
+                    for (0..len) |_| {
+                        elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
+                        elem_bit_off += elem_bits;
+                    }
+                },
+                .elems => |elems| for (elems[0..len]) |elem_val_ip| {
+                    const elem_val: Value = .fromInterned(elem_val_ip);
+                    elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
+                    elem_bit_off += elem_bits;
+                },
+                .bytes => |bytes| for (bytes.toSlice(len, ip)) |raw_byte| {
+                    std.mem.writeVarPackedInt(buffer, elem_bit_off, elem_bits, raw_byte, endian);
+                    elem_bit_off += elem_bits;
+                },
+            }
+            if (ty.sentinel(zcu)) |sentinel_val| {
+                sentinel_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
             }
         },
-        else => @panic("TODO implement writeToPackedMemory for more types"),
+        else => unreachable,
     }
 }
 
-/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.
+/// Load a Value from the contents of `buffer`, where `ty` is any integer type.
 ///
 /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
 /// the end of the value in memory.
-pub fn readUintFromMemory(
+pub fn readIntFromMemory(
     ty: Type,
     pt: Zcu.PerThread,
     buffer: []const u8,
@@ -465,23 +482,28 @@ pub fn readUintFromMemory(
     const zcu = pt.zcu;
     const endian = zcu.getTarget().cpu.arch.endian();
 
-    assert(ty.isUnsignedInt(zcu));
-    const bits = ty.intInfo(zcu).bits;
-    const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
+    const int = ty.intInfo(zcu);
+    const abi_size: usize = @intCast(ty.abiSize(zcu));
+    const exact_buf = buffer[0..abi_size];
 
-    assert(buffer.len >= byte_count);
-
-    if (bits <= 64) {
-        const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
-        const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
-        return pt.intValue(ty, result);
+    if (abi_size <= 8) {
+        const shift: u6 = @intCast(64 - int.bits);
+        switch (int.signedness) {
+            .unsigned => {
+                const x = std.mem.readVarInt(u64, exact_buf, endian);
+                return pt.intValue(ty, (x << shift) >> shift);
+            },
+            .signed => {
+                const x = std.mem.readVarInt(i64, exact_buf, endian);
+                return pt.intValue(ty, (x << shift) >> shift);
+            },
+        }
     } else {
-        const Limb = std.math.big.Limb;
-        const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
-        const limbs_buffer = try arena.alloc(Limb, limb_count);
+        const limb_count = std.math.big.int.calcTwosCompLimbCount(int.bits);
+        const limbs_buffer = try arena.alloc(std.math.big.Limb, limb_count);
 
         var bigint: BigIntMutable = .init(limbs_buffer, 0);
-        bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);
+        bigint.readTwosComplement(exact_buf, int.bits, endian, int.signedness);
         return pt.intValue_big(ty, bigint.toConst());
     }
 }
@@ -490,17 +512,17 @@ pub fn readUintFromMemory(
 ///
 /// Both the start and the end of the provided buffer must be tight, since
 /// big-endian packed memory layouts start at the end of the buffer.
+///
+/// Supports arrays and vectors, for which the value is read in logical bit
+/// order, i.e. with the first element at bit offset 0.
 pub fn readFromPackedMemory(
     ty: Type,
     pt: Zcu.PerThread,
     buffer: []const u8,
     bit_offset: usize,
-    gpa: Allocator,
-) error{
-    IllDefinedMemoryLayout,
-    OutOfMemory,
-}!Value {
+) Allocator.Error!Value {
     const zcu = pt.zcu;
+    const gpa = zcu.comp.gpa;
     const target = zcu.getTarget();
     const endian = target.cpu.arch.endian();
     switch (ty.zigTypeTag(zcu)) {
@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(
         },
         .@"enum" => {
             const int_ty = ty.intTagType(zcu);
-            const int_val = try Value.readFromPackedMemory(int_ty, pt, buffer, bit_offset, gpa);
+            const int_val: Value = try .readFromPackedMemory(int_ty, pt, buffer, bit_offset);
             return pt.getCoerced(int_val, ty);
         },
         .float => return Value.fromInterned(try pt.intern(.{ .float = .{
@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(
                 else => unreachable,
             },
         } })),
-        .vector => {
-            const elem_ty = ty.childType(zcu);
-            const elems = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
-            defer gpa.free(elems);
-
-            var bits: u16 = 0;
-            const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
-            for (elems, 0..) |_, i| {
-                // On big-endian systems, LLVM reverses the element order of vectors by default
-                const tgt_elem_i = if (endian == .big) elems.len - i - 1 else i;
-                elems[tgt_elem_i] = (try readFromPackedMemory(elem_ty, pt, buffer, bit_offset + bits, gpa)).toIntern();
-                bits += elem_bit_size;
-            }
-            return pt.aggregateValue(ty, elems);
-        },
         .@"struct", .@"union" => {
             assert(ty.containerLayout(zcu) == .@"packed");
-            const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset, gpa);
+            const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset);
             return pt.bitpackValue(ty, int_val);
         },
-        .pointer => {
-            assert(!ty.isSlice(zcu)); // No well defined layout.
-            const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
-            return pt.ptrIntValue(ty, addr);
+        .array, .vector => {
+            const elem_ty = ty.childType(zcu);
+            const elem_bits: usize = @intCast(elem_ty.bitSize(zcu));
+            const elems_buf = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
+            defer gpa.free(elems_buf);
+            var elem_bit_off: usize = bit_offset;
+            for (elems_buf) |*elem| {
+                const elem_val = try readFromPackedMemory(elem_ty, pt, buffer, elem_bit_off);
+                elem.* = elem_val.toIntern();
+                elem_bit_off += elem_bits;
+            }
+            return pt.aggregateValue(ty, elems_buf);
         },
-        .optional => {
-            assert(ty.isPtrLikeOptional(zcu));
-            const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
-            return .fromInterned(try pt.intern(.{ .opt = .{
-                .ty = ty.toIntern(),
-                .val = if (addr == 0) .none else (try pt.ptrIntValue(ty.childType(zcu), addr)).toIntern(),
-            } }));
-        },
-        else => @panic("TODO implement readFromPackedMemory for more types"),
+        else => unreachable,
     }
 }
 
@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
             const bfa = bfa_state.allocator();
             const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
             defer bfa.free(buf);
-            int_val.writeToPackedMemory(zcu, buf, 0) catch |err| switch (err) {
-                error.ReinterpretDeclRef => unreachable, // it's an integer
-                error.OutOfMemory => |e| return e,
-            };
-            return Value.readFromPackedMemory(field_ty, pt, buf, field_bit_offset, bfa) catch |err| switch (err) {
-                error.IllDefinedMemoryLayout => unreachable, // it's a bitpack
-                error.OutOfMemory => |e| return e,
-            };
+            @memset(buf, 0);
+            int_val.writeToPackedMemory(zcu, buf, 0);
+            return .readFromPackedMemory(field_ty, pt, buf, field_bit_offset);
         },
         else => unreachable,
     };
@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
         // code late in compilation. So, this error handling is too aggressive and
         // causes some false negatives, causing less-than-ideal code generation.
         error.IllDefinedMemoryLayout => return null,
-        error.Unimplemented => return null,
     };
     const first_byte = byte_buffer[0];
     for (byte_buffer[1..]) |byte| {

src/Zcu/PerThread.zig

@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e
     tracy_trace.addText(fqn.toSlice(ip));
     tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});
 
+    Air.Verify.run(pt, func_index, air);
+
     if (codegen.legalizeFeatures(pt, nav)) |features| {
         try air.legalize(pt, features);
+        // Verify the AIR again post-legalization.
+        Air.Verify.run(pt, func_index, air);
     }
 
     var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))

src/codegen/aarch64/Select.zig

@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
         .load,
         .fptrunc,
         .fpext,
-        .intcast,
-        .intcast_safe,
+        .int_cast,
+        .int_cast_safe,
         .trunc,
         .optional_payload,
         .optional_payload_ptr,
@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
             air_inst_index = air_body[air_body_index];
             continue :air_tag air_tags[@intFromEnum(air_inst_index)];
         },
-        .bitcast => {
+        .bit_cast,
+        .ptr_cast,
+        .ptr_from_int,
+        .int_from_ptr,
+        .error_cast,
+        .error_from_int,
+        .int_from_error,
+        .union_from_enum,
+        => {
             const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
             maybe_noop: {
                 if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
             }
             if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
         },
-        .bitcast => |air_tag| {
+        .bit_cast,
+        .ptr_cast,
+        .ptr_from_int,
+        .int_from_ptr,
+        .error_cast,
+        .error_from_int,
+        .int_from_error,
+        .union_from_enum,
+        => |air_tag| {
             if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
                 defer dst_vi.value.deref(isel);
                 const ty_op = air.data(air.inst_index).ty_op;
@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
             }
             if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
         },
-        .intcast => |air_tag| {
+        .int_cast => |air_tag| {
             if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
                 defer dst_vi.value.deref(isel);
 
@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
             }
             if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
         },
-        .intcast_safe => |air_tag| {
+        .int_cast_safe => |air_tag| {
             if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
                 defer dst_vi.value.deref(isel);
 
@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
     if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
     constant.writeToMemory(zcu, buffer) catch |err| switch (err) {
         error.OutOfMemory => |e| return e,
-        error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
+        error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
     };
     return true;
 }

src/codegen/aarch64/abi.zig

@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
             if (ty.containerLayout(zcu) == .@"packed") return .byval;
             if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
 
-            const bit_size = ty.bitSize(zcu);
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > 128) return .memory;
             if (bit_size > 64) return .double_integer;
             return .integer;
@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
             if (ty.containerLayout(zcu) == .@"packed") return .byval;
             if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
 
-            const bit_size = ty.bitSize(zcu);
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > 128) return .memory;
             if (bit_size > 64) return .double_integer;
             return .integer;

src/codegen/arm/abi.zig

@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
     const ip = &zcu.intern_pool;
     switch (ty.zigTypeTag(zcu)) {
         .@"struct" => {
-            const bit_size = ty.bitSize(zcu);
             if (ty.containerLayout(zcu) == .@"packed") {
-                if (bit_size > 64) return .memory;
+                if (ty.bitSize(zcu) > 64) return .memory;
                 return .byval;
             }
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_byval_size) return .memory;
             const float_count = countFloats(ty, zcu, &maybe_float_bits);
             if (float_count <= byval_float_count) return .byval;
@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
             var i: u32 = 0;
             while (i < fields) : (i += 1) {
                 const field_ty = ty.fieldType(i, zcu);
-                if (field_ty.bitSize(zcu) > 32) return Class.arrSize(bit_size, 64);
+                if (field_ty.abiSize(zcu) > 4) return Class.arrSize(bit_size, 64);
             }
             return Class.arrSize(bit_size, 32);
         },
         .@"union" => {
-            const bit_size = ty.bitSize(zcu);
             const union_obj = zcu.typeToUnion(ty).?;
             if (union_obj.layout == .@"packed") {
-                if (bit_size > 64) return .memory;
+                if (ty.bitSize(zcu) > 64) return .memory;
                 return .byval;
             }
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_byval_size) return .memory;
             const float_count = countFloats(ty, zcu, &maybe_float_bits);
             if (float_count <= byval_float_count) return .byval;
@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
             }
 
             for (union_obj.field_types.get(ip)) |field_ty| {
-                if (Type.fromInterned(field_ty).bitSize(zcu) > 32) {
+                if (Type.fromInterned(field_ty).abiSize(zcu) > 4) {
                     return Class.arrSize(bit_size, 64);
                 }
             }

src/codegen/c.zig

@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
     return comptime switch (dev.env.supports(.legalize)) {
         inline false, true => |supports_legalize| &.init(.{
             // we don't currently ask zig1 to use safe optimization modes
-            .expand_intcast_safe = supports_legalize,
+            .expand_int_cast_safe = supports_legalize,
             .expand_int_from_float_safe = supports_legalize,
             .expand_int_from_float_optimized_safe = supports_legalize,
             .expand_add_safe = supports_legalize,
@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
             .expand_packed_store = true,
             .expand_packed_struct_field_val = true,
             .expand_packed_aggregate_init = true,
+
+            .scalarize_bit_cast_array = true,
+            .scalarize_bit_cast_vector_non_elementwise = true,
         }),
     };
 }
@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
             // zig fmt: off
             .inferred_alloc, .inferred_alloc_comptime => unreachable,
 
-            // No "scalarize" legalizations are enabled, so these instructions never appear.
-            .legalize_vec_elem_val   => unreachable,
-            .legalize_vec_store_elem => unreachable,
+            // Possible because `Air.Legalize.scalarize_bit_cast_vector_non_elementwise` is enabled.
+            .legalize_vec_elem_val   => try airArrayElemVal(f, inst),
+            .legalize_vec_store_elem => try airLegalizeVecStoreElem(f, inst),
             // No soft float legalizations are enabled.
             .legalize_compiler_rt_call => unreachable,
 
@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
             .alloc            => try airAlloc(f, inst),
             .ret_ptr          => try airRetPtr(f, inst),
             .assembly         => try airAsm(f, inst),
-            .bitcast          => try airBitcast(f, inst),
-            .intcast          => try airIntCast(f, inst),
+            .ptr_cast         => try airPtrCast(f, inst),
+            .ptr_from_int     => try airSimpleCast(f, inst),
+            .int_from_ptr     => try airSimpleCast(f, inst),
+            .error_cast       => try airNopCast(f, inst),
+            .error_from_int   => try airNopCast(f, inst),
+            .int_from_error   => try airNopCast(f, inst),
+            .union_from_enum  => try airUnionFromEnum(f, inst),
+            .bit_cast         => try airBitCast(f, inst),
+            .int_cast         => try airIntCast(f, inst),
             .trunc            => try airTrunc(f, inst),
             .load             => try airLoad(f, inst),
             .store            => try airStore(f, inst, false),
@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
             .add_safe,
             .sub_safe,
             .mul_safe,
-            .intcast_safe,
+            .int_cast_safe,
             .int_from_float_safe,
             .int_from_float_optimized_safe,
             => return f.fail("TODO implement safety_checked_instructions", .{}),
@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
     return local;
 }
 
+fn airLegalizeVecStoreElem(f: *Function, inst: Air.Inst.Index) !CValue {
+    const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
+
+    const vec_ptr = try f.resolveInst(pl_op.operand);
+    const index = try f.resolveInst(extra.lhs);
+    const elem = try f.resolveInst(extra.rhs);
+    try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs });
+
+    const w = &f.code.writer;
+
+    try f.writeCValueDerefMember(w, vec_ptr, .{ .identifier = "array" });
+    try w.writeByte('[');
+    try f.writeCValue(w, index, .other);
+    try w.writeAll("] = ");
+    try f.writeCValue(w, elem, .other);
+    try w.writeByte(';');
+    try f.newline();
+
+    return .none;
+}
+
 fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
     const pt = f.dg.pt;
     const zcu = pt.zcu;
@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
 
     try reap(f, inst, &.{ty_op.operand});
 
-    const is_aligned = if (ptr_info.flags.alignment != .none)
-        ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte)
-    else
-        true;
+    const is_aligned = switch (ptr_info.flags.alignment) {
+        .none => true,
+        else => |ptr_align| ptr_align.compare(.gte, src_ty.abiAlignment(zcu)),
+    };
 
     const w = &f.code.writer;
     const local = try f.allocLocal(inst, src_ty);
-    const v = try Vectorize.start(f, inst, w, ptr_ty);
 
     if (!is_aligned) {
         try w.writeAll("memcpy(&");
         try f.writeCValue(w, local, .other);
-        try v.elem(f, w);
         try w.writeAll(", (const char *)");
-        try f.writeCValue(w, operand, .other);
-        try v.elem(f, w);
+        switch (ptr_info.flags.vector_index) {
+            .none => try f.writeCValue(w, operand, .other),
+            else => |index| {
+                try w.writeByte('&');
+                try f.writeCValue(w, operand, .other);
+                try w.print("[{d}]", .{@intFromEnum(index)});
+            },
+        }
         try w.writeAll(", sizeof(");
         try f.renderType(w, src_ty);
         try w.writeAll("))");
     } else {
         try f.writeCValue(w, local, .other);
-        try v.elem(f, w);
         try w.writeAll(" = ");
-        try f.writeCValueDeref(w, operand);
-        try v.elem(f, w);
+        switch (ptr_info.flags.vector_index) {
+            .none => try f.writeCValueDeref(w, operand),
+            else => |index| {
+                try f.writeCValue(w, operand, .other);
+                try w.print("[{d}]", .{@intFromEnum(index)});
+            },
+        }
     }
     try w.writeByte(';');
     try f.newline();
-    try v.end(f, inst, w);
 
     return local;
 }
@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
         // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
         assert(src_ty.eql(.fromInterned(ptr_info.child)));
 
-        const v = try Vectorize.start(f, inst, w, ptr_ty);
         try w.writeAll("memcpy((char *)");
-        try f.writeCValue(w, ptr_val, .other);
-        try v.elem(f, w);
+        switch (ptr_info.flags.vector_index) {
+            .none => try f.writeCValue(w, ptr_val, .other),
+            else => |index| {
+                try w.writeByte('&');
+                try f.writeCValue(w, ptr_val, .other);
+                try w.print("[{d}]", .{@intFromEnum(index)});
+            },
+        }
         try w.writeAll(", &");
         switch (src_val) {
             .constant => |val| try f.dg.renderValueAsLvalue(w, val),
             else => try f.writeCValue(w, src_val, .other),
         }
-        try v.elem(f, w);
         try w.writeAll(", sizeof(");
         try f.renderType(w, src_ty);
         try w.writeAll("));");
         try f.newline();
-        try v.end(f, inst, w);
     } else {
         switch (ptr_val) {
             .local_ref => |ptr_local_index| switch (src_val) {
@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
             },
             else => {},
         }
-        const v = try Vectorize.start(f, inst, w, ptr_ty);
-        try f.writeCValueDeref(w, ptr_val);
-        try v.elem(f, w);
+
+        switch (ptr_info.flags.vector_index) {
+            .none => try f.writeCValueDeref(w, ptr_val),
+            else => |index| {
+                try f.writeCValue(w, ptr_val, .other);
+                try w.print("[{d}]", .{@intFromEnum(index)});
+            },
+        }
         try w.writeAll(" = ");
         try f.writeCValue(w, src_val, .other);
-        try v.elem(f, w);
         try w.writeByte(';');
         try f.newline();
-        try v.end(f, inst, w);
     }
     return .none;
 }
@@ -3613,9 +3658,9 @@ fn airCmpOp(
     const lhs_ty = f.typeOf(data.lhs);
     const scalar_ty = lhs_ty.scalarType(zcu);
 
-    const scalar_bits = scalar_ty.bitSize(zcu);
-    if (scalar_ty.isInt(zcu) and scalar_bits > 64)
-        return airCmpBuiltinCall(
+    if (scalar_ty.isInt(zcu)) {
+        const scalar_bits = scalar_ty.bitSize(zcu);
+        if (scalar_bits > 64) return airCmpBuiltinCall(
             f,
             inst,
             data,
@@ -3623,6 +3668,7 @@ fn airCmpOp(
             .cmp,
             if (scalar_bits > 128) .bits else .none,
         );
+    }
     if (scalar_ty.isRuntimeFloat())
         return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);
 
@@ -3668,9 +3714,9 @@ fn airEquality(
     const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
 
     const operand_ty = f.typeOf(bin_op.lhs);
-    const operand_bits = operand_ty.bitSize(zcu);
-    if (operand_ty.isAbiInt(zcu) and operand_bits > 64)
-        return airCmpBuiltinCall(
+    if (operand_ty.isAbiInt(zcu)) {
+        const operand_bits = operand_ty.bitSize(zcu);
+        if (operand_bits > 64) return airCmpBuiltinCall(
             f,
             inst,
             bin_op,
@@ -3678,6 +3724,7 @@ fn airEquality(
             .cmp,
             if (operand_bits > 128) .bits else .none,
         );
+    }
     if (operand_ty.isRuntimeFloat())
         return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);
 
@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
     try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});
 }
 
-fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {
+fn airPtrCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
+    const zcu = f.dg.pt.zcu;
+
+    const dest_ty = f.typeOfIndex(inst);
+    const ptr_ty = switch (dest_ty.zigTypeTag(zcu)) {
+        .optional => dest_ty.childType(zcu),
+        .pointer => dest_ty,
+        else => unreachable,
+    };
+
+    if (!ptr_ty.isSlice(zcu)) {
+        return airSimpleCast(f, inst);
+    }
+
+    // For slice casts we need to assign both fields.
+
     const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
-    const inst_ty = f.typeOfIndex(inst);
+    const operand = try f.resolveInst(ty_op.operand);
+
+    const w = &f.code.writer;
+    const dest_local = try f.allocLocal(inst, dest_ty);
+
+    try f.writeCValueMember(w, dest_local, .{ .identifier = "ptr" });
+    try w.writeAll(" = (");
+    try f.renderType(w, ptr_ty.slicePtrFieldType(zcu));
+    try w.writeByte(')');
+    try f.writeCValueMember(w, operand, .{ .identifier = "ptr" });
+    try w.writeByte(';');
+    try f.newline();
+
+    try f.writeCValueMember(w, dest_local, .{ .identifier = "len" });
+    try w.writeAll(" = ");
+    try f.writeCValueMember(w, operand, .{ .identifier = "len" });
+    try w.writeByte(';');
+    try f.newline();
+
+    try reap(f, inst, &.{ty_op.operand});
+    return dest_local;
+}
+
+fn airSimpleCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
+    const zcu = f.dg.pt.zcu;
+
+    const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+    const dest_ty = f.typeOfIndex(inst);
+    const operand_ty = f.typeOf(ty_op.operand);
+    const operand = try f.resolveInst(ty_op.operand);
+
+    const w = &f.code.writer;
+    const dest_local = try f.allocLocal(inst, dest_ty);
+    const v: Vectorize = try .start(f, inst, w, operand_ty);
+    try f.writeCValue(w, dest_local, .other);
+    try v.elem(f, w);
+    try w.writeAll(" = (");
+    try f.renderType(w, dest_ty.scalarType(zcu));
+    try w.writeByte(')');
+    try f.writeCValue(w, operand, .other);
+    try v.elem(f, w);
+    try w.writeByte(';');
+    try f.newline();
+    try v.end(f, inst, w);
+
+    try reap(f, inst, &.{ty_op.operand});
+    return dest_local;
+}
+
+fn airNopCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
+    const zcu = f.dg.pt.zcu;
+
+    const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+    const dest_ty = f.typeOfIndex(inst);
+    const operand_ty = f.typeOf(ty_op.operand);
+    const operand = try f.resolveInst(ty_op.operand);
+
+    assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
+    assert(operand_ty.isAbiInt(zcu) == dest_ty.isAbiInt(zcu));
+
+    try reap(f, inst, &.{ty_op.operand});
+    return f.moveCValue(inst, dest_ty, operand);
+}
+
+fn airUnionFromEnum(f: *Function, inst: Air.Inst.Index) Error!CValue {
+    const zcu = f.dg.pt.zcu;
+
+    const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+    const dest_ty = f.typeOfIndex(inst);
+    const operand_ty = f.typeOf(ty_op.operand);
+    const operand = try f.resolveInst(ty_op.operand);
+
+    assert(dest_ty.zigTypeTag(zcu) == .@"union");
+    assert(operand_ty.zigTypeTag(zcu) == .@"enum");
+
+    const w = &f.code.writer;
+    const dest_local = try f.allocLocal(inst, dest_ty);
+    try f.writeCValueMember(w, dest_local, .{ .identifier = "tag" });
+    try w.writeAll(" = ");
+    try f.writeCValue(w, operand, .other);
+    try w.writeByte(';');
+    try f.newline();
+
+    try reap(f, inst, &.{ty_op.operand});
+    return dest_local;
+}
+
+fn airBitCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
+    const pt = f.dg.pt;
+    const zcu = pt.zcu;
+    const w = &f.code.writer;
+
+    const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+    const dest_ty = f.typeOfIndex(inst);
 
     const operand = try f.resolveInst(ty_op.operand);
     const operand_ty = f.typeOf(ty_op.operand);
 
-    const bitcasted = try bitcast(f, inst_ty, operand, operand_ty);
-    try reap(f, inst, &.{ty_op.operand});
-    return f.moveCValue(inst, inst_ty, bitcasted);
-}
+    const dest_local = try f.allocLocal(inst, dest_ty);
 
-fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CValue {
-    const pt = f.dg.pt;
-    const zcu = pt.zcu;
-    const target = &f.dg.mod.resolved_target.result;
-    const w = &f.code.writer;
+    // Because we have `scalarize_bit_cast_array` and `scalarize_bit_cast_vector_non_elementwise`
+    // enabled, we usually only see scalars here. The only case in which we may see vectors is when
+    // the operation happens elementwise, which we can handle with `Vectorize`.
+    var v: Vectorize = try .start(f, inst, w, operand_ty);
+    const operand_scalar_ty = operand_ty.scalarType(zcu);
+    const dest_scalar_ty = dest_ty.scalarType(zcu);
 
-    if (operand_ty.isAbiInt(zcu) and dest_ty.isAbiInt(zcu)) {
-        const src_info = dest_ty.intInfo(zcu);
-        const dest_info = operand_ty.intInfo(zcu);
-        if (src_info.signedness == dest_info.signedness and
-            src_info.bits == dest_info.bits) return operand;
-    }
-
-    if (dest_ty.isPtrAtRuntime(zcu) or operand_ty.isPtrAtRuntime(zcu)) {
-        const local = try f.allocLocal(null, dest_ty);
-        try f.writeCValue(w, local, .other);
+    // Some cases are handled with a simple cast:
+    // * float -> float
+    // * bool -> int
+    if ((operand_scalar_ty.isRuntimeFloat() and dest_scalar_ty.isRuntimeFloat()) or
+        (operand_scalar_ty.toIntern() == .bool_type and dest_scalar_ty.isAbiInt(zcu)))
+    {
+        try f.writeCValue(w, dest_local, .other);
+        try v.elem(f, w);
         try w.writeAll(" = (");
-        try f.renderType(w, dest_ty);
+        try f.renderType(w, dest_scalar_ty);
         try w.writeByte(')');
         try f.writeCValue(w, operand, .other);
+        try v.elem(f, w);
         try w.writeByte(';');
         try f.newline();
-        return local;
-    }
-
-    const local = try f.allocLocal(null, dest_ty);
-    // On big-endian targets, copying ABI integers with padding bits is awkward, because the padding bits are at the low bytes of the value.
-    // We need to offset the source or destination pointer appropriately and copy the right number of bytes.
-    if (target.cpu.arch.endian() == .big and dest_ty.isAbiInt(zcu) and !operand_ty.isAbiInt(zcu)) {
-        // e.g. [10]u8 -> u80. We need to offset the destination so that we copy to the least significant bits of the integer.
-        const offset = dest_ty.abiSize(zcu) - operand_ty.abiSize(zcu);
-        try w.writeAll("memcpy((char *)&");
-        try f.writeCValue(w, local, .other);
-        try w.print(" + {d}, &", .{offset});
-        switch (operand) {
-            .constant => |val| try f.dg.renderValueAsLvalue(w, val),
-            else => try f.writeCValue(w, operand, .other),
-        }
-        try w.print(", {d});", .{operand_ty.abiSize(zcu)});
-    } else if (target.cpu.arch.endian() == .big and operand_ty.isAbiInt(zcu) and !dest_ty.isAbiInt(zcu)) {
-        // e.g. u80 -> [10]u8. We need to offset the source so that we copy from the least significant bits of the integer.
-        const offset = operand_ty.abiSize(zcu) - dest_ty.abiSize(zcu);
+    } else if (dest_scalar_ty.toIntern() == .bool_type) {
+        // If the result is a boolean type, just check if the operand is non-zero.
+        assert(operand_scalar_ty.isAbiInt(zcu));
+        try f.writeCValue(w, dest_local, .other);
+        try v.elem(f, w);
+        try w.writeAll(" = ");
+        try f.writeCValue(w, operand, .other);
+        try v.elem(f, w);
+        try w.writeAll(" != 0;");
+        try f.newline();
+    } else if (dest_scalar_ty.isRuntimeFloat()) {
+        // For int->float, just do a memcpy.
+        assert(operand_scalar_ty.isAbiInt(zcu));
         try w.writeAll("memcpy(&");
-        try f.writeCValue(w, local, .other);
-        try w.writeAll(", (const char *)&");
-        switch (operand) {
-            .constant => |val| try f.dg.renderValueAsLvalue(w, val),
-            else => try f.writeCValue(w, operand, .other),
-        }
-        try w.print(" + {d}, {d});", .{ offset, dest_ty.abiSize(zcu) });
-    } else {
-        try w.writeAll("memcpy(&");
-        try f.writeCValue(w, local, .other);
+        try f.writeCValue(w, dest_local, .other);
+        try v.elem(f, w);
         try w.writeAll(", &");
         switch (operand) {
             .constant => |val| try f.dg.renderValueAsLvalue(w, val),
             else => try f.writeCValue(w, operand, .other),
         }
-        try w.print(", {d});", .{@min(dest_ty.abiSize(zcu), operand_ty.abiSize(zcu))});
-    }
+        try v.elem(f, w);
+        try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
+        try f.newline();
+    } else {
+        // The only remaining possibility is that the result is an integer. We will need to use
+        // `zig_wrap_*` to correct the "padding" bits after we populate the value bits.
+        assert(dest_scalar_ty.isAbiInt(zcu));
+        assert(operand_scalar_ty.isRuntimeFloat() or operand_scalar_ty.isAbiInt(zcu));
 
-    try f.newline();
+        // memcpy the value...
+        try w.writeAll("memcpy(&");
+        try f.writeCValue(w, dest_local, .other);
+        try v.elem(f, w);
+        try w.writeAll(", &");
+        switch (operand) {
+            .constant => |val| try f.dg.renderValueAsLvalue(w, val),
+            else => try f.writeCValue(w, operand, .other),
+        }
+        try v.elem(f, w);
+        try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
+        try f.newline();
 
-    // Ensure padding bits have the expected value.
-    if (dest_ty.isAbiInt(zcu)) {
-        switch (CType.classifyInt(dest_ty, zcu)) {
+        // ...and ensure padding bits have the correct value.
+        switch (CType.classifyInt(dest_scalar_ty, zcu)) {
             .void => unreachable, // opv
             .small => {
-                try f.writeCValue(w, local, .other);
+                try f.writeCValue(w, dest_local, .other);
+                try v.elem(f, w);
                 try w.writeAll(" = zig_wrap_");
-                try f.dg.renderTypeForBuiltinFnName(w, dest_ty);
+                try f.dg.renderTypeForBuiltinFnName(w, dest_scalar_ty);
                 try w.writeByte('(');
-                try f.writeCValue(w, local, .other);
-                try f.dg.renderBuiltinInfo(w, dest_ty, .bits);
+                try f.writeCValue(w, dest_local, .other);
+                try v.elem(f, w);
+                try f.dg.renderBuiltinInfo(w, dest_scalar_ty, .bits);
                 try w.writeAll(");");
                 try f.newline();
             },
             .big => |big| {
-                const dest_info = dest_ty.intInfo(zcu);
-                const padding_index: u16 = switch (target.cpu.arch.endian()) {
+                const dest_info = dest_scalar_ty.intInfo(zcu);
+                const padding_index: u16 = switch (f.dg.mod.resolved_target.result.cpu.arch.endian()) {
                     .little => big.limbs_len - 1,
                     .big => 0,
                 };
                 const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;
                 if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {
-                    try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
-                    try w.print("[{d}] = zig_wrap_{c}{d}(", .{
+                    try f.writeCValue(w, dest_local, .other);
+                    try v.elem(f, w);
+                    try w.print(".limbs[{d}] = zig_wrap_{c}{d}(", .{
                         padding_index,
                         signAbbrev(dest_info.signedness),
                         big.limb_size.bits(),
                     });
-                    try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
-                    try w.print("[{d}], {d});", .{ padding_index, wrap_bits });
+                    try f.writeCValue(w, dest_local, .other);
+                    try v.elem(f, w);
+                    try w.print(".limbs[{d}], {d});", .{ padding_index, wrap_bits });
                 } else {
-                    try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
-                    try w.print("[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
+                    try f.writeCValue(w, dest_local, .other);
+                    try v.elem(f, w);
+                    try w.print(".limbs[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
                         padding_index,
                     });
-                    try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
-                    try w.print("[{d}]), {d}));", .{ padding_index, wrap_bits });
+                    try f.writeCValue(w, dest_local, .other);
+                    try v.elem(f, w);
+                    try w.print(".limbs[{d}]), {d}));", .{ padding_index, wrap_bits });
                     try f.newline();
                 }
             },
         }
     }
 
-    return local;
+    try v.end(f, inst, w);
+
+    try reap(f, inst, &.{ty_op.operand});
+    return dest_local;
 }
 
 fn airTrap(f: *Function) !void {
@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
         return .none;
     }
 
-    if (elem_abi_size == 1 and !dest_ty.isVolatilePtr(zcu)) {
-        const bitcasted = try bitcast(f, .u8, value, elem_ty);
+    if (elem_abi_size == 1 and elem_ty.isAbiInt(zcu) and !dest_ty.isVolatilePtr(zcu)) {
         try w.writeAll("memset(");
         switch (dest_ty.ptrSize(zcu)) {
             .slice => {
                 try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });
-                try w.writeAll(", ");
-                try f.writeCValue(w, bitcasted, .other);
+                try w.writeAll(", *(const char *)&");
+                switch (value) {
+                    .constant => |v| try f.dg.renderValueAsLvalue(w, v),
+                    else => try f.writeCValue(w, value, .other),
+                }
                 try w.writeAll(", ");
                 try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });
             },
             .one => {
                 try f.writeCValue(w, dest_slice, .other);
-                try w.writeAll(", ");
-                try f.writeCValue(w, bitcasted, .other);
+                try w.writeAll(", *(const char *)&");
+                switch (value) {
+                    .constant => |v| try f.dg.renderValueAsLvalue(w, v),
+                    else => try f.writeCValue(w, value, .other),
+                }
                 try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});
             },
             .many, .c => unreachable,
         }
         try w.writeAll(");");
         try f.newline();
-        try f.freeCValue(inst, bitcasted);
         try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
         return .none;
     }

src/codegen/c/type.zig

@@ -514,40 +514,39 @@ pub const CType = union(enum) {
         }
     }
     fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {
-        const is_ez80 = zcu.getTarget().cpu.arch == .ez80;
-        return switch (bits) {
+        const target = zcu.getTarget();
+        return switch (std.zig.target.intByteSize(target, bits)) {
             0 => .void,
-            1...8 => switch (signedness) {
+            1 => switch (signedness) {
                 .unsigned => .{ .small = .uint8_t },
                 .signed => .{ .small = .int8_t },
             },
-            9...16 => switch (signedness) {
+            2 => switch (signedness) {
                 .unsigned => .{ .small = .uint16_t },
                 .signed => .{ .small = .int16_t },
             },
-            17...24 => switch (signedness) {
-                .unsigned => .{ .small = if (is_ez80) .uint24_t else .uint32_t },
-                .signed => .{ .small = if (is_ez80) .int24_t else .int32_t },
+            3 => switch (signedness) {
+                .unsigned => .{ .small = .uint24_t },
+                .signed => .{ .small = .int24_t },
             },
-            25...32 => switch (signedness) {
+            4 => switch (signedness) {
                 .unsigned => .{ .small = .uint32_t },
                 .signed => .{ .small = .int32_t },
             },
-            33...48 => switch (signedness) {
-                .unsigned => .{ .small = if (is_ez80) .uint48_t else .uint64_t },
-                .signed => .{ .small = if (is_ez80) .int48_t else .int64_t },
+            6 => switch (signedness) {
+                .unsigned => .{ .small = .uint48_t },
+                .signed => .{ .small = .int48_t },
             },
-            49...64 => switch (signedness) {
+            8 => switch (signedness) {
                 .unsigned => .{ .small = .uint64_t },
                 .signed => .{ .small = .int64_t },
             },
-            65...128 => switch (signedness) {
+            16 => switch (signedness) {
                 .unsigned => .{ .small = .zig_u128 },
                 .signed => .{ .small = .zig_i128 },
             },
-            else => {
+            else => |n| {
                 @branchHint(.unlikely);
-                const target = zcu.getTarget();
                 const limb_bytes = std.zig.target.intAlignment(target, bits);
                 return .{ .big = .{
                     .limb_size = switch (limb_bytes) {
@@ -558,10 +557,7 @@ pub const CType = union(enum) {
                         16 => .@"128",
                         else => unreachable,
                     },
-                    .limbs_len = @divExact(
-                        std.zig.target.intByteSize(target, bits),
-                        limb_bytes,
-                    ),
+                    .limbs_len = @divExact(n, limb_bytes),
                 } };
             },
         };

src/codegen/llvm.zig

@@ -20,10 +20,8 @@ const Value = @import("../Value.zig");
 const Zcu = @import("../Zcu.zig");
 const aarch64_c_abi = @import("aarch64/abi.zig");
 const FuncGen = @import("llvm/FuncGen.zig");
-const buildAllocaInner = FuncGen.buildAllocaInner;
 const isByRef = FuncGen.isByRef;
-const firstParamSRet = FuncGen.firstParamSRet;
-const lowerFnRetTy = FuncGen.lowerFnRetTy;
+const fnReturnStrat = FuncGen.fnReturnStrat;
 const iterateParamTypes = FuncGen.iterateParamTypes;
 const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
 
@@ -37,6 +35,11 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
     return comptime &.initMany(&.{
         .expand_int_from_float_safe,
         .expand_int_from_float_optimized_safe,
+
+        .scalarize_bit_cast_array,
+        // Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest
+        // types are vectors with equal length (hence also with equal bits-per-element).
+        .scalarize_bit_cast_vector_non_elementwise,
     });
 }
 
@@ -728,8 +731,8 @@ pub const Object = struct {
 
         // TODO: Address space
         const slice_ty = Type.slice_const_u8_sentinel_0;
-        const llvm_usize_ty = try o.lowerType(.usize);
-        const llvm_slice_ty = try o.lowerType(slice_ty);
+        const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
+        const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
         const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
 
         llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
@@ -795,7 +798,7 @@ pub const Object = struct {
         {
             if (o.errors_len_variable != .none) {
                 const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
-                const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);
+                const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
                 try o.errors_len_variable.setInitializer(init_val, &o.builder);
             }
             try o.genErrorNameTable();
@@ -1187,7 +1190,7 @@ pub const Object = struct {
         };
         {
             const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
-            global.type = try o.lowerType(fn_ty);
+            global.type = try o.lowerType(fn_ty, .in_memory);
             global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
             global.linkage = if (o.builder.strip) .private else .internal;
             global.visibility = .default;
@@ -1274,165 +1277,7 @@ pub const Object = struct {
             } }, &o.builder);
         }
 
-        var deinit_wip = true;
-        var wip = try Builder.WipFunction.init(&o.builder, .{
-            .function = llvm_function,
-            .strip = owner_mod.strip,
-        });
-        defer if (deinit_wip) wip.deinit();
-        wip.cursor = .{ .block = try wip.block(0, "Entry") };
-
-        // This is the list of args we will use that correspond directly to the AIR arg
-        // instructions. Depending on the calling convention, this list is not necessarily
-        // a bijection with the actual LLVM parameters of the function.
-        var args: std.ArrayList(Builder.Value) = .empty;
-        defer args.deinit(gpa);
-
-        const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
-            var it = iterateParamTypes(o, fn_info);
-
-            const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
-                const param = wip.arg(it.llvm_index);
-                it.llvm_index += 1;
-                break :param param;
-            } else .none;
-
-            const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
-            const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
-                const param = wip.arg(it.llvm_index);
-                it.llvm_index += 1;
-                break :param param;
-            } else .none;
-
-            while (try it.next()) |lowering| {
-                try args.ensureUnusedCapacity(gpa, 1);
-
-                switch (lowering) {
-                    .no_bits => continue,
-                    .byval => {
-                        assert(!it.byval_attr);
-                        const param_index = it.zig_index - 1;
-                        const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        if (isByRef(param_ty, zcu)) {
-                            const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                            const param_llvm_ty = param.typeOfWip(&wip);
-                            const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
-                            _ = try wip.store(.normal, param, arg_ptr, alignment);
-                            args.appendAssumeCapacity(arg_ptr);
-                        } else {
-                            args.appendAssumeCapacity(param);
-                        }
-                    },
-                    .byref => {
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(param);
-                        } else {
-                            const param_llvm_ty = try o.lowerType(param_ty);
-                            const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
-                        }
-                    },
-                    .byref_mut => {
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(param);
-                        } else {
-                            const param_llvm_ty = try o.lowerType(param_ty);
-                            const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
-                        }
-                    },
-                    .abi_sized_int => {
-                        assert(!it.byval_attr);
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        const param_llvm_ty = try o.lowerType(param_ty);
-                        const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                        const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
-                        _ = try wip.store(.normal, param, arg_ptr, alignment);
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(arg_ptr);
-                        } else {
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
-                        }
-                    },
-                    .slice => {
-                        assert(!it.byval_attr);
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        assert(!isByRef(param_ty, zcu));
-                        const slice_val = try wip.buildAggregate(
-                            try o.lowerType(param_ty),
-                            &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
-                            "",
-                        );
-                        args.appendAssumeCapacity(slice_val);
-                    },
-                    .multiple_llvm_types => {
-                        assert(!it.byval_attr);
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param_llvm_ty = try o.lowerType(param_ty);
-                        const param_alignment = param_ty.abiAlignment(zcu);
-                        const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
-                        const arg_ptr = try buildAllocaInner(&wip, llvm_ty, param_alignment.toLlvm(), target);
-                        const llvm_args_start = it.llvm_index - it.types_len;
-                        for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
-                            const param = wip.arg(@intCast(llvm_arg_index));
-                            const part_ptr = try o.ptraddConst(&wip, arg_ptr, offset);
-                            _ = try wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
-                        }
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(arg_ptr);
-                        } else {
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment.toLlvm(), ""));
-                        }
-                    },
-                    .float_array => {
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param_llvm_ty = try o.lowerType(param_ty);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                        const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
-                        _ = try wip.store(.normal, param, arg_ptr, alignment);
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(arg_ptr);
-                        } else {
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
-                        }
-                    },
-                    .i32_array, .i64_array => {
-                        const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                        const param_llvm_ty = try o.lowerType(param_ty);
-                        const param = wip.arg(it.llvm_index - 1);
-
-                        const alignment = param_ty.abiAlignment(zcu).toLlvm();
-                        const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
-                        _ = try wip.store(.normal, param, arg_ptr, alignment);
-
-                        if (isByRef(param_ty, zcu)) {
-                            args.appendAssumeCapacity(arg_ptr);
-                        } else {
-                            args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
-                        }
-                    },
-                }
-            }
-
-            break :implicit_args .{ ret_ptr, err_ret_trace };
-        };
-
-        const file, const subprogram = if (!wip.strip) debug_info: {
+        const file, const subprogram = if (!owner_mod.strip) debug_info: {
             const file = try o.getDebugFile(file_scope);
 
             const line_number = zcu.navSrcLine(func.owner_nav) + 1;
@@ -1498,11 +1343,12 @@ pub const Object = struct {
             .gpa = gpa,
             .air = air.*,
             .liveness = liveness.*.?,
-            .wip = wip,
+            .wip = try .init(&o.builder, .{
+                .function = llvm_function,
+                .strip = owner_mod.strip,
+            }),
             .is_naked = fn_info.cc == .naked,
             .fuzz = fuzz,
-            .ret_ptr = ret_ptr,
-            .args = args.items,
             .arg_index = 0,
             .arg_inline_index = 0,
             .func_inst_table = .empty,
@@ -1516,14 +1362,18 @@ pub const Object = struct {
             .base_line = zcu.navSrcLine(func.owner_nav),
             .prev_dbg_line = 0,
             .prev_dbg_column = 0,
-            .err_ret_trace = err_ret_trace,
             .disable_intrinsics = disable_intrinsics,
             .allowzero_access = false,
+
+            .ret_ptr = undefined, // populated by `genMainBody`
+            .err_ret_trace = undefined, // populated by `genMainBody`
+            .args = undefined, // populated by `genMainBody`
         };
         defer fg.deinit();
-        deinit_wip = false;
 
-        try fg.genBody(air.getMainBody(), .poi);
+        fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
+
+        try fg.genMainBody();
 
         // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
         // function as considering null pointers valid so that LLVM's optimizers don't remove these
@@ -1589,10 +1439,10 @@ pub const Object = struct {
             // represent (because it doesn't have runtime bits), we instead lower as the zero-size
             // type `[0 x i8]`. I don't think the type on an extern declaration actually does much
             // anyway.
-            if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);
+            if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
             break :ty try o.builder.arrayType(0, .i8);
         } else if (nav_ty.hasRuntimeBits(zcu)) ty: {
-            break :ty try o.lowerType(nav_ty);
+            break :ty try o.lowerType(nav_ty, .in_memory);
         } else {
             // This is a non-extern zero-bit `Nav`---we're not interested in it.
             // TODO: we might need to rethink this a little under incremental compilation. If a
@@ -1685,7 +1535,7 @@ pub const Object = struct {
             llvm_variable.setAlignment(llvm_align, &o.builder);
             llvm_variable.setSection(llvm_section, &o.builder);
             llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
-            try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);
+            try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
             llvm_variable.setThreadLocal(tl: {
                 if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
                 break :tl .default;
@@ -2283,7 +2133,7 @@ pub const Object = struct {
                 defer debug_param_types.deinit(gpa);
 
                 // Return type goes first.
-                if (firstParamSRet(fn_info, zcu, target)) {
+                if (try fnReturnStrat(o, fn_info) == .sret) {
                     // Actual return type is void, then first arg is the sret pointer.
                     const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
                     debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
@@ -2831,12 +2681,12 @@ pub const Object = struct {
         if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
 
         var it = iterateParamTypes(o, fn_info);
-        if (firstParamSRet(fn_info, zcu, target)) {
+        if (try fnReturnStrat(o, fn_info) == .sret) {
             // Sret pointers must not be address 0
             try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
             try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
 
-            const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));
+            const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type), .in_memory);
             try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
             it.llvm_index += 1;
         } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {
@@ -2878,9 +2728,7 @@ pub const Object = struct {
             },
             .byref => {
                 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                const param_llvm_ty = try o.lowerType(param_ty);
-                const alignment = param_ty.abiAlignment(zcu);
-                try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
+                try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
             },
             .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
             .slice => {
@@ -2990,14 +2838,31 @@ pub const Object = struct {
         }
     }
 
-    pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {
-        return o.builder.intType(o.zcu.errorSetBits());
+    pub const TypeRepr = enum {
+        /// The representation of the type when it is being manipulated as a value in a function.
+        /// e.g. Zig `u5` -> LLVM `i5`
+        by_value,
+        /// The representation of the type when it is stored in memory.
+        /// e.g. Zig `u5` -> LLVM `i8`
+        in_memory,
+    };
+
+    pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
+        return o.builder.intType(switch (repr) {
+            .by_value => o.zcu.errorSetBits(),
+            .in_memory => @intCast(Type.anyerror.abiSize(o.zcu) * 8),
+        });
     }
 
-    pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
+    pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
         const zcu = o.zcu;
         const target = zcu.getTarget();
         const ip = &zcu.intern_pool;
+
+        if (repr == .by_value) {
+            assert(!isByRef(t, zcu)); // by-ref types must only be manipulated in memory
+        }
+
         return switch (t.toIntern()) {
             .u0_type => unreachable, // no runtime bits
             inline .u1_type,
@@ -3013,7 +2878,10 @@ pub const Object = struct {
             .u80_type,
             .u128_type,
             .i128_type,
-            => |tag| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
+            => |tag| switch (repr) {
+                .by_value => @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
+                .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
+            },
             .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
             inline .c_char_type,
             .c_short_type,
@@ -3048,7 +2916,7 @@ pub const Object = struct {
                 return .i8;
             },
             .bool_type => .i1,
-            .anyerror_type => try o.errorIntType(),
+            .anyerror_type => try o.errorIntType(repr),
             .void_type => unreachable, // no runtime bits
             .type_type => unreachable, // no runtime bits
             .comptime_int_type => unreachable, // no runtime bits
@@ -3068,10 +2936,10 @@ pub const Object = struct {
             => .ptr,
             .slice_const_u8_type,
             .slice_const_u8_sentinel_0_type,
-            => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),
+            => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
             .anyerror_void_error_union_type,
             .adhoc_inferred_error_set_type,
-            => try o.errorIntType(),
+            => try o.errorIntType(repr),
             .generic_poison_type => unreachable,
             // values, not types
             .undef,
@@ -3097,7 +2965,10 @@ pub const Object = struct {
             .none,
             => unreachable,
             else => switch (ip.indexToKey(t.toIntern())) {
-                .int_type => |int_type| try o.builder.intType(int_type.bits),
+                .int_type => |int_type| switch (repr) {
+                    .by_value => try o.builder.intType(int_type.bits),
+                    .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
+                },
                 .ptr_type => |ptr_type| type: {
                     const ptr_ty = try o.builder.ptrType(
                         toLlvmAddressSpace(ptr_type.flags.address_space, target),
@@ -3106,18 +2977,18 @@ pub const Object = struct {
                         .one, .many, .c => ptr_ty,
                         .slice => try o.builder.structType(.normal, &.{
                             ptr_ty,
-                            try o.lowerType(.usize),
+                            try o.lowerType(.usize, repr),
                         }),
                     };
                 },
                 .array_type => |array_type| o.builder.arrayType(
                     array_type.lenIncludingSentinel(),
-                    try o.lowerType(.fromInterned(array_type.child)),
+                    try o.lowerType(.fromInterned(array_type.child), repr),
                 ),
                 .vector_type => |vector_type| o.builder.vectorType(
                     .normal,
                     vector_type.len,
-                    try o.lowerType(.fromInterned(vector_type.child)),
+                    try o.lowerType(.fromInterned(vector_type.child), .by_value),
                 ),
                 .opt_type => |child_ty| {
                     // Must stay in sync with `opt_payload` logic in `lowerPtr`.
@@ -3127,8 +2998,11 @@ pub const Object = struct {
                         .runtime, .partially_comptime => {},
                     }
 
-                    const payload_ty = try o.lowerType(.fromInterned(child_ty));
-                    if (t.optionalReprIsPayload(zcu)) return payload_ty;
+                    if (t.optionalReprIsPayload(zcu)) {
+                        return o.lowerType(.fromInterned(child_ty), repr);
+                    }
+
+                    const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
 
                     comptime assert(optional_layout_version == 3);
                     var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
@@ -3146,7 +3020,7 @@ pub const Object = struct {
                 .error_union_type => |error_union_type| {
                     // Must stay in sync with `codegen.errUnionPayloadOffset`.
                     // See logic in `lowerPtr`.
-                    const error_type = try o.errorIntType();
+                    const error_type = try o.errorIntType(repr);
 
                     switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
                         .fully_comptime => unreachable,
@@ -3154,7 +3028,7 @@ pub const Object = struct {
                         .runtime, .partially_comptime => {},
                     }
 
-                    const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));
+                    const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
 
                     const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
                     const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
@@ -3189,16 +3063,14 @@ pub const Object = struct {
                 },
                 .simple_type => unreachable,
                 .struct_type => {
-                    if (o.type_map.get(t.toIntern())) |value| return value;
-
                     const struct_type = ip.loadStructType(t.toIntern());
 
                     if (struct_type.layout == .@"packed") {
-                        const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));
-                        try o.type_map.put(o.gpa, t.toIntern(), int_ty);
-                        return int_ty;
+                        return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
                     }
 
+                    if (o.type_map.get(t.toIntern())) |value| return value;
+
                     assert(struct_type.size > 0);
 
                     var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
@@ -3232,7 +3104,7 @@ pub const Object = struct {
 
                         if (!field_ty.hasRuntimeBits(zcu)) continue;
 
-                        try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));
+                        try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
 
                         offset += field_ty.abiSize(zcu);
                     }
@@ -3288,7 +3160,7 @@ pub const Object = struct {
                         if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
                             continue;
                         }
-                        try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));
+                        try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
 
                         offset += Type.fromInterned(field_ty).abiSize(zcu);
                     }
@@ -3305,28 +3177,24 @@ pub const Object = struct {
                     return o.builder.structType(.normal, llvm_field_types.items);
                 },
                 .union_type => {
-                    if (o.type_map.get(t.toIntern())) |value| return value;
-
                     const union_obj = ip.loadUnionType(t.toIntern());
 
                     if (union_obj.layout == .@"packed") {
-                        const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));
-                        try o.type_map.put(o.gpa, t.toIntern(), int_ty);
-                        return int_ty;
+                        return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
                     }
 
-                    assert(union_obj.size > 0);
-
                     const layout = Type.getUnionLayout(union_obj, zcu);
 
                     if (layout.payload_size == 0) {
-                        const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
-                        try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
-                        return enum_tag_ty;
+                        return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
                     }
 
+                    if (o.type_map.get(t.toIntern())) |value| return value;
+
+                    assert(union_obj.size > 0);
+
                     const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
-                    const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);
+                    const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
 
                     const payload_ty = ty: {
                         if (layout.most_aligned_field_size == layout.payload_size) {
@@ -3352,7 +3220,7 @@ pub const Object = struct {
                         );
                         return ty;
                     }
-                    const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
+                    const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
 
                     // Put the tag before or after the payload depending on which one's
                     // alignment is greater.
@@ -3381,9 +3249,9 @@ pub const Object = struct {
                     return ty;
                 },
                 .opaque_type, .spirv_type => unreachable, // no runtime bits
-                .enum_type => try o.lowerType(t.intTagType(zcu)),
+                .enum_type => try o.lowerType(t.intTagType(zcu), repr),
                 .func_type => |func_type| try o.lowerFnType(t, func_type),
-                .error_set_type, .inferred_error_set_type => try o.errorIntType(),
+                .error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
                 // values, not types
                 .undef,
                 .simple_value,
@@ -3415,12 +3283,12 @@ pub const Object = struct {
 
         assert(fn_ty.fnHasRuntimeBits(zcu));
 
-        const ret_ty = try lowerFnRetTy(o, fn_info);
+        const ret_strat = try fnReturnStrat(o, fn_info);
 
         var llvm_params: std.ArrayList(Builder.Type) = .empty;
         defer llvm_params.deinit(o.gpa);
 
-        if (firstParamSRet(fn_info, zcu, target)) {
+        if (ret_strat == .sret) {
             try llvm_params.append(o.gpa, .ptr);
         }
 
@@ -3435,7 +3303,7 @@ pub const Object = struct {
             .no_bits => continue,
             .byval => {
                 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                try llvm_params.append(o.gpa, try o.lowerType(param_ty));
+                try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .in_memory else .by_value));
             },
             .byref, .byref_mut => {
                 try llvm_params.append(o.gpa, .ptr);
@@ -3450,7 +3318,7 @@ pub const Object = struct {
                 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
                 try llvm_params.appendSlice(o.gpa, &.{
                     try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
-                    try o.lowerType(.usize),
+                    try o.lowerType(.usize, .by_value),
                 });
             },
             .multiple_llvm_types => {
@@ -3458,7 +3326,7 @@ pub const Object = struct {
             },
             .float_array => |count| {
                 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
-                const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
+                const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .in_memory);
                 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
             },
             .i32_array, .i64_array => |arr_len| {
@@ -3470,14 +3338,19 @@ pub const Object = struct {
             },
         };
 
-        return o.builder.fnType(
-            ret_ty,
-            llvm_params.items,
-            if (fn_info.is_var_args) .vararg else .normal,
-        );
+        const llvm_ret_ty: Builder.Type = switch (ret_strat) {
+            .void, .sret => .void,
+            .by_val => try o.lowerType(.fromInterned(fn_info.return_type), .by_value),
+            .mem_cast => |llvm_ret_ty| llvm_ret_ty,
+        };
+        const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
+            true => .vararg,
+            false => .normal,
+        };
+        return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
     }
 
-    pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
+    pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
         const zcu = o.zcu;
         const ip = &zcu.intern_pool;
         const target = zcu.getTarget();
@@ -3509,7 +3382,7 @@ pub const Object = struct {
             .inferred_error_set_type,
             => unreachable, // types, not values
 
-            .undef => return o.builder.undefConst(try o.lowerType(ty)),
+            .undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
             .simple_value => |simple_value| switch (simple_value) {
                 .void => unreachable, // non-runtime value
                 .null => unreachable, // non-runtime value
@@ -3524,15 +3397,15 @@ pub const Object = struct {
             .int => {
                 var bigint_space: Value.BigIntSpace = undefined;
                 const bigint = val.toBigInt(&bigint_space, zcu);
-                const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);
+                const llvm_int_ty = try o.lowerType(ty, repr);
                 return o.builder.bigIntConst(llvm_int_ty, bigint);
             },
             .err => |err| {
                 const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
-                return o.builder.intConst(try o.errorIntType(), int);
+                return o.builder.intConst(try o.errorIntType(repr), int);
             },
             .error_union => |error_union| {
-                const llvm_error_ty = try o.errorIntType();
+                const llvm_error_ty = try o.errorIntType(repr);
                 const llvm_error_value = switch (error_union.val) {
                     .err_name => |name| try o.builder.intConst(
                         llvm_error_ty,
@@ -3550,8 +3423,8 @@ pub const Object = struct {
                 const payload_align = payload_type.abiAlignment(zcu);
                 const error_align = Type.errorAbiAlignment(zcu);
                 const llvm_payload_value = switch (error_union.val) {
-                    .err_name => try o.builder.undefConst(try o.lowerType(payload_type)),
-                    .payload => |payload| try o.lowerValue(payload),
+                    .err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
+                    .payload => |payload| try o.lowerValue(payload, repr),
                 };
 
                 var fields: [3]Builder.Type = undefined;
@@ -3566,7 +3439,7 @@ pub const Object = struct {
                 fields[0] = vals[0].typeOf(&o.builder);
                 fields[1] = vals[1].typeOf(&o.builder);
 
-                const llvm_ty = try o.lowerType(ty);
+                const llvm_ty = try o.lowerType(ty, repr);
                 const llvm_ty_fields = llvm_ty.structFields(&o.builder);
                 if (llvm_ty_fields.len > 2) {
                     assert(llvm_ty_fields.len == 3);
@@ -3578,7 +3451,7 @@ pub const Object = struct {
                     fields[0..llvm_ty_fields.len],
                 ), vals[0..llvm_ty_fields.len]);
             },
-            .enum_tag => |enum_tag| o.lowerValue(enum_tag.int),
+            .enum_tag => |enum_tag| o.lowerValue(enum_tag.int, repr),
             .float => switch (ty.floatBits(target)) {
                 16 => if (backendSupportsF16(target))
                     try o.builder.halfConst(val.toFloat(f16, zcu))
@@ -3594,9 +3467,9 @@ pub const Object = struct {
                 else => unreachable,
             },
             .ptr => try o.lowerPtr(arg_val, 0),
-            .slice => |slice| return o.builder.structConst(try o.lowerType(ty), &.{
-                try o.lowerValue(slice.ptr),
-                try o.lowerValue(slice.len),
+            .slice => |slice| return o.builder.structConst(try o.lowerType(ty, repr), &.{
+                try o.lowerValue(slice.ptr, repr),
+                try o.lowerValue(slice.len, repr),
             }),
             .opt => |opt| {
                 comptime assert(optional_layout_version == 3);
@@ -3606,7 +3479,7 @@ pub const Object = struct {
                 if (!payload_ty.hasRuntimeBits(zcu)) {
                     return non_null_bit;
                 }
-                const llvm_ty = try o.lowerType(ty);
+                const llvm_ty = try o.lowerType(ty, repr);
                 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
                     .none => switch (llvm_ty.tag(&o.builder)) {
                         .integer => try o.builder.intConst(llvm_ty, 0),
@@ -3614,15 +3487,15 @@ pub const Object = struct {
                         .structure => try o.builder.zeroInitConst(llvm_ty),
                         else => unreachable,
                     },
-                    else => |payload| try o.lowerValue(payload),
+                    else => |payload| try o.lowerValue(payload, repr),
                 };
                 assert(payload_ty.zigTypeTag(zcu) != .@"fn");
 
                 var fields: [3]Builder.Type = undefined;
                 var vals: [3]Builder.Constant = undefined;
                 vals[0] = switch (opt.val) {
-                    .none => try o.builder.undefConst(try o.lowerType(payload_ty)),
-                    else => |payload| try o.lowerValue(payload),
+                    .none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
+                    else => |payload| try o.lowerValue(payload, repr),
                 };
                 vals[1] = non_null_bit;
                 fields[0] = vals[0].typeOf(&o.builder);
@@ -3639,14 +3512,14 @@ pub const Object = struct {
                     fields[0..llvm_ty_fields.len],
                 ), vals[0..llvm_ty_fields.len]);
             },
-            .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val),
+            .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val, repr),
             .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {
                 .array_type => |array_type| switch (aggregate.storage) {
                     .bytes => |bytes| try o.builder.stringConst(try o.builder.string(
                         bytes.toSlice(array_type.lenIncludingSentinel(), ip),
                     )),
                     .elems => |elems| {
-                        const array_ty = try o.lowerType(ty);
+                        const array_ty = try o.lowerType(ty, repr);
                         const elem_ty = array_ty.childType(&o.builder);
                         assert(elems.len == array_ty.aggregateLen(&o.builder));
 
@@ -3664,7 +3537,7 @@ pub const Object = struct {
 
                         var need_unnamed = false;
                         for (vals, fields, elems) |*result_val, *result_field, elem| {
-                            result_val.* = try o.lowerValue(elem);
+                            result_val.* = try o.lowerValue(elem, repr);
                             result_field.* = result_val.typeOf(&o.builder);
                             if (result_field.* != elem_ty) need_unnamed = true;
                         }
@@ -3676,7 +3549,7 @@ pub const Object = struct {
                     .repeated_elem => |elem| {
                         const len: usize = @intCast(array_type.len);
                         const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
-                        const array_ty = try o.lowerType(ty);
+                        const array_ty = try o.lowerType(ty, repr);
                         const elem_ty = array_ty.childType(&o.builder);
 
                         const ExpectedContents = extern struct {
@@ -3692,12 +3565,12 @@ pub const Object = struct {
                         defer allocator.free(fields);
 
                         var need_unnamed = false;
-                        @memset(vals[0..len], try o.lowerValue(elem));
+                        @memset(vals[0..len], try o.lowerValue(elem, repr));
                         @memset(fields[0..len], vals[0].typeOf(&o.builder));
                         if (fields[0] != elem_ty) need_unnamed = true;
 
                         if (array_type.sentinel != .none) {
-                            vals[len] = try o.lowerValue(array_type.sentinel);
+                            vals[len] = try o.lowerValue(array_type.sentinel, repr);
                             fields[len] = vals[len].typeOf(&o.builder);
                             if (fields[len] != elem_ty) need_unnamed = true;
                         }
@@ -3709,7 +3582,7 @@ pub const Object = struct {
                     },
                 },
                 .vector_type => |vector_type| {
-                    const vector_ty = try o.lowerType(ty);
+                    const vector_ty = try o.lowerType(ty, repr);
                     switch (aggregate.storage) {
                         .bytes, .elems => {
                             const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
@@ -3724,7 +3597,7 @@ pub const Object = struct {
                                     result_val.* = try o.builder.intConst(.i8, byte);
                                 },
                                 .elems => |elems| for (vals, elems) |*result_val, elem| {
-                                    result_val.* = try o.lowerValue(elem);
+                                    result_val.* = try o.lowerValue(elem, .by_value);
                                 },
                                 .repeated_elem => unreachable,
                             }
@@ -3732,12 +3605,12 @@ pub const Object = struct {
                         },
                         .repeated_elem => |elem| return o.builder.splatConst(
                             vector_ty,
-                            try o.lowerValue(elem),
+                            try o.lowerValue(elem, .by_value),
                         ),
                     }
                 },
                 .tuple_type => |tuple| {
-                    const struct_ty = try o.lowerType(ty);
+                    const struct_ty = try o.lowerType(ty, repr);
                     const llvm_len = struct_ty.aggregateLen(&o.builder);
 
                     const ExpectedContents = extern struct {
@@ -3782,8 +3655,8 @@ pub const Object = struct {
 
                         vals[llvm_index] = switch (aggregate.storage) {
                             .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
-                            .elems => |elems| try o.lowerValue(elems[field_index]),
-                            .repeated_elem => |elem| try o.lowerValue(elem),
+                            .elems => |elems| try o.lowerValue(elems[field_index], repr),
+                            .repeated_elem => |elem| try o.lowerValue(elem, repr),
                         };
                         fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
                         if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
@@ -3812,7 +3685,7 @@ pub const Object = struct {
                 },
                 .struct_type => {
                     const struct_type = ip.loadStructType(ty.toIntern());
-                    const struct_ty = try o.lowerType(ty);
+                    const struct_ty = try o.lowerType(ty, repr);
                     assert(struct_type.layout != .@"packed");
                     const llvm_len = struct_ty.aggregateLen(&o.builder);
 
@@ -3856,8 +3729,8 @@ pub const Object = struct {
 
                         vals[llvm_index] = switch (aggregate.storage) {
                             .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
-                            .elems => |elems| try o.lowerValue(elems[field_index]),
-                            .repeated_elem => |elem| try o.lowerValue(elem),
+                            .elems => |elems| try o.lowerValue(elems[field_index], repr),
+                            .repeated_elem => |elem| try o.lowerValue(elem, repr),
                         };
                         fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
                         if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
@@ -3887,9 +3760,9 @@ pub const Object = struct {
                 else => unreachable,
             },
             .un => |un| {
-                const union_ty = try o.lowerType(ty);
+                const union_ty = try o.lowerType(ty, repr);
                 const layout = ty.unionGetLayout(zcu);
-                if (layout.payload_size == 0) return o.lowerValue(un.tag);
+                if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
 
                 const union_obj = zcu.typeToUnion(ty).?;
                 const container_layout = union_obj.layout;
@@ -3910,7 +3783,7 @@ pub const Object = struct {
                         const padding_len = layout.payload_size;
                         break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
                     }
-                    const payload = try o.lowerValue(un.val);
+                    const payload = try o.lowerValue(un.val, repr);
                     const payload_ty = payload.typeOf(&o.builder);
                     if (payload_ty != union_ty.structFields(&o.builder)[
                         @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
@@ -3925,7 +3798,7 @@ pub const Object = struct {
                     );
                 } else p: {
                     assert(layout.tag_size == 0);
-                    const union_val = try o.lowerValue(un.val);
+                    const union_val = try o.lowerValue(un.val, repr);
                     need_unnamed = true;
                     break :p union_val;
                 };
@@ -3935,7 +3808,7 @@ pub const Object = struct {
                     try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
                 else
                     union_ty, &.{payload});
-                const tag = try o.lowerValue(un.tag);
+                const tag = try o.lowerValue(un.tag, repr);
                 const tag_ty = tag.typeOf(&o.builder);
                 var fields: [3]Builder.Type = undefined;
                 var vals: [3]Builder.Constant = undefined;
@@ -3989,8 +3862,8 @@ pub const Object = struct {
             },
             .int => try o.builder.castConst(
                 .inttoptr,
-                try o.builder.intConst(try o.lowerType(.usize), offset),
-                try o.lowerType(.fromInterned(ptr.ty)),
+                try o.builder.intConst(try o.lowerType(.usize, .by_value), offset),
+                try o.lowerType(.fromInterned(ptr.ty), .by_value),
             ),
             .eu_payload => |eu_ptr| try o.lowerPtr(
                 eu_ptr,
@@ -4037,7 +3910,7 @@ pub const Object = struct {
         @"addrspace": std.lang.AddressSpace,
     ) Allocator.Error!Builder.Constant {
         const addr: u64 = @"align".toByteUnits().?;
-        const llvm_usize = try o.lowerType(.usize);
+        const llvm_usize = try o.lowerType(.usize, .by_value);
         const llvm_addr = try o.builder.intConst(llvm_usize, addr);
         const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
         return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
@@ -4074,17 +3947,18 @@ pub const Object = struct {
         if (gop.found_existing) {
             // Keep the greater of the two alignments.
             const llvm_variable = gop.value_ptr.*;
-            const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));
-            llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);
+            const llvm_old_align = llvm_variable.getAlignment(&o.builder);
+            const llvm_new_align = llvm_old_align.max(@"align".toLlvm());
+            llvm_variable.setAlignment(llvm_new_align, &o.builder);
             return llvm_variable.ptrConst(&o.builder).global.toConst();
         }
         errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
 
-        const llvm_ty = try o.lowerType(uav_ty);
+        const llvm_ty = try o.lowerType(uav_ty, .in_memory);
         const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
         const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
         gop.value_ptr.* = llvm_variable;
-        try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);
+        try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
         llvm_variable.setMutability(.constant, &o.builder);
         llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
         const llvm_global = llvm_variable.ptrConst(&o.builder).global;
@@ -4156,7 +4030,7 @@ pub const Object = struct {
                 .x86_64_interrupt,
                 .x86_interrupt,
                 => {
-                    const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));
+                    const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
                     try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
                 },
             }
@@ -4178,14 +4052,15 @@ pub const Object = struct {
         o: *Object,
         attributes: *Builder.FunctionAttributes.Wip,
         llvm_arg_i: u32,
-        alignment: Builder.Alignment,
         byval: bool,
-        param_llvm_ty: Builder.Type,
+        param_ty: Type,
     ) Allocator.Error!void {
+        const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
+        const alignment = param_ty.abiAlignment(o.zcu).toLlvm();
         try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
         try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
         try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);
-        if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
+        if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
     }
 
     pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
@@ -4210,7 +4085,7 @@ pub const Object = struct {
     pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
         const builder = &o.builder;
         if (o.errors_len_variable == .none) {
-            const llvm_err_int_ty = try o.errorIntType();
+            const llvm_err_int_ty = try o.errorIntType(.in_memory);
             const name = try builder.strtabString("__zig_errors_len");
             const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
             variable_index.setMutability(.constant, builder);
@@ -4250,9 +4125,9 @@ pub const Object = struct {
         const ip = &zcu.intern_pool;
         const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
 
-        const llvm_usize_ty = try o.lowerType(.usize);
-        const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);
-        const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
+        const llvm_usize_ty = try o.lowerType(.usize, .by_value);
+        const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .by_value);
+        const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
 
         function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
             try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
@@ -4301,7 +4176,7 @@ pub const Object = struct {
             const return_block = try wip.block(1, "Name");
             const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
                 .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
-                else => |tag_val_ip| try o.lowerValue(tag_val_ip),
+                else => |tag_val_ip| try o.lowerValue(tag_val_ip, .by_value),
             };
             try wip_switch.addCase(llvm_tag_val, return_block, &wip);
 
@@ -4347,7 +4222,7 @@ pub const Object = struct {
         const ip = &zcu.intern_pool;
         const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
 
-        const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
+        const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
         function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
             try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
 
@@ -4374,7 +4249,7 @@ pub const Object = struct {
 
         if (loaded_enum.field_values.len > 0) {
             for (loaded_enum.field_values.get(ip)) |tag_val_ip| {
-                const llvm_tag_val = try o.lowerValue(tag_val_ip);
+                const llvm_tag_val = try o.lowerValue(tag_val_ip, .by_value);
                 try wip_switch.addCase(llvm_tag_val, named_block, &wip);
             }
         } else {
@@ -4411,13 +4286,6 @@ pub const Object = struct {
             toLlvmAddressSpace(.generic, o.zcu.getTarget()),
         );
     }
-
-    pub fn ptraddConst(o: *Object, wip: *Builder.WipFunction, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
-        if (offset == 0) return ptr;
-        const llvm_usize_ty = try o.lowerType(.usize);
-        const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
-        return wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
-    }
 };
 
 const CallingConventionInfo = struct {

src/codegen/mips/abi.zig

@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
     const max_direct_size = target.ptrBitWidth() * 2;
     switch (ty.zigTypeTag(zcu)) {
         .@"struct" => {
-            const bit_size = ty.bitSize(zcu);
             if (ty.containerLayout(zcu) == .@"packed") {
-                if (bit_size > max_direct_size) return .memory;
+                if (ty.bitSize(zcu) > max_direct_size) return .memory;
                 return .byval;
             }
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_direct_size) return .memory;
             // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
             const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
             return .{ .i32_array = count };
         },
         .@"union" => {
-            const bit_size = ty.bitSize(zcu);
             if (ty.containerLayout(zcu) == .@"packed") {
-                if (bit_size > max_direct_size) return .memory;
+                if (ty.bitSize(zcu) > max_direct_size) return .memory;
                 return .byval;
             }
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_direct_size) return .memory;
-
             return .byval;
         },
         .bool => return .byval,

src/codegen/riscv64/CodeGen.zig

@@ -51,7 +51,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};
 
 pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
     return comptime &.initMany(&.{
-        .expand_intcast_safe,
+        .expand_int_cast_safe,
         .expand_int_from_float_safe,
         .expand_int_from_float_optimized_safe,
         .expand_add_safe,
@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
             .add_safe,
             .sub_safe,
             .mul_safe,
-            .intcast_safe,
+            .int_cast_safe,
             .int_from_float_safe,
             .int_from_float_optimized_safe,
             => return func.fail("TODO implement safety_checked_instructions", .{}),
@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
             .ret_ptr         => try func.airRetPtr(inst),
             .arg             => try func.airArg(inst),
             .assembly        => try func.airAsm(inst),
-            .bitcast         => try func.airBitCast(inst),
+            .bit_cast        => try func.airBitCast(inst),
+            .ptr_cast        => try func.airBitCast(inst),
+            .ptr_from_int    => try func.airBitCast(inst),
+            .int_from_ptr    => try func.airBitCast(inst),
+            .error_cast      => try func.airBitCast(inst),
+            .error_from_int  => try func.airBitCast(inst),
+            .int_from_error  => try func.airBitCast(inst),
+            .union_from_enum => try func.airBitCast(inst),
             .block           => try func.airBlock(inst),
             .br              => try func.airBr(inst),
             .repeat          => try func.airRepeat(inst),
@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
             .dbg_empty_stmt  => func.finishAirBookkeeping(),
             .fptrunc         => try func.airFptrunc(inst),
             .fpext           => try func.airFpext(inst),
-            .intcast         => try func.airIntCast(inst),
+            .int_cast        => try func.airIntCast(inst),
             .trunc           => try func.airTrunc(inst),
             .is_non_null     => try func.airIsNonNull(inst),
             .is_non_null_ptr => try func.airIsNonNullPtr(inst),
@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
         const elem_ptr_ty = func.typeOfIndex(inst);
         const base_ptr_ty = func.typeOf(extra.lhs);
 
-        if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
-            @panic("audit");
-        }
+        assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
 
         const base_ptr_mcv = try func.resolveInst(extra.lhs);
         const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {

src/codegen/riscv64/abi.zig

@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
     const max_byval_size = target.ptrBitWidth() * 2;
     switch (ty.zigTypeTag(zcu)) {
         .@"struct" => {
-            const bit_size = ty.bitSize(zcu);
             if (ty.containerLayout(zcu) == .@"packed") {
-                if (bit_size > max_byval_size) return .memory;
+                if (ty.bitSize(zcu) > max_byval_size) return .memory;
                 return .byval;
             }
 
@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
             }
 
             // TODO this doesn't exactly match what clang produces but its better than nothing
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_byval_size) return .memory;
             if (bit_size > max_byval_size / 2) return .double_integer;
             return .integer;
         },
         .@"union" => {
-            const bit_size = ty.bitSize(zcu);
             if (ty.containerLayout(zcu) == .@"packed") {
-                if (bit_size > max_byval_size) return .memory;
+                if (ty.bitSize(zcu) > max_byval_size) return .memory;
                 return .byval;
             }
             // TODO this doesn't exactly match what clang produces but its better than nothing
+            const bit_size = ty.abiSize(zcu) * 8;
             if (bit_size > max_byval_size) return .memory;
             if (bit_size > max_byval_size / 2) return .double_integer;
             return .integer;
@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {
         },
         .error_union => {
             const payload_ty = ty.errorUnionPayload(zcu);
-            const payload_bits = payload_ty.bitSize(zcu);
 
             // the error union itself
             result[0] = .integer;
 
             // anyerror!void can fit into one register
-            if (payload_bits == 0) return result;
+            if (!payload_ty.hasRuntimeBits(zcu)) return result;
 
             return memory_class;
         },

src/codegen/sparc64/CodeGen.zig

@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .ret_ptr         => try self.airRetPtr(inst),
             .arg             => try self.airArg(inst),
             .assembly        => try self.airAsm(inst),
-            .bitcast         => try self.airBitCast(inst),
+            .bit_cast        => try self.airBitCast(inst),
+            .ptr_cast        => try self.airBitCast(inst),
+            .ptr_from_int    => try self.airBitCast(inst),
+            .int_from_ptr    => try self.airBitCast(inst),
+            .error_cast      => try self.airBitCast(inst),
+            .error_from_int  => try self.airBitCast(inst),
+            .int_from_error  => try self.airBitCast(inst),
+            .union_from_enum => try self.airBitCast(inst),
             .block           => try self.airBlock(inst),
             .br              => try self.airBr(inst),
             .repeat          => return self.fail("TODO implement `repeat`", .{}),
@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .cond_br         => try self.airCondBr(inst),
             .fptrunc         => @panic("TODO try self.airFptrunc(inst)"),
             .fpext           => @panic("TODO try self.airFpext(inst)"),
-            .intcast         => try self.airIntCast(inst),
+            .int_cast        => try self.airIntCast(inst),
             .trunc           => try self.airTrunc(inst),
             .is_non_null     => try self.airIsNonNull(inst),
             .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .add_safe,
             .sub_safe,
             .mul_safe,
-            .intcast_safe,
+            .int_cast_safe,
             .int_from_float_safe,
             .int_from_float_optimized_safe,
             => @panic("TODO implement safety_checked_instructions"),
@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
     const info_a = operand_ty.intInfo(zcu);
     const info_b = self.typeOfIndex(inst).intInfo(zcu);
     if (info_a.signedness != info_b.signedness)
-        return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
+        return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
 
     if (info_a.bits == info_b.bits)
         return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });

src/codegen/spirv/CodeGen.zig

@@ -34,7 +34,7 @@ const CodeGen = @This();
 
 pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
     return comptime &.initMany(&.{
-        .expand_intcast_safe,
+        .expand_int_cast_safe,
         .expand_int_from_float_safe,
         .expand_int_from_float_optimized_safe,
         .expand_add_safe,
@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
         .pointer => {
             const ptr_info = ty.ptrInfo(zcu);
 
-            const child_ty: Type = .fromInterned(ptr_info.child);
+            const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
+                0 => .fromInterned(ptr_info.child),
+                else => switch (ptr_info.flags.vector_index) {
+                    // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
+                    .none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
+                    else => try pt.vectorType(.{
+                        .child = ptr_info.child,
+                        .len = ptr_info.packed_offset.host_size,
+                    }),
+                },
+            };
             const child_ty_id = try cg.resolveType(child_ty, .indirect);
             const storage_class = cg.module.storageClass(ptr_info.flags.address_space);
             const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);
@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
             .min => try cg.airMinMax(inst, .min),
             .max => try cg.airMinMax(inst, .max),
 
-            .bitcast         => try cg.airBitCast(inst),
-            .intcast, .trunc => try cg.airIntCast(inst),
-            .float_from_int  => try cg.airFloatFromInt(inst),
-            .int_from_float  => try cg.airIntFromFloat(inst),
-            .fpext, .fptrunc => try cg.airFloatCast(inst),
-            .not             => try cg.airNot(inst),
+            .bit_cast         => try cg.airBitCast(inst),
+            .ptr_cast         => try cg.airBitCast(inst),
+            .ptr_from_int     => try cg.airBitCast(inst),
+            .int_from_ptr     => try cg.airBitCast(inst),
+            .error_cast       => try cg.airBitCast(inst),
+            .error_from_int   => try cg.airBitCast(inst),
+            .int_from_error   => try cg.airBitCast(inst),
+            .union_from_enum  => try cg.airBitCast(inst),
+            .int_cast, .trunc => try cg.airIntCast(inst),
+            .float_from_int   => try cg.airFloatFromInt(inst),
+            .int_from_float   => try cg.airIntFromFloat(inst),
+            .fpext, .fptrunc  => try cg.airFloatCast(inst),
+            .not              => try cg.airNot(inst),
 
             .array_to_slice => try cg.airArrayToSlice(inst),
             .slice          => try cg.airSlice(inst),
@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
     const zcu = cg.module.zcu;
     const pt = cg.pt;
     const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
-    const ptr_ty = cg.typeOf(ty_op.operand);
-    const ptr_info = ptr_ty.ptrInfo(zcu);
-    const elem_ty = cg.typeOfIndex(inst);
-    const operand = try cg.resolve(ty_op.operand);
-    if (!ptr_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
 
-    if (cg.virtual_allocas.get(operand)) |stored| return stored.?;
+    const ptr_info = cg.typeOf(ty_op.operand).ptrInfo(zcu);
+
+    const elem_ty = cg.typeOfIndex(inst);
+    const operand_ptr_id = try cg.resolve(ty_op.operand);
+
+    assert(ptr_info.child == elem_ty.toIntern());
+
+    if (cg.virtual_allocas.get(operand_ptr_id)) |stored| return stored.?;
 
     if (ptr_info.packed_offset.host_size != 0 and
         ptr_info.flags.vector_index == .none)
@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
         const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
         const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
         const host_int_ty = try pt.intType(.unsigned, host_bits);
-        const host_val = try cg.load(host_int_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
+        const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
         const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
         const field_int_ty = try pt.intType(signedness, elem_bit_size);
         const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
         return try cg.bitCast(elem_ty, field_int_ty, result_id);
     }
 
-    return try cg.load(elem_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
+    const ptr_id = switch (ptr_info.flags.vector_index) {
+        .none => operand_ptr_id,
+        else => |index| ptr_id: {
+            const elem_ptr_ty_id = try cg.module.ptrType(
+                try cg.resolveType(elem_ty, .indirect),
+                cg.module.storageClass(ptr_info.flags.address_space),
+            );
+            break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
+        },
+    };
+    return try cg.load(elem_ty, ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
 }
 
 fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
     const zcu = cg.module.zcu;
     const pt = cg.pt;
     const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
-    const ptr_ty = cg.typeOf(bin_op.lhs);
-    const ptr_info = ptr_ty.ptrInfo(zcu);
-    const elem_ty = ptr_ty.childType(zcu);
-    const ptr = try cg.resolve(bin_op.lhs);
-    const value = try cg.resolve(bin_op.rhs);
+    const ptr_info = cg.typeOf(bin_op.lhs).ptrInfo(zcu);
+    const elem_ty: Type = .fromInterned(ptr_info.child);
+    const operand_ptr_id = try cg.resolve(bin_op.lhs);
+    const value_id = try cg.resolve(bin_op.rhs);
 
-    if (cg.virtual_allocas.getPtr(ptr)) |slot| {
-        slot.* = value;
+    if (cg.virtual_allocas.getPtr(operand_ptr_id)) |slot| {
+        slot.* = value_id;
         return;
     }
 
@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
     {
         const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
         const host_int_ty = try pt.intType(.unsigned, host_bits);
-        const host_val = try cg.load(host_int_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
+        const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
         const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
         const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
         const field_int_ty = try pt.intType(signedness, elem_bit_size);
 
         var value_as_int: Id = undefined;
         if (elem_ty.ip_index == .bool_type) {
-            value_as_int = try cg.convertToIndirect(.bool, value);
+            value_as_int = try cg.convertToIndirect(.bool, value_id);
             value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
         } else if (elem_ty.isInt(zcu)) {
-            value_as_int = value;
+            value_as_int = value_id;
         } else {
-            value_as_int = try cg.bitCast(field_int_ty, elem_ty, value);
+            value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
         }
 
         const extended = blk: {
@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
         const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
         const combined_id = try combined.materialize(cg);
 
-        try cg.store(host_int_ty, ptr, combined_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
+        try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
         return;
     }
 
-    try cg.store(elem_ty, ptr, value, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
+    const ptr_id = switch (ptr_info.flags.vector_index) {
+        .none => operand_ptr_id,
+        else => |index| ptr_id: {
+            const elem_ptr_ty_id = try cg.module.ptrType(
+                try cg.resolveType(elem_ty, .indirect),
+                cg.module.storageClass(ptr_info.flags.address_space),
+            );
+            break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
+        },
+    };
+
+    try cg.store(elem_ty, ptr_id, value_id, .{ .is_volatile = ptr_info.flags.is_volatile });
 }
 
 fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {

src/codegen/wasm/CodeGen.zig

@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
 
 pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
     return comptime &.initMany(&.{
-        .expand_intcast_safe,
+        .expand_int_cast_safe,
         .expand_int_from_float_safe,
         .expand_int_from_float_optimized_safe,
         .expand_add_safe,
@@ -83,7 +83,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
         .scalarize_shl_sat,
         .scalarize_xor,
         .scalarize_not,
-        .scalarize_bitcast,
         .scalarize_clz,
         .scalarize_ctz,
         .scalarize_popcount,
@@ -109,7 +108,10 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
         .scalarize_cmp_vector_optimized,
         .scalarize_fptrunc,
         .scalarize_fpext,
-        .scalarize_intcast,
+        .scalarize_int_cast,
+        .scalarize_ptr_cast,
+        .scalarize_ptr_from_int,
+        .scalarize_int_from_ptr,
         .scalarize_trunc,
         .scalarize_int_from_float,
         .scalarize_int_from_float_optimized,
@@ -120,6 +122,8 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
         .scalarize_shuffle_two,
         .scalarize_select,
         .scalarize_mul_add,
+
+        .scalarize_bit_cast_padded_elems,
     });
 }
 
@@ -1550,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
             try cg.finishAir(inst, result, &.{ty_op.operand});
         },
 
-        .bitcast => cg.airBitcast(inst),
+        .ptr_cast => cg.airNopCast(inst),
+        .error_cast => cg.airNopCast(inst),
+        .error_from_int => cg.airNopCast(inst),
+        .int_from_error => cg.airNopCast(inst),
+        .ptr_from_int => cg.airNopCast(inst),
+        .int_from_ptr => cg.airIntFromPtr(inst),
 
-        .intcast => {
+        .bit_cast => cg.airBitcast(inst),
+        .union_from_enum => cg.airBitcast(inst),
+
+        .int_cast => {
             const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
 
             const dest_ty = ty_op.ty.toType();
@@ -1560,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
             const src_ty = cg.typeOf(ty_op.operand);
 
             if (dest_ty.zigTypeTag(zcu) == .vector) {
-                return cg.fail("TODO: implement AIR op: intcast for vectors", .{});
+                return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
             }
 
             const src_int_ty: IntType = .fromType(cg, src_ty);
@@ -1875,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         .add_safe,
         .sub_safe,
         .mul_safe,
-        .intcast_safe,
+        .int_cast_safe,
         .int_from_float_safe,
         .int_from_float_optimized_safe,
         => return cg.fail("TODO implement safety_checked_instructions", .{}),
@@ -2099,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
     const rhs = try cg.resolveInst(bin_op.rhs);
     const ptr_ty = cg.typeOf(bin_op.lhs);
     const ptr_info = ptr_ty.ptrInfo(zcu);
-    const ty = ptr_ty.childType(zcu);
+    const elem_ty = ptr_ty.childType(zcu);
 
     if (!safety and bin_op.rhs == .undef) {
         return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
     }
 
-    assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_store
-
-    try cg.store(lhs, rhs, ty, 0);
-
+    const offset: u32 = switch (ptr_info.flags.vector_index) {
+        .none => offset: {
+            assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
+            break :offset 0;
+        },
+        else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
+    };
+    try cg.store(lhs, rhs, elem_ty, offset);
     return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
 }
 
@@ -2121,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
     if (!ty.hasRuntimeBits(zcu)) return;
 
     if (isByRef(ty, zcu, cg.target)) {
-        return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });
+        const offset_ptr: WValue = switch (offset + lhs.offset()) {
+            0 => lhs,
+            else => |total_offset| ptr: {
+                try cg.emitWValue(lhs);
+                try cg.addImm32(total_offset);
+                try cg.addTag(.i32_add);
+                break :ptr .stack;
+            },
+        };
+        return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
     }
 
     if (ty.zigTypeTag(zcu) == .vector) {
@@ -2133,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
         try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
             @intFromEnum(std.wasm.SimdOpcode.v128_store),
             offset + lhs.offset(),
-            @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
+            @intCast(ty.abiAlignment(zcu).toByteUnits().?),
         });
         return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
     }
@@ -2174,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
     const zcu = pt.zcu;
     const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
     const operand = try cg.resolveInst(ty_op.operand);
-    const ty = ty_op.ty.toType();
+    const elem_ty = ty_op.ty.toType();
     const ptr_ty = cg.typeOf(ty_op.operand);
     const ptr_info = ptr_ty.ptrInfo(zcu);
 
-    if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});
+    assert(elem_ty.hasRuntimeBits(zcu));
 
-    assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_load
-
-    const result = try cg.load(operand, ty, 0);
+    const offset: u32 = switch (ptr_info.flags.vector_index) {
+        .none => offset: {
+            assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
+            break :offset 0;
+        },
+        else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
+    };
+    const result = try cg.load(operand, elem_ty, offset);
     return cg.finishAir(inst, result, &.{ty_op.operand});
 }
 
@@ -2191,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
 fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
     const zcu = cg.pt.zcu;
     if (isByRef(ty, zcu, cg.target)) {
-        const val = try cg.allocStack(ty);
-        try cg.store(val, try operand.toLocal(cg, .usize), ty, 0);
-        return val;
+        const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
+            0 => operand,
+            else => |total_offset| ptr: {
+                try cg.emitWValue(operand);
+                try cg.addImm32(total_offset);
+                try cg.addTag(.i32_add);
+                break :ptr .stack;
+            },
+        };
+        const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
+        const new_ptr = try cg.allocStack(ty);
+        try cg.store(new_ptr, src_ptr, ty, 0);
+        return new_ptr;
     }
 
     // load local's value from memory by its stack position
@@ -5235,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
     return cg.finishAir(inst, .none, &.{});
 }
 
+fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
+    const zcu = cg.pt.zcu;
+    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+
+    const operand_ty = cg.typeOf(ty_op.operand);
+    const dest_ty = cg.typeOfIndex(inst);
+    assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
+    assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
+    assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
+
+    const operand = try cg.resolveInst(ty_op.operand);
+    const result = cg.reuseOperand(ty_op.operand, operand);
+    return cg.finishAir(inst, result, &.{ty_op.operand});
+}
+
+fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
+    const zcu = cg.pt.zcu;
+    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
+
+    const operand_ty = cg.typeOf(ty_op.operand);
+    const dest_ty = cg.typeOfIndex(inst);
+    assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
+    assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
+    assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
+
+    const operand = try cg.resolveInst(ty_op.operand);
+    const result = switch (operand) {
+        .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
+        else => cg.reuseOperand(ty_op.operand, operand),
+    };
+    return cg.finishAir(inst, result, &.{ty_op.operand});
+}
+
 fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
     const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
     const operand = try cg.resolveInst(ty_op.operand);
@@ -6340,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         }
     }
 
-    const elem_result = if (isByRef(elem_ty, zcu, cg.target))
+    const result = if (isByRef(elem_ty, zcu, cg.target))
         .stack
     else
         try cg.load(.stack, elem_ty, 0);
-
-    return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
+    return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
 }
 
 fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {

src/codegen/x86_64/CodeGen.zig

@@ -47,7 +47,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
         .scalarize_shl,
         .scalarize_shl_exact,
         .scalarize_shl_sat,
-        .scalarize_bitcast,
         .scalarize_ctz,
         .scalarize_popcount,
         .scalarize_byte_swap,
@@ -58,11 +57,13 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
         .scalarize_shuffle_two,
         .scalarize_select,
 
-        //.unsplat_shift_rhs,
-        .reduce_one_elem_to_bitcast,
-        .splat_one_elem_to_bitcast,
+        .scalarize_bit_cast_padded_elems,
 
-        .expand_intcast_safe,
+        //.unsplat_shift_rhs,
+        .reduce_one_elem_to_bit_cast,
+        .splat_one_elem_to_bit_cast,
+
+        .expand_int_cast_safe,
         .expand_int_from_float_safe,
         .expand_int_from_float_optimized_safe,
         .expand_add_safe,
@@ -67433,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
                 };
                 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
             },
-            .bitcast => try cg.airBitCast(inst),
+            .bit_cast,
+            .ptr_cast,
+            .ptr_from_int,
+            .int_from_ptr,
+            .error_cast,
+            .error_from_int,
+            .int_from_error,
+            .union_from_enum,
+            => try cg.airBitCast(inst),
             .block => {
                 const block = cg.air.unwrapBlock(inst);
                 if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
@@ -93374,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
                 };
                 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
             },
-            .intcast => |air_tag| {
+            .int_cast => |air_tag| {
                 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
                 const dst_ty = ty_op.ty.toType();
                 const src_ty = cg.typeOf(ty_op.operand);
@@ -98132,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
                 };
                 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
             },
-            .intcast_safe => unreachable,
+            .int_cast_safe => unreachable,
             .trunc => |air_tag| {
                 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
                 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
@@ -104350,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
                 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
                 try ops[0].toSlicePtr(cg);
                 const dst_ty = ty_pl.ty.toType();
-                if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
+                zero_offset: {
                     const elem_size = dst_ty.childType(zcu).abiSize(zcu);
                     if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;
                     while (true) for (&ops) |*op| {
@@ -179053,8 +179062,11 @@ fn genSetReg(
     const zcu = pt.zcu;
     const abi_size: u32 = @intCast(ty.abiSize(zcu));
     const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));
-    if (ty.bitSize(zcu) > dst_alias.size().bitSize(cg.target))
-        return cg.fail("genSetReg called with a value larger than dst_reg", .{});
+    {
+        const ty_bit_size = if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else 8 * abi_size;
+        if (ty_bit_size > dst_alias.size().bitSize(cg.target))
+            return cg.fail("genSetReg called with a value larger than dst_reg", .{});
+    }
     switch (src_mcv) {
         .none,
         .unreach,
@@ -180127,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
             break :dst dst_mcv;
         };
 
-        if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
+        switch (dst_ty.zigTypeTag(zcu)) {
+            .float, .error_union, .error_set, .vector => break :result dst_mcv,
+            .@"struct", .@"union" => if (dst_ty.containerLayout(zcu) != .@"packed") break :result dst_mcv,
+            .optional, .pointer => if (!dst_ty.isPtrAtRuntime(zcu)) break :result dst_mcv,
+            else => {},
+        }
 
         if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
             dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
 
         const abi_size = dst_ty.abiSize(zcu);
         const bit_size = dst_ty.bitSize(zcu);
-        if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
+        if (abi_size * 8 <= bit_size) break :result dst_mcv;
 
         const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
         const high_mcv: MCValue = switch (dst_mcv) {
@@ -182410,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
     const zcu = pt.zcu;
     const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
         .signedness = .unsigned,
-        .bits = @intCast(ty.bitSize(zcu)),
+        .bits = @intCast(if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else ty.abiSize(zcu) * 8),
     };
     const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
     try self.spillEflagsIfOccupied();
@@ -182450,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {
     };
 }
 
-fn regExtraBits(self: *CodeGen, ty: Type) u64 {
-    return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
-}
-
 fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
     return switch (feature) {
         .@"64bit" => switch (cg.target.cpu.arch) {
@@ -182569,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {
             .bool_type => vector_type.len,
             else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),
         },
-        else => @intCast(ty.bitSize(zcu)),
+        else => if (ty.hasBitRepresentation(zcu) or ty.isAbiInt(zcu)) @intCast(ty.bitSize(zcu)) else @intCast(ty.abiSize(zcu) * 8),
     };
 }
 
@@ -192242,7 +192255,8 @@ const Select = struct {
                 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
                 .@"8_size_sub_bit_size" => {
                     const ty = op.flags.base.ref.typeOf(s);
-                    break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));
+                    const bit_size = s.cg.intInfo(ty).?.bits;
+                    break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - bit_size);
                 },
                 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
                 .elem_limbs => @intCast(@divExact(

stage1/zig.h

@@ -11,8 +11,6 @@
 #elif defined(__GNUC__)
 #define zig_gcc
 #define zig_gnuc
-#elif defined(__IBMC__)
-#define zig_xlc
 #elif defined(__TINYC__)
 #define zig_tinyc
 #elif defined(__slimcc__)
@@ -28,8 +26,18 @@
 #define zig_arm
 #elif defined(__arm__)
 #define zig_arm
+#elif defined(__arc__)
+#define zig_arc
+#elif defined(__csky__)
+#define zig_csky
 #elif defined(__hexagon__)
 #define zig_hexagon
+#elif defined(__hppa__) && defined(_LP64)
+#define zig_hppa64
+#define zig_hppa
+#elif defined(__hppa__)
+#define zig_hppa32
+#define zig_hppa
 #elif defined(__kvx__)
 #define zig_kvx
 #elif defined(__loongarch32)
@@ -42,6 +50,8 @@
 #define zig_m68k
 #elif defined(__m88k__)
 #define zig_m88k
+#elif defined(__microblaze__)
+#define zig_microblaze
 #elif defined(__mips64)
 #define zig_mips64
 #define zig_mips
@@ -64,6 +74,8 @@
 #define zig_riscv
 #elif defined(__s390x__)
 #define zig_s390x
+#elif defined(__sh__)
+#define zig_sh
 #elif defined(__sparc__) && defined(__arch64__)
 #define zig_sparc64
 #define zig_sparc
@@ -100,32 +112,30 @@
 #define zig_big_endian 1
 #endif
 
-#if defined(__MACH__)
+#if defined(__APPLE__)
 #define zig_darwin
 #elif defined(__DragonFly__)
 #define zig_dragonfly
-#define zig_bsd
 #elif defined(__EMSCRIPTEN__)
 #define zig_emscripten
 #elif defined(__FreeBSD__)
 #define zig_freebsd
-#define zig_bsd
 #elif defined(__Fuchsia__)
 #define zig_fuchsia
 #elif defined(__HAIKU__)
 #define zig_haiku
 #elif defined(__gnu_hurd__)
 #define zig_hurd
+#elif defined(__illumos__)
+#define zig_illumos
 #elif defined(__linux__)
 #define zig_linux
 #elif defined(__NetBSD__)
 #define zig_netbsd
-#define zig_bsd
 #elif defined(__OpenBSD__)
 #define zig_openbsd
-#define zig_bsd
-#elif defined(__SVR4)
-#define zig_solaris
+#elif defined(__serenity__)
+#define zig_serenity
 #elif defined(__wasi__)
 #define zig_wasi
 #elif defined(_WIN32)
@@ -404,14 +414,22 @@
 #define zig_trap() __asm__ volatile("udf #0xfe")
 #elif defined(zig_arm) || defined(zig_aarch64)
 #define zig_trap() __asm__ volatile("udf #0xfdee")
+#elif defined(zig_arc)
+#define zig_trap() __asm__ volatile("unimp_s")
+#elif defined(zig_csky)
+#define zig_trap() __asm__ volatile(".word 0x3fff")
 #elif defined(zig_hexagon)
 #define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)")
+#elif defined(zig_hppa)
+#define zig_trap() __asm__ volatile("iitlbp %r0, (%sr0, %r0)")
 #elif defined(zig_kvx) || defined(zig_loongarch) || defined(zig_powerpc)
 #define zig_trap() __asm__ volatile(".word 0x0")
 #elif defined(zig_m68k)
 #define zig_trap() __asm__ volatile("illegal")
 #elif defined(zig_m88k)
 #define zig_trap() __asm__ volatile("tb0 0, %%r0, 511")
+#elif defined(zig_microblaze)
+#define zig_trap() __asm__ volatile("getd r0, r0")
 #elif defined(zig_mips)
 #define zig_trap() __asm__ volatile(".word 0x3d")
 #elif defined(zig_or1k)
@@ -420,6 +438,8 @@
 #define zig_trap() __asm__ volatile("unimp")
 #elif defined(zig_s390x)
 #define zig_trap() __asm__ volatile("j 0x2")
+#elif defined(zig_sh)
+#define zig_trap() __asm__ volatile(".word 0x0001")
 #elif defined(zig_sparc)
 #define zig_trap() __asm__ volatile("illtrap")
 #elif defined(zig_x86_16)
@@ -446,16 +466,22 @@
 
 #if defined(zig_alpha)
 #define zig_breakpoint() __asm__ volatile("call_pal 0x000080")
-#elif defined(zig_arm)
+#elif defined(zig_arm) || defined(zig_csky)
 #define zig_breakpoint() __asm__ volatile("bkpt #0x0")
 #elif defined(zig_aarch64)
 #define zig_breakpoint() __asm__ volatile("brk #0xf000")
+#elif defined(zig_arc)
+#define zig_breakpoint() __asm__ volatile("brk_s")
 #elif defined(zig_hexagon)
 #define zig_breakpoint() __asm__ volatile("brkpt")
+#elif defined(zig_hppa)
+#define zig_breakpoint() __asm__ volatile("break 0x04, 0x0008")
 #elif defined(zig_kvx) || defined(zig_loongarch)
 #define zig_breakpoint() __asm__ volatile("break 0x0")
 #elif defined(zig_m88k)
 #define zig_breakpoint() __asm__ volatile("illop1")
+#elif defined(zig_microblaze)
+#define zig_breakpoint() __asm__ volatile("brki r16, 0x0018")
 #elif defined(zig_mips)
 #define zig_breakpoint() __asm__ volatile("break")
 #elif defined(zig_or1k)
@@ -466,6 +492,8 @@
 #define zig_breakpoint() __asm__ volatile("ebreak")
 #elif defined(zig_s390x)
 #define zig_breakpoint() __asm__ volatile("j 0x6")
+#elif defined(zig_sh)
+#define zig_breakpoint() __asm__ volatile("trapa #0xc3")
 #elif defined(zig_sparc)
 #define zig_breakpoint() __asm__ volatile("ta 0x1")
 #elif defined(zig_x86)
@@ -4529,9 +4557,12 @@ static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 a
 #include <intrin.h>
 #endif
 
+static inline void* zig_e_zig_windows_teb(void) zig_mangled(zig_e_zig_windows_teb, "zig_windows_teb");
+static inline void* zig_e_zig_windows_peb(void) zig_mangled(zig_e_zig_windows_peb, "zig_windows_peb");
+
 #if defined(zig_thumb)
 
-static inline void* zig_windows_teb(void) {
+static inline void* zig_e_zig_windows_teb(void) {
     void* teb = 0;
 #if defined(zig_msvc)
     teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2);
@@ -4543,7 +4574,7 @@ static inline void* zig_windows_teb(void) {
 
 #elif defined(zig_aarch64)
 
-static inline void* zig_windows_teb(void) {
+static inline void* zig_e_zig_windows_teb(void) {
     void* teb = 0;
 #if defined(zig_msvc)
     teb = (void*)__readx18qword(0x0);
@@ -4555,7 +4586,7 @@ static inline void* zig_windows_teb(void) {
 
 #elif defined(zig_x86_32)
 
-static inline void* zig_windows_teb(void) {
+static inline void* zig_e_zig_windows_teb(void) {
     void* teb = 0;
 #if defined(zig_msvc)
     teb = (void*)__readfsdword(0x18);
@@ -4565,7 +4596,7 @@ static inline void* zig_windows_teb(void) {
     return teb;
 }
 
-static inline void* zig_windows_peb(void) {
+static inline void* zig_e_zig_windows_peb(void) {
     void* peb = 0;
 #if defined(zig_msvc)
     peb = (void*)__readfsdword(0x30);
@@ -4577,7 +4608,7 @@ static inline void* zig_windows_peb(void) {
 
 #elif defined(zig_x86_64)
 
-static inline void* zig_windows_teb(void) {
+static inline void* zig_e_zig_windows_teb(void) {
     void* teb = 0;
 #if defined(zig_msvc)
     teb = (void*)__readgsqword(0x30);
@@ -4587,7 +4618,7 @@ static inline void* zig_windows_teb(void) {
     return teb;
 }
 
-static inline void* zig_windows_peb(void) {
+static inline void* zig_e_zig_windows_peb(void) {
     void* peb = 0;
 #if defined(zig_msvc)
     peb = (void*)__readgsqword(0x60);
@@ -4601,7 +4632,9 @@ static inline void* zig_windows_peb(void) {
 
 #if defined(zig_loongarch)
 
-static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
+static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) zig_mangled(zig_e_zig_loongarch_cpucfg, "zig_loongarch_cpucfg");
+
+static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
 #if defined(zig_gnuc_asm)
     __asm__("cpucfg %[result], %[word]" : [result] "=r" (result) : [word] "r" (word));
 #else
@@ -4611,7 +4644,10 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
 
 #elif defined(zig_x86) && !defined(zig_x86_16)
 
-static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
+static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) zig_mangled(zig_e_zig_x86_cpuid, "zig_x86_cpuid");
+static inline uint32_t zig_e_zig_x86_get_xcr0(void) zig_mangled(zig_e_zig_x86_get_xcr0, "zig_x86_get_xcr0");
+
+static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
 #if defined(zig_msvc)
     int cpu_info[4];
     __cpuidex(cpu_info, leaf_id, subid);
@@ -4629,7 +4665,7 @@ static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax
 #endif
 }
 
-static inline uint32_t zig_x86_get_xcr0(void) {
+static inline uint32_t zig_e_zig_x86_get_xcr0(void) {
 #if defined(zig_msvc)
     return (uint32_t)_xgetbv(0);
 #elif defined(zig_gnuc_asm)

test/behavior/asm.zig

@@ -209,42 +209,6 @@ test "packed output types (x86_64)" {
     }
 }
 
-test "extern output types (x86_64)" {
-    if (builtin.target.cpu.arch != .x86_64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly
-    if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/31531
-
-    const S = extern struct { x: u32 };
-    {
-        const s: S = asm volatile ("mov $123, %[ret]"
-            : [ret] "=r" (-> S),
-        );
-        try expect(s.x == 123);
-    }
-    {
-        var s: S = undefined;
-        asm volatile ("mov $123, %[ret]"
-            : [ret] "=r" (s),
-        );
-        try expect(s.x == 123);
-    }
-
-    const U = extern union { x: u32 };
-    {
-        const u: U = asm volatile ("mov $123, %[ret]"
-            : [ret] "=r" (-> U),
-        );
-        try expect(u.x == 123);
-    }
-    {
-        var u: U = undefined;
-        asm volatile ("mov $123, %[ret]"
-            : [ret] "=r" (u),
-        );
-        try expect(u.x == 123);
-    }
-}
-
 test "abi register aliases as clobbers (RISC-V)" {
     if (!builtin.target.cpu.arch.isRISCV()) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;

test/behavior/atomics.zig

@@ -358,7 +358,6 @@ test "atomics with different types" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
-    if (builtin.target.cpu.arch.endian() == .big) return error.SkipZigTest; // #24282
 
     try testAtomicsWithType(bool, true, false);
 

test/behavior/bitcast.zig

@@ -74,55 +74,6 @@ fn conv_uN(comptime N: usize, x: @Int(.unsigned, N)) @Int(.signed, N) {
     return @as(@Int(.signed, N), @bitCast(x));
 }
 
-test "bitcast uX to bytes" {
-    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
-
-    const bit_values = [_]usize{ 1, 48, 27, 512, 493, 293, 125, 204, 112 };
-    inline for (bit_values) |bits| {
-        try testBitCast(bits);
-        try comptime testBitCast(bits);
-    }
-}
-
-fn testBitCastuXToBytes(comptime N: usize) !void {
-
-    // The location of padding bits in these layouts are technically not defined
-    // by LLVM, but we currently allow exotic integers to be cast (at comptime)
-    // to types that expose their padding bits anyway.
-    //
-    // This test at least makes sure those bits are matched by the runtime behavior
-    // on the platforms we target. If the above behavior is restricted after all,
-    // this test should be deleted.
-
-    const T = @Int(.unsigned, N);
-    for ([_]T{ 0, ~@as(T, 0) }) |init_value| {
-        var x: T = init_value;
-        const bytes = std.mem.asBytes(&x);
-
-        const byte_count = (N + 7) / 8;
-        switch (native_endian) {
-            .little => {
-                var byte_i = 0;
-                while (byte_i < (byte_count - 1)) : (byte_i += 1) {
-                    try expect(bytes[byte_i] == 0xff);
-                }
-                try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
-            },
-            .big => {
-                var byte_i = byte_count - 1;
-                while (byte_i > 0) : (byte_i -= 1) {
-                    try expect(bytes[byte_i] == 0xff);
-                }
-                try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
-            },
-        }
-    }
-}
-
 test "nested bitcast" {
     const S = struct {
         fn moo(x: isize) !void {
@@ -183,39 +134,6 @@ test "@bitCast packed structs at runtime and comptime" {
     try comptime S.doTheTest();
 }
 
-test "@bitCast extern structs at runtime and comptime" {
-    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
-
-    const Full = extern struct {
-        number: u16,
-    };
-    const TwoHalves = extern struct {
-        half1: u8,
-        half2: u8,
-    };
-    const S = struct {
-        fn doTheTest() !void {
-            var full = Full{ .number = 0x1234 };
-            _ = &full;
-            const two_halves: TwoHalves = @bitCast(full);
-            switch (native_endian) {
-                .big => {
-                    try expect(two_halves.half1 == 0x12);
-                    try expect(two_halves.half2 == 0x34);
-                },
-                .little => {
-                    try expect(two_halves.half1 == 0x34);
-                    try expect(two_halves.half2 == 0x12);
-                },
-            }
-        }
-    };
-    try S.doTheTest();
-    try comptime S.doTheTest();
-}
-
 test "bitcast packed struct to integer and back" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
@@ -363,32 +281,12 @@ test "comptime @bitCast packed struct to int and back" {
     }
 }
 
-test "comptime bitcast with fields following f80" {
-    if (true) {
-        // https://github.com/ziglang/zig/issues/19387
-        return error.SkipZigTest;
-    }
-
-    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-
-    const FloatT = extern struct { f: f80, x: u128 align(16) };
-    const x: FloatT = .{ .f = 0.5, .x = 123 };
-    var x_as_uint: u256 = comptime @as(u256, @bitCast(x));
-    _ = &x_as_uint;
-
-    try expect(x.f == @as(FloatT, @bitCast(x_as_uint)).f);
-    try expect(x.x == @as(FloatT, @bitCast(x_as_uint)).x);
-}
-
 test "bitcast vector to integer and back" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
-    if (builtin.cpu.arch.endian() == .big and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
 
     var vec: @Vector(16, bool) = @splat(true);
     vec[1] = false;
@@ -524,75 +422,6 @@ test "@bitCast of packed struct of bools all false" {
     try expect(@as(u8, @as(u4, @bitCast(p))) == 0);
 }
 
-test "@bitCast of extern struct containing pointer" {
-    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; // TODO
-
-    const S = struct {
-        const A = extern struct {
-            ptr: *const u32,
-        };
-
-        const B = extern struct {
-            ptr: *const i32,
-        };
-
-        fn doTheTest() !void {
-            const x: u32 = 123;
-            var a: A = undefined;
-            a = .{ .ptr = &x };
-            const b: B = @bitCast(a);
-            try expect(b.ptr.* == 123);
-        }
-    };
-
-    try S.doTheTest();
-    try comptime S.doTheTest();
-}
-
-test "@bitCast of extern struct to float" {
-    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
-
-    const S = struct {
-        const S = extern struct {
-            x: u16,
-            y: u16,
-        };
-        fn doTheTest() !void {
-            var s: S = .{ .x = 0, .y = 0 };
-            _ = &s;
-            const a: f32 = @bitCast(s);
-            try expect(a == 0);
-        }
-    };
-
-    try S.doTheTest();
-    try comptime S.doTheTest();
-}
-
-test "@bitCast of float to extern struct" {
-    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
-
-    const S = struct {
-        const S = extern struct {
-            x: u32,
-        };
-        fn doTheTest() !void {
-            var a: f32 = -0.0;
-            _ = &a;
-            const s: S = @bitCast(a);
-            try expect(s.x == 0x80000000);
-        }
-    };
-
-    try S.doTheTest();
-    try comptime S.doTheTest();
-}
-
 test "@bitCast of packed struct with void field to integer" {
     const S = packed struct(u8) {
         v: void,
@@ -608,3 +437,148 @@ test "@bitCast of packed struct with void field to integer" {
     try S.doTheTest(123);
     try comptime S.doTheTest(123);
 }
+
+test "@bitCast vector to array with different element size" {
+    const static = struct {
+        fn doTheTest(v: @Vector(4, u5)) !void {
+            const result: [5]u4 = @bitCast(v);
+            // See the definition of `v` in the test proper for these values.
+            try expect(result[0] == 0b0010);
+            try expect(result[1] == 0b1110);
+            try expect(result[2] == 0b0101);
+            try expect(result[3] == 0b0110);
+            try expect(result[4] == 0b0000);
+        }
+    };
+    // The strange digit groupings here are to indicate how this maps to `expected` above.
+    const v: @Vector(4, u5) = .{
+        0b0_0010,
+        0b01_111,
+        0b110_01,
+        0b0000_0,
+    };
+    try static.doTheTest(v);
+    try comptime static.doTheTest(v);
+}
+
+test "@bitCast packed struct to array of bits" {
+    const S = packed struct(u16) {
+        foo: u5,
+        bar: i7,
+        baz: u3,
+        qux: bool,
+        fn doTheTest(val: @This(), comptime Bits: type) !void {
+            const bits: Bits = @bitCast(val);
+
+            // foo
+            try expect(bits[0] == 1);
+            try expect(bits[1] == 0);
+            try expect(bits[2] == 0);
+            try expect(bits[3] == 1);
+            try expect(bits[4] == 0);
+            // bar
+            try expect(bits[5] == 0);
+            try expect(bits[6] == 1);
+            try expect(bits[7] == 1);
+            try expect(bits[8] == 1);
+            try expect(bits[9] == 1);
+            try expect(bits[10] == 1);
+            try expect(bits[11] == 1);
+            // baz
+            try expect(bits[12] == 0);
+            try expect(bits[13] == 1);
+            try expect(bits[14] == 0);
+            // qux
+            try expect(bits[15] == 1);
+        }
+    };
+
+    const val: S = .{
+        .foo = 0b01001,
+        .bar = -2,
+        .baz = 0b010,
+        .qux = true,
+    };
+
+    try val.doTheTest(@Vector(16, u1));
+    try val.doTheTest([16]u1);
+
+    try comptime val.doTheTest(@Vector(16, u1));
+    try comptime val.doTheTest([16]u1);
+}
+
+test "@bitCast nested arrays of vectors" {
+    const Src = [2][2]@Vector(4, u5);
+    const Dest = [5]@Vector(2, u8);
+
+    // The strange digit groupings here are to indicate how this maps to the output.
+    const src: Src = .{ .{
+        .{ 0b00011, 0b00_100, 0b11100, 0b0010_1 },
+        .{ 0b1_0110, 0b11011, 0b101_10, 0b10101 },
+    }, .{
+        .{ 0b10101, 0b00_001, 0b01011, 0b0001_0 },
+        .{ 0b0_0001, 0b01111, 0b111_10, 0b00001 },
+    } };
+
+    const expected: Dest = .{
+        .{ 0b10000011, 0b11110000 },
+        .{ 0b01100010, 0b10110111 },
+        .{ 0b10101101, 0b00110101 },
+        .{ 0b00101100, 0b00010001 },
+        .{ 0b10011110, 0b00001111 },
+    };
+
+    const static = struct {
+        fn doTheTest(src_arg: Src) !void {
+            const actual: Dest = @bitCast(src_arg);
+            for (actual, expected) |actual_vec, expected_vec| {
+                try expect(actual_vec[0] == expected_vec[0]);
+                try expect(actual_vec[1] == expected_vec[1]);
+            }
+        }
+    };
+
+    try static.doTheTest(src);
+    try comptime static.doTheTest(src);
+}
+
+test "@bitCast nested arrays of bool to scalar" {
+    const static = struct {
+        fn doTheTest(src: [4][4]bool) !void {
+            const result: u16 = @bitCast(src);
+            try expect(result == 0b1100_0101_1010_0011);
+        }
+    };
+    const src: [4][4]bool = .{
+        .{ true, true, false, false }, // 0b0011
+        .{ false, true, false, true }, // 0b1010
+        .{ true, false, true, false }, // 0b0101
+        .{ false, false, true, true }, // 0b1100
+    };
+    try static.doTheTest(src);
+    try comptime static.doTheTest(src);
+}
+
+test "@bitCast deeply nested arrays to scalar" {
+    const static = struct {
+        fn doTheTest(src: [2][1][3][5]u4) !void {
+            const signed: i120 = @bitCast(src);
+            try expect(signed < 0); // top nibble is 0x8 so sign bit is 1
+            const unsigned: u120 = @bitCast(src);
+            try expect(unsigned == 0x8873B_5BF6F_F4020_0E7AC_1EFED_40F51);
+            try expect(@as(i120, @bitCast(unsigned)) == signed);
+            try expect(@as(u120, @bitCast(signed)) == unsigned);
+        }
+    };
+    const src: [2][1][3][5]u4 = .{ .{.{
+        .{ 0x1, 0x5, 0xF, 0x0, 0x4 },
+        .{ 0xD, 0xE, 0xF, 0xE, 0x1 },
+        .{ 0xC, 0xA, 0x7, 0xE, 0x0 },
+    }}, .{.{
+        .{ 0x0, 0x2, 0x0, 0x4, 0xF },
+        .{ 0xF, 0x6, 0xF, 0xB, 0x5 },
+        .{ 0xB, 0x3, 0x7, 0x8, 0x8 },
+    }} };
+    try static.doTheTest(src);
+    try comptime static.doTheTest(src);
+}

test/behavior/comptime_memory.zig

@@ -583,3 +583,19 @@ test "comptime store to extern struct reinterpreted as byte array" {
 
     comptime std.debug.assert(val.x == 0);
 }
+
+test "reinterpret sentinel-terminated array as packed struct" {
+    const S = packed struct(u16) { lo: u8, hi: u8 };
+    const data: [2:0]u8 = .{ 0x12, 0x34 };
+    const ptr: *align(1) const S = @ptrCast(&data);
+    switch (endian) {
+        .little => {
+            try testing.expect(ptr.lo == 0x12);
+            try testing.expect(ptr.hi == 0x34);
+        },
+        .big => {
+            try testing.expect(ptr.lo == 0x34);
+            try testing.expect(ptr.hi == 0x12);
+        },
+    }
+}

test/behavior/sizeof_and_typeof.zig

@@ -151,9 +151,6 @@ test "branching logic inside @TypeOf" {
 test "@bitSizeOf" {
     try expect(@bitSizeOf(u2) == 2);
     try expect(@bitSizeOf(u8) == @sizeOf(u8) * 8);
-    try expect(@bitSizeOf(struct {
-        a: u2,
-    }) == 8);
     try expect(@bitSizeOf(packed struct {
         a: u2,
     }) == 2);
@@ -281,14 +278,6 @@ test "@offsetOf zero-bit field" {
     try expect(@offsetOf(S, "b") == @offsetOf(S, "c"));
 }
 
-test "@bitSizeOf on array of structs" {
-    const S = struct {
-        foo: u64,
-    };
-
-    try expectEqual(128, @bitSizeOf([2]S));
-}
-
 test "lazy abi size used in comparison" {
     const S = struct { a: usize };
     var rhs: i32 = 100;

test/behavior/vector.zig

@@ -1536,12 +1536,12 @@ test "vector pointer is indexable" {
     const V = @Vector(2, u32);
 
     const x: V = .{ 123, 456 };
-    comptime assert(@TypeOf(&(&x)[0]) == *const u32); // validate constness
+    comptime assert(@typeInfo(@TypeOf(&(&x)[0])).pointer.attrs.@"const");
     try expectEqual(@as(u32, 123), (&x)[0]);
     try expectEqual(@as(u32, 456), (&x)[1]);
 
     var y: V = .{ 123, 456 };
-    comptime assert(@TypeOf(&(&y)[0]) == *u32); // validate constness
+    comptime assert(!@typeInfo(@TypeOf(&(&y)[0])).pointer.attrs.@"const");
     try expectEqual(@as(u32, 123), (&y)[0]);
     try expectEqual(@as(u32, 456), (&y)[1]);
 
@@ -1620,6 +1620,7 @@ test "bitcast vector to array of smaller vectors" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
 
     const u8x32 = @Vector(32, u8);
     const u8x64 = @Vector(64, u8);

test/c_abi/main.zig

@@ -14501,6 +14501,7 @@ test "@Vector(4, f64)" {
     if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
     if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
     if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
+    if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
 
     const v = c_ret_vector_4_f64();
     try expect(v[0] == 33);
@@ -14570,6 +14571,7 @@ test "@Vector(8, f64)" {
     if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
     if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
     if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
+    if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
 
     const v = c_ret_vector_8_f64();
     try expect(v[0] == 81);
@@ -17109,7 +17111,8 @@ const byval_tail_callsite_attr = struct {
         }
 
         fn cast(self: MyRect) struct_Rect {
-            return @bitCast(self);
+            const ptr: *const struct_Rect = @ptrCast(&self);
+            return ptr.*;
         }
 
         extern fn c_byval_tail_callsite_attr(struct_Rect) f64;

test/cases/compile_errors/asm_output_type_no_guaranteed_in_memory_layout.zig

@@ -25,14 +25,38 @@ export fn entry4() void {
         : [_] "=r" (u),
     );
 }
+const ES = extern struct { x: u32 };
+export fn entry5() void {
+    var es: ES = undefined;
+    asm volatile (""
+        : [_] "=r" (es),
+    );
+}
+const EU = extern union { x: u32 };
+export fn entry6() void {
+    var eu: EU = undefined;
+    asm volatile (""
+        : [_] "=r" (eu),
+    );
+}
 
 // error
 //
-// :4:24: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout
+// :4:24: error: invalid inline assembly output type 'tmp.S'
+// :4:24: note: struct types cannot be passed to inline assembly
 // :1:11: note: struct declared here
-// :11:21: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout
+// :11:21: error: invalid inline assembly output type 'tmp.S'
+// :11:21: note: struct types cannot be passed to inline assembly
 // :1:11: note: struct declared here
-// :18:24: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout
+// :18:24: error: invalid inline assembly output type 'tmp.U'
+// :18:24: note: union types cannot be passed to inline assembly
 // :15:11: note: union declared here
-// :25:21: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout
+// :25:21: error: invalid inline assembly output type 'tmp.U'
+// :25:21: note: union types cannot be passed to inline assembly
 // :15:11: note: union declared here
+// :32:21: error: invalid inline assembly output type 'tmp.ES'
+// :32:21: note: struct types cannot be passed to inline assembly
+// :28:19: note: struct declared here
+// :39:21: error: invalid inline assembly output type 'tmp.EU'
+// :39:21: note: union types cannot be passed to inline assembly
+// :35:19: note: union declared here

test/cases/compile_errors/bitCast_extern_struct.zig

@@ -0,0 +1,10 @@
+const S = extern struct { x: u32 };
+export fn foo(s: S) void {
+    const as_int: u32 = @bitCast(s);
+    _ = as_int;
+}
+
+// error
+//
+// :3:34: error: cannot @bitCast from 'tmp.S'
+// :1:18: note: struct declared here

test/cases/compile_errors/bitCast_to_enum_type.zig

@@ -1,10 +0,0 @@
-export fn entry() void {
-    const E = enum(u32) { a, b };
-    const y: E = @bitCast(@as(u32, 3));
-    _ = y;
-}
-
-// error
-//
-// :3:18: error: cannot @bitCast to 'tmp.entry.E'
-// :3:18: note: use @enumFromInt to cast from 'u32'

test/cases/compile_errors/bitCast_vector_of_pointer.zig

@@ -0,0 +1,9 @@
+export fn foo(p: *u32) void {
+    const vec: @Vector(2, *u32) = .{ p, p };
+    const raw: [2]usize = @bitCast(vec);
+    _ = raw;
+}
+
+// error
+//
+// :3:36: error: cannot @bitCast from '@Vector(2, *u32)'

test/cases/compile_errors/bitCast_with_invalid_array_element_type.zig

@@ -18,9 +18,6 @@ export fn baz() void {
 
 // error
 //
-// :5:29: error: cannot @bitCast from '[1]tmp.foo.S'
-// :5:29: note: array element type 'tmp.foo.S' does not have a guaranteed in-memory layout
+// :5:42: error: cannot @bitCast from '[1]tmp.foo.S'
 // :12:19: error: cannot @bitCast to '[1]tmp.bar.S'
-// :12:19: note: array element type 'tmp.bar.S' does not have a guaranteed in-memory layout
-// :16:21: error: cannot @bitCast from '[1]comptime_int'
-// :16:21: note: array element type 'comptime_int' does not have a guaranteed in-memory layout
+// :16:45: error: cannot @bitCast from '[1]comptime_int'

test/cases/compile_errors/error_in_nested_declaration.zig

@@ -23,7 +23,6 @@ pub export fn entry2() void {
 }
 
 // error
-// backend=selfhosted,llvm
 //
 // :6:20: error: cannot @bitCast to '[]i32'
 // :6:20: note: use @ptrCast to cast from '[]u32'