[Flint] adding minimalist IR builder, adding pseudo instructions,

improving edge validations

src/intel/compiler/compiler.zig

@@ -5,11 +5,13 @@ pub const device = @import("device.zig");
 pub const ir = @import("ir/ir.zig");
 pub const lower = @import("lower/lower.zig");
 
+pub const Builder = ir.Builder;
 pub const id = ir.id;
 pub const instruction = ir.instruction;
 pub const operand = ir.operand;
 pub const printer = ir.printer;
 pub const program = ir.program;
+pub const pseudo = ir.pseudo;
 pub const validator = ir.validator;
 
 pub const Program = ir.Program;
@@ -17,7 +19,7 @@ pub const Stage = ir.Stage;
 
 const std = @import("std");
 
-test "Flint IR foundation" {
+test "[ir] basic shader" {
     // ; Flint program:
     // ;   .stage: vertex
     // ;   .generation: gen9
@@ -44,8 +46,9 @@ test "Flint IR foundation" {
 
     var shader = Program.init(std.testing.allocator, .vertex, device_info, .simd8);
     defer shader.deinit();
+    var builder = Builder.init(&shader);
 
-    const position = try shader.addVirtualRegister(.{
+    const position = try builder.addVirtualRegister(.{
         .size_bytes = 32,
         .alignment_bytes = 32,
         .element_type = .f32,
@@ -53,7 +56,7 @@ test "Flint IR foundation" {
         .class = .varying,
         .name = "position",
     });
-    const urb_payload = try shader.addVirtualRegister(.{
+    const urb_payload = try builder.addVirtualRegister(.{
         .size_bytes = 64,
         .alignment_bytes = 32,
         .element_type = .u32,
@@ -62,10 +65,10 @@ test "Flint IR foundation" {
         .spillable = false,
         .name = "urb_payload",
     });
-    const entry = try shader.addBlock("entry");
-    try shader.setEntryBlock(entry);
+    const entry = try builder.addBlock("entry");
+    try builder.setEntryBlock(entry);
 
-    _ = try shader.appendInstruction(entry, .simd8, null, .{
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
         .load_input = .{
             .destination = .{
                 .register = .{ .virtual = position },
@@ -78,7 +81,7 @@ test "Flint IR foundation" {
             },
         },
     });
-    _ = try shader.appendInstruction(entry, .simd8, null, .{
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
         .binary = .{
             .opcode = .multiply,
             .destination = .{
@@ -99,7 +102,7 @@ test "Flint IR foundation" {
             },
         },
     });
-    _ = try shader.appendInstruction(entry, .simd8, null, .{
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
         .move = .{
             .destination = .{
                 .register = .{ .virtual = position },
@@ -117,7 +120,7 @@ test "Flint IR foundation" {
             },
         },
     });
-    _ = try shader.appendInstruction(entry, .simd8, null, .{
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
         .store_output = .{
             .semantic = .{
                 .builtin = .{ .builtin = .position },
@@ -129,7 +132,7 @@ test "Flint IR foundation" {
             },
         },
     });
-    _ = try shader.appendInstruction(entry, .simd8, null, .{
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
         .send = .{
             .message = .{
                 .urb_write = .{
@@ -143,7 +146,7 @@ test "Flint IR foundation" {
             },
         },
     });
-    try shader.setTerminator(entry, .end_thread);
+    try builder.setTerminator(entry, .end_thread);
 
     shader.properties.instructions_selected = true;
     try validator.validate(&shader);
@@ -165,7 +168,7 @@ test "Flint IR foundation" {
     try std.testing.expect(std.mem.indexOf(u8, text, "send urb_write[offset(0), channels(xyzw), end_of_thread], payload(%urb_payload[2])") != null);
 }
 
-test "ID stability after removal" {
+test "[ir] ID stability after removal" {
     var store: id.Store(id.VirtualFlagId, operand.VirtualFlag) = .{};
     defer store.entries.deinit(std.testing.allocator);
 

src/intel/compiler/ir/Builder.zig

@@ -0,0 +1,178 @@
+const std = @import("std");
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+const instruction = @import("instruction.zig");
+const operand = @import("operand.zig");
+const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
+
+const Self = @This();
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidBlock,
+    InvalidInsertionIndex,
+    TerminatorAlreadySet,
+};
+
+program: *program_ir.Program,
+
+pub fn init(program: *program_ir.Program) Self {
+    return .{ .program = program };
+}
+
+pub fn addVirtualRegister(self: *Self, register: operand.VirtualRegister) Error!ids.VirtualRegisterId {
+    return self.program.addVirtualRegister(register);
+}
+
+pub fn addVirtualFlag(self: *Self, flag: operand.VirtualFlag) Error!ids.VirtualFlagId {
+    return self.program.addVirtualFlag(flag);
+}
+
+pub fn addBlock(self: *Self, name: ?[]const u8) Error!ids.BlockId {
+    return self.program.addBlock(name);
+}
+
+pub fn setEntryBlock(self: *Self, block_id: ids.BlockId) Error!void {
+    return self.program.setEntryBlock(block_id);
+}
+
+pub fn addBlockParameter(self: *Self, block_id: ids.BlockId, parameter: pseudo.BlockParameter) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    try block.parameters.append(self.program.allocator(), parameter);
+}
+
+pub fn clearBlockParameters(self: *Self, block_id: ids.BlockId) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.parameters.clearRetainingCapacity();
+}
+
+pub fn edge(self: *Self, target: ids.BlockId, arguments: []const pseudo.EdgeArgument) Error!instruction.Edge {
+    if (!self.program.blocks.isLive(target))
+        return Error.InvalidBlock;
+    return .{
+        .target = target,
+        .arguments = try self.program.allocator().dupe(pseudo.EdgeArgument, arguments),
+    };
+}
+
+pub fn appendInstruction(
+    self: *Self,
+    block_id: ids.BlockId,
+    execution_size: device.ExecutionSize,
+    predicate: ?operand.Predicate,
+    operation: instruction.Operation,
+) Error!ids.InstructionId {
+    const block = self.program.blocks.get(block_id) orelse return Error.InvalidBlock;
+    return self.insertInstruction(block_id, block.instructions.items.len, execution_size, predicate, operation);
+}
+
+pub fn insertInstruction(
+    self: *Self,
+    block_id: ids.BlockId,
+    index: usize,
+    execution_size: device.ExecutionSize,
+    predicate: ?operand.Predicate,
+    operation: instruction.Operation,
+) Error!ids.InstructionId {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    if (index > block.instructions.items.len)
+        return Error.InvalidInsertionIndex;
+
+    const owned_operation = try instruction.cloneOperation(self.program.allocator(), operation);
+    const instruction_id = try self.program.instructions.add(self.program.allocator(), .{
+        .parent_block = block_id,
+        .execution_size = execution_size,
+        .predicate = predicate,
+        .operation = owned_operation,
+    });
+    errdefer std.debug.assert(self.program.instructions.remove(instruction_id));
+
+    try block.instructions.insert(self.program.allocator(), index, instruction_id);
+    return instruction_id;
+}
+
+pub fn setStructuredControl(self: *Self, block_id: ids.BlockId, control: instruction.StructuredControl) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.structured_control = control;
+}
+
+pub fn setTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void {
+    return self.program.setTerminator(block_id, terminator);
+}
+
+pub fn replaceTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.terminator = try instruction.cloneTerminator(self.program.allocator(), terminator);
+}
+
+fn moveImmediate(register_id: ids.VirtualRegisterId, value: u32) instruction.Operation {
+    return .{
+        .move = .{
+            .destination = .{
+                .register = .{ .virtual = register_id },
+                .type = .u32,
+            },
+            .source = .{
+                .register = .{ .immediate = .{ .u32 = value } },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            },
+        },
+    };
+}
+
+test "[ir] Builder: construction and ordered insertion" {
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8);
+    defer program.deinit();
+    var builder = Self.init(&program);
+
+    var register_name = [_]u8{ 'v', 'a', 'l', 'u', 'e' };
+    const register_id = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = &register_name,
+    });
+    register_name[0] = 'x';
+    try std.testing.expectEqualStrings("value", program.virtual_registers.get(register_id).?.name.?);
+
+    const flag_id = try builder.addVirtualFlag(.{ .name = "condition" });
+    try std.testing.expectEqualStrings("condition", program.virtual_flags.get(flag_id).?.name.?);
+
+    const entry = try builder.addBlock("entry");
+    const exit = try builder.addBlock("exit");
+    try builder.setEntryBlock(entry);
+
+    const second = try builder.appendInstruction(entry, .simd8, null, moveImmediate(register_id, 2));
+    const first = try builder.insertInstruction(entry, 0, .simd8, null, moveImmediate(register_id, 1));
+
+    const entry_block = program.blocks.get(entry).?;
+    try std.testing.expectEqualSlices(ids.InstructionId, &.{ first, second }, entry_block.instructions.items);
+    try std.testing.expectEqual(entry, program.instructions.get(first).?.parent_block);
+    try std.testing.expectEqual(entry, program.instructions.get(second).?.parent_block);
+
+    try builder.setStructuredControl(entry, .{ .selection = .{ .merge_block = exit } });
+    try builder.setTerminator(entry, .{ .jump = try builder.edge(exit, &.{}) });
+    try builder.setTerminator(exit, .end_thread);
+    try std.testing.expectError(Error.TerminatorAlreadySet, builder.setTerminator(entry, .end_thread));
+
+    const instruction_count = program.instructions.entries.items.len;
+    try std.testing.expectError(
+        Error.InvalidInsertionIndex,
+        builder.insertInstruction(entry, 3, .simd8, null, moveImmediate(register_id, 3)),
+    );
+    try std.testing.expectEqual(instruction_count, program.instructions.entries.items.len);
+
+    try validator.validate(&program);
+}

src/intel/compiler/ir/instruction.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const device = @import("../device.zig");
 const ids = @import("id.zig");
 const operand = @import("operand.zig");
+const pseudo = @import("pseudo.zig");
 
 pub const Builtin = enum {
     position,
@@ -98,8 +99,21 @@ pub const Operation = union(enum) {
     binary: Binary,
     compare: Compare,
     send: Send,
+    parallel_copy: pseudo.ParallelCopy,
 };
 
+pub fn cloneOperation(allocator: std.mem.Allocator, operation: Operation) std.mem.Allocator.Error!Operation {
+    return switch (operation) {
+        .parallel_copy => |copy| .{
+            .parallel_copy = .{
+                .register_copies = try allocator.dupe(pseudo.RegisterCopy, copy.register_copies),
+                .flag_copies = try allocator.dupe(pseudo.FlagCopy, copy.flag_copies),
+            },
+        },
+        else => operation,
+    };
+}
+
 pub const Instruction = struct {
     parent_block: ids.BlockId,
     execution_size: device.ExecutionSize,
@@ -107,17 +121,43 @@ pub const Instruction = struct {
     operation: Operation,
 };
 
+pub const Edge = struct {
+    target: ids.BlockId,
+    arguments: []const pseudo.EdgeArgument,
+};
+
 pub const Terminator = union(enum) {
-    jump: ids.BlockId,
+    jump: Edge,
     conditional_branch: struct {
         predicate: operand.Predicate,
-        true_block: ids.BlockId,
-        false_block: ids.BlockId,
+        true_edge: Edge,
+        false_edge: Edge,
     },
     end_thread,
     @"unreachable",
 };
 
+pub fn cloneEdge(allocator: std.mem.Allocator, edge: Edge) std.mem.Allocator.Error!Edge {
+    return .{
+        .target = edge.target,
+        .arguments = try allocator.dupe(pseudo.EdgeArgument, edge.arguments),
+    };
+}
+
+pub fn cloneTerminator(allocator: std.mem.Allocator, terminator: Terminator) std.mem.Allocator.Error!Terminator {
+    return switch (terminator) {
+        .jump => |edge| .{ .jump = try cloneEdge(allocator, edge) },
+        .conditional_branch => |branch| .{
+            .conditional_branch = .{
+                .predicate = branch.predicate,
+                .true_edge = try cloneEdge(allocator, branch.true_edge),
+                .false_edge = try cloneEdge(allocator, branch.false_edge),
+            },
+        },
+        else => terminator,
+    };
+}
+
 pub const StructuredControl = union(enum) {
     none,
     selection: struct {
@@ -130,6 +170,7 @@ pub const StructuredControl = union(enum) {
 };
 
 pub const Block = struct {
+    parameters: std.ArrayList(pseudo.BlockParameter) = .empty,
     instructions: std.ArrayList(ids.InstructionId) = .empty,
     terminator: ?Terminator = null,
     structured_control: StructuredControl = .none,

src/intel/compiler/ir/ir.zig

@@ -1,8 +1,10 @@
+pub const Builder = @import("Builder.zig");
 pub const id = @import("id.zig");
 pub const instruction = @import("instruction.zig");
 pub const operand = @import("operand.zig");
 pub const printer = @import("printer.zig");
 pub const program = @import("program.zig");
+pub const pseudo = @import("pseudo.zig");
 pub const validator = @import("validator.zig");
 
 pub const Program = program.Program;

src/intel/compiler/ir/printer.zig

@@ -4,6 +4,7 @@ const ids = @import("id.zig");
 const inst_ir = @import("instruction.zig");
 const operand = @import("operand.zig");
 const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
 
 const indent = "    ";
 
@@ -40,6 +41,15 @@ pub fn write(program: *const program_ir.Program, writer: *std.Io.Writer) std.Io.
         const block_id = ids.BlockId.fromIndex(block_index);
 
         try writeBlockRef(program, writer, block_id);
+        if (block.parameters.items.len != 0) {
+            try writer.writeByte('(');
+            for (block.parameters.items, 0..) |parameter, index| {
+                if (index != 0)
+                    try writer.writeAll(", ");
+                try writeBlockParameter(program, writer, parameter);
+            }
+            try writer.writeByte(')');
+        }
         try writer.writeAll(":\n");
 
         switch (block.structured_control) {
@@ -127,6 +137,7 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex
             try writer.writeAll(", ");
             try writeSource(program, writer, execution_size, op.rhs);
         },
+        .parallel_copy => |op| try writeParallelCopy(program, writer, execution_size, op),
         .send => |op| {
             try writer.writeAll("send ");
             if (op.response) |response| {
@@ -141,25 +152,81 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex
     }
 }
 
+fn writeParallelCopy(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, copy: pseudo.ParallelCopy) !void {
+    try writer.writeAll("parallel_copy [");
+    var needs_separator = false;
+
+    for (copy.register_copies) |item| {
+        if (needs_separator)
+            try writer.writeAll(", ");
+        try writeDestination(program, writer, execution_size, item.destination);
+        try writer.writeAll(" <- ");
+        try writeSource(program, writer, execution_size, item.source);
+        needs_separator = true;
+    }
+
+    for (copy.flag_copies) |item| {
+        if (needs_separator)
+            try writer.writeAll(", ");
+        try writeVirtualFlagRef(program, writer, item.destination);
+        try writer.writeAll(" <- ");
+        switch (item.source) {
+            .constant => |value| try writer.writeAll(if (value) "true" else "false"),
+            .dynamic => |predicate| try writePredicate(program, writer, predicate),
+        }
+        needs_separator = true;
+    }
+
+    try writer.writeByte(']');
+}
+
 fn writeTerminator(program: *const program_ir.Program, writer: *std.Io.Writer, terminator: inst_ir.Terminator) !void {
     switch (terminator) {
-        .jump => |target| {
+        .jump => |edge| {
             try writer.writeAll("jump ");
-            try writeBlockRef(program, writer, target);
+            try writeEdge(program, writer, edge);
         },
         .conditional_branch => |branch| {
             try writer.writeAll("conditional_branch ");
             try writePredicate(program, writer, branch.predicate);
             try writer.writeAll(", ");
-            try writeBlockRef(program, writer, branch.true_block);
+            try writeEdge(program, writer, branch.true_edge);
             try writer.writeAll(", ");
-            try writeBlockRef(program, writer, branch.false_block);
+            try writeEdge(program, writer, branch.false_edge);
         },
         .end_thread => try writer.writeAll("end_thread"),
         .@"unreachable" => try writer.writeAll("unreachable"),
     }
 }
 
+fn writeBlockParameter(program: *const program_ir.Program, writer: *std.Io.Writer, parameter: pseudo.BlockParameter) !void {
+    switch (parameter) {
+        .register => |register_id| try writeVirtualRegisterRef(program, writer, register_id),
+        .flag => |flag_id| try writeVirtualFlagRef(program, writer, flag_id),
+    }
+}
+
+fn writeEdge(program: *const program_ir.Program, writer: *std.Io.Writer, edge: inst_ir.Edge) !void {
+    try writeBlockRef(program, writer, edge.target);
+    if (edge.arguments.len == 0)
+        return;
+
+    const execution_size: device.ExecutionSize = @enumFromInt(@intFromEnum(program.dispatch_width));
+    try writer.writeByte('(');
+    for (edge.arguments, 0..) |argument, index| {
+        if (index != 0)
+            try writer.writeAll(", ");
+        switch (argument) {
+            .source => |source| try writeSource(program, writer, execution_size, source),
+            .predicate => |predicate_value| switch (predicate_value) {
+                .constant => |value| try writer.writeAll(if (value) "true" else "false"),
+                .dynamic => |predicate| try writePredicate(program, writer, predicate),
+            },
+        }
+    }
+    try writer.writeByte(')');
+}
+
 fn writeSource(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, source: operand.Source) !void {
     if (source.negate)
         try writer.writeByte('-');

src/intel/compiler/ir/program.zig

@@ -10,6 +10,7 @@ pub const Stage = shared_ir.Stage;
 pub const Properties = packed struct {
     instructions_selected: bool = false,
     block_parameters_lowered: bool = false,
+    parallel_copies_lowered: bool = false,
 
     stage_io_lowered: bool = false,
     resources_lowered: bool = false,
@@ -23,7 +24,7 @@ pub const Properties = packed struct {
     flags_allocated: bool = false,
     branches_resolved: bool = false,
 
-    _padding: u21 = 0,
+    _padding: u20 = 0,
 };
 
 pub const VertexPayload = struct {
@@ -116,12 +117,15 @@ pub const Program = struct {
     pub fn appendInstruction(self: *Program, block_id: ids.BlockId, execution_size: device.ExecutionSize, predicate: ?operand.Predicate, operation: instructions.Operation) !ids.InstructionId {
         const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock;
 
+        const owned_operation = try instructions.cloneOperation(self.allocator(), operation);
         const instruction_id = try self.instructions.add(self.allocator(), .{
             .parent_block = block_id,
             .execution_size = execution_size,
             .predicate = predicate,
-            .operation = operation,
+            .operation = owned_operation,
         });
+        errdefer std.debug.assert(self.instructions.remove(instruction_id));
+
         try block.instructions.append(self.allocator(), instruction_id);
         return instruction_id;
     }
@@ -130,6 +134,6 @@ pub const Program = struct {
         const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock;
         if (block.terminator != null)
             return error.TerminatorAlreadySet;
-        block.terminator = terminator;
+        block.terminator = try instructions.cloneTerminator(self.allocator(), terminator);
     }
 };

src/intel/compiler/ir/pseudo.zig

@@ -0,0 +1,183 @@
+const ids = @import("id.zig");
+const operand = @import("operand.zig");
+
+pub const PredicateValue = union(enum) {
+    constant: bool,
+    dynamic: operand.Predicate,
+};
+
+pub const BlockParameter = union(enum) {
+    register: ids.VirtualRegisterId,
+    flag: ids.VirtualFlagId,
+};
+
+pub const EdgeArgument = union(enum) {
+    source: operand.Source,
+    predicate: PredicateValue,
+};
+
+pub const RegisterCopy = struct {
+    destination: operand.Destination,
+    source: operand.Source,
+};
+
+pub const FlagCopy = struct {
+    destination: ids.VirtualFlagId,
+    source: PredicateValue,
+};
+
+/// A simultaneous assignment: every source is read before any destination is
+/// written. This pseudo-operation must be eliminated before machine emission.
+pub const ParallelCopy = struct {
+    register_copies: []const RegisterCopy,
+    flag_copies: []const FlagCopy,
+};
+
+test "[ir] pseudo: parallel copy ownership and printing" {
+    const std = @import("std");
+    const Builder = @import("Builder.zig");
+    const device = @import("../device.zig");
+    const printer = @import("printer.zig");
+    const program_ir = @import("program.zig");
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const source_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = "source",
+    });
+    const destination_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = "destination",
+    });
+    const source_flag = try builder.addVirtualFlag(.{ .name = "source_flag" });
+    const destination_flag = try builder.addVirtualFlag(.{ .name = "destination_flag" });
+    const entry = try builder.addBlock("entry");
+
+    var register_copies = [_]RegisterCopy{.{
+        .destination = .{
+            .register = .{ .virtual = destination_register },
+            .type = .u32,
+        },
+        .source = .{
+            .register = .{ .virtual = source_register },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    }};
+    var flag_copies = [_]FlagCopy{.{
+        .destination = destination_flag,
+        .source = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } },
+    }};
+
+    const copy_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .parallel_copy = .{
+            .register_copies = &register_copies,
+            .flag_copies = &flag_copies,
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+
+    const stored = program.instructions.get(copy_id).?.operation.parallel_copy;
+    try std.testing.expect(stored.register_copies.ptr != register_copies[0..].ptr);
+    try std.testing.expect(stored.flag_copies.ptr != flag_copies[0..].ptr);
+
+    register_copies[0].source.register = .{ .immediate = .{ .u32 = 42 } };
+    flag_copies[0].source = .{ .constant = false };
+    try std.testing.expect(stored.register_copies[0].source.register == .virtual);
+    try std.testing.expect(stored.flag_copies[0].source == .dynamic);
+
+    try validator.validate(&program);
+
+    const text = try printer.allocPrint(std.testing.allocator, &program);
+    defer std.testing.allocator.free(text);
+
+    try std.testing.expect(std.mem.indexOf(
+        u8,
+        text,
+        "parallel_copy [%destination:u32 <- %source:u32, %destination_flag <- (+%source_flag)]",
+    ) != null);
+
+    program.properties.parallel_copies_lowered = true;
+    try std.testing.expectError(error.UnloweredParallelCopy, validator.validate(&program));
+}
+
+test "[ir] pseudo: validator rejects invalid parallel copies" {
+    const std = @import("std");
+    const Builder = @import("Builder.zig");
+    const device = @import("../device.zig");
+    const program_ir = @import("program.zig");
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const register_id = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const flag_id = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const instruction_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .parallel_copy = .{
+            .register_copies = &.{},
+            .flag_copies = &.{},
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+
+    try std.testing.expectError(error.EmptyParallelCopy, validator.validate(&program));
+
+    const inst = program.instructions.getMut(instruction_id).?;
+    inst.predicate = .{ .flag = .{ .virtual = flag_id } };
+    try std.testing.expectError(error.PredicatedParallelCopy, validator.validate(&program));
+    inst.predicate = null;
+
+    const duplicate_copy: RegisterCopy = .{
+        .destination = .{
+            .register = .{ .virtual = register_id },
+            .type = .u32,
+        },
+        .source = .{
+            .register = .{ .immediate = .{ .u32 = 1 } },
+            .type = .u32,
+            .region = operand.Region.broadcast(),
+        },
+    };
+    inst.operation = .{
+        .parallel_copy = .{
+            .register_copies = &.{ duplicate_copy, duplicate_copy },
+            .flag_copies = &.{},
+        },
+    };
+    try std.testing.expectError(error.DuplicateParallelCopyDestination, validator.validate(&program));
+}

src/intel/compiler/ir/validator.zig

@@ -2,6 +2,7 @@ const ids = @import("id.zig");
 const instruction = @import("instruction.zig");
 const operand = @import("operand.zig");
 const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
 
 pub const Error = error{
     UnsupportedGeneration,
@@ -24,42 +25,61 @@ pub const Error = error{
     InvalidDestination,
     InvalidImmediateType,
     InvalidRegisterSpan,
+    EmptyParallelCopy,
+    InvalidParallelCopyDestination,
+    ParallelCopyTypeMismatch,
+    DuplicateParallelCopyDestination,
+    PredicatedParallelCopy,
+    UnloweredParallelCopy,
+    EntryBlockHasParameters,
+    DuplicateBlockParameter,
+    EdgeArgumentCountMismatch,
+    EdgeArgumentKindMismatch,
+    EdgeArgumentTypeMismatch,
+    UnloweredBlockParameter,
 };
 
 pub fn validate(program: *const program_ir.Program) Error!void {
     if (program.device_info.generation != .gen9)
-        return error.UnsupportedGeneration;
+        return Error.UnsupportedGeneration;
     if (program.stage != .vertex)
-        return error.UnsupportedStage;
+        return Error.UnsupportedStage;
     if (program.dispatch_width != .simd8 or !program.device_info.supportsDispatch(.simd8))
-        return error.UnsupportedDispatchWidth;
+        return Error.UnsupportedDispatchWidth;
 
-    const entry_block = program.entry_block orelse return error.MissingEntryBlock;
+    const entry_block = program.entry_block orelse return Error.MissingEntryBlock;
     if (!program.blocks.isLive(entry_block))
-        return error.InvalidBlock;
+        return Error.InvalidBlock;
 
     for (program.virtual_registers.entries.items) |entry| {
         const register = entry orelse continue;
         if (register.size_bytes == 0)
-            return error.InvalidRegisterSize;
+            return Error.InvalidRegisterSize;
         if (register.alignment_bytes == 0 or
             (register.alignment_bytes & (register.alignment_bytes - 1)) != 0)
-            return error.InvalidRegisterAlignment;
+            return Error.InvalidRegisterAlignment;
         if (register.lane_count == 0)
-            return error.InvalidLaneCount;
+            return Error.InvalidLaneCount;
         try validateType(register.element_type);
     }
 
     for (program.blocks.entries.items, 0..) |entry, block_index| {
         const block = entry orelse continue;
         if (block.terminator == null)
-            return error.MissingTerminator;
+            return Error.MissingTerminator;
 
         const block_id = ids.BlockId.fromIndex(block_index);
+        if (block_id == entry_block and block.parameters.items.len != 0)
+            return Error.EntryBlockHasParameters;
+        if (program.properties.block_parameters_lowered and block.parameters.items.len != 0)
+            return Error.UnloweredBlockParameter;
+        for (block.parameters.items, 0..) |parameter, parameter_index|
+            try validateBlockParameter(program, block_index, parameter_index, parameter);
+
         for (block.instructions.items) |instruction_id| {
-            const inst = program.instructions.get(instruction_id) orelse return error.InvalidInstruction;
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidInstruction;
             if (inst.parent_block != block_id)
-                return error.InvalidInstruction;
+                return Error.InvalidInstruction;
             try validateInstruction(program, inst.*);
         }
 
@@ -68,10 +88,37 @@ pub fn validate(program: *const program_ir.Program) Error!void {
     }
 }
 
+fn validateBlockParameter(program: *const program_ir.Program, block_index: usize, parameter_index: usize, parameter: pseudo.BlockParameter) Error!void {
+    switch (parameter) {
+        .register => |register_id| if (!program.virtual_registers.isLive(register_id))
+            return Error.InvalidVirtualRegister,
+        .flag => |flag_id| if (!program.virtual_flags.isLive(flag_id))
+            return Error.InvalidVirtualFlag,
+    }
+
+    for (program.blocks.entries.items, 0..) |entry, candidate_block_index| {
+        const block = entry orelse continue;
+        if (candidate_block_index > block_index)
+            break;
+        const limit = if (candidate_block_index == block_index) parameter_index else block.parameters.items.len;
+        for (block.parameters.items[0..limit]) |candidate| {
+            if (blockParametersEqual(parameter, candidate))
+                return Error.DuplicateBlockParameter;
+        }
+    }
+}
+
+fn blockParametersEqual(a: pseudo.BlockParameter, b: pseudo.BlockParameter) bool {
+    return switch (a) {
+        .register => |register_id| b == .register and b.register == register_id,
+        .flag => |flag_id| b == .flag and b.flag == flag_id,
+    };
+}
+
 fn validateInstruction(program: *const program_ir.Program, inst: instruction.Instruction) Error!void {
     switch (inst.execution_size) {
         .simd1, .simd8 => {},
-        else => return error.UnsupportedExecutionSize,
+        else => return Error.UnsupportedExecutionSize,
     }
 
     if (inst.predicate) |predicate|
@@ -99,18 +146,91 @@ fn validateInstruction(program: *const program_ir.Program, inst: instruction.Ins
             if (op.response) |response|
                 try validateSpan(program, response);
         },
+        .parallel_copy => |op| {
+            if (program.properties.parallel_copies_lowered)
+                return Error.UnloweredParallelCopy;
+            if (inst.predicate != null)
+                return Error.PredicatedParallelCopy;
+            try validateParallelCopy(program, op);
+        },
     }
 }
 
+fn validateParallelCopy(program: *const program_ir.Program, copy: pseudo.ParallelCopy) Error!void {
+    if (copy.register_copies.len == 0 and copy.flag_copies.len == 0)
+        return Error.EmptyParallelCopy;
+
+    for (copy.register_copies, 0..) |item, index| {
+        try validateDestination(program, item.destination);
+        try validateSource(program, item.source);
+
+        if (item.destination.type != item.source.type)
+            return Error.ParallelCopyTypeMismatch;
+        if (item.destination.region.byte_offset != 0 or item.destination.region.horizontal_stride != 1)
+            return Error.InvalidParallelCopyDestination;
+
+        const destination_id = switch (item.destination.register) {
+            .virtual => |register_id| register_id,
+            else => return Error.InvalidParallelCopyDestination,
+        };
+        const destination_register = program.virtual_registers.get(destination_id) orelse
+            return Error.InvalidVirtualRegister;
+        if (destination_register.element_type != item.destination.type)
+            return Error.ParallelCopyTypeMismatch;
+
+        switch (item.source.register) {
+            .virtual => |source_id| {
+                const source_register = program.virtual_registers.get(source_id) orelse
+                    return Error.InvalidVirtualRegister;
+                if (source_register.element_type != item.source.type)
+                    return Error.ParallelCopyTypeMismatch;
+                if (!isBroadcast(item.source.region) and
+                    (source_register.size_bytes != destination_register.size_bytes or
+                        source_register.lane_count != destination_register.lane_count))
+                    return Error.ParallelCopyTypeMismatch;
+            },
+            .null => return Error.ParallelCopyTypeMismatch,
+            else => {},
+        }
+
+        for (copy.register_copies[0..index]) |previous| {
+            const previous_id = switch (previous.destination.register) {
+                .virtual => |register_id| register_id,
+                else => unreachable,
+            };
+            if (previous_id == destination_id)
+                return Error.DuplicateParallelCopyDestination;
+        }
+    }
+
+    for (copy.flag_copies, 0..) |item, index| {
+        if (!program.virtual_flags.isLive(item.destination))
+            return Error.InvalidVirtualFlag;
+        switch (item.source) {
+            .constant => {},
+            .dynamic => |predicate| try validateFlag(program, predicate.flag),
+        }
+
+        for (copy.flag_copies[0..index]) |previous| {
+            if (previous.destination == item.destination)
+                return Error.DuplicateParallelCopyDestination;
+        }
+    }
+}
+
+fn isBroadcast(region: operand.Region) bool {
+    return region.vertical_stride == 0 and region.width == 1 and region.horizontal_stride == 0;
+}
+
 fn validateType(data_type: operand.DataType) Error!void {
     if (!data_type.isInitialTargetType())
-        return error.UnsupportedDataType;
+        return Error.UnsupportedDataType;
 }
 
 fn validateSource(program: *const program_ir.Program, source: operand.Source) Error!void {
     try validateType(source.type);
     if (source.region.width == 0)
-        return error.InvalidRegion;
+        return Error.InvalidRegion;
     try validateRegisterRef(program, source.register);
 
     if (source.register == .immediate) {
@@ -120,16 +240,16 @@ fn validateSource(program: *const program_ir.Program, source: operand.Source) Er
             .f32 => source.type == .f32,
         };
         if (!matches)
-            return error.InvalidImmediateType;
+            return Error.InvalidImmediateType;
     }
 }
 
 fn validateDestination(program: *const program_ir.Program, destination: operand.Destination) Error!void {
     try validateType(destination.type);
     if (destination.region.horizontal_stride == 0)
-        return error.InvalidRegion;
+        return Error.InvalidRegion;
     switch (destination.register) {
-        .immediate, .null => return error.InvalidDestination,
+        .immediate, .null => return Error.InvalidDestination,
         else => try validateRegisterRef(program, destination.register),
     }
 }
@@ -137,11 +257,11 @@ fn validateDestination(program: *const program_ir.Program, destination: operand.
 fn validateRegisterRef(program: *const program_ir.Program, register: operand.RegisterRef) Error!void {
     switch (register) {
         .virtual => |id| if (!program.virtual_registers.isLive(id))
-            return error.InvalidVirtualRegister,
+            return Error.InvalidVirtualRegister,
         .physical_grf => |physical| {
             if (physical.number >= program.device_info.grf_count or
                 physical.byte_offset >= program.device_info.grf_size_bytes)
-                return error.InvalidPhysicalRegister;
+                return Error.InvalidPhysicalRegister;
         },
         .architecture, .immediate, .null => {},
     }
@@ -150,33 +270,82 @@ fn validateRegisterRef(program: *const program_ir.Program, register: operand.Reg
 fn validateFlag(program: *const program_ir.Program, flag: operand.FlagRef) Error!void {
     switch (flag) {
         .virtual => |id| if (!program.virtual_flags.isLive(id))
-            return error.InvalidVirtualFlag,
+            return Error.InvalidVirtualFlag,
         .physical => |physical| if (physical.register != 0 or physical.subregister > 1)
-            return error.InvalidPhysicalFlag,
+            return Error.InvalidPhysicalFlag,
     }
 }
 
 fn validateSpan(program: *const program_ir.Program, span: operand.RegisterSpan) Error!void {
     if (span.register_count == 0)
-        return error.InvalidRegisterSpan;
+        return Error.InvalidRegisterSpan;
     switch (span.base) {
         .virtual, .physical_grf => try validateRegisterRef(program, span.base),
-        else => return error.InvalidRegisterSpan,
+        else => return Error.InvalidRegisterSpan,
     }
 }
 
 fn validateTerminator(program: *const program_ir.Program, terminator: instruction.Terminator) Error!void {
     switch (terminator) {
-        .jump => |target| try validateBlockTarget(program, target),
+        .jump => |edge| try validateEdge(program, edge),
         .conditional_branch => |branch| {
             try validateFlag(program, branch.predicate.flag);
-            try validateBlockTarget(program, branch.true_block);
-            try validateBlockTarget(program, branch.false_block);
+            try validateEdge(program, branch.true_edge);
+            try validateEdge(program, branch.false_edge);
         },
         .end_thread, .@"unreachable" => {},
     }
 }
 
+fn validateEdge(program: *const program_ir.Program, edge: instruction.Edge) Error!void {
+    try validateBlockTarget(program, edge.target);
+    const target = program.blocks.get(edge.target).?;
+
+    if (program.properties.block_parameters_lowered and edge.arguments.len != 0)
+        return Error.UnloweredBlockParameter;
+    if (edge.arguments.len != target.parameters.items.len)
+        return Error.EdgeArgumentCountMismatch;
+
+    for (target.parameters.items, edge.arguments) |parameter, argument| {
+        switch (parameter) {
+            .register => |destination_id| switch (argument) {
+                .source => |source| try validateRegisterEdgeArgument(program, destination_id, source),
+                .predicate => return Error.EdgeArgumentKindMismatch,
+            },
+            .flag => switch (argument) {
+                .source => return Error.EdgeArgumentKindMismatch,
+                .predicate => |predicate_value| switch (predicate_value) {
+                    .constant => {},
+                    .dynamic => |predicate| try validateFlag(program, predicate.flag),
+                },
+            },
+        }
+    }
+}
+
+fn validateRegisterEdgeArgument(program: *const program_ir.Program, destination_id: ids.VirtualRegisterId, source: operand.Source) Error!void {
+    const destination = program.virtual_registers.get(destination_id) orelse
+        return Error.InvalidVirtualRegister;
+    try validateSource(program, source);
+    if (source.type != destination.element_type)
+        return Error.EdgeArgumentTypeMismatch;
+
+    switch (source.register) {
+        .virtual => |source_id| {
+            const source_register = program.virtual_registers.get(source_id) orelse
+                return Error.InvalidVirtualRegister;
+            if (source_register.element_type != source.type)
+                return Error.EdgeArgumentTypeMismatch;
+            if (!isBroadcast(source.region) and
+                (source_register.size_bytes != destination.size_bytes or
+                    source_register.lane_count != destination.lane_count))
+                return Error.EdgeArgumentTypeMismatch;
+        },
+        .null => return Error.EdgeArgumentTypeMismatch,
+        else => {},
+    }
+}
+
 fn validateStructuredControl(program: *const program_ir.Program, control: instruction.StructuredControl) Error!void {
     switch (control) {
         .none => {},
@@ -190,5 +359,5 @@ fn validateStructuredControl(program: *const program_ir.Program, control: instru
 
 fn validateBlockTarget(program: *const program_ir.Program, block_id: ids.BlockId) Error!void {
     if (!program.blocks.isLive(block_id))
-        return error.InvalidBlock;
+        return Error.InvalidBlock;
 }

src/intel/compiler/lower/block_arguments.zig

@@ -0,0 +1,274 @@
+const std = @import("std");
+const Builder = @import("../ir/Builder.zig");
+const ids = @import("../ir/id.zig");
+const instruction = @import("../ir/instruction.zig");
+const operand = @import("../ir/operand.zig");
+const program_ir = @import("../ir/program.zig");
+const pseudo = @import("../ir/pseudo.zig");
+const validator = @import("../ir/validator.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidProgram,
+};
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    validator.validate(program) catch return error.InvalidProgram;
+    if (program.properties.block_parameters_lowered)
+        return;
+
+    var builder = Builder.init(program);
+    var original_blocks: std.ArrayList(ids.BlockId) = .empty;
+    defer original_blocks.deinit(allocator);
+
+    for (program.blocks.entries.items, 0..) |entry, index| {
+        _ = entry orelse continue;
+        try original_blocks.append(allocator, ids.BlockId.fromIndex(index));
+    }
+
+    var emitted_parallel_copy = false;
+    for (original_blocks.items) |block_id| {
+        const block = program.blocks.get(block_id) orelse return error.InvalidProgram;
+        const terminator = block.terminator orelse return error.InvalidProgram;
+        const rewritten: instruction.Terminator = switch (terminator) {
+            .jump => |edge| .{ .jump = try rewriteEdge(
+                allocator,
+                &builder,
+                edge,
+                &emitted_parallel_copy,
+            ) },
+            .conditional_branch => |branch| .{ .conditional_branch = .{
+                .predicate = branch.predicate,
+                .true_edge = try rewriteEdge(
+                    allocator,
+                    &builder,
+                    branch.true_edge,
+                    &emitted_parallel_copy,
+                ),
+                .false_edge = try rewriteEdge(
+                    allocator,
+                    &builder,
+                    branch.false_edge,
+                    &emitted_parallel_copy,
+                ),
+            } },
+            else => terminator,
+        };
+        builder.replaceTerminator(block_id, rewritten) catch |err| return mapBuilderError(err);
+    }
+
+    for (original_blocks.items) |block_id|
+        builder.clearBlockParameters(block_id) catch |err| return mapBuilderError(err);
+
+    program.properties.block_parameters_lowered = true;
+    if (emitted_parallel_copy)
+        program.properties.parallel_copies_lowered = false;
+}
+
+fn rewriteEdge(
+    allocator: std.mem.Allocator,
+    builder: *Builder,
+    edge: instruction.Edge,
+    emitted_parallel_copy: *bool,
+) Error!instruction.Edge {
+    if (edge.arguments.len == 0)
+        return .{ .target = edge.target, .arguments = &.{} };
+
+    const target = builder.program.blocks.get(edge.target) orelse return error.InvalidProgram;
+    if (target.parameters.items.len != edge.arguments.len)
+        return error.InvalidProgram;
+
+    var register_copies: std.ArrayList(pseudo.RegisterCopy) = .empty;
+    defer register_copies.deinit(allocator);
+    var flag_copies: std.ArrayList(pseudo.FlagCopy) = .empty;
+    defer flag_copies.deinit(allocator);
+
+    for (target.parameters.items, edge.arguments) |parameter, argument| {
+        switch (parameter) {
+            .register => |destination_id| {
+                const source = switch (argument) {
+                    .source => |value| value,
+                    .predicate => return error.InvalidProgram,
+                };
+                const destination = builder.program.virtual_registers.get(destination_id) orelse
+                    return error.InvalidProgram;
+                try register_copies.append(allocator, .{
+                    .destination = .{
+                        .register = .{ .virtual = destination_id },
+                        .type = destination.element_type,
+                    },
+                    .source = source,
+                });
+            },
+            .flag => |destination_id| {
+                const source = switch (argument) {
+                    .source => return error.InvalidProgram,
+                    .predicate => |value| value,
+                };
+                try flag_copies.append(allocator, .{
+                    .destination = destination_id,
+                    .source = source,
+                });
+            },
+        }
+    }
+
+    const edge_block = builder.addBlock(null) catch |err| return mapBuilderError(err);
+    _ = builder.appendInstruction(edge_block, executionSize(builder.program.dispatch_width), null, .{
+        .parallel_copy = .{
+            .register_copies = register_copies.items,
+            .flag_copies = flag_copies.items,
+        },
+    }) catch |err| return mapBuilderError(err);
+    builder.setTerminator(edge_block, .{ .jump = .{
+        .target = edge.target,
+        .arguments = &.{},
+    } }) catch |err| return mapBuilderError(err);
+
+    emitted_parallel_copy.* = true;
+    return .{ .target = edge_block, .arguments = &.{} };
+}
+
+fn executionSize(dispatch_width: @import("../device.zig").DispatchWidth) @import("../device.zig").ExecutionSize {
+    return @enumFromInt(@intFromEnum(dispatch_width));
+}
+
+fn mapBuilderError(err: anyerror) Error {
+    return switch (err) {
+        error.OutOfMemory => error.OutOfMemory,
+        else => error.InvalidProgram,
+    };
+}
+
+test "[ir] block arguments: lower register and flag parameters" {
+    const device = @import("../device.zig");
+    const printer = @import("../ir/printer.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const source_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const destination_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const source_flag = try builder.addVirtualFlag(.{});
+    const destination_flag = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const merge = try builder.addBlock("merge");
+
+    try builder.addBlockParameter(merge, .{ .register = destination_register });
+    try builder.addBlockParameter(merge, .{ .flag = destination_flag });
+    try builder.setTerminator(entry, .{ .jump = try builder.edge(merge, &.{
+        .{ .source = .{
+            .register = .{ .virtual = source_register },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        } },
+        .{ .predicate = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } } },
+    }) });
+    try builder.setTerminator(merge, .end_thread);
+
+    try validator.validate(&program);
+    const before = try printer.allocPrint(std.testing.allocator, &program);
+    defer std.testing.allocator.free(before);
+    try std.testing.expect(std.mem.indexOf(u8, before, ".merge(%v1, %f1):") != null);
+    try std.testing.expect(std.mem.indexOf(u8, before, "jump .merge(%v0:u32, (+%f0))") != null);
+
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    try std.testing.expect(program.properties.block_parameters_lowered);
+    try std.testing.expect(!program.properties.parallel_copies_lowered);
+    try std.testing.expectEqual(@as(usize, 0), program.blocks.get(merge).?.parameters.items.len);
+
+    const edge_block_id = program.blocks.get(entry).?.terminator.?.jump.target;
+    try std.testing.expect(edge_block_id != merge);
+    try std.testing.expectEqual(@as(usize, 0), program.blocks.get(entry).?.terminator.?.jump.arguments.len);
+
+    const edge_block = program.blocks.get(edge_block_id).?;
+    try std.testing.expectEqual(@as(usize, 1), edge_block.instructions.items.len);
+    const copy = program.instructions.get(edge_block.instructions.items[0]).?.operation.parallel_copy;
+    try std.testing.expectEqual(@as(usize, 1), copy.register_copies.len);
+    try std.testing.expectEqual(destination_register, copy.register_copies[0].destination.register.virtual);
+    try std.testing.expectEqual(source_register, copy.register_copies[0].source.register.virtual);
+    try std.testing.expectEqual(@as(usize, 1), copy.flag_copies.len);
+    try std.testing.expectEqual(destination_flag, copy.flag_copies[0].destination);
+    try std.testing.expectEqual(source_flag, copy.flag_copies[0].source.dynamic.flag.virtual);
+    try std.testing.expectEqual(merge, edge_block.terminator.?.jump.target);
+}
+
+test "[ir] block arguments: split same-target conditional edges" {
+    const device = @import("../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const destination = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const condition = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const merge = try builder.addBlock("merge");
+    try builder.addBlockParameter(merge, .{ .register = destination });
+
+    const one: pseudo.EdgeArgument = .{ .source = .{
+        .register = .{ .immediate = .{ .u32 = 1 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    } };
+    const two: pseudo.EdgeArgument = .{ .source = .{
+        .register = .{ .immediate = .{ .u32 = 2 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    } };
+    try builder.setTerminator(entry, .{ .conditional_branch = .{
+        .predicate = .{ .flag = .{ .virtual = condition } },
+        .true_edge = try builder.edge(merge, &.{one}),
+        .false_edge = try builder.edge(merge, &.{two}),
+    } });
+    try builder.setTerminator(merge, .end_thread);
+
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    const branch = program.blocks.get(entry).?.terminator.?.conditional_branch;
+    try std.testing.expect(branch.true_edge.target != branch.false_edge.target);
+    try std.testing.expect(branch.true_edge.target != merge);
+    try std.testing.expect(branch.false_edge.target != merge);
+
+    const true_block = program.blocks.get(branch.true_edge.target).?;
+    const false_block = program.blocks.get(branch.false_edge.target).?;
+    const true_copy = program.instructions.get(true_block.instructions.items[0]).?.operation.parallel_copy;
+    const false_copy = program.instructions.get(false_block.instructions.items[0]).?.operation.parallel_copy;
+    try std.testing.expectEqual(@as(u32, 1), true_copy.register_copies[0].source.register.immediate.u32);
+    try std.testing.expectEqual(@as(u32, 2), false_copy.register_copies[0].source.register.immediate.u32);
+}

src/intel/compiler/lower/lower.zig

@@ -1,13 +1,17 @@
 const std = @import("std");
 const shader_ir = @import("shader_ir").ir;
 const device = @import("../device.zig");
+const Builder = @import("../ir/Builder.zig");
 const ids = @import("../ir/id.zig");
 const instruction = @import("../ir/instruction.zig");
 const operand = @import("../ir/operand.zig");
 const printer = @import("../ir/printer.zig");
+const pseudo = @import("../ir/pseudo.zig");
 const program_ir = @import("../ir/program.zig");
 const validator = @import("../ir/validator.zig");
 
+pub const block_arguments = @import("block_arguments.zig");
+
 pub const Options = struct {
     dispatch_width: device.DispatchWidth = .simd8,
 };
@@ -27,10 +31,7 @@ pub const Error = std.mem.Allocator.Error || error{
     UnsupportedTerminator,
 };
 
-const PredicateValue = union(enum) {
-    constant: bool,
-    dynamic: operand.Predicate,
-};
+const PredicateValue = pseudo.PredicateValue;
 
 const ValueLocation = union(enum) {
     source: operand.Source,
@@ -39,7 +40,7 @@ const ValueLocation = union(enum) {
 
 const LoweringState = struct {
     lowerer: *Lowerer,
-    program: *program_ir.Program,
+    builder: Builder,
     block_map: []?ids.BlockId,
     value_locations: []?ValueLocation,
 
@@ -77,7 +78,7 @@ const LoweringState = struct {
     }
 
     fn addRegister(self: *LoweringState, data_type: operand.DataType, class: operand.RegisterClass, name: ?[]const u8) Error!ids.VirtualRegisterId {
-        return self.program.addVirtualRegister(.{
+        return self.builder.addVirtualRegister(.{
             .size_bytes = @as(u32, data_type.sizeBytes()) * @intFromEnum(self.lowerer.options.dispatch_width),
             .alignment_bytes = self.lowerer.device_info.grf_size_bytes,
             .element_type = data_type,
@@ -195,7 +196,7 @@ const LoweringState = struct {
     }
 
     fn appendInstruction(self: *LoweringState, block_id: ids.BlockId, predicate_value: ?operand.Predicate, operation: instruction.Operation) Error!void {
-        _ = self.program.appendInstruction(block_id, .simd8, predicate_value, operation) catch |err|
+        _ = self.builder.appendInstruction(block_id, .simd8, predicate_value, operation) catch |err|
             return mapProgramError(err);
     }
 
@@ -224,7 +225,7 @@ const LoweringState = struct {
             const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule;
             if (source_block.parent_function != source_function_id)
                 return Error.InvalidModule;
-            const target_block_id = self.program.addBlock(source_block.name) catch |err|
+            const target_block_id = self.builder.addBlock(source_block.name) catch |err|
                 return mapProgramError(err);
             if (source_block_id.index() >= self.block_map.len or self.block_map[source_block_id.index()] != null)
                 return Error.InvalidModule;
@@ -232,14 +233,36 @@ const LoweringState = struct {
         }
 
         const source_entry = function.entry_block orelse return Error.InvalidModule;
-        self.program.setEntryBlock(try self.mappedBlock(source_entry)) catch |err| return mapProgramError(err);
+        self.builder.setEntryBlock(try self.mappedBlock(source_entry)) catch |err| return mapProgramError(err);
     }
 
     fn lowerParameters(self: *LoweringState) Error!void {
-        for (self.lowerer.module.blocks.entries.items) |entry| {
-            const block = entry orelse continue;
-            for (block.parameters.items) |parameter_id|
-                _ = try self.addRegisterLocation(parameter_id, .temporary);
+        const source_entry = try self.sourceEntryFunction();
+        const function = source_entry[1];
+
+        for (function.blocks.items) |source_block_id| {
+            const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule;
+            const target_block_id = try self.mappedBlock(source_block_id);
+
+            for (source_block.parameters.items) |parameter_id| {
+                const value = self.lowerer.module.values.get(parameter_id) orelse return Error.InvalidModule;
+                if (try self.isBoolean(value.type)) {
+                    const flag_id = self.builder.addVirtualFlag(.{ .name = value.name }) catch |err|
+                        return mapProgramError(err);
+                    const predicate_value: operand.Predicate = .{ .flag = .{ .virtual = flag_id } };
+                    try self.putLocation(parameter_id, .{ .predicate = .{ .dynamic = predicate_value } });
+                    self.builder.addBlockParameter(target_block_id, .{ .flag = flag_id }) catch |err|
+                        return mapProgramError(err);
+                } else {
+                    const parameter_source = try self.addRegisterLocation(parameter_id, .temporary);
+                    const register_id = switch (parameter_source.register) {
+                        .virtual => |id| id,
+                        else => return Error.InvalidLoweredProgram,
+                    };
+                    self.builder.addBlockParameter(target_block_id, .{ .register = register_id }) catch |err|
+                        return mapProgramError(err);
+                }
+            }
         }
     }
 
@@ -437,7 +460,7 @@ const LoweringState = struct {
             => return Error.UnsupportedOperation,
         };
 
-        const flag_id = self.program.addVirtualFlag(.{ .name = result_value.name }) catch |err|
+        const flag_id = self.builder.addVirtualFlag(.{ .name = result_value.name }) catch |err|
             return mapProgramError(err);
 
         const predicate_value: operand.Predicate = .{ .flag = .{ .virtual = flag_id } };
@@ -551,8 +574,7 @@ const LoweringState = struct {
         for (function.blocks.items) |source_block_id| {
             const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule;
             const target_block_id = try self.mappedBlock(source_block_id);
-            const target_block = self.program.blocks.getMut(target_block_id) orelse return Error.InvalidLoweredProgram;
-            target_block.structured_control = switch (source_block.structured_control) {
+            const structured_control: instruction.StructuredControl = switch (source_block.structured_control) {
                 .none => .none,
                 .selection => |selection| .{ .selection = .{
                     .merge_block = try self.mappedBlock(selection.merge_block),
@@ -562,30 +584,26 @@ const LoweringState = struct {
                     .continue_block = try self.mappedBlock(loop.continue_block),
                 } },
             };
+            self.builder.setStructuredControl(target_block_id, structured_control) catch |err|
+                return mapProgramError(err);
 
             const source_terminator = source_block.terminator orelse return Error.InvalidModule;
             const target_terminator: instruction.Terminator = switch (source_terminator) {
-                .branch => |edge| branch: {
-                    try self.lowerEdgeCopies(allocator, target_block_id, edge, null);
-                    break :branch .{ .jump = try self.mappedBlock(edge.target) };
-                },
+                .branch => |edge| .{ .jump = try self.lowerEdge(allocator, edge) },
                 .conditional_branch => |branch| conditional: {
                     switch (try self.predicate(branch.condition)) {
                         .constant => |condition| {
                             const edge = if (condition) branch.true_edge else branch.false_edge;
-                            try self.lowerEdgeCopies(allocator, target_block_id, edge, null);
-                            break :conditional .{ .jump = try self.mappedBlock(edge.target) };
+                            break :conditional .{ .jump = try self.lowerEdge(allocator, edge) };
                         },
                         .dynamic => |condition| {
-                            try self.lowerEdgeCopies(allocator, target_block_id, branch.true_edge, condition);
-                            try self.lowerEdgeCopies(allocator, target_block_id, branch.false_edge, .{
-                                .flag = condition.flag,
-                                .inverse = !condition.inverse,
-                            });
+                            const true_edge = try self.lowerEdge(allocator, branch.true_edge);
+                            errdefer allocator.free(true_edge.arguments);
+                            const false_edge = try self.lowerEdge(allocator, branch.false_edge);
                             break :conditional .{ .conditional_branch = .{
                                 .predicate = condition,
-                                .true_block = try self.mappedBlock(branch.true_edge.target),
-                                .false_block = try self.mappedBlock(branch.false_edge.target),
+                                .true_edge = true_edge,
+                                .false_edge = false_edge,
                             } };
                         },
                     }
@@ -595,48 +613,44 @@ const LoweringState = struct {
                 .discard => return Error.UnsupportedTerminator,
                 .@"unreachable" => .@"unreachable",
             };
-            self.program.setTerminator(target_block_id, target_terminator) catch |err|
+            defer freeTerminatorArguments(allocator, target_terminator);
+            self.builder.setTerminator(target_block_id, target_terminator) catch |err|
                 return mapProgramError(err);
         }
     }
 
-    fn lowerEdgeCopies(
-        self: *LoweringState,
-        allocator: std.mem.Allocator,
-        source_block: ids.BlockId,
-        edge: shader_ir.module.Edge,
-        predicate_value: ?operand.Predicate,
-    ) Error!void {
+    fn lowerEdge(self: *LoweringState, allocator: std.mem.Allocator, edge: shader_ir.module.Edge) Error!instruction.Edge {
         const target_source_block = self.lowerer.module.blocks.get(edge.target) orelse return Error.InvalidModule;
-
         if (edge.arguments.len != target_source_block.parameters.items.len)
             return Error.InvalidModule;
 
-        if (edge.arguments.len == 0)
-            return;
-
-        const temporaries = try allocator.alloc(ids.VirtualRegisterId, edge.arguments.len);
-        defer allocator.free(temporaries);
-
-        // Capture every source before writing any destination so loop backedges and
-        // swaps retain the parallel-copy semantics of shared-IR block arguments.
-        for (edge.arguments, 0..) |argument_id, index| {
-            const argument = try self.source(argument_id);
-            const temporary = try self.addRegister(argument.type, .temporary, null);
-            temporaries[index] = temporary;
-            try self.appendMove(source_block, predicate_value, .{
-                .register = .{ .virtual = temporary },
-                .type = argument.type,
-            }, argument);
+        const arguments = try allocator.alloc(pseudo.EdgeArgument, edge.arguments.len);
+        errdefer allocator.free(arguments);
+        for (edge.arguments, arguments) |argument_id, *argument| {
+            argument.* = switch (try self.location(argument_id)) {
+                .source => |source_value| .{ .source = source_value },
+                .predicate => |predicate_value| .{ .predicate = predicate_value },
+            };
         }
 
-        for (target_source_block.parameters.items, temporaries) |parameter_id, temporary| {
-            const destination_value = try self.destination(parameter_id);
-            try self.appendMove(source_block, predicate_value, destination_value, self.registerSource(temporary, destination_value.type));
-        }
+        return .{
+            .target = try self.mappedBlock(edge.target),
+            .arguments = arguments,
+        };
     }
 };
 
+fn freeTerminatorArguments(allocator: std.mem.Allocator, terminator: instruction.Terminator) void {
+    switch (terminator) {
+        .jump => |edge| allocator.free(edge.arguments),
+        .conditional_branch => |branch| {
+            allocator.free(branch.true_edge.arguments);
+            allocator.free(branch.false_edge.arguments);
+        },
+        else => {},
+    }
+}
+
 pub const Lowerer = struct {
     module: *shader_ir.module.Module,
     device_info: device.DeviceInfo,
@@ -690,7 +704,7 @@ pub const Lowerer = struct {
 
         var state: LoweringState = .{
             .lowerer = self,
-            .program = &program,
+            .builder = Builder.init(&program),
             .block_map = block_map,
             .value_locations = value_locations,
         };
@@ -701,7 +715,12 @@ pub const Lowerer = struct {
         try state.lowerControlAndTerminators(allocator);
 
         program.properties.instructions_selected = true;
-        program.properties.block_parameters_lowered = true;
+        validator.validate(&program) catch return Error.InvalidLoweredProgram;
+
+        block_arguments.run(allocator, &program) catch |err| return switch (err) {
+            error.OutOfMemory => Error.OutOfMemory,
+            else => Error.InvalidLoweredProgram,
+        };
         validator.validate(&program) catch return Error.InvalidLoweredProgram;
         return program;
     }
@@ -805,7 +824,7 @@ fn expectLoweringError(source: []const u8, expected: Error) !void {
     return error.TestExpectedError;
 }
 
-test "Lower: basic shader" {
+test "[ir] Lower: basic shader" {
     const source =
         \\shader vertex @main
         \\{
@@ -838,8 +857,6 @@ test "Lower: basic shader" {
         \\
         \\%value: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
         \\%sum: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
-        \\%v2: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
-        \\%v3: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
         \\%condition: vflag
         \\
         \\.entry:
@@ -848,26 +865,30 @@ test "Lower: basic shader" {
         \\    conditional_branch (+%condition), .left, .right
         \\
         \\.left:
-        \\    [simd8] mov %v2:u32, %sum:u32
-        \\    [simd8] mov %value:u32, %v2:u32
-        \\    jump .merge
+        \\    jump .b4
         \\
         \\.right:
-        \\    [simd8] mov %v3:u32, 2:u32
-        \\    [simd8] mov %value:u32, %v3:u32
-        \\    jump .merge
+        \\    jump .b5
         \\
         \\.merge:
         \\    [simd8] store_output location(0), component(0), %value:u32
         \\    end_thread
         \\
+        \\.b4:
+        \\    [simd8] parallel_copy [%value:u32 <- %sum:u32]
+        \\    jump .merge
+        \\
+        \\.b5:
+        \\    [simd8] parallel_copy [%value:u32 <- 2:u32]
+        \\    jump .merge
+        \\
         \\
     ;
 
     try expectLowered(source, expected);
 }
 
-test "Lower: control flow" {
+test "[ir] Lower: control flow" {
     const source =
         \\shader vertex @main
         \\{
@@ -913,7 +934,7 @@ test "Lower: control flow" {
     try expectLowered(source, expected);
 }
 
-test "Lower: function call" {
+test "[ir] Lower: function call" {
     const source =
         \\shader vertex @main
         \\{
@@ -940,7 +961,6 @@ test "Lower: function call" {
         \\;   .dispatch_width: simd8
         \\
         \\%result: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
-        \\%v1: vgrf u32[8], class(temporary), size(32), alignment(32), spillable
         \\
         \\.entry:
         \\    jump .b2
@@ -949,8 +969,10 @@ test "Lower: function call" {
         \\    end_thread
         \\
         \\.b2:
-        \\    [simd8] mov %v1:u32, 1:u32
-        \\    [simd8] mov %result:u32, %v1:u32
+        \\    jump .b3
+        \\
+        \\.b3:
+        \\    [simd8] parallel_copy [%result:u32 <- 1:u32]
         \\    jump .b1
         \\
         \\
@@ -959,7 +981,7 @@ test "Lower: function call" {
     try expectLowered(source, expected);
 }
 
-test "Lower: unary/binary operations" {
+test "[ir] Lower: unary/binary operations" {
     const source =
         \\shader vertex @main
         \\{
@@ -1008,7 +1030,7 @@ test "Lower: unary/binary operations" {
     }, &.{});
 }
 
-test "Lower: selects and bitcasts" {
+test "[ir] Lower: selects and bitcasts" {
     const source =
         \\shader vertex @main
         \\{
@@ -1050,7 +1072,7 @@ test "Lower: selects and bitcasts" {
     });
 }
 
-test "Lower: vertex interfaces" {
+test "[ir] Lower: vertex interfaces" {
     const source =
         \\shader vertex @main
         \\{
@@ -1079,7 +1101,7 @@ test "Lower: vertex interfaces" {
     }, &.{});
 }
 
-test "Lower: constant conditional branch" {
+test "[ir] Lower: constant conditional branch" {
     const source =
         \\shader vertex @main
         \\{
@@ -1106,7 +1128,42 @@ test "Lower: constant conditional branch" {
     });
 }
 
-test "Lower: unsupported operations" {
+test "[ir] Lower: boolean block parameter" {
+    const source =
+        \\shader vertex @main
+        \\{
+        \\    %one: constant u32 = bits(0x1)
+        \\    %two: constant u32 = bits(0x2)
+        \\    %never: constant bool = false
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %condition: bool = cmp_unsigned_less %one, %two
+        \\            conditional_branch %condition, .left(), .right()
+        \\        .left():
+        \\            branch .merge(%condition)
+        \\        .right():
+        \\            branch .merge(%never)
+        \\        .merge(%merged: bool):
+        \\            conditional_branch %merged, .taken(), .not_taken()
+        \\        .taken():
+        \\            return
+        \\        .not_taken():
+        \\            return
+        \\    }
+        \\}
+    ;
+
+    try expectLoweredFragments(source, &.{
+        "%condition: vflag",
+        "%merged: vflag",
+        "parallel_copy [%merged <- (+%condition)]",
+        "parallel_copy [%merged <- false]",
+        ".merge:\n    conditional_branch (+%merged), .taken, .not_taken",
+    }, &.{});
+}
+
+test "[ir] Lower: unsupported operations" {
     try expectLoweringError(
         \\shader vertex @main
         \\{
@@ -1149,7 +1206,7 @@ test "Lower: unsupported operations" {
     , Error.UnsupportedType);
 }
 
-test "Lower: unreachable terminator" {
+test "[ir] Lower: unreachable terminator" {
     const source =
         \\shader vertex @main
         \\{
@@ -1166,7 +1223,7 @@ test "Lower: unreachable terminator" {
     }, &.{});
 }
 
-test "Lower: unsupported target configuration" {
+test "[ir] Lower: unsupported target configuration" {
     var module = try shader_ir.parser.parseString(std.testing.allocator,
         \\shader vertex @main
         \\{