diff --git a/build.zig b/build.zig index dd9a622..71df7d7 100644 --- a/build.zig +++ b/build.zig @@ -614,12 +614,7 @@ fn optionsPhi(b: *std.Build, options: *Step.Options) !void { options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix); } -fn addPhiCardDaemon( - b: *std.Build, - optimize: std.builtin.OptimizeMode, - cc: []const u8, - sysroot: ?[]const u8, -) !std.Build.LazyPath { +fn addPhiCardDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u8, sysroot: ?[]const u8) !std.Build.LazyPath { const cmd = b.addSystemCommand(&.{cc}); cmd.addArgs(&.{ diff --git a/src/intel/compiler/compiler.zig b/src/intel/compiler/compiler.zig index 2a48e91..b05f481 100644 --- a/src/intel/compiler/compiler.zig +++ b/src/intel/compiler/compiler.zig @@ -5,11 +5,13 @@ pub const device = @import("device.zig"); pub const ir = @import("ir/ir.zig"); pub const lower = @import("lower/lower.zig"); +pub const Builder = ir.Builder; pub const id = ir.id; pub const instruction = ir.instruction; pub const operand = ir.operand; pub const printer = ir.printer; pub const program = ir.program; +pub const pseudo = ir.pseudo; pub const validator = ir.validator; pub const Program = ir.Program; @@ -17,7 +19,7 @@ pub const Stage = ir.Stage; const std = @import("std"); -test "Flint IR foundation" { +test "[ir] basic shader" { // ; Flint program: // ; .stage: vertex // ; .generation: gen9 @@ -44,8 +46,9 @@ test "Flint IR foundation" { var shader = Program.init(std.testing.allocator, .vertex, device_info, .simd8); defer shader.deinit(); + var builder = Builder.init(&shader); - const position = try shader.addVirtualRegister(.{ + const position = try builder.addVirtualRegister(.{ .size_bytes = 32, .alignment_bytes = 32, .element_type = .f32, @@ -53,7 +56,7 @@ test "Flint IR foundation" { .class = .varying, .name = "position", }); - const urb_payload = try shader.addVirtualRegister(.{ + const urb_payload = try builder.addVirtualRegister(.{ .size_bytes = 64, .alignment_bytes = 32, .element_type = .u32, @@ -62,10 +65,10 @@ test "Flint IR foundation" { .spillable = false, .name = "urb_payload", }); - const entry = try shader.addBlock("entry"); - try shader.setEntryBlock(entry); + const entry = try builder.addBlock("entry"); + try builder.setEntryBlock(entry); - _ = try shader.appendInstruction(entry, .simd8, null, .{ + _ = try builder.appendInstruction(entry, .simd8, null, .{ .load_input = .{ .destination = .{ .register = .{ .virtual = position }, @@ -78,7 +81,7 @@ test "Flint IR foundation" { }, }, }); - _ = try shader.appendInstruction(entry, .simd8, null, .{ + _ = try builder.appendInstruction(entry, .simd8, null, .{ .binary = .{ .opcode = .multiply, .destination = .{ @@ -99,7 +102,7 @@ test "Flint IR foundation" { }, }, }); - _ = try shader.appendInstruction(entry, .simd8, null, .{ + _ = try builder.appendInstruction(entry, .simd8, null, .{ .move = .{ .destination = .{ .register = .{ .virtual = position }, @@ -117,7 +120,7 @@ test "Flint IR foundation" { }, }, }); - _ = try shader.appendInstruction(entry, .simd8, null, .{ + _ = try builder.appendInstruction(entry, .simd8, null, .{ .store_output = .{ .semantic = .{ .builtin = .{ .builtin = .position }, @@ -129,7 +132,7 @@ test "Flint IR foundation" { }, }, }); - _ = try shader.appendInstruction(entry, .simd8, null, .{ + _ = try builder.appendInstruction(entry, .simd8, null, .{ .send = .{ .message = .{ .urb_write = .{ @@ -143,7 +146,7 @@ test "Flint IR foundation" { }, }, }); - try shader.setTerminator(entry, .end_thread); + try builder.setTerminator(entry, .end_thread); shader.properties.instructions_selected = true; try validator.validate(&shader); @@ -165,7 +168,7 @@ test "Flint IR foundation" { try std.testing.expect(std.mem.indexOf(u8, text, "send urb_write[offset(0), channels(xyzw), end_of_thread], payload(%urb_payload[2])") != null); } -test "ID stability after removal" { +test "[ir] ID stability after removal" { var store: id.Store(id.VirtualFlagId, operand.VirtualFlag) = .{}; defer store.entries.deinit(std.testing.allocator); diff --git a/src/intel/compiler/ir/Builder.zig b/src/intel/compiler/ir/Builder.zig new file mode 100644 index 0000000..9f98759 --- /dev/null +++ b/src/intel/compiler/ir/Builder.zig @@ -0,0 +1,178 @@ +const std = @import("std"); +const device = @import("../device.zig"); +const ids = @import("id.zig"); +const instruction = @import("instruction.zig"); +const operand = @import("operand.zig"); +const program_ir = @import("program.zig"); +const pseudo = @import("pseudo.zig"); + +const Self = @This(); + +pub const Error = std.mem.Allocator.Error || error{ + InvalidBlock, + InvalidInsertionIndex, + TerminatorAlreadySet, +}; + +program: *program_ir.Program, + +pub fn init(program: *program_ir.Program) Self { + return .{ .program = program }; +} + +pub fn addVirtualRegister(self: *Self, register: operand.VirtualRegister) Error!ids.VirtualRegisterId { + return self.program.addVirtualRegister(register); +} + +pub fn addVirtualFlag(self: *Self, flag: operand.VirtualFlag) Error!ids.VirtualFlagId { + return self.program.addVirtualFlag(flag); +} + +pub fn addBlock(self: *Self, name: ?[]const u8) Error!ids.BlockId { + return self.program.addBlock(name); +} + +pub fn setEntryBlock(self: *Self, block_id: ids.BlockId) Error!void { + return self.program.setEntryBlock(block_id); +} + +pub fn addBlockParameter(self: *Self, block_id: ids.BlockId, parameter: pseudo.BlockParameter) Error!void { + const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock; + try block.parameters.append(self.program.allocator(), parameter); +} + +pub fn clearBlockParameters(self: *Self, block_id: ids.BlockId) Error!void { + const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock; + block.parameters.clearRetainingCapacity(); +} + +pub fn edge(self: *Self, target: ids.BlockId, arguments: []const pseudo.EdgeArgument) Error!instruction.Edge { + if (!self.program.blocks.isLive(target)) + return Error.InvalidBlock; + return .{ + .target = target, + .arguments = try self.program.allocator().dupe(pseudo.EdgeArgument, arguments), + }; +} + +pub fn appendInstruction( + self: *Self, + block_id: ids.BlockId, + execution_size: device.ExecutionSize, + predicate: ?operand.Predicate, + operation: instruction.Operation, +) Error!ids.InstructionId { + const block = self.program.blocks.get(block_id) orelse return Error.InvalidBlock; + return self.insertInstruction(block_id, block.instructions.items.len, execution_size, predicate, operation); +} + +pub fn insertInstruction( + self: *Self, + block_id: ids.BlockId, + index: usize, + execution_size: device.ExecutionSize, + predicate: ?operand.Predicate, + operation: instruction.Operation, +) Error!ids.InstructionId { + const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock; + if (index > block.instructions.items.len) + return Error.InvalidInsertionIndex; + + const owned_operation = try instruction.cloneOperation(self.program.allocator(), operation); + const instruction_id = try self.program.instructions.add(self.program.allocator(), .{ + .parent_block = block_id, + .execution_size = execution_size, + .predicate = predicate, + .operation = owned_operation, + }); + errdefer std.debug.assert(self.program.instructions.remove(instruction_id)); + + try block.instructions.insert(self.program.allocator(), index, instruction_id); + return instruction_id; +} + +pub fn setStructuredControl(self: *Self, block_id: ids.BlockId, control: instruction.StructuredControl) Error!void { + const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock; + block.structured_control = control; +} + +pub fn setTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void { + return self.program.setTerminator(block_id, terminator); +} + +pub fn replaceTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void { + const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock; + block.terminator = try instruction.cloneTerminator(self.program.allocator(), terminator); +} + +fn moveImmediate(register_id: ids.VirtualRegisterId, value: u32) instruction.Operation { + return .{ + .move = .{ + .destination = .{ + .register = .{ .virtual = register_id }, + .type = .u32, + }, + .source = .{ + .register = .{ .immediate = .{ .u32 = value } }, + .type = .u32, + .region = operand.Region.broadcast(), + }, + }, + }; +} + +test "[ir] Builder: construction and ordered insertion" { + const validator = @import("validator.zig"); + + const device_info: device.DeviceInfo = .{ + .generation = .gen9, + .platform = .skylake, + .pci_device_id = 0x1912, + .grf_count = 128, + }; + + var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8); + defer program.deinit(); + var builder = Self.init(&program); + + var register_name = [_]u8{ 'v', 'a', 'l', 'u', 'e' }; + const register_id = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + .name = ®ister_name, + }); + register_name[0] = 'x'; + try std.testing.expectEqualStrings("value", program.virtual_registers.get(register_id).?.name.?); + + const flag_id = try builder.addVirtualFlag(.{ .name = "condition" }); + try std.testing.expectEqualStrings("condition", program.virtual_flags.get(flag_id).?.name.?); + + const entry = try builder.addBlock("entry"); + const exit = try builder.addBlock("exit"); + try builder.setEntryBlock(entry); + + const second = try builder.appendInstruction(entry, .simd8, null, moveImmediate(register_id, 2)); + const first = try builder.insertInstruction(entry, 0, .simd8, null, moveImmediate(register_id, 1)); + + const entry_block = program.blocks.get(entry).?; + try std.testing.expectEqualSlices(ids.InstructionId, &.{ first, second }, entry_block.instructions.items); + try std.testing.expectEqual(entry, program.instructions.get(first).?.parent_block); + try std.testing.expectEqual(entry, program.instructions.get(second).?.parent_block); + + try builder.setStructuredControl(entry, .{ .selection = .{ .merge_block = exit } }); + try builder.setTerminator(entry, .{ .jump = try builder.edge(exit, &.{}) }); + try builder.setTerminator(exit, .end_thread); + try std.testing.expectError(Error.TerminatorAlreadySet, builder.setTerminator(entry, .end_thread)); + + const instruction_count = program.instructions.entries.items.len; + try std.testing.expectError( + Error.InvalidInsertionIndex, + builder.insertInstruction(entry, 3, .simd8, null, moveImmediate(register_id, 3)), + ); + try std.testing.expectEqual(instruction_count, program.instructions.entries.items.len); + + try validator.validate(&program); +} diff --git a/src/intel/compiler/ir/instruction.zig b/src/intel/compiler/ir/instruction.zig index 8e9658c..c4d4afa 100644 --- a/src/intel/compiler/ir/instruction.zig +++ b/src/intel/compiler/ir/instruction.zig @@ -2,6 +2,7 @@ const std = @import("std"); const device = @import("../device.zig"); const ids = @import("id.zig"); const operand = @import("operand.zig"); +const pseudo = @import("pseudo.zig"); pub const Builtin = enum { position, @@ -98,8 +99,21 @@ pub const Operation = union(enum) { binary: Binary, compare: Compare, send: Send, + parallel_copy: pseudo.ParallelCopy, }; +pub fn cloneOperation(allocator: std.mem.Allocator, operation: Operation) std.mem.Allocator.Error!Operation { + return switch (operation) { + .parallel_copy => |copy| .{ + .parallel_copy = .{ + .register_copies = try allocator.dupe(pseudo.RegisterCopy, copy.register_copies), + .flag_copies = try allocator.dupe(pseudo.FlagCopy, copy.flag_copies), + }, + }, + else => operation, + }; +} + pub const Instruction = struct { parent_block: ids.BlockId, execution_size: device.ExecutionSize, @@ -107,17 +121,43 @@ pub const Instruction = struct { operation: Operation, }; +pub const Edge = struct { + target: ids.BlockId, + arguments: []const pseudo.EdgeArgument, +}; + pub const Terminator = union(enum) { - jump: ids.BlockId, + jump: Edge, conditional_branch: struct { predicate: operand.Predicate, - true_block: ids.BlockId, - false_block: ids.BlockId, + true_edge: Edge, + false_edge: Edge, }, end_thread, @"unreachable", }; +pub fn cloneEdge(allocator: std.mem.Allocator, edge: Edge) std.mem.Allocator.Error!Edge { + return .{ + .target = edge.target, + .arguments = try allocator.dupe(pseudo.EdgeArgument, edge.arguments), + }; +} + +pub fn cloneTerminator(allocator: std.mem.Allocator, terminator: Terminator) std.mem.Allocator.Error!Terminator { + return switch (terminator) { + .jump => |edge| .{ .jump = try cloneEdge(allocator, edge) }, + .conditional_branch => |branch| .{ + .conditional_branch = .{ + .predicate = branch.predicate, + .true_edge = try cloneEdge(allocator, branch.true_edge), + .false_edge = try cloneEdge(allocator, branch.false_edge), + }, + }, + else => terminator, + }; +} + pub const StructuredControl = union(enum) { none, selection: struct { @@ -130,6 +170,7 @@ pub const StructuredControl = union(enum) { }; pub const Block = struct { + parameters: std.ArrayList(pseudo.BlockParameter) = .empty, instructions: std.ArrayList(ids.InstructionId) = .empty, terminator: ?Terminator = null, structured_control: StructuredControl = .none, diff --git a/src/intel/compiler/ir/ir.zig b/src/intel/compiler/ir/ir.zig index 154b7af..7770dee 100644 --- a/src/intel/compiler/ir/ir.zig +++ b/src/intel/compiler/ir/ir.zig @@ -1,8 +1,10 @@ +pub const Builder = @import("Builder.zig"); pub const id = @import("id.zig"); pub const instruction = @import("instruction.zig"); pub const operand = @import("operand.zig"); pub const printer = @import("printer.zig"); pub const program = @import("program.zig"); +pub const pseudo = @import("pseudo.zig"); pub const validator = @import("validator.zig"); pub const Program = program.Program; diff --git a/src/intel/compiler/ir/printer.zig b/src/intel/compiler/ir/printer.zig index e81606c..e673abf 100644 --- a/src/intel/compiler/ir/printer.zig +++ b/src/intel/compiler/ir/printer.zig @@ -4,6 +4,7 @@ const ids = @import("id.zig"); const inst_ir = @import("instruction.zig"); const operand = @import("operand.zig"); const program_ir = @import("program.zig"); +const pseudo = @import("pseudo.zig"); const indent = " "; @@ -40,6 +41,15 @@ pub fn write(program: *const program_ir.Program, writer: *std.Io.Writer) std.Io. const block_id = ids.BlockId.fromIndex(block_index); try writeBlockRef(program, writer, block_id); + if (block.parameters.items.len != 0) { + try writer.writeByte('('); + for (block.parameters.items, 0..) |parameter, index| { + if (index != 0) + try writer.writeAll(", "); + try writeBlockParameter(program, writer, parameter); + } + try writer.writeByte(')'); + } try writer.writeAll(":\n"); switch (block.structured_control) { @@ -127,6 +137,7 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex try writer.writeAll(", "); try writeSource(program, writer, execution_size, op.rhs); }, + .parallel_copy => |op| try writeParallelCopy(program, writer, execution_size, op), .send => |op| { try writer.writeAll("send "); if (op.response) |response| { @@ -141,25 +152,81 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex } } +fn writeParallelCopy(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, copy: pseudo.ParallelCopy) !void { + try writer.writeAll("parallel_copy ["); + var needs_separator = false; + + for (copy.register_copies) |item| { + if (needs_separator) + try writer.writeAll(", "); + try writeDestination(program, writer, execution_size, item.destination); + try writer.writeAll(" <- "); + try writeSource(program, writer, execution_size, item.source); + needs_separator = true; + } + + for (copy.flag_copies) |item| { + if (needs_separator) + try writer.writeAll(", "); + try writeVirtualFlagRef(program, writer, item.destination); + try writer.writeAll(" <- "); + switch (item.source) { + .constant => |value| try writer.writeAll(if (value) "true" else "false"), + .dynamic => |predicate| try writePredicate(program, writer, predicate), + } + needs_separator = true; + } + + try writer.writeByte(']'); +} + fn writeTerminator(program: *const program_ir.Program, writer: *std.Io.Writer, terminator: inst_ir.Terminator) !void { switch (terminator) { - .jump => |target| { + .jump => |edge| { try writer.writeAll("jump "); - try writeBlockRef(program, writer, target); + try writeEdge(program, writer, edge); }, .conditional_branch => |branch| { try writer.writeAll("conditional_branch "); try writePredicate(program, writer, branch.predicate); try writer.writeAll(", "); - try writeBlockRef(program, writer, branch.true_block); + try writeEdge(program, writer, branch.true_edge); try writer.writeAll(", "); - try writeBlockRef(program, writer, branch.false_block); + try writeEdge(program, writer, branch.false_edge); }, .end_thread => try writer.writeAll("end_thread"), .@"unreachable" => try writer.writeAll("unreachable"), } } +fn writeBlockParameter(program: *const program_ir.Program, writer: *std.Io.Writer, parameter: pseudo.BlockParameter) !void { + switch (parameter) { + .register => |register_id| try writeVirtualRegisterRef(program, writer, register_id), + .flag => |flag_id| try writeVirtualFlagRef(program, writer, flag_id), + } +} + +fn writeEdge(program: *const program_ir.Program, writer: *std.Io.Writer, edge: inst_ir.Edge) !void { + try writeBlockRef(program, writer, edge.target); + if (edge.arguments.len == 0) + return; + + const execution_size: device.ExecutionSize = @enumFromInt(@intFromEnum(program.dispatch_width)); + try writer.writeByte('('); + for (edge.arguments, 0..) |argument, index| { + if (index != 0) + try writer.writeAll(", "); + switch (argument) { + .source => |source| try writeSource(program, writer, execution_size, source), + .predicate => |predicate_value| switch (predicate_value) { + .constant => |value| try writer.writeAll(if (value) "true" else "false"), + .dynamic => |predicate| try writePredicate(program, writer, predicate), + }, + } + } + try writer.writeByte(')'); +} + fn writeSource(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, source: operand.Source) !void { if (source.negate) try writer.writeByte('-'); diff --git a/src/intel/compiler/ir/program.zig b/src/intel/compiler/ir/program.zig index f0db2b0..c2d3716 100644 --- a/src/intel/compiler/ir/program.zig +++ b/src/intel/compiler/ir/program.zig @@ -10,6 +10,7 @@ pub const Stage = shared_ir.Stage; pub const Properties = packed struct { instructions_selected: bool = false, block_parameters_lowered: bool = false, + parallel_copies_lowered: bool = false, stage_io_lowered: bool = false, resources_lowered: bool = false, @@ -23,7 +24,7 @@ pub const Properties = packed struct { flags_allocated: bool = false, branches_resolved: bool = false, - _padding: u21 = 0, + _padding: u20 = 0, }; pub const VertexPayload = struct { @@ -116,12 +117,15 @@ pub const Program = struct { pub fn appendInstruction(self: *Program, block_id: ids.BlockId, execution_size: device.ExecutionSize, predicate: ?operand.Predicate, operation: instructions.Operation) !ids.InstructionId { const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock; + const owned_operation = try instructions.cloneOperation(self.allocator(), operation); const instruction_id = try self.instructions.add(self.allocator(), .{ .parent_block = block_id, .execution_size = execution_size, .predicate = predicate, - .operation = operation, + .operation = owned_operation, }); + errdefer std.debug.assert(self.instructions.remove(instruction_id)); + try block.instructions.append(self.allocator(), instruction_id); return instruction_id; } @@ -130,6 +134,6 @@ pub const Program = struct { const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock; if (block.terminator != null) return error.TerminatorAlreadySet; - block.terminator = terminator; + block.terminator = try instructions.cloneTerminator(self.allocator(), terminator); } }; diff --git a/src/intel/compiler/ir/pseudo.zig b/src/intel/compiler/ir/pseudo.zig new file mode 100644 index 0000000..54a5b80 --- /dev/null +++ b/src/intel/compiler/ir/pseudo.zig @@ -0,0 +1,183 @@ +const ids = @import("id.zig"); +const operand = @import("operand.zig"); + +pub const PredicateValue = union(enum) { + constant: bool, + dynamic: operand.Predicate, +}; + +pub const BlockParameter = union(enum) { + register: ids.VirtualRegisterId, + flag: ids.VirtualFlagId, +}; + +pub const EdgeArgument = union(enum) { + source: operand.Source, + predicate: PredicateValue, +}; + +pub const RegisterCopy = struct { + destination: operand.Destination, + source: operand.Source, +}; + +pub const FlagCopy = struct { + destination: ids.VirtualFlagId, + source: PredicateValue, +}; + +/// A simultaneous assignment: every source is read before any destination is +/// written. This pseudo-operation must be eliminated before machine emission. +pub const ParallelCopy = struct { + register_copies: []const RegisterCopy, + flag_copies: []const FlagCopy, +}; + +test "[ir] pseudo: parallel copy ownership and printing" { + const std = @import("std"); + const Builder = @import("Builder.zig"); + const device = @import("../device.zig"); + const printer = @import("printer.zig"); + const program_ir = @import("program.zig"); + const validator = @import("validator.zig"); + + const device_info: device.DeviceInfo = .{ + .generation = .gen9, + .platform = .skylake, + .pci_device_id = 0x1912, + .grf_count = 128, + }; + + var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8); + defer program.deinit(); + var builder = Builder.init(&program); + + const source_register = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + .name = "source", + }); + const destination_register = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + .name = "destination", + }); + const source_flag = try builder.addVirtualFlag(.{ .name = "source_flag" }); + const destination_flag = try builder.addVirtualFlag(.{ .name = "destination_flag" }); + const entry = try builder.addBlock("entry"); + + var register_copies = [_]RegisterCopy{.{ + .destination = .{ + .register = .{ .virtual = destination_register }, + .type = .u32, + }, + .source = .{ + .register = .{ .virtual = source_register }, + .type = .u32, + .region = operand.Region.contiguous(.simd8), + }, + }}; + var flag_copies = [_]FlagCopy{.{ + .destination = destination_flag, + .source = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } }, + }}; + + const copy_id = try builder.appendInstruction(entry, .simd8, null, .{ + .parallel_copy = .{ + .register_copies = ®ister_copies, + .flag_copies = &flag_copies, + }, + }); + try builder.setTerminator(entry, .end_thread); + + const stored = program.instructions.get(copy_id).?.operation.parallel_copy; + try std.testing.expect(stored.register_copies.ptr != register_copies[0..].ptr); + try std.testing.expect(stored.flag_copies.ptr != flag_copies[0..].ptr); + + register_copies[0].source.register = .{ .immediate = .{ .u32 = 42 } }; + flag_copies[0].source = .{ .constant = false }; + try std.testing.expect(stored.register_copies[0].source.register == .virtual); + try std.testing.expect(stored.flag_copies[0].source == .dynamic); + + try validator.validate(&program); + + const text = try printer.allocPrint(std.testing.allocator, &program); + defer std.testing.allocator.free(text); + + try std.testing.expect(std.mem.indexOf( + u8, + text, + "parallel_copy [%destination:u32 <- %source:u32, %destination_flag <- (+%source_flag)]", + ) != null); + + program.properties.parallel_copies_lowered = true; + try std.testing.expectError(error.UnloweredParallelCopy, validator.validate(&program)); +} + +test "[ir] pseudo: validator rejects invalid parallel copies" { + const std = @import("std"); + const Builder = @import("Builder.zig"); + const device = @import("../device.zig"); + const program_ir = @import("program.zig"); + const validator = @import("validator.zig"); + + const device_info: device.DeviceInfo = .{ + .generation = .gen9, + .platform = .skylake, + .pci_device_id = 0x1912, + .grf_count = 128, + }; + + var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8); + defer program.deinit(); + var builder = Builder.init(&program); + + const register_id = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + }); + const flag_id = try builder.addVirtualFlag(.{}); + const entry = try builder.addBlock("entry"); + const instruction_id = try builder.appendInstruction(entry, .simd8, null, .{ + .parallel_copy = .{ + .register_copies = &.{}, + .flag_copies = &.{}, + }, + }); + try builder.setTerminator(entry, .end_thread); + + try std.testing.expectError(error.EmptyParallelCopy, validator.validate(&program)); + + const inst = program.instructions.getMut(instruction_id).?; + inst.predicate = .{ .flag = .{ .virtual = flag_id } }; + try std.testing.expectError(error.PredicatedParallelCopy, validator.validate(&program)); + inst.predicate = null; + + const duplicate_copy: RegisterCopy = .{ + .destination = .{ + .register = .{ .virtual = register_id }, + .type = .u32, + }, + .source = .{ + .register = .{ .immediate = .{ .u32 = 1 } }, + .type = .u32, + .region = operand.Region.broadcast(), + }, + }; + inst.operation = .{ + .parallel_copy = .{ + .register_copies = &.{ duplicate_copy, duplicate_copy }, + .flag_copies = &.{}, + }, + }; + try std.testing.expectError(error.DuplicateParallelCopyDestination, validator.validate(&program)); +} diff --git a/src/intel/compiler/ir/validator.zig b/src/intel/compiler/ir/validator.zig index 201735a..a4dfc58 100644 --- a/src/intel/compiler/ir/validator.zig +++ b/src/intel/compiler/ir/validator.zig @@ -2,6 +2,7 @@ const ids = @import("id.zig"); const instruction = @import("instruction.zig"); const operand = @import("operand.zig"); const program_ir = @import("program.zig"); +const pseudo = @import("pseudo.zig"); pub const Error = error{ UnsupportedGeneration, @@ -24,42 +25,61 @@ pub const Error = error{ InvalidDestination, InvalidImmediateType, InvalidRegisterSpan, + EmptyParallelCopy, + InvalidParallelCopyDestination, + ParallelCopyTypeMismatch, + DuplicateParallelCopyDestination, + PredicatedParallelCopy, + UnloweredParallelCopy, + EntryBlockHasParameters, + DuplicateBlockParameter, + EdgeArgumentCountMismatch, + EdgeArgumentKindMismatch, + EdgeArgumentTypeMismatch, + UnloweredBlockParameter, }; pub fn validate(program: *const program_ir.Program) Error!void { if (program.device_info.generation != .gen9) - return error.UnsupportedGeneration; + return Error.UnsupportedGeneration; if (program.stage != .vertex) - return error.UnsupportedStage; + return Error.UnsupportedStage; if (program.dispatch_width != .simd8 or !program.device_info.supportsDispatch(.simd8)) - return error.UnsupportedDispatchWidth; + return Error.UnsupportedDispatchWidth; - const entry_block = program.entry_block orelse return error.MissingEntryBlock; + const entry_block = program.entry_block orelse return Error.MissingEntryBlock; if (!program.blocks.isLive(entry_block)) - return error.InvalidBlock; + return Error.InvalidBlock; for (program.virtual_registers.entries.items) |entry| { const register = entry orelse continue; if (register.size_bytes == 0) - return error.InvalidRegisterSize; + return Error.InvalidRegisterSize; if (register.alignment_bytes == 0 or (register.alignment_bytes & (register.alignment_bytes - 1)) != 0) - return error.InvalidRegisterAlignment; + return Error.InvalidRegisterAlignment; if (register.lane_count == 0) - return error.InvalidLaneCount; + return Error.InvalidLaneCount; try validateType(register.element_type); } for (program.blocks.entries.items, 0..) |entry, block_index| { const block = entry orelse continue; if (block.terminator == null) - return error.MissingTerminator; + return Error.MissingTerminator; const block_id = ids.BlockId.fromIndex(block_index); + if (block_id == entry_block and block.parameters.items.len != 0) + return Error.EntryBlockHasParameters; + if (program.properties.block_parameters_lowered and block.parameters.items.len != 0) + return Error.UnloweredBlockParameter; + for (block.parameters.items, 0..) |parameter, parameter_index| + try validateBlockParameter(program, block_index, parameter_index, parameter); + for (block.instructions.items) |instruction_id| { - const inst = program.instructions.get(instruction_id) orelse return error.InvalidInstruction; + const inst = program.instructions.get(instruction_id) orelse return Error.InvalidInstruction; if (inst.parent_block != block_id) - return error.InvalidInstruction; + return Error.InvalidInstruction; try validateInstruction(program, inst.*); } @@ -68,10 +88,37 @@ pub fn validate(program: *const program_ir.Program) Error!void { } } +fn validateBlockParameter(program: *const program_ir.Program, block_index: usize, parameter_index: usize, parameter: pseudo.BlockParameter) Error!void { + switch (parameter) { + .register => |register_id| if (!program.virtual_registers.isLive(register_id)) + return Error.InvalidVirtualRegister, + .flag => |flag_id| if (!program.virtual_flags.isLive(flag_id)) + return Error.InvalidVirtualFlag, + } + + for (program.blocks.entries.items, 0..) |entry, candidate_block_index| { + const block = entry orelse continue; + if (candidate_block_index > block_index) + break; + const limit = if (candidate_block_index == block_index) parameter_index else block.parameters.items.len; + for (block.parameters.items[0..limit]) |candidate| { + if (blockParametersEqual(parameter, candidate)) + return Error.DuplicateBlockParameter; + } + } +} + +fn blockParametersEqual(a: pseudo.BlockParameter, b: pseudo.BlockParameter) bool { + return switch (a) { + .register => |register_id| b == .register and b.register == register_id, + .flag => |flag_id| b == .flag and b.flag == flag_id, + }; +} + fn validateInstruction(program: *const program_ir.Program, inst: instruction.Instruction) Error!void { switch (inst.execution_size) { .simd1, .simd8 => {}, - else => return error.UnsupportedExecutionSize, + else => return Error.UnsupportedExecutionSize, } if (inst.predicate) |predicate| @@ -99,18 +146,91 @@ fn validateInstruction(program: *const program_ir.Program, inst: instruction.Ins if (op.response) |response| try validateSpan(program, response); }, + .parallel_copy => |op| { + if (program.properties.parallel_copies_lowered) + return Error.UnloweredParallelCopy; + if (inst.predicate != null) + return Error.PredicatedParallelCopy; + try validateParallelCopy(program, op); + }, } } +fn validateParallelCopy(program: *const program_ir.Program, copy: pseudo.ParallelCopy) Error!void { + if (copy.register_copies.len == 0 and copy.flag_copies.len == 0) + return Error.EmptyParallelCopy; + + for (copy.register_copies, 0..) |item, index| { + try validateDestination(program, item.destination); + try validateSource(program, item.source); + + if (item.destination.type != item.source.type) + return Error.ParallelCopyTypeMismatch; + if (item.destination.region.byte_offset != 0 or item.destination.region.horizontal_stride != 1) + return Error.InvalidParallelCopyDestination; + + const destination_id = switch (item.destination.register) { + .virtual => |register_id| register_id, + else => return Error.InvalidParallelCopyDestination, + }; + const destination_register = program.virtual_registers.get(destination_id) orelse + return Error.InvalidVirtualRegister; + if (destination_register.element_type != item.destination.type) + return Error.ParallelCopyTypeMismatch; + + switch (item.source.register) { + .virtual => |source_id| { + const source_register = program.virtual_registers.get(source_id) orelse + return Error.InvalidVirtualRegister; + if (source_register.element_type != item.source.type) + return Error.ParallelCopyTypeMismatch; + if (!isBroadcast(item.source.region) and + (source_register.size_bytes != destination_register.size_bytes or + source_register.lane_count != destination_register.lane_count)) + return Error.ParallelCopyTypeMismatch; + }, + .null => return Error.ParallelCopyTypeMismatch, + else => {}, + } + + for (copy.register_copies[0..index]) |previous| { + const previous_id = switch (previous.destination.register) { + .virtual => |register_id| register_id, + else => unreachable, + }; + if (previous_id == destination_id) + return Error.DuplicateParallelCopyDestination; + } + } + + for (copy.flag_copies, 0..) |item, index| { + if (!program.virtual_flags.isLive(item.destination)) + return Error.InvalidVirtualFlag; + switch (item.source) { + .constant => {}, + .dynamic => |predicate| try validateFlag(program, predicate.flag), + } + + for (copy.flag_copies[0..index]) |previous| { + if (previous.destination == item.destination) + return Error.DuplicateParallelCopyDestination; + } + } +} + +fn isBroadcast(region: operand.Region) bool { + return region.vertical_stride == 0 and region.width == 1 and region.horizontal_stride == 0; +} + fn validateType(data_type: operand.DataType) Error!void { if (!data_type.isInitialTargetType()) - return error.UnsupportedDataType; + return Error.UnsupportedDataType; } fn validateSource(program: *const program_ir.Program, source: operand.Source) Error!void { try validateType(source.type); if (source.region.width == 0) - return error.InvalidRegion; + return Error.InvalidRegion; try validateRegisterRef(program, source.register); if (source.register == .immediate) { @@ -120,16 +240,16 @@ fn validateSource(program: *const program_ir.Program, source: operand.Source) Er .f32 => source.type == .f32, }; if (!matches) - return error.InvalidImmediateType; + return Error.InvalidImmediateType; } } fn validateDestination(program: *const program_ir.Program, destination: operand.Destination) Error!void { try validateType(destination.type); if (destination.region.horizontal_stride == 0) - return error.InvalidRegion; + return Error.InvalidRegion; switch (destination.register) { - .immediate, .null => return error.InvalidDestination, + .immediate, .null => return Error.InvalidDestination, else => try validateRegisterRef(program, destination.register), } } @@ -137,11 +257,11 @@ fn validateDestination(program: *const program_ir.Program, destination: operand. fn validateRegisterRef(program: *const program_ir.Program, register: operand.RegisterRef) Error!void { switch (register) { .virtual => |id| if (!program.virtual_registers.isLive(id)) - return error.InvalidVirtualRegister, + return Error.InvalidVirtualRegister, .physical_grf => |physical| { if (physical.number >= program.device_info.grf_count or physical.byte_offset >= program.device_info.grf_size_bytes) - return error.InvalidPhysicalRegister; + return Error.InvalidPhysicalRegister; }, .architecture, .immediate, .null => {}, } @@ -150,33 +270,82 @@ fn validateRegisterRef(program: *const program_ir.Program, register: operand.Reg fn validateFlag(program: *const program_ir.Program, flag: operand.FlagRef) Error!void { switch (flag) { .virtual => |id| if (!program.virtual_flags.isLive(id)) - return error.InvalidVirtualFlag, + return Error.InvalidVirtualFlag, .physical => |physical| if (physical.register != 0 or physical.subregister > 1) - return error.InvalidPhysicalFlag, + return Error.InvalidPhysicalFlag, } } fn validateSpan(program: *const program_ir.Program, span: operand.RegisterSpan) Error!void { if (span.register_count == 0) - return error.InvalidRegisterSpan; + return Error.InvalidRegisterSpan; switch (span.base) { .virtual, .physical_grf => try validateRegisterRef(program, span.base), - else => return error.InvalidRegisterSpan, + else => return Error.InvalidRegisterSpan, } } fn validateTerminator(program: *const program_ir.Program, terminator: instruction.Terminator) Error!void { switch (terminator) { - .jump => |target| try validateBlockTarget(program, target), + .jump => |edge| try validateEdge(program, edge), .conditional_branch => |branch| { try validateFlag(program, branch.predicate.flag); - try validateBlockTarget(program, branch.true_block); - try validateBlockTarget(program, branch.false_block); + try validateEdge(program, branch.true_edge); + try validateEdge(program, branch.false_edge); }, .end_thread, .@"unreachable" => {}, } } +fn validateEdge(program: *const program_ir.Program, edge: instruction.Edge) Error!void { + try validateBlockTarget(program, edge.target); + const target = program.blocks.get(edge.target).?; + + if (program.properties.block_parameters_lowered and edge.arguments.len != 0) + return Error.UnloweredBlockParameter; + if (edge.arguments.len != target.parameters.items.len) + return Error.EdgeArgumentCountMismatch; + + for (target.parameters.items, edge.arguments) |parameter, argument| { + switch (parameter) { + .register => |destination_id| switch (argument) { + .source => |source| try validateRegisterEdgeArgument(program, destination_id, source), + .predicate => return Error.EdgeArgumentKindMismatch, + }, + .flag => switch (argument) { + .source => return Error.EdgeArgumentKindMismatch, + .predicate => |predicate_value| switch (predicate_value) { + .constant => {}, + .dynamic => |predicate| try validateFlag(program, predicate.flag), + }, + }, + } + } +} + +fn validateRegisterEdgeArgument(program: *const program_ir.Program, destination_id: ids.VirtualRegisterId, source: operand.Source) Error!void { + const destination = program.virtual_registers.get(destination_id) orelse + return Error.InvalidVirtualRegister; + try validateSource(program, source); + if (source.type != destination.element_type) + return Error.EdgeArgumentTypeMismatch; + + switch (source.register) { + .virtual => |source_id| { + const source_register = program.virtual_registers.get(source_id) orelse + return Error.InvalidVirtualRegister; + if (source_register.element_type != source.type) + return Error.EdgeArgumentTypeMismatch; + if (!isBroadcast(source.region) and + (source_register.size_bytes != destination.size_bytes or + source_register.lane_count != destination.lane_count)) + return Error.EdgeArgumentTypeMismatch; + }, + .null => return Error.EdgeArgumentTypeMismatch, + else => {}, + } +} + fn validateStructuredControl(program: *const program_ir.Program, control: instruction.StructuredControl) Error!void { switch (control) { .none => {}, @@ -190,5 +359,5 @@ fn validateStructuredControl(program: *const program_ir.Program, control: instru fn validateBlockTarget(program: *const program_ir.Program, block_id: ids.BlockId) Error!void { if (!program.blocks.isLive(block_id)) - return error.InvalidBlock; + return Error.InvalidBlock; } diff --git a/src/intel/compiler/lower/block_arguments.zig b/src/intel/compiler/lower/block_arguments.zig new file mode 100644 index 0000000..39592b1 --- /dev/null +++ b/src/intel/compiler/lower/block_arguments.zig @@ -0,0 +1,274 @@ +const std = @import("std"); +const Builder = @import("../ir/Builder.zig"); +const ids = @import("../ir/id.zig"); +const instruction = @import("../ir/instruction.zig"); +const operand = @import("../ir/operand.zig"); +const program_ir = @import("../ir/program.zig"); +const pseudo = @import("../ir/pseudo.zig"); +const validator = @import("../ir/validator.zig"); + +pub const Error = std.mem.Allocator.Error || error{ + InvalidProgram, +}; + +pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void { + validator.validate(program) catch return error.InvalidProgram; + if (program.properties.block_parameters_lowered) + return; + + var builder = Builder.init(program); + var original_blocks: std.ArrayList(ids.BlockId) = .empty; + defer original_blocks.deinit(allocator); + + for (program.blocks.entries.items, 0..) |entry, index| { + _ = entry orelse continue; + try original_blocks.append(allocator, ids.BlockId.fromIndex(index)); + } + + var emitted_parallel_copy = false; + for (original_blocks.items) |block_id| { + const block = program.blocks.get(block_id) orelse return error.InvalidProgram; + const terminator = block.terminator orelse return error.InvalidProgram; + const rewritten: instruction.Terminator = switch (terminator) { + .jump => |edge| .{ .jump = try rewriteEdge( + allocator, + &builder, + edge, + &emitted_parallel_copy, + ) }, + .conditional_branch => |branch| .{ .conditional_branch = .{ + .predicate = branch.predicate, + .true_edge = try rewriteEdge( + allocator, + &builder, + branch.true_edge, + &emitted_parallel_copy, + ), + .false_edge = try rewriteEdge( + allocator, + &builder, + branch.false_edge, + &emitted_parallel_copy, + ), + } }, + else => terminator, + }; + builder.replaceTerminator(block_id, rewritten) catch |err| return mapBuilderError(err); + } + + for (original_blocks.items) |block_id| + builder.clearBlockParameters(block_id) catch |err| return mapBuilderError(err); + + program.properties.block_parameters_lowered = true; + if (emitted_parallel_copy) + program.properties.parallel_copies_lowered = false; +} + +fn rewriteEdge( + allocator: std.mem.Allocator, + builder: *Builder, + edge: instruction.Edge, + emitted_parallel_copy: *bool, +) Error!instruction.Edge { + if (edge.arguments.len == 0) + return .{ .target = edge.target, .arguments = &.{} }; + + const target = builder.program.blocks.get(edge.target) orelse return error.InvalidProgram; + if (target.parameters.items.len != edge.arguments.len) + return error.InvalidProgram; + + var register_copies: std.ArrayList(pseudo.RegisterCopy) = .empty; + defer register_copies.deinit(allocator); + var flag_copies: std.ArrayList(pseudo.FlagCopy) = .empty; + defer flag_copies.deinit(allocator); + + for (target.parameters.items, edge.arguments) |parameter, argument| { + switch (parameter) { + .register => |destination_id| { + const source = switch (argument) { + .source => |value| value, + .predicate => return error.InvalidProgram, + }; + const destination = builder.program.virtual_registers.get(destination_id) orelse + return error.InvalidProgram; + try register_copies.append(allocator, .{ + .destination = .{ + .register = .{ .virtual = destination_id }, + .type = destination.element_type, + }, + .source = source, + }); + }, + .flag => |destination_id| { + const source = switch (argument) { + .source => return error.InvalidProgram, + .predicate => |value| value, + }; + try flag_copies.append(allocator, .{ + .destination = destination_id, + .source = source, + }); + }, + } + } + + const edge_block = builder.addBlock(null) catch |err| return mapBuilderError(err); + _ = builder.appendInstruction(edge_block, executionSize(builder.program.dispatch_width), null, .{ + .parallel_copy = .{ + .register_copies = register_copies.items, + .flag_copies = flag_copies.items, + }, + }) catch |err| return mapBuilderError(err); + builder.setTerminator(edge_block, .{ .jump = .{ + .target = edge.target, + .arguments = &.{}, + } }) catch |err| return mapBuilderError(err); + + emitted_parallel_copy.* = true; + return .{ .target = edge_block, .arguments = &.{} }; +} + +fn executionSize(dispatch_width: @import("../device.zig").DispatchWidth) @import("../device.zig").ExecutionSize { + return @enumFromInt(@intFromEnum(dispatch_width)); +} + +fn mapBuilderError(err: anyerror) Error { + return switch (err) { + error.OutOfMemory => error.OutOfMemory, + else => error.InvalidProgram, + }; +} + +test "[ir] block arguments: lower register and flag parameters" { + const device = @import("../device.zig"); + const printer = @import("../ir/printer.zig"); + + const device_info: device.DeviceInfo = .{ + .generation = .gen9, + .platform = .skylake, + .pci_device_id = 0x1912, + .grf_count = 128, + }; + + var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8); + defer program.deinit(); + var builder = Builder.init(&program); + + const source_register = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + }); + const destination_register = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + }); + const source_flag = try builder.addVirtualFlag(.{}); + const destination_flag = try builder.addVirtualFlag(.{}); + const entry = try builder.addBlock("entry"); + const merge = try builder.addBlock("merge"); + + try builder.addBlockParameter(merge, .{ .register = destination_register }); + try builder.addBlockParameter(merge, .{ .flag = destination_flag }); + try builder.setTerminator(entry, .{ .jump = try builder.edge(merge, &.{ + .{ .source = .{ + .register = .{ .virtual = source_register }, + .type = .u32, + .region = operand.Region.contiguous(.simd8), + } }, + .{ .predicate = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } } }, + }) }); + try builder.setTerminator(merge, .end_thread); + + try validator.validate(&program); + const before = try printer.allocPrint(std.testing.allocator, &program); + defer std.testing.allocator.free(before); + try std.testing.expect(std.mem.indexOf(u8, before, ".merge(%v1, %f1):") != null); + try std.testing.expect(std.mem.indexOf(u8, before, "jump .merge(%v0:u32, (+%f0))") != null); + + try run(std.testing.allocator, &program); + try validator.validate(&program); + + try std.testing.expect(program.properties.block_parameters_lowered); + try std.testing.expect(!program.properties.parallel_copies_lowered); + try std.testing.expectEqual(@as(usize, 0), program.blocks.get(merge).?.parameters.items.len); + + const edge_block_id = program.blocks.get(entry).?.terminator.?.jump.target; + try std.testing.expect(edge_block_id != merge); + try std.testing.expectEqual(@as(usize, 0), program.blocks.get(entry).?.terminator.?.jump.arguments.len); + + const edge_block = program.blocks.get(edge_block_id).?; + try std.testing.expectEqual(@as(usize, 1), edge_block.instructions.items.len); + const copy = program.instructions.get(edge_block.instructions.items[0]).?.operation.parallel_copy; + try std.testing.expectEqual(@as(usize, 1), copy.register_copies.len); + try std.testing.expectEqual(destination_register, copy.register_copies[0].destination.register.virtual); + try std.testing.expectEqual(source_register, copy.register_copies[0].source.register.virtual); + try std.testing.expectEqual(@as(usize, 1), copy.flag_copies.len); + try std.testing.expectEqual(destination_flag, copy.flag_copies[0].destination); + try std.testing.expectEqual(source_flag, copy.flag_copies[0].source.dynamic.flag.virtual); + try std.testing.expectEqual(merge, edge_block.terminator.?.jump.target); +} + +test "[ir] block arguments: split same-target conditional edges" { + const device = @import("../device.zig"); + + const device_info: device.DeviceInfo = .{ + .generation = .gen9, + .platform = .skylake, + .pci_device_id = 0x1912, + .grf_count = 128, + }; + + var program = program_ir.Program.init(std.testing.allocator, .vertex, device_info, .simd8); + defer program.deinit(); + var builder = Builder.init(&program); + + const destination = try builder.addVirtualRegister(.{ + .size_bytes = 32, + .alignment_bytes = 32, + .element_type = .u32, + .lane_count = 8, + .class = .temporary, + }); + const condition = try builder.addVirtualFlag(.{}); + const entry = try builder.addBlock("entry"); + const merge = try builder.addBlock("merge"); + try builder.addBlockParameter(merge, .{ .register = destination }); + + const one: pseudo.EdgeArgument = .{ .source = .{ + .register = .{ .immediate = .{ .u32 = 1 } }, + .type = .u32, + .region = operand.Region.broadcast(), + } }; + const two: pseudo.EdgeArgument = .{ .source = .{ + .register = .{ .immediate = .{ .u32 = 2 } }, + .type = .u32, + .region = operand.Region.broadcast(), + } }; + try builder.setTerminator(entry, .{ .conditional_branch = .{ + .predicate = .{ .flag = .{ .virtual = condition } }, + .true_edge = try builder.edge(merge, &.{one}), + .false_edge = try builder.edge(merge, &.{two}), + } }); + try builder.setTerminator(merge, .end_thread); + + try run(std.testing.allocator, &program); + try validator.validate(&program); + + const branch = program.blocks.get(entry).?.terminator.?.conditional_branch; + try std.testing.expect(branch.true_edge.target != branch.false_edge.target); + try std.testing.expect(branch.true_edge.target != merge); + try std.testing.expect(branch.false_edge.target != merge); + + const true_block = program.blocks.get(branch.true_edge.target).?; + const false_block = program.blocks.get(branch.false_edge.target).?; + const true_copy = program.instructions.get(true_block.instructions.items[0]).?.operation.parallel_copy; + const false_copy = program.instructions.get(false_block.instructions.items[0]).?.operation.parallel_copy; + try std.testing.expectEqual(@as(u32, 1), true_copy.register_copies[0].source.register.immediate.u32); + try std.testing.expectEqual(@as(u32, 2), false_copy.register_copies[0].source.register.immediate.u32); +} diff --git a/src/intel/compiler/lower/lower.zig b/src/intel/compiler/lower/lower.zig index bc75bf5..db0fbbb 100644 --- a/src/intel/compiler/lower/lower.zig +++ b/src/intel/compiler/lower/lower.zig @@ -1,13 +1,17 @@ const std = @import("std"); const shader_ir = @import("shader_ir").ir; const device = @import("../device.zig"); +const Builder = @import("../ir/Builder.zig"); const ids = @import("../ir/id.zig"); const instruction = @import("../ir/instruction.zig"); const operand = @import("../ir/operand.zig"); const printer = @import("../ir/printer.zig"); +const pseudo = @import("../ir/pseudo.zig"); const program_ir = @import("../ir/program.zig"); const validator = @import("../ir/validator.zig"); +pub const block_arguments = @import("block_arguments.zig"); + pub const Options = struct { dispatch_width: device.DispatchWidth = .simd8, }; @@ -27,10 +31,7 @@ pub const Error = std.mem.Allocator.Error || error{ UnsupportedTerminator, }; -const PredicateValue = union(enum) { - constant: bool, - dynamic: operand.Predicate, -}; +const PredicateValue = pseudo.PredicateValue; const ValueLocation = union(enum) { source: operand.Source, @@ -39,7 +40,7 @@ const ValueLocation = union(enum) { const LoweringState = struct { lowerer: *Lowerer, - program: *program_ir.Program, + builder: Builder, block_map: []?ids.BlockId, value_locations: []?ValueLocation, @@ -77,7 +78,7 @@ const LoweringState = struct { } fn addRegister(self: *LoweringState, data_type: operand.DataType, class: operand.RegisterClass, name: ?[]const u8) Error!ids.VirtualRegisterId { - return self.program.addVirtualRegister(.{ + return self.builder.addVirtualRegister(.{ .size_bytes = @as(u32, data_type.sizeBytes()) * @intFromEnum(self.lowerer.options.dispatch_width), .alignment_bytes = self.lowerer.device_info.grf_size_bytes, .element_type = data_type, @@ -195,7 +196,7 @@ const LoweringState = struct { } fn appendInstruction(self: *LoweringState, block_id: ids.BlockId, predicate_value: ?operand.Predicate, operation: instruction.Operation) Error!void { - _ = self.program.appendInstruction(block_id, .simd8, predicate_value, operation) catch |err| + _ = self.builder.appendInstruction(block_id, .simd8, predicate_value, operation) catch |err| return mapProgramError(err); } @@ -224,7 +225,7 @@ const LoweringState = struct { const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule; if (source_block.parent_function != source_function_id) return Error.InvalidModule; - const target_block_id = self.program.addBlock(source_block.name) catch |err| + const target_block_id = self.builder.addBlock(source_block.name) catch |err| return mapProgramError(err); if (source_block_id.index() >= self.block_map.len or self.block_map[source_block_id.index()] != null) return Error.InvalidModule; @@ -232,14 +233,36 @@ const LoweringState = struct { } const source_entry = function.entry_block orelse return Error.InvalidModule; - self.program.setEntryBlock(try self.mappedBlock(source_entry)) catch |err| return mapProgramError(err); + self.builder.setEntryBlock(try self.mappedBlock(source_entry)) catch |err| return mapProgramError(err); } fn lowerParameters(self: *LoweringState) Error!void { - for (self.lowerer.module.blocks.entries.items) |entry| { - const block = entry orelse continue; - for (block.parameters.items) |parameter_id| - _ = try self.addRegisterLocation(parameter_id, .temporary); + const source_entry = try self.sourceEntryFunction(); + const function = source_entry[1]; + + for (function.blocks.items) |source_block_id| { + const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule; + const target_block_id = try self.mappedBlock(source_block_id); + + for (source_block.parameters.items) |parameter_id| { + const value = self.lowerer.module.values.get(parameter_id) orelse return Error.InvalidModule; + if (try self.isBoolean(value.type)) { + const flag_id = self.builder.addVirtualFlag(.{ .name = value.name }) catch |err| + return mapProgramError(err); + const predicate_value: operand.Predicate = .{ .flag = .{ .virtual = flag_id } }; + try self.putLocation(parameter_id, .{ .predicate = .{ .dynamic = predicate_value } }); + self.builder.addBlockParameter(target_block_id, .{ .flag = flag_id }) catch |err| + return mapProgramError(err); + } else { + const parameter_source = try self.addRegisterLocation(parameter_id, .temporary); + const register_id = switch (parameter_source.register) { + .virtual => |id| id, + else => return Error.InvalidLoweredProgram, + }; + self.builder.addBlockParameter(target_block_id, .{ .register = register_id }) catch |err| + return mapProgramError(err); + } + } } } @@ -437,7 +460,7 @@ const LoweringState = struct { => return Error.UnsupportedOperation, }; - const flag_id = self.program.addVirtualFlag(.{ .name = result_value.name }) catch |err| + const flag_id = self.builder.addVirtualFlag(.{ .name = result_value.name }) catch |err| return mapProgramError(err); const predicate_value: operand.Predicate = .{ .flag = .{ .virtual = flag_id } }; @@ -551,8 +574,7 @@ const LoweringState = struct { for (function.blocks.items) |source_block_id| { const source_block = self.lowerer.module.blocks.get(source_block_id) orelse return Error.InvalidModule; const target_block_id = try self.mappedBlock(source_block_id); - const target_block = self.program.blocks.getMut(target_block_id) orelse return Error.InvalidLoweredProgram; - target_block.structured_control = switch (source_block.structured_control) { + const structured_control: instruction.StructuredControl = switch (source_block.structured_control) { .none => .none, .selection => |selection| .{ .selection = .{ .merge_block = try self.mappedBlock(selection.merge_block), @@ -562,30 +584,26 @@ const LoweringState = struct { .continue_block = try self.mappedBlock(loop.continue_block), } }, }; + self.builder.setStructuredControl(target_block_id, structured_control) catch |err| + return mapProgramError(err); const source_terminator = source_block.terminator orelse return Error.InvalidModule; const target_terminator: instruction.Terminator = switch (source_terminator) { - .branch => |edge| branch: { - try self.lowerEdgeCopies(allocator, target_block_id, edge, null); - break :branch .{ .jump = try self.mappedBlock(edge.target) }; - }, + .branch => |edge| .{ .jump = try self.lowerEdge(allocator, edge) }, .conditional_branch => |branch| conditional: { switch (try self.predicate(branch.condition)) { .constant => |condition| { const edge = if (condition) branch.true_edge else branch.false_edge; - try self.lowerEdgeCopies(allocator, target_block_id, edge, null); - break :conditional .{ .jump = try self.mappedBlock(edge.target) }; + break :conditional .{ .jump = try self.lowerEdge(allocator, edge) }; }, .dynamic => |condition| { - try self.lowerEdgeCopies(allocator, target_block_id, branch.true_edge, condition); - try self.lowerEdgeCopies(allocator, target_block_id, branch.false_edge, .{ - .flag = condition.flag, - .inverse = !condition.inverse, - }); + const true_edge = try self.lowerEdge(allocator, branch.true_edge); + errdefer allocator.free(true_edge.arguments); + const false_edge = try self.lowerEdge(allocator, branch.false_edge); break :conditional .{ .conditional_branch = .{ .predicate = condition, - .true_block = try self.mappedBlock(branch.true_edge.target), - .false_block = try self.mappedBlock(branch.false_edge.target), + .true_edge = true_edge, + .false_edge = false_edge, } }; }, } @@ -595,48 +613,44 @@ const LoweringState = struct { .discard => return Error.UnsupportedTerminator, .@"unreachable" => .@"unreachable", }; - self.program.setTerminator(target_block_id, target_terminator) catch |err| + defer freeTerminatorArguments(allocator, target_terminator); + self.builder.setTerminator(target_block_id, target_terminator) catch |err| return mapProgramError(err); } } - fn lowerEdgeCopies( - self: *LoweringState, - allocator: std.mem.Allocator, - source_block: ids.BlockId, - edge: shader_ir.module.Edge, - predicate_value: ?operand.Predicate, - ) Error!void { + fn lowerEdge(self: *LoweringState, allocator: std.mem.Allocator, edge: shader_ir.module.Edge) Error!instruction.Edge { const target_source_block = self.lowerer.module.blocks.get(edge.target) orelse return Error.InvalidModule; - if (edge.arguments.len != target_source_block.parameters.items.len) return Error.InvalidModule; - if (edge.arguments.len == 0) - return; - - const temporaries = try allocator.alloc(ids.VirtualRegisterId, edge.arguments.len); - defer allocator.free(temporaries); - - // Capture every source before writing any destination so loop backedges and - // swaps retain the parallel-copy semantics of shared-IR block arguments. - for (edge.arguments, 0..) |argument_id, index| { - const argument = try self.source(argument_id); - const temporary = try self.addRegister(argument.type, .temporary, null); - temporaries[index] = temporary; - try self.appendMove(source_block, predicate_value, .{ - .register = .{ .virtual = temporary }, - .type = argument.type, - }, argument); + const arguments = try allocator.alloc(pseudo.EdgeArgument, edge.arguments.len); + errdefer allocator.free(arguments); + for (edge.arguments, arguments) |argument_id, *argument| { + argument.* = switch (try self.location(argument_id)) { + .source => |source_value| .{ .source = source_value }, + .predicate => |predicate_value| .{ .predicate = predicate_value }, + }; } - for (target_source_block.parameters.items, temporaries) |parameter_id, temporary| { - const destination_value = try self.destination(parameter_id); - try self.appendMove(source_block, predicate_value, destination_value, self.registerSource(temporary, destination_value.type)); - } + return .{ + .target = try self.mappedBlock(edge.target), + .arguments = arguments, + }; } }; +fn freeTerminatorArguments(allocator: std.mem.Allocator, terminator: instruction.Terminator) void { + switch (terminator) { + .jump => |edge| allocator.free(edge.arguments), + .conditional_branch => |branch| { + allocator.free(branch.true_edge.arguments); + allocator.free(branch.false_edge.arguments); + }, + else => {}, + } +} + pub const Lowerer = struct { module: *shader_ir.module.Module, device_info: device.DeviceInfo, @@ -690,7 +704,7 @@ pub const Lowerer = struct { var state: LoweringState = .{ .lowerer = self, - .program = &program, + .builder = Builder.init(&program), .block_map = block_map, .value_locations = value_locations, }; @@ -701,7 +715,12 @@ pub const Lowerer = struct { try state.lowerControlAndTerminators(allocator); program.properties.instructions_selected = true; - program.properties.block_parameters_lowered = true; + validator.validate(&program) catch return Error.InvalidLoweredProgram; + + block_arguments.run(allocator, &program) catch |err| return switch (err) { + error.OutOfMemory => Error.OutOfMemory, + else => Error.InvalidLoweredProgram, + }; validator.validate(&program) catch return Error.InvalidLoweredProgram; return program; } @@ -805,7 +824,7 @@ fn expectLoweringError(source: []const u8, expected: Error) !void { return error.TestExpectedError; } -test "Lower: basic shader" { +test "[ir] Lower: basic shader" { const source = \\shader vertex @main \\{ @@ -838,8 +857,6 @@ test "Lower: basic shader" { \\ \\%value: vgrf u32[8], class(temporary), size(32), alignment(32), spillable \\%sum: vgrf u32[8], class(temporary), size(32), alignment(32), spillable - \\%v2: vgrf u32[8], class(temporary), size(32), alignment(32), spillable - \\%v3: vgrf u32[8], class(temporary), size(32), alignment(32), spillable \\%condition: vflag \\ \\.entry: @@ -848,26 +865,30 @@ test "Lower: basic shader" { \\ conditional_branch (+%condition), .left, .right \\ \\.left: - \\ [simd8] mov %v2:u32, %sum:u32 - \\ [simd8] mov %value:u32, %v2:u32 - \\ jump .merge + \\ jump .b4 \\ \\.right: - \\ [simd8] mov %v3:u32, 2:u32 - \\ [simd8] mov %value:u32, %v3:u32 - \\ jump .merge + \\ jump .b5 \\ \\.merge: \\ [simd8] store_output location(0), component(0), %value:u32 \\ end_thread \\ + \\.b4: + \\ [simd8] parallel_copy [%value:u32 <- %sum:u32] + \\ jump .merge + \\ + \\.b5: + \\ [simd8] parallel_copy [%value:u32 <- 2:u32] + \\ jump .merge + \\ \\ ; try expectLowered(source, expected); } -test "Lower: control flow" { +test "[ir] Lower: control flow" { const source = \\shader vertex @main \\{ @@ -913,7 +934,7 @@ test "Lower: control flow" { try expectLowered(source, expected); } -test "Lower: function call" { +test "[ir] Lower: function call" { const source = \\shader vertex @main \\{ @@ -940,7 +961,6 @@ test "Lower: function call" { \\; .dispatch_width: simd8 \\ \\%result: vgrf u32[8], class(temporary), size(32), alignment(32), spillable - \\%v1: vgrf u32[8], class(temporary), size(32), alignment(32), spillable \\ \\.entry: \\ jump .b2 @@ -949,8 +969,10 @@ test "Lower: function call" { \\ end_thread \\ \\.b2: - \\ [simd8] mov %v1:u32, 1:u32 - \\ [simd8] mov %result:u32, %v1:u32 + \\ jump .b3 + \\ + \\.b3: + \\ [simd8] parallel_copy [%result:u32 <- 1:u32] \\ jump .b1 \\ \\ @@ -959,7 +981,7 @@ test "Lower: function call" { try expectLowered(source, expected); } -test "Lower: unary/binary operations" { +test "[ir] Lower: unary/binary operations" { const source = \\shader vertex @main \\{ @@ -1008,7 +1030,7 @@ test "Lower: unary/binary operations" { }, &.{}); } -test "Lower: selects and bitcasts" { +test "[ir] Lower: selects and bitcasts" { const source = \\shader vertex @main \\{ @@ -1050,7 +1072,7 @@ test "Lower: selects and bitcasts" { }); } -test "Lower: vertex interfaces" { +test "[ir] Lower: vertex interfaces" { const source = \\shader vertex @main \\{ @@ -1079,7 +1101,7 @@ test "Lower: vertex interfaces" { }, &.{}); } -test "Lower: constant conditional branch" { +test "[ir] Lower: constant conditional branch" { const source = \\shader vertex @main \\{ @@ -1106,7 +1128,42 @@ test "Lower: constant conditional branch" { }); } -test "Lower: unsupported operations" { +test "[ir] Lower: boolean block parameter" { + const source = + \\shader vertex @main + \\{ + \\ %one: constant u32 = bits(0x1) + \\ %two: constant u32 = bits(0x2) + \\ %never: constant bool = false + \\ fn @main() -> void + \\ { + \\ .entry(): + \\ %condition: bool = cmp_unsigned_less %one, %two + \\ conditional_branch %condition, .left(), .right() + \\ .left(): + \\ branch .merge(%condition) + \\ .right(): + \\ branch .merge(%never) + \\ .merge(%merged: bool): + \\ conditional_branch %merged, .taken(), .not_taken() + \\ .taken(): + \\ return + \\ .not_taken(): + \\ return + \\ } + \\} + ; + + try expectLoweredFragments(source, &.{ + "%condition: vflag", + "%merged: vflag", + "parallel_copy [%merged <- (+%condition)]", + "parallel_copy [%merged <- false]", + ".merge:\n conditional_branch (+%merged), .taken, .not_taken", + }, &.{}); +} + +test "[ir] Lower: unsupported operations" { try expectLoweringError( \\shader vertex @main \\{ @@ -1149,7 +1206,7 @@ test "Lower: unsupported operations" { , Error.UnsupportedType); } -test "Lower: unreachable terminator" { +test "[ir] Lower: unreachable terminator" { const source = \\shader vertex @main \\{ @@ -1166,7 +1223,7 @@ test "Lower: unreachable terminator" { }, &.{}); } -test "Lower: unsupported target configuration" { +test "[ir] Lower: unsupported target configuration" { var module = try shader_ir.parser.parseString(std.testing.allocator, \\shader vertex @main \\{