[Flint] adding gen9 eu encoding

src/intel/compiler/targets/gen9/compute/abi.zig

@@ -0,0 +1,60 @@
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = error{
+    InvalidPayloadLayout,
+};
+
+const thread_header: operand.PhysicalGrf = .{
+    .number = 0,
+    .byte_offset = 0,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (program.properties.compute_abi_lowered)
+        return;
+
+    if (program.payload.header_grf) |header| {
+        if (header.number != thread_header.number or header.byte_offset != thread_header.byte_offset)
+            return Error.InvalidPayloadLayout;
+    }
+    if (program.program_data.payload_grf_count > 1)
+        return Error.InvalidPayloadLayout;
+
+    program.payload.header_grf = thread_header;
+    program.program_data.payload_grf_count = 1;
+    program.properties.compute_abi_lowered = true;
+}
+
+const std = @import("std");
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+test "[gen9] compute ABI: reserve thread header" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    try run(&program);
+
+    try std.testing.expectEqual(thread_header, program.payload.header_grf.?);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+
+    try run(&program);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+}
+
+test "[gen9] compute ABI: reject conflicting payload" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+    program.payload.header_grf = .{ .number = 1 };
+
+    try std.testing.expectError(Error.InvalidPayloadLayout, run(&program));
+    try std.testing.expect(!program.properties.compute_abi_lowered);
+}

src/intel/compiler/targets/gen9/compute/compute.zig

@@ -1,6 +1,11 @@
 const std = @import("std");
 
+pub const abi = @import("abi.zig");
+pub const dispatch = @import("dispatch.zig");
+pub const eu_encoder = @import("eu_encoder.zig");
+pub const kernel_encoder = @import("kernel_encoder.zig");
 pub const message_addresses = @import("message_addresses.zig");
+pub const message_descriptor = @import("message_descriptor.zig");
 pub const message_lowering = @import("message_lowering.zig");
 pub const message_payloads = @import("message_payloads.zig");
 pub const resource_layout = @import("resource_layout.zig");

src/intel/compiler/targets/gen9/compute/dispatch.zig

@@ -0,0 +1,166 @@
+const std = @import("std");
+
+pub const max_surfaces: usize = 4;
+pub const page_size: usize = 4096;
+pub const surface_state_size: usize = 64;
+pub const interface_descriptor_size: usize = 32;
+
+const mocs: u32 = 0x78;
+pub const base_address_delta: u32 = 1 | (mocs << 4);
+const raw_surface_format: u32 = 0x1ff;
+
+pub const Error = error{
+    EmptyBuffer,
+    StateTooLarge,
+    UnsupportedBufferSize,
+    TooManySurfaces,
+};
+
+pub const StateLayout = struct {
+    size: usize,
+    kernel_offset: u32,
+    surface_offsets: [max_surfaces]u32,
+    surface_address_offsets: [max_surfaces]u32,
+    surface_count: u8,
+    binding_table_offset: u32,
+    interface_descriptor_offset: u32,
+};
+
+pub fn writeState(destination: []u8, kernel: []const u8, buffer_sizes: []const u64) Error!StateLayout {
+    if (buffer_sizes.len > max_surfaces)
+        return Error.TooManySurfaces;
+
+    var layout: StateLayout = .{
+        .size = 0,
+        .kernel_offset = 0,
+        .surface_offsets = @splat(0),
+        .surface_address_offsets = @splat(0),
+        .surface_count = @intCast(buffer_sizes.len),
+        .binding_table_offset = 0,
+        .interface_descriptor_offset = 0,
+    };
+
+    var cursor = alignForward(kernel.len, 64);
+    for (buffer_sizes, 0..) |size, index| {
+        cursor = alignForward(cursor, surface_state_size);
+        layout.surface_offsets[index] = @intCast(cursor);
+        layout.surface_address_offsets[index] = @intCast(cursor + 8 * @sizeOf(u32));
+        cursor += surface_state_size;
+        if (size == 0)
+            return Error.EmptyBuffer;
+    }
+
+    cursor = alignForward(cursor, 32);
+    layout.binding_table_offset = @intCast(cursor);
+    cursor += buffer_sizes.len * @sizeOf(u32);
+
+    cursor = alignForward(cursor, 64);
+    layout.interface_descriptor_offset = @intCast(cursor);
+    cursor += interface_descriptor_size;
+    layout.size = alignForward(cursor, page_size);
+    if (layout.size > destination.len or layout.size > page_size)
+        return Error.StateTooLarge;
+
+    @memset(destination[0..layout.size], 0);
+    @memcpy(destination[layout.kernel_offset .. layout.kernel_offset + kernel.len], kernel);
+
+    for (buffer_sizes, 0..) |size, index| {
+        _ = try encodeRawBufferSurface(destination, layout.surface_offsets[index], size);
+        putU32(destination, layout.binding_table_offset + @as(u32, @intCast(index * @sizeOf(u32))), layout.surface_offsets[index]);
+    }
+
+    const idd = layout.interface_descriptor_offset;
+    putU32(destination, idd + 0, layout.kernel_offset);
+    putU32(destination, idd + 4, 0);
+    putU32(destination, idd + 4 * @sizeOf(u32), @as(u32, @intCast(buffer_sizes.len)) | layout.binding_table_offset);
+    putU32(destination, idd + 6 * @sizeOf(u32), 1);
+
+    return layout;
+}
+
+fn encodeRawBufferSurface(destination: []u8, offset: u32, byte_size: u64) Error!void {
+    if (byte_size == 0)
+        return Error.EmptyBuffer;
+
+    const aligned_size = std.mem.alignForward(u64, byte_size, 4);
+    const padded_size = aligned_size + (aligned_size - byte_size);
+    if (padded_size == 0 or padded_size > (@as(u64, 1) << 32))
+        return Error.UnsupportedBufferSize;
+    const length_minus_one: u32 = @intCast(padded_size - 1);
+
+    putU32(destination, offset + 0, (4 << 29) |
+        (raw_surface_format << 18) |
+        (1 << 16) |
+        (1 << 14));
+    putU32(destination, offset + 1 * @sizeOf(u32), mocs << 24);
+    putU32(destination, offset + 2 * @sizeOf(u32), (length_minus_one & 0x7f) |
+        (((length_minus_one >> 7) & 0x3fff) << 16));
+    putU32(destination, offset + 3 * @sizeOf(u32), ((length_minus_one >> 21) & 0x7ff) << 21);
+}
+
+pub const ccStatePointers = [_]u32{
+    0x780e0000,
+    0,
+};
+
+pub const pipelineSelectGpgpu = [_]u32{0x69040302};
+
+pub fn pipeControl(bits: u32) [6]u32 {
+    return .{ 0x7a000004, bits, 0, 0, 0, 0 };
+}
+
+pub const pipe_control = struct {
+    pub const state_invalidate: u32 = 1 << 2;
+    pub const constant_invalidate: u32 = 1 << 3;
+    pub const dc_flush: u32 = 1 << 5;
+    pub const texture_invalidate: u32 = 1 << 10;
+    pub const instruction_invalidate: u32 = 1 << 11;
+    pub const render_target_flush: u32 = 1 << 12;
+    pub const depth_flush: u32 = 1 << 0;
+    pub const cs_stall: u32 = 1 << 20;
+};
+
+pub fn stateBaseAddress() [19]u32 {
+    var words: [19]u32 = @splat(0);
+    words[0] = 0x61010011;
+    words[3] = mocs << 16;
+    words[4] = base_address_delta;
+    words[6] = base_address_delta;
+    words[10] = base_address_delta;
+    words[13] = (1 << 12) | 1;
+    words[15] = (1 << 12) | 1;
+    return words;
+}
+
+pub fn mediaVfeState() [9]u32 {
+    var words: [9]u32 = @splat(0);
+    words[0] = 0x70000007;
+    words[3] = (1 << 16) | (2 << 8);
+    words[5] = 2 << 16;
+    return words;
+}
+
+pub fn interfaceDescriptorLoad(offset: u32) [4]u32 {
+    return .{ 0x70020002, 0, interface_descriptor_size, offset };
+}
+
+pub fn gpgpuWalker(group_count: [3]u32, right_mask: u32) [15]u32 {
+    var words: [15]u32 = @splat(0);
+    words[0] = 0x7105000d;
+    words[7] = group_count[0];
+    words[10] = group_count[1];
+    words[12] = group_count[2];
+    words[13] = right_mask;
+    words[14] = 0xffffffff;
+    return words;
+}
+
+pub const mediaStateFlush = [_]u32{ 0x70040000, 0 };
+
+fn alignForward(value: usize, alignment: usize) usize {
+    return std.mem.alignForward(usize, value, alignment);
+}
+
+fn putU32(destination: []u8, offset: u32, value: u32) void {
+    std.mem.writeInt(u32, destination[offset..][0..@sizeOf(u32)], value, .little);
+}

src/intel/compiler/targets/gen9/compute/eu_encoder.zig

@@ -0,0 +1,250 @@
+const std = @import("std");
+const device = @import("../../../device.zig");
+const ir_instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const message_descriptor = @import("message_descriptor.zig");
+
+pub const Error = error{
+    UnsupportedExecutionSize,
+    UnsupportedDataType,
+    UnsupportedOperand,
+    InvalidRegister,
+    InvalidRegion,
+};
+
+pub const eot_payload_grf: u8 = 112;
+
+pub const EncodedInstruction = struct {
+    words: [2]u64 = .{ 0, 0 },
+
+    pub fn setBits(self: *EncodedInstruction, high: u7, low: u7, value: u64) void {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        const mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width)) << word_low;
+        self.words[word] = (self.words[word] & ~mask) | ((value << word_low) & mask);
+    }
+
+    pub fn bits(self: EncodedInstruction, high: u7, low: u7) u64 {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        return (self.words[word] >> word_low) & (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width));
+    }
+};
+
+const RegisterFile = enum(u2) {
+    architecture = 0,
+    grf = 1,
+    immediate = 3,
+};
+
+const HardwareType = enum(u4) {
+    unsigned_dword = 0,
+    signed_dword = 1,
+    unsigned_word = 2,
+    float = 7,
+};
+
+const Grf = struct {
+    number: u8,
+    byte_offset: u5,
+};
+
+pub fn encodeMove(execution_size: device.ExecutionSize, move: ir_instruction.Move) Error!EncodedInstruction {
+    var encoded = try instructionHeader(1, execution_size);
+    const destination = try resolveGrf(move.destination.register, move.destination.region.byte_offset);
+
+    setDestination(
+        &encoded,
+        .grf,
+        try hardwareType(move.destination.type),
+        destination,
+        try horizontalStride(move.destination.region.horizontal_stride),
+    );
+
+    switch (move.source.register) {
+        .physical_grf => {
+            const source = try resolveGrf(move.source.register, move.source.region.byte_offset);
+            try setSource0Register(&encoded, move.source, source);
+        },
+        .immediate => |immediate| {
+            if (move.source.negate or move.source.absolute)
+                return Error.UnsupportedOperand;
+            setSource0Immediate(&encoded, try hardwareType(move.source.type), immediate);
+        },
+        else => return Error.UnsupportedOperand,
+    }
+
+    return encoded;
+}
+
+pub fn encodeEndThread(header: operand.PhysicalGrf) Error![2]EncodedInstruction {
+    if (header.number != 0 or header.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    var copy = try instructionHeader(1, .simd8);
+    copy.setBits(34, 34, 1); // NoMask
+    setDestination(&copy, .grf, .unsigned_dword, .{ .number = eot_payload_grf, .byte_offset = 0 }, 1);
+    copy.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    copy.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    copy.setBits(76, 69, header.number);
+    copy.setBits(81, 80, 1);
+    copy.setBits(84, 82, 3);
+    copy.setBits(88, 85, 4);
+
+    var send = try instructionHeader(49, .simd8);
+    send.setBits(34, 34, 1); // NoMask
+    setDestination(&send, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    send.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    send.setBits(46, 43, @intFromEnum(HardwareType.unsigned_word));
+    send.setBits(76, 69, eot_payload_grf);
+    send.setBits(81, 80, 1);
+    send.setBits(84, 82, 3);
+    send.setBits(88, 85, 4);
+    send.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    send.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    send.setBits(124, 96, 0x02000010); // mlen=1, no response, do not dereference URB
+    send.setBits(27, 24, 7); // Thread Spawner
+    send.setBits(127, 127, 1);
+
+    return .{ copy, send };
+}
+
+pub fn encodeSurfaceMessage(execution_size: device.ExecutionSize, message: ir_instruction.SurfaceMessage) Error!EncodedInstruction {
+    var encoded = try instructionHeader(49, execution_size);
+    const descriptor = message_descriptor.encode(message);
+    const payload = try resolveGrf(message.payload.base, 0);
+    if (payload.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    if (message.response) |response| {
+        const destination = try resolveGrf(response.base, 0);
+        if (destination.byte_offset != 0)
+            return Error.InvalidRegister;
+        setDestination(&encoded, .grf, .unsigned_word, destination, 1);
+    } else {
+        setDestination(&encoded, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    }
+
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(76, 69, payload.number);
+    encoded.setBits(68, 64, payload.byte_offset);
+    encoded.setBits(81, 80, 1); // horizontal stride 1
+    encoded.setBits(84, 82, 3); // width 8
+    encoded.setBits(88, 85, 4); // vertical stride 8
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(124, 96, descriptor.value);
+    encoded.setBits(27, 24, descriptor.sfid);
+    return encoded;
+}
+
+fn instructionHeader(opcode: u7, execution_size: device.ExecutionSize) Error!EncodedInstruction {
+    var encoded: EncodedInstruction = .{};
+    encoded.setBits(6, 0, opcode);
+    encoded.setBits(23, 21, try executionSize(execution_size));
+    return encoded;
+}
+
+fn setDestination(encoded: *EncodedInstruction, file: RegisterFile, data_type: HardwareType, register: Grf, horizontal_stride: u2) void {
+    encoded.setBits(36, 35, @intFromEnum(file));
+    encoded.setBits(40, 37, @intFromEnum(data_type));
+    encoded.setBits(52, 48, register.byte_offset);
+    encoded.setBits(60, 53, register.number);
+    encoded.setBits(62, 61, horizontal_stride);
+}
+
+fn setSource0Register(encoded: *EncodedInstruction, source: operand.Source, register: Grf) Error!void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(try hardwareType(source.type)));
+    encoded.setBits(68, 64, register.byte_offset);
+    encoded.setBits(76, 69, register.number);
+    encoded.setBits(77, 77, @intFromBool(source.absolute));
+    encoded.setBits(78, 78, @intFromBool(source.negate));
+    encoded.setBits(81, 80, try horizontalStride(source.region.horizontal_stride));
+    encoded.setBits(84, 82, try regionWidth(source.region.width));
+    encoded.setBits(88, 85, try verticalStride(source.region.vertical_stride));
+}
+
+fn setSource0Immediate(encoded: *EncodedInstruction, data_type: HardwareType, immediate: operand.Immediate) void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(46, 43, @intFromEnum(data_type));
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.architecture));
+    encoded.setBits(94, 91, @intFromEnum(data_type));
+    encoded.setBits(127, 96, switch (immediate) {
+        .u32 => |value| value,
+        .i32 => |value| @as(u32, @bitCast(value)),
+        .f32 => |value| @as(u32, @bitCast(value)),
+    });
+}
+
+fn resolveGrf(register: operand.RegisterRef, region_byte_offset: u16) Error!Grf {
+    const physical = switch (register) {
+        .physical_grf => |value| value,
+        else => return Error.UnsupportedOperand,
+    };
+
+    const byte_address = @as(u32, physical.number) * 32 + physical.byte_offset + region_byte_offset;
+    const number = byte_address / 32;
+
+    if (number >= 128)
+        return Error.InvalidRegister;
+
+    return .{
+        .number = @intCast(number),
+        .byte_offset = @intCast(byte_address % 32),
+    };
+}
+
+fn hardwareType(data_type: operand.DataType) Error!HardwareType {
+    return switch (data_type) {
+        .u32 => .unsigned_dword,
+        .i32 => .signed_dword,
+        .f32 => .float,
+        else => Error.UnsupportedDataType,
+    };
+}
+
+fn executionSize(size: device.ExecutionSize) Error!u3 {
+    return switch (size) {
+        .simd1 => 0,
+        .simd8 => 3,
+        else => Error.UnsupportedExecutionSize,
+    };
+}
+
+fn horizontalStride(stride: u8) Error!u2 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn regionWidth(width: u8) Error!u3 {
+    return switch (width) {
+        1 => 0,
+        2 => 1,
+        4 => 2,
+        8 => 3,
+        16 => 4,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn verticalStride(stride: u8) Error!u4 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        8 => 4,
+        16 => 5,
+        32 => 6,
+        else => Error.InvalidRegion,
+    };
+}

src/intel/compiler/targets/gen9/compute/kernel_encoder.zig

@@ -0,0 +1,67 @@
+const std = @import("std");
+
+const eu = @import("eu_encoder.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || eu.Error || error{
+    InvalidProgram,
+    UnsupportedControlFlow,
+    UnsupportedOperation,
+    UnsupportedPredication,
+    EotRegisterUnavailable,
+};
+
+pub fn encode(allocator: std.mem.Allocator, program: *program_ir.Program) Error![]u8 {
+    if (!program.properties.registers_allocated)
+        return Error.InvalidProgram;
+    if (program.program_data.total_grf_count > eu.eot_payload_grf)
+        return Error.EotRegisterUnavailable;
+
+    const entry_id = program.entry_block orelse return Error.InvalidProgram;
+    const entry = program.blocks.get(entry_id) orelse return Error.InvalidProgram;
+    var live_block_count: usize = 0;
+    for (program.blocks.entries.items) |block| {
+        if (block != null)
+            live_block_count += 1;
+    }
+
+    if (live_block_count != 1)
+        return Error.UnsupportedControlFlow;
+
+    var kernel: std.ArrayList(u8) = .empty;
+    errdefer kernel.deinit(allocator);
+
+    for (entry.instructions.items) |instruction_id| {
+        const instruction = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+        if (instruction.predicate != null)
+            return Error.UnsupportedPredication;
+
+        const encoded = switch (instruction.operation) {
+            .move => |move| try eu.encodeMove(instruction.execution_size, move),
+            .surface_message => |message| try eu.encodeSurfaceMessage(instruction.execution_size, message),
+            else => return Error.UnsupportedOperation,
+        };
+        try appendInstruction(allocator, &kernel, encoded);
+    }
+
+    const terminator = entry.terminator orelse return Error.InvalidProgram;
+    switch (terminator) {
+        .end_thread => {
+            const header = program.payload.header_grf orelse return Error.InvalidProgram;
+            const instructions = try eu.encodeEndThread(header);
+            for (instructions) |instruction|
+                try appendInstruction(allocator, &kernel, instruction);
+            program.program_data.total_grf_count = eu.eot_payload_grf + 1;
+        },
+        else => return Error.UnsupportedControlFlow,
+    }
+
+    return kernel.toOwnedSlice(allocator);
+}
+
+fn appendInstruction(allocator: std.mem.Allocator, kernel: *std.ArrayList(u8), instruction: eu.EncodedInstruction) std.mem.Allocator.Error!void {
+    var bytes: [16]u8 = undefined;
+    std.mem.writeInt(u64, bytes[0..8], instruction.words[0], .little);
+    std.mem.writeInt(u64, bytes[8..16], instruction.words[1], .little);
+    try kernel.appendSlice(allocator, &bytes);
+}

src/intel/compiler/targets/gen9/compute/message_addresses.zig

@@ -18,7 +18,7 @@ const AddressAdjustment = struct {
 
 pub fn run(program: *program_ir.Program) Error!void {
     if (!program.properties.messages_lowered)
-        return error.MessagesNotLowered;
+        return Error.MessagesNotLowered;
     if (program.properties.message_addresses_lowered)
         return;
 
@@ -29,18 +29,18 @@ pub fn run(program: *program_ir.Program) Error!void {
         var instruction_index: usize = 0;
 
         while (true) {
-            const block = program.blocks.get(block_id) orelse return error.InvalidProgram;
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
             if (instruction_index >= block.instructions.items.len)
                 break;
 
             const instruction_id = block.instructions.items[instruction_index];
-            const inst = program.instructions.get(instruction_id) orelse return error.InvalidProgram;
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
             const adjustment = addressAdjustment(inst.operation) orelse {
                 instruction_index += 1;
                 continue;
             };
             if (adjustment.address.type != .u32)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
 
             if (adjustment.immediate_offset == 0) {
                 instruction_index += 1;
@@ -51,15 +51,18 @@ pub fn run(program: *program_ir.Program) Error!void {
                 .immediate => |immediate| {
                     const base = switch (immediate) {
                         .u32 => |value| value,
-                        else => return error.InvalidProgram,
+                        else => return Error.InvalidProgram,
                     };
-                    const mutable = program.instructions.getMut(instruction_id) orelse return error.InvalidProgram;
-                    const address = messageAddressMut(&mutable.operation) orelse return error.InvalidProgram;
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
                     address.source.register = .{ .immediate = .{ .u32 = base +% adjustment.immediate_offset } };
                     address.immediate_offset.* = 0;
                     instruction_index += 1;
                 },
-                .virtual, .physical_grf, .architecture => {
+                .virtual,
+                .physical_grf,
+                .architecture,
+                => {
                     const execution_width: u32 = @intFromEnum(inst.execution_size);
                     const size_bytes = execution_width * @sizeOf(u32);
                     const address_register = builder.addVirtualRegister(.{
@@ -82,8 +85,8 @@ pub fn run(program: *program_ir.Program) Error!void {
                         },
                     }) catch |err| return mapBuilderError(err);
 
-                    const mutable = program.instructions.getMut(instruction_id) orelse return error.InvalidProgram;
-                    const address = messageAddressMut(&mutable.operation) orelse return error.InvalidProgram;
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
                     address.source.* = .{
                         .register = .{ .virtual = address_register },
                         .type = .u32,
@@ -92,7 +95,7 @@ pub fn run(program: *program_ir.Program) Error!void {
                     address.immediate_offset.* = 0;
                     instruction_index += 2;
                 },
-                .null => return error.InvalidProgram,
+                .null => return Error.InvalidProgram,
             }
         }
     }
@@ -131,8 +134,8 @@ fn immediateSource(value: u32) operand.Source {
 
 fn mapBuilderError(err: Builder.Error) Error {
     return switch (err) {
-        error.OutOfMemory => error.OutOfMemory,
-        else => error.InvalidProgram,
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
     };
 }
 

src/intel/compiler/targets/gen9/compute/message_descriptor.zig

@@ -0,0 +1,90 @@
+const instruction = @import("../../../ir/instruction.zig");
+
+pub const Descriptor = struct {
+    sfid: u8,
+    value: u32,
+    message_length: u8,
+    response_length: u8,
+};
+
+const dc1_sfid: u8 = 12;
+const simd8_one_channel_control: u8 = 0x2e;
+
+const MessageType = enum(u8) {
+    untyped_surface_read = 1,
+    untyped_surface_write = 9,
+};
+
+pub fn encode(message: instruction.SurfaceMessage) Descriptor {
+    const lengths: struct { message: u8, response: u8 } = switch (message.kind) {
+        .read => .{ .message = 1, .response = 1 },
+        .write => .{ .message = 2, .response = 0 },
+    };
+    const message_type: MessageType = switch (message.kind) {
+        .read => .untyped_surface_read,
+        .write => .untyped_surface_write,
+    };
+
+    return .{
+        .sfid = dc1_sfid,
+        .value = makeDescriptor(
+            message.binding_table,
+            simd8_one_channel_control,
+            message_type,
+            lengths.message,
+            lengths.response,
+        ),
+        .message_length = lengths.message,
+        .response_length = lengths.response,
+    };
+}
+
+fn makeDescriptor(binding_table: u8, message_control: u8, message_type: MessageType, message_length: u8, response_length: u8) u32 {
+    return @as(u32, binding_table) |
+        (@as(u32, message_control) << 8) |
+        (@as(u32, @intFromEnum(message_type)) << 14) |
+        (@as(u32, response_length) << 20) |
+        (@as(u32, message_length) << 25);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface read" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .read,
+        .binding_table = 3,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 1 },
+        .response = .{ .base = .{ .physical_grf = .{ .number = 2 } }, .register_count = 1 },
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 3), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x02106e03), descriptor.value);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface write" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .write,
+        .binding_table = 7,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 2 },
+        .response = null,
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 2), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 0), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 7), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 9), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 0), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 2), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x04026e07), descriptor.value);
+}

src/intel/compiler/targets/gen9/compute/message_lowering.zig

@@ -9,7 +9,7 @@ pub const Error = error{
 
 pub fn run(program: *program_ir.Program) Error!void {
     if (!program.properties.resources_lowered)
-        return error.ResourcesNotLowered;
+        return Error.ResourcesNotLowered;
     if (program.properties.messages_lowered)
         return;
 
@@ -18,12 +18,12 @@ pub fn run(program: *program_ir.Program) Error!void {
         inst.operation = switch (inst.operation) {
             .load_buffer => |op| .{ .surface_read = .{
                 .destination = op.destination,
-                .binding_table = bindingTableIndex(op.buffer) orelse return error.InvalidProgram,
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
                 .address = op.byte_offset,
                 .immediate_offset = op.immediate_offset,
             } },
             .store_buffer => |op| .{ .surface_write = .{
-                .binding_table = bindingTableIndex(op.buffer) orelse return error.InvalidProgram,
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
                 .address = op.byte_offset,
                 .immediate_offset = op.immediate_offset,
                 .data = op.source,
@@ -106,5 +106,5 @@ test "[gen9] compute message lowering: reject unresolved resources" {
     var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
     defer program.deinit();
 
-    try std.testing.expectError(error.ResourcesNotLowered, run(&program));
+    try std.testing.expectError(Error.ResourcesNotLowered, run(&program));
 }

src/intel/compiler/targets/gen9/compute/message_payloads.zig

@@ -14,11 +14,11 @@ pub const Error = std.mem.Allocator.Error || error{
 
 pub fn run(program: *program_ir.Program) Error!void {
     if (!program.properties.message_addresses_lowered)
-        return error.MessageAddressesNotLowered;
+        return Error.MessageAddressesNotLowered;
     if (program.properties.message_payloads_lowered)
         return;
     if (program.device_info.grf_size_bytes != 32)
-        return error.InvalidProgram;
+        return Error.InvalidProgram;
 
     var builder = Builder.init(program);
     for (program.blocks.entries.items, 0..) |entry, block_index| {
@@ -27,17 +27,17 @@ pub fn run(program: *program_ir.Program) Error!void {
         var instruction_index: usize = 0;
 
         while (true) {
-            const block = program.blocks.get(block_id) orelse return error.InvalidProgram;
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
             if (instruction_index >= block.instructions.items.len)
                 break;
 
             const instruction_id = block.instructions.items[instruction_index];
-            const inst = program.instructions.get(instruction_id) orelse return error.InvalidProgram;
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
             const execution_size = inst.execution_size;
             switch (inst.operation) {
                 .surface_read => |op| {
                     if (op.immediate_offset != 0 or op.address.type != .u32)
-                        return error.InvalidProgram;
+                        return Error.InvalidProgram;
                     const response = try responseSpan(op.destination);
                     const payload = try addPayloadRegister(&builder, execution_size, 1);
                     _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
@@ -45,7 +45,7 @@ pub fn run(program: *program_ir.Program) Error!void {
                         .source = op.address,
                     } }) catch |err| return mapBuilderError(err);
 
-                    const mutable = program.instructions.getMut(instruction_id) orelse return error.InvalidProgram;
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
                     mutable.operation = .{ .surface_message = .{
                         .kind = .read,
                         .binding_table = op.binding_table,
@@ -57,7 +57,7 @@ pub fn run(program: *program_ir.Program) Error!void {
                 },
                 .surface_write => |op| {
                     if (op.immediate_offset != 0 or op.address.type != .u32)
-                        return error.InvalidProgram;
+                        return Error.InvalidProgram;
                     const payload = try addPayloadRegister(&builder, execution_size, 2);
                     _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
                         .destination = payloadDestination(payload, 0, .u32),
@@ -68,7 +68,7 @@ pub fn run(program: *program_ir.Program) Error!void {
                         .source = op.data,
                     } }) catch |err| return mapBuilderError(err);
 
-                    const mutable = program.instructions.getMut(instruction_id) orelse return error.InvalidProgram;
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
                     mutable.operation = .{ .surface_message = .{
                         .kind = .write,
                         .binding_table = op.binding_table,
@@ -107,20 +107,20 @@ fn payloadDestination(register: ids.VirtualRegisterId, byte_offset: u16, data_ty
 
 fn responseSpan(destination: operand.Destination) Error!operand.RegisterSpan {
     if (destination.region.byte_offset != 0 or destination.region.horizontal_stride != 1)
-        return error.InvalidProgram;
+        return Error.InvalidProgram;
     return switch (destination.register) {
         .virtual, .physical_grf => .{
             .base = destination.register,
             .register_count = 1,
         },
-        else => error.InvalidProgram,
+        else => Error.InvalidProgram,
     };
 }
 
 fn mapBuilderError(err: Builder.Error) Error {
     return switch (err) {
-        error.OutOfMemory => error.OutOfMemory,
-        else => error.InvalidProgram,
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
     };
 }
 

src/intel/compiler/targets/gen9/compute/pipeline.zig

@@ -10,25 +10,42 @@ const flag_allocation = @import("../flag_allocation.zig");
 const register_allocation = @import("../register_allocation.zig");
 
 const compute = @import("compute.zig");
+const abi = @import("abi.zig");
+const kernel_encoder = @import("kernel_encoder.zig");
 const message_addresses = @import("message_addresses.zig");
 const message_lowering = @import("message_lowering.zig");
 const message_payloads = @import("message_payloads.zig");
 const resource_layout = @import("resource_layout.zig");
 const resource_lowering = @import("resource_lowering.zig");
 
-pub const Error = common_ir.Error || block_arguments.Error || parallel_copies.Error ||
-    message_addresses.Error || message_lowering.Error || message_payloads.Error || resource_layout.Error || resource_lowering.Error || flag_allocation.Error || register_allocation.Error || compute.Error || error{
-    UnsupportedGeneration,
-    UnsupportedStage,
-    UnsupportedDispatchWidth,
-    UnsupportedGrfSize,
-};
+pub const Error = common_ir.Error ||
+    block_arguments.Error ||
+    parallel_copies.Error ||
+    abi.Error ||
+    kernel_encoder.Error ||
+    message_addresses.Error ||
+    message_lowering.Error ||
+    message_payloads.Error ||
+    resource_layout.Error ||
+    resource_lowering.Error ||
+    flag_allocation.Error ||
+    register_allocation.Error ||
+    compute.Error ||
+    error{
+        UnsupportedGeneration,
+        UnsupportedStage,
+        UnsupportedDispatchWidth,
+        UnsupportedGrfSize,
+    };
 
 pub const Artifact = struct {
     program: program_ir.Program,
     resources: resource_layout.Layout,
+    kernel: ?[]u8,
 
     pub fn deinit(self: *Artifact, allocator: std.mem.Allocator) void {
+        if (self.kernel) |kernel|
+            allocator.free(kernel);
         self.resources.deinit(allocator);
         self.program.deinit();
         self.* = undefined;
@@ -55,6 +72,7 @@ pub fn compile(allocator: std.mem.Allocator, module: *shader_ir.module.Module, d
     );
     errdefer program.deinit();
 
+    try abi.run(&program);
     try block_arguments.run(allocator, &program);
     try parallel_copies.run(allocator, &program);
 
@@ -71,8 +89,22 @@ pub fn compile(allocator: std.mem.Allocator, module: *shader_ir.module.Module, d
     try flag_allocation.run(allocator, &program);
     try register_allocation.run(allocator, &program);
 
+    const kernel = kernel_encoder.encode(allocator, &program) catch |err| switch (err) {
+        error.UnsupportedControlFlow,
+        error.UnsupportedOperation,
+        error.UnsupportedPredication,
+        error.UnsupportedExecutionSize,
+        error.UnsupportedDataType,
+        error.UnsupportedOperand,
+        error.EotRegisterUnavailable,
+        => null,
+        else => return err,
+    };
+    errdefer if (kernel) |bytes| allocator.free(bytes);
+
     return .{
         .program = program,
         .resources = resources,
+        .kernel = kernel,
     };
 }

src/intel/compiler/targets/gen9/flag_allocation.zig

@@ -18,15 +18,15 @@ const physical_flag_count = 2;
 
 pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
     if (!program.properties.block_parameters_lowered)
-        return error.BlockParametersNotLowered;
+        return Error.BlockParametersNotLowered;
 
     if (!program.properties.parallel_copies_lowered)
-        return error.ParallelCopiesNotLowered;
+        return Error.ParallelCopiesNotLowered;
 
     if (program.properties.flags_allocated)
         return;
 
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
 
     const allocations = try allocator.alloc(?operand.PhysicalFlag, program.virtual_flags.entries.items.len);
     defer allocator.free(allocations);
@@ -38,9 +38,9 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
     for (allocations) |*allocation| {
         const marker = allocation.* orelse continue;
         if (marker.subregister != std.math.maxInt(u8))
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
 
-        const subregister = std.mem.indexOfScalar(bool, &occupied, false) orelse return error.OutOfFlagRegisters;
+        const subregister = std.mem.indexOfScalar(bool, &occupied, false) orelse return Error.OutOfFlagRegisters;
 
         allocation.* = .{
             .register = 0,
@@ -51,7 +51,7 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
 
     try visitProgramFlags(program, allocations, &occupied, true);
     program.properties.flags_allocated = true;
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
 }
 
 fn visitProgramFlags(
@@ -63,14 +63,14 @@ fn visitProgramFlags(
     for (program.instructions.entries.items, 0..) |entry, instruction_index| {
         _ = entry orelse continue;
         const inst = program.instructions.getMut(ids.InstructionId.fromIndex(instruction_index)) orelse
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
 
         if (inst.predicate) |*predicate|
             try visitFlagRef(program, &predicate.flag, allocations, occupied, rewrite);
 
         switch (inst.operation) {
             .compare => |*compare| try visitFlagRef(program, &compare.destination, allocations, occupied, rewrite),
-            .parallel_copy => return error.ParallelCopiesNotLowered,
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
             else => {},
         }
     }
@@ -78,8 +78,8 @@ fn visitProgramFlags(
     for (program.blocks.entries.items, 0..) |entry, block_index| {
         _ = entry orelse continue;
         const block = program.blocks.getMut(ids.BlockId.fromIndex(block_index)) orelse
-            return error.InvalidProgram;
-        const terminator = if (block.terminator) |*value| value else return error.InvalidProgram;
+            return Error.InvalidProgram;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
 
         switch (terminator.*) {
             .jump => |*edge| try visitEdge(program, edge, allocations, occupied, rewrite),
@@ -129,7 +129,7 @@ fn visitFlagRef(
     switch (flag.*) {
         .virtual => |virtual| {
             if (!program.virtual_flags.isLive(virtual) or virtual.index() >= allocations.len)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
 
             if (!rewrite) {
                 // Mark this virtual flag as referenced without assigning a physical
@@ -139,14 +139,14 @@ fn visitFlagRef(
                 return;
             }
 
-            const physical = allocations[virtual.index()] orelse return error.InvalidProgram;
+            const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
             if (physical.subregister >= physical_flag_count)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
             flag.* = .{ .physical = physical };
         },
         .physical => |physical| {
             if (physical.register != 0 or physical.subregister >= physical_flag_count)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
             occupied[physical.subregister] = true;
         },
     }
@@ -251,7 +251,7 @@ test "[gen9] flag allocation: report exhaustion without rewriting" {
     try program.setTerminator(entry, .end_thread);
     markPrerequisites(&program);
 
-    try std.testing.expectError(error.OutOfFlagRegisters, run(std.testing.allocator, &program));
+    try std.testing.expectError(Error.OutOfFlagRegisters, run(std.testing.allocator, &program));
     try std.testing.expect(!program.properties.flags_allocated);
     try std.testing.expectEqual(first, program.instructions.get(first_compare).?.operation.compare.destination.virtual);
 }

src/intel/compiler/targets/gen9/gen9.zig

@@ -99,6 +99,9 @@ test "[gen9] target: lower 256 KiB SSBO copy loop" {
     const resources = &artifact.resources;
 
     try std.testing.expect(program.properties.common_ir_lowered);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expectEqual(@as(u16, 0), program.payload.header_grf.?.number);
     try std.testing.expect(program.properties.block_parameters_lowered);
     try std.testing.expect(program.properties.parallel_copies_lowered);
     try std.testing.expect(program.properties.flags_allocated);

src/intel/compiler/targets/gen9/register_allocation.zig

@@ -14,15 +14,15 @@ pub const Error = std.mem.Allocator.Error || error{
 
 pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
     if (!program.properties.block_parameters_lowered)
-        return error.BlockParametersNotLowered;
+        return Error.BlockParametersNotLowered;
     if (!program.properties.parallel_copies_lowered)
-        return error.ParallelCopiesNotLowered;
+        return Error.ParallelCopiesNotLowered;
     if (program.properties.registers_allocated)
         return;
 
     const grf_size = program.device_info.grf_size_bytes;
     if (grf_size == 0)
-        return error.InvalidProgram;
+        return Error.InvalidProgram;
 
     const allocations = try allocator.alloc(?operand.PhysicalGrf, program.virtual_registers.entries.items.len);
     defer allocator.free(allocations);
@@ -35,9 +35,9 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
     for (program.virtual_registers.entries.items, 0..) |entry, index| {
         const register = entry orelse continue;
         const start = std.mem.alignForward(usize, next_byte, register.alignment_bytes);
-        const end = std.math.add(usize, start, register.size_bytes) catch return error.OutOfRegisters;
+        const end = std.math.add(usize, start, register.size_bytes) catch return Error.OutOfRegisters;
         if (end > capacity)
-            return error.OutOfRegisters;
+            return Error.OutOfRegisters;
 
         allocations[index] = .{
             .number = @intCast(start / grf_size),
@@ -47,7 +47,7 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
     }
 
     try rewriteProgram(program, allocations);
-    program.program_data.total_grf_count = @intCast(std.math.divCeil(usize, next_byte, grf_size) catch return error.InvalidProgram);
+    program.program_data.total_grf_count = @intCast(std.math.divCeil(usize, next_byte, grf_size) catch return Error.InvalidProgram);
     program.properties.registers_allocated = true;
 }
 
@@ -94,7 +94,7 @@ fn reserveExistingPhysicalRegisters(program: *const program_ir.Program, initial:
                 reserveRegister(&next_byte, op.lhs.register, grf_size);
                 reserveRegister(&next_byte, op.rhs.register, grf_size);
             },
-            .parallel_copy => return error.ParallelCopiesNotLowered,
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
         }
     }
     return next_byte;
@@ -151,15 +151,15 @@ fn rewriteProgram(program: *program_ir.Program, allocations: []const ?operand.Ph
                 try rewriteSource(program, &op.lhs, allocations);
                 try rewriteSource(program, &op.rhs, allocations);
             },
-            .parallel_copy => return error.ParallelCopiesNotLowered,
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
         }
     }
 
     for (program.blocks.entries.items) |*entry| {
         const block = if (entry.*) |*value| value else continue;
         if (block.parameters.items.len != 0)
-            return error.BlockParametersNotLowered;
-        const terminator = if (block.terminator) |*value| value else return error.InvalidProgram;
+            return Error.BlockParametersNotLowered;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
         switch (terminator.*) {
             .jump => |*edge| try rewriteEdge(program, edge, allocations),
             .conditional_branch => |*branch| {
@@ -192,8 +192,8 @@ fn rewriteRegister(program: *const program_ir.Program, register: *operand.Regist
         else => return,
     };
     if (!program.virtual_registers.isLive(virtual) or virtual.index() >= allocations.len)
-        return error.InvalidProgram;
-    const physical = allocations[virtual.index()] orelse return error.InvalidProgram;
+        return Error.InvalidProgram;
+    const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
     register.* = .{ .physical_grf = physical };
 }
 
@@ -262,6 +262,6 @@ test "[gen9] register allocation: report GRF exhaustion" {
     try program.setTerminator(entry, .end_thread);
     markPrerequisites(&program);
 
-    try std.testing.expectError(error.OutOfRegisters, run(std.testing.allocator, &program));
+    try std.testing.expectError(Error.OutOfRegisters, run(std.testing.allocator, &program));
     try std.testing.expect(!program.properties.registers_allocated);
 }