VulkanDriver/src/intel/i915/kmd.zig

const std = @import("std");
const vk = @import("vulkan");
const base = @import("base");

const _i915 = @import("i915.zig");
const common_kmd = @import("../kmd.zig");

const VkError = base.VkError;

const RelocationGroup = struct {
    source_handle: u32,
    entries: std.ArrayList(_i915.RelocationEntry) = .empty,
};

const Mapping = struct {
    bytes: []align(std.heap.page_size_min) u8,

    inline fn slice(self: Mapping, offset: vk.DeviceSize, size: vk.DeviceSize) VkError![]u8 {
        const start: usize = @intCast(offset);
        const len: usize = @intCast(size);
        return self.bytes[start .. start + len];
    }
};

pub const Device = struct {
    card: base.drm.Card,

    pub fn open(io: std.Io, node_path: []const u8) VkError!Device {
        return .{
            .card = base.drm.Card.open(io, node_path) catch return VkError.InitializationFailed,
        };
    }

    pub fn close(self: *Device, io: std.Io) void {
        self.card.close(io);
    }

    pub fn allocateMemory(self: *Device, io: std.Io, size: vk.DeviceSize) VkError!Memory {
        var create = _i915.GemCreate{
            .size = size,
            .handle = 0,
            .pad = 0,
        };
        base.utils.ioctl(
            self.card.handle,
            io,
            common_kmd.drmIoctlIowr(_i915.command_base + _i915.gem_create, _i915.GemCreate),
            &create,
        ) catch return VkError.OutOfDeviceMemory;

        var memory = Memory{
            .handle = create.handle,
            .size = create.size,
            .mapping = null,
        };
        errdefer memory.deinit(self, io);

        try memory.setDomain(self, io, _i915.gem_domain_cpu, 0);
        return memory;
    }

    pub fn submitBatch(
        self: *Device,
        io: std.Io,
        allocator: std.mem.Allocator,
        engine: common_kmd.Engine,
        commands: []const u32,
        relocations: []const common_kmd.Relocation,
        syncs: []const common_kmd.SyncDependency,
    ) VkError!void {
        const trailer_words: usize = switch (engine) {
            .blitter => 6,
            .render => if (commands.len % 2 == 0) 2 else 1,
        };
        const batch_size = (commands.len + trailer_words) * @sizeOf(u32);
        var batch = try self.allocateMemory(io, batch_size);
        defer batch.deinit(self, io);

        {
            const batch_map = try batch.map(self, io, 0, batch_size);
            const batch_words = std.mem.bytesAsSlice(u32, batch_map);
            @memcpy(batch_words[0..commands.len], commands);
            @memset(batch_words[commands.len..], 0);
            switch (engine) {
                .blitter => {
                    batch_words[commands.len] = _i915.mi_flush_dw;
                    batch_words[commands.len + 5] = _i915.mi_batch_buffer_end;
                },
                .render => batch_words[commands.len] = _i915.mi_batch_buffer_end,
            }
            batch.unmap();
        }
        try batch.flushRange(self, io, 0, batch_size);

        var object_handles = std.ArrayList(u32).empty;
        defer object_handles.deinit(allocator);
        for (relocations) |relocation| {
            if (relocation.source_handle) |source| {
                if (std.mem.indexOfScalar(u32, object_handles.items, source) == null)
                    object_handles.append(allocator, source) catch return VkError.OutOfHostMemory;
            }
            if (std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle) == null)
                object_handles.append(allocator, relocation.target_handle) catch return VkError.OutOfHostMemory;
        }
        if (std.mem.indexOfScalar(u32, object_handles.items, batch.handle) == null)
            object_handles.append(allocator, batch.handle) catch return VkError.OutOfHostMemory;

        var groups = std.ArrayList(RelocationGroup).empty;
        defer {
            for (groups.items) |*group| group.entries.deinit(allocator);
            groups.deinit(allocator);
        }
        for (relocations) |relocation| {
            const source = relocation.source_handle orelse batch.handle;
            var group_index = std.mem.indexOfScalar(u32, object_handles.items, source) orelse return VkError.DeviceLost;
            for (groups.items, 0..) |group, index| {
                if (group.source_handle == source) {
                    group_index = index;
                    break;
                }
            } else {
                groups.append(allocator, .{ .source_handle = source }) catch return VkError.OutOfHostMemory;
                group_index = groups.items.len - 1;
            }

            groups.items[group_index].entries.append(allocator, relocationEntry(relocation)) catch return VkError.OutOfHostMemory;
        }

        var objects = std.ArrayList(_i915.ExecObject2).empty;
        defer objects.deinit(allocator);
        for (object_handles.items) |handle| {
            var flags: u64 = 0;
            for (relocations) |relocation| {
                if (relocation.target_handle == handle and relocation.write)
                    flags |= _i915.exec_object_write;
            }

            var relocation_count: u32 = 0;
            var relocs_ptr: u64 = 0;
            for (groups.items) |group| {
                if (group.source_handle == handle) {
                    relocation_count = @intCast(group.entries.items.len);
                    relocs_ptr = @intFromPtr(group.entries.items.ptr);
                    break;
                }
            }
            objects.append(allocator, .{
                .handle = handle,
                .relocation_count = relocation_count,
                .relocs_ptr = relocs_ptr,
                .alignment = 0,
                .offset = 0,
                .flags = flags,
                .rsvd1 = 0,
                .rsvd2 = 0,
            }) catch return VkError.OutOfHostMemory;
        }

        var exec_fences = std.ArrayList(_i915.ExecFence).empty;
        defer exec_fences.deinit(allocator);
        for (syncs) |sync| {
            exec_fences.append(allocator, .{
                .handle = sync.handle,
                .flags = (if (sync.wait) _i915.exec_fence_wait else 0) | (if (sync.signal) _i915.exec_fence_signal else 0),
            }) catch return VkError.OutOfHostMemory;
        }

        var execbuffer = _i915.ExecBuffer2{
            .buffers_ptr = @intFromPtr(objects.items.ptr),
            .buffer_count = @intCast(objects.items.len),
            .batch_start_offset = 0,
            .batch_len = @intCast(batch_size),
            .DR1 = 0,
            .DR4 = 0,
            .num_cliprects = @intCast(exec_fences.items.len),
            .cliprects_ptr = if (exec_fences.items.len == 0) 0 else @intFromPtr(exec_fences.items.ptr),
            .flags = @as(u64, switch (engine) {
                .blitter => _i915.exec_blt,
                .render => _i915.exec_render,
            }) | (if (exec_fences.items.len == 0) 0 else _i915.exec_fence_array),
            .rsvd1 = 0,
            .rsvd2 = 0,
        };
        base.utils.ioctl(
            self.card.handle,
            io,
            common_kmd.drmIoctlIowr(_i915.command_base + _i915.gem_execbuffer2, _i915.ExecBuffer2),
            &execbuffer,
        ) catch return VkError.DeviceLost;
    }
};

fn relocationEntry(relocation: common_kmd.Relocation) _i915.RelocationEntry {
    const domain: u32 = switch (relocation.domain) {
        .none => 0,
        .render => _i915.gem_domain_render,
        .instruction => _i915.gem_domain_instruction,
    };
    return .{
        .target_handle = relocation.target_handle,
        .delta = relocation.delta,
        .offset = relocation.offset,
        // GPU address zero is valid. These locations have not been patched yet,
        // so never let i915 skip the initial relocation, including its delta.
        .presumed_offset = std.math.maxInt(u64),
        .read_domains = if (relocation.read) domain else 0,
        .write_domain = if (relocation.write) domain else 0,
    };
}

pub const Memory = struct {
    handle: u32,
    size: vk.DeviceSize,
    mapping: ?Mapping,

    pub fn deinit(self: *Memory, device: *Device, io: std.Io) void {
        self.unmap();

        var close = _i915.GemClose{
            .handle = self.handle,
            .pad = 0,
        };
        base.utils.ioctl(device.card.handle, io, common_kmd.drmIoctlIow(_i915.gem_close, _i915.GemClose), &close) catch @panic("Caught an error while handling an error");

        self.* = undefined;
    }

    pub fn map(self: *Memory, device: *Device, io: std.Io, offset: vk.DeviceSize, size: vk.DeviceSize) VkError![]u8 {
        if (offset > self.size) return VkError.MemoryMapFailed;
        const available = self.size - offset;
        const map_size = if (size == vk.WHOLE_SIZE) available else size;
        if (map_size > available) return VkError.MemoryMapFailed;
        if (map_size > std.math.maxInt(usize)) return VkError.MemoryMapFailed;

        if (self.mapping) |mapping| {
            return mapping.slice(offset, map_size);
        }

        var mmap_offset = _i915.GemMmapOffset{
            .handle = self.handle,
            .pad = 0,
            .offset = 0,
            .flags = _i915.mmap_offset_wb,
            .extensions = 0,
        };
        base.utils.ioctl(
            device.card.handle,
            io,
            common_kmd.drmIoctlIowr(_i915.command_base + _i915.gem_mmap_gtt, _i915.GemMmapOffset),
            &mmap_offset,
        ) catch return VkError.MemoryMapFailed;

        if (self.size > std.math.maxInt(usize)) return VkError.MemoryMapFailed;
        const full_size: usize = @intCast(self.size);
        const bytes = std.posix.mmap(
            null,
            full_size,
            .{ .READ = true, .WRITE = true },
            .{ .TYPE = .SHARED },
            device.card.handle.handle,
            @intCast(mmap_offset.offset),
        ) catch return VkError.MemoryMapFailed;

        self.mapping = .{ .bytes = bytes };
        return self.mapping.?.slice(offset, map_size);
    }

    pub fn unmap(self: *Memory) void {
        if (self.mapping) |mapping| {
            std.posix.munmap(mapping.bytes);
            self.mapping = null;
        }
    }

    pub fn flushRange(self: *Memory, device: *Device, io: std.Io, offset: vk.DeviceSize, size: vk.DeviceSize) VkError!void {
        _ = offset;
        _ = size;
        try self.setDomain(device, io, _i915.gem_domain_cpu, 0);
    }

    pub fn invalidateRange(self: *Memory, device: *Device, io: std.Io, offset: vk.DeviceSize, size: vk.DeviceSize) VkError!void {
        _ = offset;
        _ = size;
        try self.setDomain(device, io, _i915.gem_domain_cpu, 0);
    }

    fn setDomain(self: *Memory, device: *Device, io: std.Io, read_domains: u32, write_domain: u32) VkError!void {
        var domain = _i915.GemSetDomain{
            .handle = self.handle,
            .read_domains = read_domains,
            .write_domain = write_domain,
        };
        base.utils.ioctl(
            device.card.handle,
            io,
            common_kmd.drmIoctlIow(_i915.command_base + _i915.gem_set_domain, _i915.GemSetDomain),
            &domain,
        ) catch return VkError.DeviceLost;
    }
};

test "[i915] initial relocations force patching even at GPU address zero" {
    const entry = relocationEntry(.{
        .source_handle = 4,
        .target_handle = 4,
        .offset = 64,
        .delta = 1920,
        .read = true,
        .write = false,
        .domain = .render,
    });
    try std.testing.expectEqual(std.math.maxInt(u64), entry.presumed_offset);
    try std.testing.expectEqual(@as(u32, 1920), entry.delta);
    try std.testing.expectEqual(@as(u64, 64), entry.offset);
    try std.testing.expectEqual(@as(u32, 4), entry.target_handle);
    try std.testing.expectEqual(_i915.gem_domain_render, entry.read_domains);
    try std.testing.expectEqual(@as(u32, 0), entry.write_domain);
}