From b3c4248cae3b87951c8349f97992015427d7614f Mon Sep 17 00:00:00 2001 From: Kbz-8 Date: Mon, 13 Jul 2026 18:50:13 +0200 Subject: [PATCH] [Flint] implementing fences and improving image/buffer copy --- README.md | 26 +++--- src/intel/FlintCommandBuffer.zig | 11 ++- src/intel/FlintDeviceMemory.zig | 4 +- src/intel/FlintFence.zig | 137 ++++++++++++++++++++++++++++-- src/intel/FlintQueue.zig | 75 +++++++++++++++-- src/intel/blitter.zig | 99 ++++++++++++++++++---- src/intel/copy_commands.zig | 139 +++++++++++++++++++++++++++++++ src/intel/i915/drm.zig | 9 +- src/intel/i915/kmd.zig | 48 +++++------ src/intel/kmd.zig | 29 ++++++- src/intel/xe/kmd.zig | 8 +- src/vulkan/utils.zig | 18 ++-- 12 files changed, 505 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index bec73ac..d040c8e 100644 --- a/README.md +++ b/README.md @@ -455,7 +455,7 @@ vkAllocateDescriptorSets | ⚙️ WIP vkAllocateMemory | ✅ Implemented vkBeginCommandBuffer | ✅ Implemented vkBindBufferMemory | ✅ Implemented -vkBindImageMemory | ⚙️ WIP +vkBindImageMemory | ✅ Implemented vkCmdBeginQuery | ⚙️ WIP vkCmdBeginRenderPass | ⚙️ WIP vkCmdBindDescriptorSets | ⚙️ WIP @@ -467,9 +467,9 @@ vkCmdClearAttachments | ⚙️ WIP vkCmdClearColorImage | ⚙️ WIP vkCmdClearDepthStencilImage | ⚙️ WIP vkCmdCopyBuffer | ✅ Implemented -vkCmdCopyBufferToImage | ⚙️ WIP -vkCmdCopyImage | ⚙️ WIP -vkCmdCopyImageToBuffer | ⚙️ WIP +vkCmdCopyBufferToImage | ✅ Implemented +vkCmdCopyImage | ✅ Implemented +vkCmdCopyImageToBuffer | ✅ Implemented vkCmdCopyQueryPoolResults | ⚙️ WIP vkCmdDispatch | ⚙️ WIP vkCmdDispatchBaseKHR | ⚙️ WIP @@ -510,10 +510,10 @@ vkCreateDescriptorPool | ⚙️ WIP vkCreateDescriptorSetLayout | ⚙️ WIP vkCreateDevice | ✅ Implemented vkCreateEvent | ⚙️ WIP -vkCreateFence | ⚙️ WIP +vkCreateFence | ✅ Implemented vkCreateFramebuffer | ⚙️ WIP vkCreateGraphicsPipelines | ⚙️ WIP -vkCreateImage | ⚙️ WIP +vkCreateImage | ✅ Implemented vkCreateImageView | ⚙️ WIP vkCreateInstance | ✅ Implemented vkCreatePipelineCache | ⚙️ WIP @@ -535,9 +535,9 @@ vkDestroyDescriptorPool | ⚙️ WIP vkDestroyDescriptorSetLayout | ⚙️ WIP vkDestroyDevice | ✅ Implemented vkDestroyEvent | ⚙️ WIP -vkDestroyFence | ⚙️ WIP +vkDestroyFence | ✅ Implemented vkDestroyFramebuffer | ⚙️ WIP -vkDestroyImage | ⚙️ WIP +vkDestroyImage | ✅ Implemented vkDestroyImageView | ⚙️ WIP vkDestroyInstance | ✅ Implemented vkDestroyPipeline | ⚙️ WIP @@ -550,7 +550,7 @@ vkDestroySemaphore | ⚙️ WIP vkDestroyShaderModule | ⚙️ WIP vkDestroySurfaceKHR | ⚙️ WIP vkDestroySwapchainKHR | ⚙️ WIP -vkDeviceWaitIdle | ⚙️ WIP +vkDeviceWaitIdle | ✅ Implemented vkEndCommandBuffer | ✅ Implemented vkEnumerateDeviceExtensionProperties | ✅ Implemented vkEnumerateDeviceLayerProperties | ⚙️ WIP @@ -573,7 +573,7 @@ vkGetDeviceMemoryCommitment | ⚙️ WIP vkGetDeviceProcAddr | ✅ Implemented vkGetDeviceQueue | ✅ Implemented vkGetEventStatus | ⚙️ WIP -vkGetFenceStatus | ⚙️ WIP +vkGetFenceStatus | ✅ Implemented vkGetImageMemoryRequirements | ⚙️ WIP vkGetImageSparseMemoryRequirements | ⚙️ WIP vkGetImageSubresourceLayout | ⚙️ WIP @@ -603,17 +603,17 @@ vkMergePipelineCaches | ⚙️ WIP vkQueueBindSparse | ⚙️ WIP vkQueuePresentKHR | ⚙️ WIP vkQueueSubmit | ✅ Implemented -vkQueueWaitIdle | ⚙️ WIP +vkQueueWaitIdle | ✅ Implemented vkResetCommandBuffer | ✅ Implemented vkResetCommandPool | ✅ Implemented vkResetDescriptorPool | ⚙️ WIP vkResetEvent | ⚙️ WIP -vkResetFences | ⚙️ WIP +vkResetFences | ✅ Implemented vkResetQueryPool | ⚙️ WIP vkSetEvent | ⚙️ WIP vkUnmapMemory | ✅ Implemented vkUpdateDescriptorSets | ⚙️ WIP -vkWaitForFences | ⚙️ WIP +vkWaitForFences | ✅ Implemented [Herein](https://vulkan-driver.kbz8.me/cts/flint/) is the reckoning of the Vulkan 1.0 conformance trials laid bare. Judge it as thou wilt. diff --git a/src/intel/FlintCommandBuffer.zig b/src/intel/FlintCommandBuffer.zig index 4a59d30..d2914a2 100644 --- a/src/intel/FlintCommandBuffer.zig +++ b/src/intel/FlintCommandBuffer.zig @@ -94,7 +94,7 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void { allocator.destroy(self); } -pub fn submitGpuBatch(self: *Self) VkError!void { +pub fn submitGpuBatch(self: *Self, syncs: []const kmd.SyncDependency) VkError!void { try self.interface.submit(); defer self.interface.finish() catch {}; @@ -102,7 +102,7 @@ pub fn submitGpuBatch(self: *Self) VkError!void { const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner)); const allocator = self.interface.host_allocator.allocator(); - try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.batch.items, self.relocations.items); + try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.batch.items, self.relocations.items, syncs); } pub fn begin(interface: *Interface, info: *const vk.CommandBufferBeginInfo) VkError!void { @@ -245,12 +245,11 @@ pub fn copyBufferToImage(interface: *Interface, src: *base.Buffer, dst: *base.Im } pub fn copyImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageCopy) VkError!void { - _ = interface; - _ = src; + const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); _ = src_layout; - _ = dst; _ = dst_layout; - _ = regions; + for (regions) |region| + try copy.copyImage(self, src, dst, region); } pub fn copyImageToBuffer(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Buffer, regions: []const vk.BufferImageCopy) VkError!void { diff --git a/src/intel/FlintDeviceMemory.zig b/src/intel/FlintDeviceMemory.zig index 6caebad..7513eb1 100644 --- a/src/intel/FlintDeviceMemory.zig +++ b/src/intel/FlintDeviceMemory.zig @@ -3,7 +3,7 @@ const vk = @import("vulkan"); const base = @import("base"); const FlintDevice = @import("FlintDevice.zig"); -const Kmd = @import("kmd.zig"); +const kmd = @import("kmd.zig"); const VkError = base.VkError; @@ -11,7 +11,7 @@ const Self = @This(); pub const Interface = base.DeviceMemory; interface: Interface, -allocation: Kmd.Memory, +allocation: kmd.Memory, pub fn create(device: *FlintDevice, allocator: std.mem.Allocator, size: vk.DeviceSize, memory_type_index: u32) VkError!*Self { const self = allocator.create(Self) catch return VkError.OutOfHostMemory; diff --git a/src/intel/FlintFence.zig b/src/intel/FlintFence.zig index 440a251..16659f8 100644 --- a/src/intel/FlintFence.zig +++ b/src/intel/FlintFence.zig @@ -1,20 +1,73 @@ const std = @import("std"); const vk = @import("vulkan"); const base = @import("base"); +const kmd = @import("kmd.zig"); const VkError = base.VkError; const Device = base.Device; +const FlintDevice = @import("FlintDevice.zig"); + +const drm_syncobj_create = 0xbf; +const drm_syncobj_destroy = 0xc0; +const drm_syncobj_wait = 0xc3; +const drm_syncobj_reset = 0xc4; +const drm_syncobj_signal = 0xc5; + +const syncobj_create_signaled: u32 = 1 << 0; +const syncobj_wait_all: u32 = 1 << 0; +const syncobj_wait_for_submit: u32 = 1 << 1; + +const SyncObjCreate = extern struct { + handle: u32, + flags: u32, +}; + +const SyncObjDestroy = extern struct { + handle: u32, + pad: u32, +}; + +const SyncObjWait = extern struct { + handles: u64, + timeout_nsec: i64, + count_handles: u32, + flags: u32, + first_signaled: u32, + pad: u32, + deadline_nsec: u64, +}; + +const SyncObjArray = extern struct { + handles: u64, + count_handles: u32, + pad: u32, +}; const Self = @This(); pub const Interface = base.Fence; interface: Interface, +handle: u32, pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.FenceCreateInfo) VkError!*Self { const self = allocator.create(Self) catch return VkError.OutOfHostMemory; errdefer allocator.destroy(self); var interface = try Interface.init(device, allocator, info); + const flint_device: *FlintDevice = @alignCast(@fieldParentPtr("interface", device)); + var create_info = SyncObjCreate{ + .handle = 0, + .flags = if (info.flags.signaled_bit) syncobj_create_signaled else 0, + }; + + base.utils.ioctl( + try flint_device.kmd.file(), + device.io(), + kmd.drmIoctlIowr(drm_syncobj_create, SyncObjCreate), + &create_info, + ) catch return VkError.DeviceLost; + + errdefer destroyHandle(flint_device, device.io(), create_info.handle); interface.vtable = &.{ .destroy = destroy, @@ -26,32 +79,104 @@ pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.Fen self.* = .{ .interface = interface, + .handle = create_info.handle, }; return self; } pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void { const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); + const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", interface.owner)); + destroyHandle(device, interface.owner.io(), self.handle); allocator.destroy(self); } pub fn getStatus(interface: *Interface) VkError!void { - const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); - _ = self; + wait(interface, 0) catch |err| switch (err) { + VkError.Timeout => return VkError.NotReady, + else => return err, + }; } pub fn reset(interface: *Interface) VkError!void { const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); - _ = self; + const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", interface.owner)); + var handles = [_]u32{self.handle}; + var reset_info = SyncObjArray{ + .handles = @intFromPtr(&handles), + .count_handles = handles.len, + .pad = 0, + }; + base.utils.ioctl( + try device.kmd.file(), + interface.owner.io(), + kmd.drmIoctlIowr(drm_syncobj_reset, SyncObjArray), + &reset_info, + ) catch return VkError.DeviceLost; } pub fn signal(interface: *Interface) VkError!void { const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); - _ = self; + const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", interface.owner)); + var handles = [_]u32{self.handle}; + var signal_info = SyncObjArray{ + .handles = @intFromPtr(&handles), + .count_handles = handles.len, + .pad = 0, + }; + base.utils.ioctl( + try device.kmd.file(), + interface.owner.io(), + kmd.drmIoctlIowr(drm_syncobj_signal, SyncObjArray), + &signal_info, + ) catch return VkError.DeviceLost; } pub fn wait(interface: *Interface, timeout: u64) VkError!void { const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); - _ = self; - _ = timeout; + const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", interface.owner)); + var handles = [_]u32{self.handle}; + var wait_info = SyncObjWait{ + .handles = @intFromPtr(&handles), + .timeout_nsec = absoluteTimeout(interface.owner.io(), timeout), + .count_handles = handles.len, + .flags = syncobj_wait_all | syncobj_wait_for_submit, + .first_signaled = 0, + .pad = 0, + .deadline_nsec = 0, + }; + + const errno = base.utils.ioctlErrno( + try device.kmd.file(), + interface.owner.io(), + kmd.drmIoctlIowr(drm_syncobj_wait, SyncObjWait), + &wait_info, + ) catch return VkError.DeviceLost; + + return switch (errno) { + .SUCCESS => {}, + .TIME => VkError.Timeout, + else => VkError.DeviceLost, + }; +} + +fn destroyHandle(device: *FlintDevice, io: std.Io, handle: u32) void { + var destroy_info = SyncObjDestroy{ + .handle = handle, + .pad = 0, + }; + base.utils.ioctl( + device.kmd.file() catch return, + io, + kmd.drmIoctlIowr(drm_syncobj_destroy, SyncObjDestroy), + &destroy_info, + ) catch {}; +} + +fn absoluteTimeout(io: std.Io, timeout: u64) i64 { + if (timeout == std.math.maxInt(u64)) return std.math.maxInt(i64); + + const now = std.Io.Clock.awake.now(io).nanoseconds; + const deadline: i96 = now + @as(i96, timeout); + return @intCast(@min(deadline, std.math.maxInt(i64))); } diff --git a/src/intel/FlintQueue.zig b/src/intel/FlintQueue.zig index 798b516..7e4d3da 100644 --- a/src/intel/FlintQueue.zig +++ b/src/intel/FlintQueue.zig @@ -3,6 +3,9 @@ const vk = @import("vulkan"); const base = @import("base"); const FlintCommandBuffer = @import("FlintCommandBuffer.zig"); +const FlintDevice = @import("FlintDevice.zig"); +const FlintFence = @import("FlintFence.zig"); +const kmd = @import("kmd.zig"); const VkError = base.VkError; @@ -10,24 +13,35 @@ const Self = @This(); pub const Interface = base.Queue; interface: Interface, +completion: *FlintFence, pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, family_index: u32, flags: vk.DeviceQueueCreateFlags) VkError!*Interface { const self = allocator.create(Self) catch return VkError.OutOfHostMemory; errdefer allocator.destroy(self); var interface = try Interface.init(allocator, device, index, family_index, flags); + const completion = try FlintFence.create(device, allocator, &.{ + .s_type = .fence_create_info, + .p_next = null, + .flags = .{ .signaled_bit = true }, + }); + errdefer completion.interface.destroy(allocator); interface.dispatch_table = &.{ .bindSparse = bindSparse, .submit = submit, .waitIdle = waitIdle, }; - self.* = .{ .interface = interface }; + self.* = .{ + .interface = interface, + .completion = completion, + }; return &self.interface; } pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) VkError!void { const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); + self.completion.interface.destroy(allocator); allocator.destroy(self); } @@ -39,7 +53,20 @@ pub fn bindSparse(interface: *Interface, info: []const vk.BindSparseInfo, fence: } pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*base.Fence) VkError!void { - _ = interface; + const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); + const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", interface.owner)); + + var remaining_batches: usize = 0; + for (infos) |info| { + for (info.command_buffers.items) |command_buffer| { + const intel_command_buffer: *FlintCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer)); + if (intel_command_buffer.batch.items.len != 0) remaining_batches += 1; + } + } + const batch_count = remaining_batches; + + try self.completion.interface.reset(); + for (infos) |info| { for (info.wait_semaphores.items) |semaphore| { try semaphore.wait(); @@ -47,18 +74,54 @@ pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*bas for (info.command_buffers.items) |command_buffer| { const intel_command_buffer: *FlintCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer)); - try intel_command_buffer.submitGpuBatch(); + if (intel_command_buffer.batch.items.len == 0) { + try intel_command_buffer.submitGpuBatch(&.{}); + continue; + } + + remaining_batches -= 1; + var syncs: [2]kmd.SyncDependency = undefined; + var sync_count: usize = 0; + if (remaining_batches == 0) { + syncs[sync_count] = .{ .handle = self.completion.handle, .signal = true }; + sync_count += 1; + if (fence) |base_fence| { + const flint_fence: *FlintFence = @alignCast(@fieldParentPtr("interface", base_fence)); + syncs[sync_count] = .{ .handle = flint_fence.handle, .signal = true }; + sync_count += 1; + } + } + try intel_command_buffer.submitGpuBatch(syncs[0..sync_count]); } for (info.signal_semaphores.items) |semaphore| { try semaphore.signal(); } } - if (fence) |value| { - try value.signal(); + + if (batch_count == 0) { + var syncs: [2]kmd.SyncDependency = undefined; + var sync_count: usize = 1; + syncs[0] = .{ .handle = self.completion.handle, .signal = true }; + if (fence) |base_fence| { + const flint_fence: *FlintFence = @alignCast(@fieldParentPtr("interface", base_fence)); + syncs[sync_count] = .{ .handle = flint_fence.handle, .signal = true }; + sync_count += 1; + } + + // A real no-op request preserves queue order: its output fence cannot + // signal ahead of an earlier request that is still running. + try device.kmd.submitBatch( + interface.owner.io(), + interface.host_allocator.allocator(), + &.{}, + &.{}, + syncs[0..sync_count], + ); } } pub fn waitIdle(interface: *Interface) VkError!void { - _ = interface; + const self: *Self = @alignCast(@fieldParentPtr("interface", interface)); + try self.completion.interface.wait(std.math.maxInt(u64)); } diff --git a/src/intel/blitter.zig b/src/intel/blitter.zig index ba1b5d1..78109b4 100644 --- a/src/intel/blitter.zig +++ b/src/intel/blitter.zig @@ -11,6 +11,18 @@ const MemoryRange = @import("MemoryRange.zig"); const copy = @import("copy_commands.zig"); pub fn blitImageRegion(cmd: *FlintCommandBuffer, src: *base.Image, dst: *base.Image, region: vk.ImageBlit) VkError!void { + if (region.src_subresource.mip_level >= src.mip_levels or + region.dst_subresource.mip_level >= dst.mip_levels) + return VkError.ValidationFailed; + if (src.samples.toInt() != 1 or dst.samples.toInt() != 1) + return VkError.ValidationFailed; + if (src.image_type != dst.image_type) + return VkError.FeatureNotPresent; + try validateAspect(src, region.src_subresource.aspect_mask); + try validateAspect(dst, region.dst_subresource.aspect_mask); + if (region.src_subresource.aspect_mask != region.dst_subresource.aspect_mask) + return VkError.ValidationFailed; + const src_image: *FlintImage = @alignCast(@fieldParentPtr("interface", src)); const dst_image: *FlintImage = @alignCast(@fieldParentPtr("interface", dst)); const src_format = src.formatFromAspect(region.src_subresource.aspect_mask); @@ -18,9 +30,29 @@ pub fn blitImageRegion(cmd: *FlintCommandBuffer, src: *base.Image, dst: *base.Im const src_texel_size = base.format.texelSize(src_format); const dst_texel_size = base.format.texelSize(dst_format); - if (src_format != dst_format or base.format.isCompressed(src_format) or base.format.isCompressed(dst_format) or src_texel_size != dst_texel_size) + if (base.format.isCompressed(src_format) or base.format.isCompressed(dst_format)) return VkError.FormatNotSupported; + // XY_SRC_COPY_BLT does not perform component conversion. Different Vulkan + // names are safe only when they describe the same bytes and values. + if (!bitwiseCompatibleFormats(src_format, dst_format) or src_texel_size != dst_texel_size) + return VkError.FormatNotSupported; + + if ((base.format.isDepth(src.format) or base.format.isStencil(src.format) or + base.format.isDepth(dst.format) or base.format.isStencil(dst.format)) and + src.format != dst.format) + return VkError.FormatNotSupported; + + const src_extent = src_image.getMipLevelExtent(region.src_subresource.mip_level); + const dst_extent = dst_image.getMipLevelExtent(region.dst_subresource.mip_level); + try validateOffsets(region.src_offsets, src_extent); + try validateOffsets(region.dst_offsets, dst_extent); + + const src_layer_count = try resolveLayerCount(src, region.src_subresource); + const dst_layer_count = try resolveLayerCount(dst, region.dst_subresource); + if (src_layer_count != dst_layer_count) + return VkError.ValidationFailed; + var src_0 = region.src_offsets[0]; var src_1 = region.src_offsets[1]; var dst_0 = region.dst_offsets[0]; @@ -32,23 +64,12 @@ pub fn blitImageRegion(cmd: *FlintCommandBuffer, src: *base.Image, dst: *base.Im } } - if (dst_0.x < 0 or dst_0.y < 0 or dst_0.z < 0 or - dst_0.x == dst_1.x or dst_0.y == dst_1.y or dst_0.z == dst_1.z) - return VkError.ValidationFailed; - - const src_extent = src_image.getMipLevelExtent(region.src_subresource.mip_level); - const layer_count = if (region.dst_subresource.layer_count == vk.REMAINING_ARRAY_LAYERS) - dst.array_layers - region.dst_subresource.base_array_layer - else - region.dst_subresource.layer_count; - - if (layer_count == 0 or - layer_count > src.array_layers - region.src_subresource.base_array_layer or - layer_count > dst.array_layers - region.dst_subresource.base_array_layer) + if (dst_0.x == dst_1.x or dst_0.y == dst_1.y or dst_0.z == dst_1.z or + src_0.x == src_1.x or src_0.y == src_1.y or src_0.z == src_1.z) return VkError.ValidationFailed; const copy_whole_rows = src_1.x - src_0.x == dst_1.x - dst_0.x and src_1.x > src_0.x; - for (0..layer_count) |layer| { + for (0..src_layer_count) |layer| { const src_layer = region.src_subresource.base_array_layer + @as(u32, @intCast(layer)); const dst_layer = region.dst_subresource.base_array_layer + @as(u32, @intCast(layer)); @@ -87,6 +108,54 @@ pub fn blitImageRegion(cmd: *FlintCommandBuffer, src: *base.Image, dst: *base.Im } } +fn validateAspect(image: *const base.Image, aspect: vk.ImageAspectFlags) VkError!void { + const valid_aspects = base.format.toAspect(image.format); + if (aspect.toInt() == 0 or @popCount(aspect.toInt()) != 1 or + aspect.subtract(valid_aspects).toInt() != 0) + return VkError.ValidationFailed; +} + +fn validateOffsets(offsets: [2]vk.Offset3D, extent: vk.Extent3D) VkError!void { + for (offsets) |offset| { + if (offset.x < 0 or offset.y < 0 or offset.z < 0 or + offset.x > extent.width or offset.y > extent.height or offset.z > extent.depth) + return VkError.ValidationFailed; + } +} + +fn resolveLayerCount(image: *const base.Image, subresource: vk.ImageSubresourceLayers) VkError!u32 { + if (subresource.base_array_layer >= image.array_layers) + return VkError.ValidationFailed; + const available = image.array_layers - subresource.base_array_layer; + const count = if (subresource.layer_count == vk.REMAINING_ARRAY_LAYERS) + available + else + subresource.layer_count; + if (count == 0 or count > available) + return VkError.ValidationFailed; + return count; +} + +fn bitwiseCompatibleFormats(src: vk.Format, dst: vk.Format) bool { + if (src == dst) return true; + + // On little-endian the packed A8B8G8R8 formats have the same byte + // representation and component interpretation as their R8G8B8A8 peers. + return switch (src) { + .r8g8b8a8_unorm => dst == .a8b8g8r8_unorm_pack32, + .a8b8g8r8_unorm_pack32 => dst == .r8g8b8a8_unorm, + .r8g8b8a8_snorm => dst == .a8b8g8r8_snorm_pack32, + .a8b8g8r8_snorm_pack32 => dst == .r8g8b8a8_snorm, + .r8g8b8a8_uint => dst == .a8b8g8r8_uint_pack32, + .a8b8g8r8_uint_pack32 => dst == .r8g8b8a8_uint, + .r8g8b8a8_sint => dst == .a8b8g8r8_sint_pack32, + .a8b8g8r8_sint_pack32 => dst == .r8g8b8a8_sint, + .r8g8b8a8_srgb => dst == .a8b8g8r8_srgb_pack32, + .a8b8g8r8_srgb_pack32 => dst == .r8g8b8a8_srgb, + else => false, + }; +} + fn nearestBlitCoordinate(src_0: i32, src_1: i32, dst_0: i32, dst_1: i32, dst: i32, extent: u32) usize { const numerator = @as(i64, 2 * (dst - dst_0) + 1) * @as(i64, src_1 - src_0); const denominator = @as(i64, 2 * (dst_1 - dst_0)); diff --git a/src/intel/copy_commands.zig b/src/intel/copy_commands.zig index 1796e34..ee4487a 100644 --- a/src/intel/copy_commands.zig +++ b/src/intel/copy_commands.zig @@ -90,6 +90,145 @@ pub fn copyBufferImage(cmd: *FlintCommandBuffer, buffer: *base.Buffer, image_int } } +pub fn copyImage(cmd: *FlintCommandBuffer, src_interface: *base.Image, dst_interface: *base.Image, region: vk.ImageCopy) VkError!void { + const depth_stencil: vk.ImageAspectFlags = .{ .depth_bit = true, .stencil_bit = true }; + if (region.src_subresource.aspect_mask == depth_stencil and region.dst_subresource.aspect_mask == depth_stencil) { + var single_aspect_region = region; + single_aspect_region.src_subresource.aspect_mask = .{ .depth_bit = true }; + single_aspect_region.dst_subresource.aspect_mask = .{ .depth_bit = true }; + try copyImageSingleAspect(cmd, src_interface, dst_interface, single_aspect_region); + + single_aspect_region.src_subresource.aspect_mask = .{ .stencil_bit = true }; + single_aspect_region.dst_subresource.aspect_mask = .{ .stencil_bit = true }; + try copyImageSingleAspect(cmd, src_interface, dst_interface, single_aspect_region); + return; + } + + try copyImageSingleAspect(cmd, src_interface, dst_interface, region); +} + +fn copyImageSingleAspect(cmd: *FlintCommandBuffer, src_interface: *base.Image, dst_interface: *base.Image, region: vk.ImageCopy) VkError!void { + if (region.extent.width == 0 or region.extent.height == 0 or region.extent.depth == 0) + return; + if (region.src_offset.x < 0 or region.src_offset.y < 0 or region.src_offset.z < 0 or + region.dst_offset.x < 0 or region.dst_offset.y < 0 or region.dst_offset.z < 0) + return VkError.ValidationFailed; + if (@popCount(region.src_subresource.aspect_mask.toInt()) != 1 or + @popCount(region.dst_subresource.aspect_mask.toInt()) != 1) + return VkError.ValidationFailed; + if (region.src_subresource.aspect_mask.subtract(base.format.toAspect(src_interface.format)).toInt() != 0 or + region.dst_subresource.aspect_mask.subtract(base.format.toAspect(dst_interface.format)).toInt() != 0) + return VkError.ValidationFailed; + if (src_interface.samples.toInt() != dst_interface.samples.toInt()) + return VkError.ValidationFailed; + + const src: *FlintImage = @alignCast(@fieldParentPtr("interface", src_interface)); + const dst: *FlintImage = @alignCast(@fieldParentPtr("interface", dst_interface)); + const src_format = src_interface.formatFromAspect(region.src_subresource.aspect_mask); + const dst_format = dst_interface.formatFromAspect(region.dst_subresource.aspect_mask); + const bytes_per_block = base.format.texelSize(src_format); + + if (bytes_per_block != base.format.texelSize(dst_format)) + return VkError.FormatNotSupported; + + const src_block_width = base.format.blockWidth(src_format); + const src_block_height = base.format.blockHeight(src_format); + const dst_block_width = base.format.blockWidth(dst_format); + const dst_block_height = base.format.blockHeight(dst_format); + if (base.format.isCompressed(src_format) and base.format.isCompressed(dst_format) and + (src_block_width != dst_block_width or src_block_height != dst_block_height)) + return VkError.FormatNotSupported; + + const src_x: usize = @intCast(region.src_offset.x); + const src_y: usize = @intCast(region.src_offset.y); + const src_z: usize = @intCast(region.src_offset.z); + const dst_x: usize = @intCast(region.dst_offset.x); + const dst_y: usize = @intCast(region.dst_offset.y); + const dst_z: usize = @intCast(region.dst_offset.z); + if (@mod(src_x, src_block_width) != 0 or @mod(src_y, src_block_height) != 0 or + @mod(dst_x, dst_block_width) != 0 or @mod(dst_y, dst_block_height) != 0) + return VkError.ValidationFailed; + + if (region.src_subresource.mip_level >= src_interface.mip_levels or + region.dst_subresource.mip_level >= dst_interface.mip_levels) + return VkError.ValidationFailed; + const src_extent = src.getMipLevelExtent(region.src_subresource.mip_level); + const dst_extent = dst.getMipLevelExtent(region.dst_subresource.mip_level); + const copy_blocks_x = base.format.blockCountX(src_format, region.extent.width); + const copy_blocks_y = base.format.blockCountY(src_format, region.extent.height); + const src_block_x = src_x / src_block_width; + const src_block_y = src_y / src_block_height; + const dst_block_x = dst_x / dst_block_width; + const dst_block_y = dst_y / dst_block_height; + if (src_block_x + copy_blocks_x > base.format.blockCountX(src_format, src_extent.width) or + src_block_y + copy_blocks_y > base.format.blockCountY(src_format, src_extent.height) or + dst_block_x + copy_blocks_x > base.format.blockCountX(dst_format, dst_extent.width) or + dst_block_y + copy_blocks_y > base.format.blockCountY(dst_format, dst_extent.height) or + src_z + region.extent.depth > src_extent.depth or + dst_z + region.extent.depth > dst_extent.depth) + return VkError.ValidationFailed; + + const src_layer_count = try resolveLayerCount(src_interface, region.src_subresource); + const dst_layer_count = try resolveLayerCount(dst_interface, region.dst_subresource); + if (src_layer_count != dst_layer_count) + return VkError.ValidationFailed; + + const src_row_pitch = src_interface.getRowPitchMemSizeForMipLevel(region.src_subresource.aspect_mask, region.src_subresource.mip_level); + const src_slice_pitch = src_interface.getSliceMemSizeForMipLevel(region.src_subresource.aspect_mask, region.src_subresource.mip_level); + const src_sample_pitch = src.getMipLevelSize(region.src_subresource.aspect_mask, region.src_subresource.mip_level); + const dst_row_pitch = dst_interface.getRowPitchMemSizeForMipLevel(region.dst_subresource.aspect_mask, region.dst_subresource.mip_level); + const dst_slice_pitch = dst_interface.getSliceMemSizeForMipLevel(region.dst_subresource.aspect_mask, region.dst_subresource.mip_level); + const dst_sample_pitch = dst.getMipLevelSize(region.dst_subresource.aspect_mask, region.dst_subresource.mip_level); + const copy_row_size = copy_blocks_x * bytes_per_block; + + for (0..src_layer_count) |layer| { + const src_subresource_offset = try src.getSubresourceOffset( + region.src_subresource.aspect_mask, + region.src_subresource.mip_level, + region.src_subresource.base_array_layer + @as(u32, @intCast(layer)), + ); + const dst_subresource_offset = try dst.getSubresourceOffset( + region.dst_subresource.aspect_mask, + region.dst_subresource.mip_level, + region.dst_subresource.base_array_layer + @as(u32, @intCast(layer)), + ); + + for (0..src_interface.samples.toInt()) |sample| { + for (0..region.extent.depth) |z| { + for (0..copy_blocks_y) |row| { + const src_offset = src_subresource_offset + + sample * src_sample_pitch + + (src_z + z) * src_slice_pitch + + (src_block_y + row) * src_row_pitch + + src_block_x * bytes_per_block; + const dst_offset = dst_subresource_offset + + sample * dst_sample_pitch + + (dst_z + z) * dst_slice_pitch + + (dst_block_y + row) * dst_row_pitch + + dst_block_x * bytes_per_block; + try emitLinearCopy( + cmd, + try MemoryRange.fromImage(src_interface, src_offset, copy_row_size), + try MemoryRange.fromImage(dst_interface, dst_offset, copy_row_size), + ); + } + } + } + } +} + +fn resolveLayerCount(image: *const base.Image, subresource: vk.ImageSubresourceLayers) VkError!u32 { + if (subresource.base_array_layer >= image.array_layers) + return VkError.ValidationFailed; + const layer_count = if (subresource.layer_count == vk.REMAINING_ARRAY_LAYERS) + image.array_layers - subresource.base_array_layer + else + subresource.layer_count; + if (layer_count == 0 or layer_count > image.array_layers - subresource.base_array_layer) + return VkError.ValidationFailed; + return layer_count; +} + pub fn copyRangeFromRegion(buffer: *base.Buffer, offset: vk.DeviceSize, size: vk.DeviceSize) VkError!MemoryRange { return MemoryRange.fromBuffer(buffer, offset, size); } diff --git a/src/intel/i915/drm.zig b/src/intel/i915/drm.zig index 9026fe6..fddd156 100644 --- a/src/intel/i915/drm.zig +++ b/src/intel/i915/drm.zig @@ -5,13 +5,15 @@ pub const i915_gem_create = 0x1b; pub const i915_gem_mmap_gtt = 0x24; pub const i915_gem_set_domain = 0x1f; pub const i915_gem_execbuffer2 = 0x29; -pub const i915_gem_wait = 0x2c; pub const gem_close = 0x09; pub const i915_mmap_offset_wb = 2; pub const i915_gem_domain_cpu = 0x00000001; pub const i915_gem_domain_gtt = 0x00000040; pub const i915_exec_blt = 3 << 0; +pub const i915_exec_fence_array: u64 = 1 << 19; +pub const i915_exec_fence_wait: u32 = 1 << 0; +pub const i915_exec_fence_signal: u32 = 1 << 1; pub const exec_object_write = 1 << 2; pub const mi_flush_dw: u32 = (0x26 << 23) | 3; @@ -74,8 +76,7 @@ pub const ExecBuffer2 = extern struct { rsvd2: u64, }; -pub const GemWait = extern struct { - bo_handle: u32, +pub const ExecFence = extern struct { + handle: u32, flags: u32, - timeout_ns: i64, }; diff --git a/src/intel/i915/kmd.zig b/src/intel/i915/kmd.zig index 4a20ab6..39931e8 100644 --- a/src/intel/i915/kmd.zig +++ b/src/intel/i915/kmd.zig @@ -38,7 +38,7 @@ pub const Device = struct { .handle = 0, .pad = 0, }; - base.utils.ioctl(self.card.handle, io, drmIoctlIowr(drm.command_base + drm.i915_gem_create, drm.GemCreate), &create) catch return VkError.OutOfDeviceMemory; + base.utils.ioctl(self.card.handle, io, common_kmd.drmIoctlIowr(drm.command_base + drm.i915_gem_create, drm.GemCreate), &create) catch return VkError.OutOfDeviceMemory; var memory = Memory{ .handle = create.handle, @@ -51,7 +51,7 @@ pub const Device = struct { return memory; } - pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const common_kmd.Relocation) VkError!void { + pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const common_kmd.Relocation, syncs: []const common_kmd.SyncDependency) VkError!void { const trailer_words = 6; const batch_size = (commands.len + trailer_words) * @sizeOf(u32); var batch = try self.allocateMemory(io, batch_size); @@ -121,6 +121,16 @@ pub const Device = struct { .rsvd2 = 0, }) catch return VkError.OutOfHostMemory; + var exec_fences = std.ArrayList(drm.ExecFence).empty; + defer exec_fences.deinit(allocator); + for (syncs) |sync| { + exec_fences.append(allocator, .{ + .handle = sync.handle, + .flags = (if (sync.wait) drm.i915_exec_fence_wait else 0) | + (if (sync.signal) drm.i915_exec_fence_signal else 0), + }) catch return VkError.OutOfHostMemory; + } + var execbuffer = drm.ExecBuffer2{ .buffers_ptr = @intFromPtr(objects.items.ptr), .buffer_count = @intCast(objects.items.len), @@ -128,25 +138,13 @@ pub const Device = struct { .batch_len = @intCast(batch_size), .DR1 = 0, .DR4 = 0, - .num_cliprects = 0, - .cliprects_ptr = 0, - .flags = drm.i915_exec_blt, + .num_cliprects = @intCast(exec_fences.items.len), + .cliprects_ptr = if (exec_fences.items.len == 0) 0 else @intFromPtr(exec_fences.items.ptr), + .flags = drm.i915_exec_blt | (if (exec_fences.items.len == 0) 0 else drm.i915_exec_fence_array), .rsvd1 = 0, .rsvd2 = 0, }; - base.utils.ioctl(self.card.handle, io, drmIoctlIowr(drm.command_base + drm.i915_gem_execbuffer2, drm.ExecBuffer2), &execbuffer) catch return VkError.DeviceLost; - - for (object_handles.items) |handle| { - const object = objects.items[std.mem.indexOfScalar(u32, object_handles.items, handle).?]; - if (object.flags & drm.exec_object_write == 0) continue; - - var wait = drm.GemWait{ - .bo_handle = handle, - .flags = 0, - .timeout_ns = -1, - }; - base.utils.ioctl(self.card.handle, io, drmIoctlIowr(drm.command_base + drm.i915_gem_wait, drm.GemWait), &wait) catch return VkError.DeviceLost; - } + base.utils.ioctl(self.card.handle, io, common_kmd.drmIoctlIowr(drm.command_base + drm.i915_gem_execbuffer2, drm.ExecBuffer2), &execbuffer) catch return VkError.DeviceLost; } }; @@ -162,7 +160,7 @@ pub const Memory = struct { .handle = self.handle, .pad = 0, }; - base.utils.ioctl(device.card.handle, io, drmIoctlIow(drm.gem_close, drm.GemClose), &close) catch {}; + base.utils.ioctl(device.card.handle, io, common_kmd.drmIoctlIow(drm.gem_close, drm.GemClose), &close) catch {}; self.* = undefined; } @@ -185,7 +183,7 @@ pub const Memory = struct { .flags = drm.i915_mmap_offset_wb, .extensions = 0, }; - base.utils.ioctl(device.card.handle, io, drmIoctlIowr(drm.command_base + drm.i915_gem_mmap_gtt, drm.GemMmapOffset), &mmap_offset) catch return VkError.MemoryMapFailed; + base.utils.ioctl(device.card.handle, io, common_kmd.drmIoctlIowr(drm.command_base + drm.i915_gem_mmap_gtt, drm.GemMmapOffset), &mmap_offset) catch return VkError.MemoryMapFailed; if (self.size > std.math.maxInt(usize)) return VkError.MemoryMapFailed; const full_size: usize = @intCast(self.size); @@ -227,14 +225,6 @@ pub const Memory = struct { .read_domains = read_domains, .write_domain = write_domain, }; - base.utils.ioctl(device.card.handle, io, drmIoctlIow(drm.command_base + drm.i915_gem_set_domain, drm.GemSetDomain), &domain) catch return VkError.DeviceLost; + base.utils.ioctl(device.card.handle, io, common_kmd.drmIoctlIow(drm.command_base + drm.i915_gem_set_domain, drm.GemSetDomain), &domain) catch return VkError.DeviceLost; } }; - -inline fn drmIoctlIow(nr: u8, comptime T: type) u32 { - return IOCTL.IOW('d', nr, T); -} - -inline fn drmIoctlIowr(nr: u8, comptime T: type) u32 { - return IOCTL.IOWR('d', nr, T); -} diff --git a/src/intel/kmd.zig b/src/intel/kmd.zig index 22d27aa..43afdb0 100644 --- a/src/intel/kmd.zig +++ b/src/intel/kmd.zig @@ -8,6 +8,7 @@ const i915_kmd = @import("i915/kmd.zig"); const xe = @import("xe/kmd.zig"); const VkError = base.VkError; +const IOCTL = std.os.linux.IOCTL; pub const xy_src_copy_blt: u32 = (2 << 29) | (0x53 << 22) | 8; pub const xy_blt_write_alpha: u32 = 1 << 21; @@ -25,6 +26,12 @@ pub const Relocation = struct { write: bool = false, }; +pub const SyncDependency = struct { + handle: u32, + wait: bool = false, + signal: bool = false, +}; + pub const Device = union(lib.KmdType) { Invalid: void, I915: i915_kmd.Device, @@ -55,10 +62,18 @@ pub const Device = union(lib.KmdType) { }; } - pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const Relocation) VkError!void { + pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const Relocation, syncs: []const SyncDependency) VkError!void { return switch (self.*) { - .I915 => |*device| device.submitBatch(io, allocator, commands, relocations), - .Xe => |*device| device.submitBatch(io, allocator, commands, relocations), + .I915 => |*device| device.submitBatch(io, allocator, commands, relocations, syncs), + .Xe => |*device| device.submitBatch(io, allocator, commands, relocations, syncs), + .Invalid => VkError.DeviceLost, + }; + } + + pub fn file(self: *Device) VkError!std.Io.File { + return switch (self.*) { + .I915 => |*device| device.card.handle, + .Xe => |*device| device.card.handle, .Invalid => VkError.DeviceLost, }; } @@ -142,3 +157,11 @@ pub const Memory = union(lib.KmdType) { }; } }; + +pub inline fn drmIoctlIow(nr: u8, comptime T: type) u32 { + return IOCTL.IOW('d', nr, T); +} + +pub inline fn drmIoctlIowr(nr: u8, comptime T: type) u32 { + return IOCTL.IOWR('d', nr, T); +} diff --git a/src/intel/xe/kmd.zig b/src/intel/xe/kmd.zig index 751e259..fbae03c 100644 --- a/src/intel/xe/kmd.zig +++ b/src/intel/xe/kmd.zig @@ -2,6 +2,8 @@ const std = @import("std"); const vk = @import("vulkan"); const base = @import("base"); +const common_kmd = @import("../kmd.zig"); + const VkError = base.VkError; pub const Device = struct { @@ -21,7 +23,7 @@ pub const Device = struct { return VkError.OutOfDeviceMemory; } - pub fn submitBatch(_: *Device, _: std.Io, _: std.mem.Allocator, _: []const u32, _: []const @import("../kmd.zig").Relocation) VkError!void { + pub fn submitBatch(_: *Device, _: std.Io, _: std.mem.Allocator, _: []const u32, _: []const common_kmd.Relocation, _: []const common_kmd.SyncDependency) VkError!void { return VkError.FeatureNotPresent; } }; @@ -43,7 +45,3 @@ pub const Memory = struct { return VkError.FeatureNotPresent; } }; - -test { - std.testing.refAllDecls(@This()); -} diff --git a/src/vulkan/utils.zig b/src/vulkan/utils.zig index f9a6a98..2dcb8f6 100644 --- a/src/vulkan/utils.zig +++ b/src/vulkan/utils.zig @@ -30,19 +30,19 @@ pub fn writePacked(comptime T: type, bytes: []u8, value: T) void { } pub fn ioctl(file: std.Io.File, io: std.Io, request: u32, arg: ?*anyopaque) (std.Io.Cancelable || std.posix.UnexpectedError)!void { + return switch (try ioctlErrno(file, io, request, arg)) { + .SUCCESS => {}, + else => |e| std.posix.unexpectedErrno(e), + }; +} + +pub fn ioctlErrno(file: std.Io.File, io: std.Io, request: u32, arg: ?*anyopaque) std.Io.Cancelable!std.posix.E { const result = try io.operate(.{ .device_io_control = .{ .file = file, .code = request, .arg = arg, } }); - const rc = if (@import("builtin").link_libc) - @as(c_int, @intCast(result.device_io_control)) - else - @as(usize, @bitCast(@as(isize, @intCast(result.device_io_control)))); - - return switch (std.posix.errno(rc)) { - .SUCCESS => {}, - else => |e| std.posix.unexpectedErrno(e), - }; + const rc = result.device_io_control; + return if (rc < 0) @enumFromInt(-rc) else .SUCCESS; }