reworking command buffers, adding soft compute routines

src/soft/SoftBuffer.zig

@@ -46,3 +46,35 @@ pub fn getMemoryRequirements(interface: *Interface, requirements: *vk.MemoryRequ
         requirements.alignment = @max(requirements.alignment, lib.MIN_UNIFORM_BUFFER_ALIGNMENT);
     }
 }
+
+pub fn copyBuffer(self: *const Self, dst: *Self, regions: []const vk.BufferCopy) VkError!void {
+    for (regions) |region| {
+        const src_memory = if (self.interface.memory) |memory| memory else return VkError.InvalidDeviceMemoryDrv;
+        const dst_memory = if (dst.interface.memory) |memory| memory else return VkError.InvalidDeviceMemoryDrv;
+
+        const src_map: []u8 = @as([*]u8, @ptrCast(try src_memory.map(region.src_offset, region.size)))[0..region.size];
+        const dst_map: []u8 = @as([*]u8, @ptrCast(try dst_memory.map(region.dst_offset, region.size)))[0..region.size];
+
+        @memcpy(dst_map, src_map);
+
+        src_memory.unmap();
+        dst_memory.unmap();
+    }
+}
+
+pub fn fillBuffer(self: *Self, offset: vk.DeviceSize, size: vk.DeviceSize, data: u32) VkError!void {
+    const memory = if (self.interface.memory) |memory| memory else return VkError.InvalidDeviceMemoryDrv;
+    var memory_map: []u32 = @as([*]u32, @ptrCast(@alignCast(try memory.map(offset, size))))[0..size];
+
+    var bytes = if (size == vk.WHOLE_SIZE) memory.size - offset else size;
+
+    var i: usize = 0;
+    while (bytes >= 4) : ({
+        bytes -= 4;
+        i += 1;
+    }) {
+        memory_map[i] = data;
+    }
+
+    memory.unmap();
+}

src/soft/SoftCommandBuffer.zig

@@ -1,15 +1,32 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
+const lib = @import("lib.zig");
+
+const InterfaceFactory = @import("interface").Interface;
 
 const VkError = base.VkError;
 const Device = base.Device;
 
+const SoftBuffer = @import("SoftBuffer.zig");
+const SoftImage = @import("SoftImage.zig");
+const SoftPipeline = @import("SoftPipeline.zig");
+const SoftDescriptorSet = @import("SoftDescriptorSet.zig");
+
+const ExecutionDevice = @import("device/Device.zig");
+
 const Self = @This();
 pub const Interface = base.CommandBuffer;
 
+const Command = InterfaceFactory(.{
+    .execute = fn (*ExecutionDevice) VkError!void,
+}, null);
+
 interface: Interface,
 
+command_allocator: std.heap.ArenaAllocator,
+commands: std.ArrayList(Command),
+
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.CommandBufferAllocateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
     errdefer allocator.destroy(self);
@@ -28,6 +45,7 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
         .copyBuffer = copyBuffer,
         .copyImage = copyImage,
         .copyImageToBuffer = copyImageToBuffer,
+        .dispatch = dispatch,
         .end = end,
         .fillBuffer = fillBuffer,
         .reset = reset,
@@ -38,7 +56,10 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
 
     self.* = .{
         .interface = interface,
+        .command_allocator = undefined,
+        .commands = .empty,
     };
+    self.command_allocator = .init(self.interface.host_allocator.allocator());
     return self;
 }
 
@@ -47,10 +68,16 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     allocator.destroy(self);
 }
 
-pub fn begin(interface: *Interface, info: *const vk.CommandBufferBeginInfo) VkError!void {
-    // No-op
-    _ = interface;
-    _ = info;
+pub fn execute(self: *Self, device: *ExecutionDevice) VkError!void {
+    self.interface.submit() catch return;
+    for (self.commands.items) |command| {
+        try command.vtable.execute(command.ptr, device);
+    }
+}
+
+pub fn begin(interface: *Interface, _: *const vk.CommandBufferBeginInfo) VkError!void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    self.command_allocator.deinit();
 }
 
 pub fn end(interface: *Interface) VkError!void {
@@ -58,76 +85,236 @@ pub fn end(interface: *Interface) VkError!void {
     _ = interface;
 }
 
-pub fn reset(interface: *Interface, flags: vk.CommandBufferResetFlags) VkError!void {
-    // No-op
-    _ = interface;
-    _ = flags;
+pub fn reset(interface: *Interface, _: vk.CommandBufferResetFlags) VkError!void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+    self.commands.clearAndFree(allocator);
+    if (!self.command_allocator.reset(.{ .retain_with_limit = 16_384 }))
+        return VkError.OutOfHostMemory;
 }
 
 // Commands ====================================================================================================
 
 pub fn bindDescriptorSets(interface: *Interface, bind_point: vk.PipelineBindPoint, first_set: u32, sets: [base.VULKAN_MAX_DESCRIPTOR_SETS]?*base.DescriptorSet, dynamic_offsets: []const u32) VkError!void {
-    // No-op
-    _ = interface;
-    _ = bind_point;
-    _ = first_set;
-    _ = sets;
-    _ = dynamic_offsets;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        bind_point: vk.PipelineBindPoint,
+        first_set: u32,
+        sets: [base.VULKAN_MAX_DESCRIPTOR_SETS]?*base.DescriptorSet,
+        dynamic_offsets: []const u32,
+
+        pub fn execute(impl: *const Impl, device: *ExecutionDevice) VkError!void {
+            for (impl.first_set.., impl.sets[0..]) |i, set| {
+                if (set == null)
+                    break;
+                device.pipeline_states[@intCast(@intFromEnum(impl.bind_point))].sets[i] = @alignCast(@fieldParentPtr("interface", set.?));
+            }
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .bind_point = bind_point,
+        .first_set = first_set,
+        .sets = sets,
+        .dynamic_offsets = dynamic_offsets,
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn bindPipeline(interface: *Interface, bind_point: vk.PipelineBindPoint, pipeline: *base.Pipeline) VkError!void {
-    _ = interface;
-    _ = pipeline;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
 
-    if (bind_point != .graphics and bind_point != .compute) {
-        std.log.warn("Software driver does not support bind point {s}", .{@tagName(bind_point)});
-        return VkError.ValidationFailed;
-    }
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        bind_point: vk.PipelineBindPoint,
+        pipeline: *SoftPipeline,
+
+        pub fn execute(impl: *const Impl, device: *ExecutionDevice) VkError!void {
+            device.pipeline_states[@intCast(@intFromEnum(impl.bind_point))].pipeline = impl.pipeline;
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .bind_point = bind_point,
+        .pipeline = @alignCast(@fieldParentPtr("interface", pipeline)),
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn clearColorImage(interface: *Interface, image: *base.Image, layout: vk.ImageLayout, color: *const vk.ClearColorValue, range: vk.ImageSubresourceRange) VkError!void {
-    // No-op
-    _ = interface;
-    _ = image;
-    _ = layout;
-    _ = color;
-    _ = range;
-}
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
 
-pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.DeviceSize, size: vk.DeviceSize, data: u32) VkError!void {
-    // No-op
-    _ = interface;
-    _ = buffer;
-    _ = offset;
-    _ = size;
-    _ = data;
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        image: *SoftImage,
+        layout: vk.ImageLayout,
+        clear_color: vk.ClearColorValue,
+        range: vk.ImageSubresourceRange,
+
+        pub fn execute(impl: *const Impl, _: *ExecutionDevice) VkError!void {
+            impl.image.clearRange(impl.clear_color, impl.range);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .image = @alignCast(@fieldParentPtr("interface", image)),
+        .layout = layout,
+        .clear_color = color.*,
+        .range = range,
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn copyBuffer(interface: *Interface, src: *base.Buffer, dst: *base.Buffer, regions: []const vk.BufferCopy) VkError!void {
-    // No-op
-    _ = interface;
-    _ = src;
-    _ = dst;
-    _ = regions;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        src: *const SoftBuffer,
+        dst: *SoftBuffer,
+        regions: []const vk.BufferCopy,
+
+        pub fn execute(impl: *const Impl, _: *ExecutionDevice) VkError!void {
+            try impl.src.copyBuffer(impl.dst, impl.regions);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .src = @alignCast(@fieldParentPtr("interface", src)),
+        .dst = @alignCast(@fieldParentPtr("interface", dst)),
+        .regions = allocator.dupe(vk.BufferCopy, regions) catch return VkError.OutOfHostMemory, // Will be freed on cmdbuf reset or destroy
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn copyImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageCopy) VkError!void {
-    // No-op
-    _ = interface;
-    _ = src;
-    _ = src_layout;
-    _ = dst;
-    _ = dst_layout;
-    _ = regions;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        src: *const SoftImage,
+        src_layout: vk.ImageLayout,
+        dst: *SoftImage,
+        dst_layout: vk.ImageLayout,
+        regions: []const vk.ImageCopy,
+
+        pub fn execute(impl: *const Impl, _: *ExecutionDevice) VkError!void {
+            try impl.src.copyImage(impl.src_layout, impl.dst, impl.dst_layout, impl.regions);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .src = @alignCast(@fieldParentPtr("interface", src)),
+        .src_layout = src_layout,
+        .dst = @alignCast(@fieldParentPtr("interface", dst)),
+        .dst_layout = dst_layout,
+        .regions = allocator.dupe(vk.ImageCopy, regions) catch return VkError.OutOfHostMemory, // Will be freed on cmdbuf reset or destroy
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn copyImageToBuffer(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Buffer, regions: []const vk.BufferImageCopy) VkError!void {
-    // No-op
-    _ = interface;
-    _ = src;
-    _ = src_layout;
-    _ = dst;
-    _ = regions;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        src: *const SoftImage,
+        src_layout: vk.ImageLayout,
+        dst: *SoftBuffer,
+        regions: []const vk.BufferImageCopy,
+
+        pub fn execute(impl: *const Impl, _: *ExecutionDevice) VkError!void {
+            try impl.src.copyImageToBuffer(impl.src_layout, impl.dst, impl.regions);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .src = @alignCast(@fieldParentPtr("interface", src)),
+        .src_layout = src_layout,
+        .dst = @alignCast(@fieldParentPtr("interface", dst)),
+        .regions = allocator.dupe(vk.BufferImageCopy, regions) catch return VkError.OutOfHostMemory, // Will be freed on cmdbuf reset or destroy
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
+}
+
+pub fn dispatch(interface: *Interface, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        group_count_x: u32,
+        group_count_y: u32,
+        group_count_z: u32,
+
+        pub fn execute(impl: *const Impl, device: *ExecutionDevice) VkError!void {
+            try device.compute_routines.dispatch(impl.group_count_x, impl.group_count_y, impl.group_count_z);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .group_count_x = group_count_x,
+        .group_count_y = group_count_y,
+        .group_count_z = group_count_z,
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
+}
+
+pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.DeviceSize, size: vk.DeviceSize, data: u32) VkError!void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const allocator = self.command_allocator.allocator();
+
+    const CommandImpl = struct {
+        const Impl = @This();
+
+        buffer: *SoftBuffer,
+        offset: vk.DeviceSize,
+        size: vk.DeviceSize,
+        data: u32,
+
+        pub fn execute(impl: *const Impl, _: *ExecutionDevice) VkError!void {
+            try impl.buffer.fillBuffer(impl.offset, impl.size, impl.data);
+        }
+    };
+
+    const cmd = allocator.create(CommandImpl) catch return VkError.OutOfHostMemory;
+    errdefer allocator.destroy(cmd);
+    cmd.* = .{
+        .buffer = @alignCast(@fieldParentPtr("interface", buffer)),
+        .offset = offset,
+        .size = size,
+        .data = data,
+    };
+    self.commands.append(allocator, Command.from(cmd)) catch return VkError.OutOfHostMemory;
 }
 
 pub fn resetEvent(interface: *Interface, event: *base.Event, stage: vk.PipelineStageFlags) VkError!void {

src/soft/SoftDescriptorSet.zig

@@ -77,6 +77,9 @@ pub fn write(interface: *Interface, write_data: vk.WriteDescriptorSet) VkError!v
                 if (buffer_info.buffer != .null_handle) {
                     const buffer = try NonDispatchable(Buffer).fromHandleObject(buffer_info.buffer);
                     desc.buffer.object = @as(*SoftBuffer, @alignCast(@fieldParentPtr("interface", buffer)));
+                    if (desc.buffer.size == vk.WHOLE_SIZE) {
+                        desc.buffer.size = if (buffer.memory) |memory| memory.size - desc.buffer.offset else return VkError.InvalidDeviceMemoryDrv;
+                    }
                 }
             }
         },

src/soft/SoftImage.zig

@@ -7,6 +7,7 @@ const lib = @import("lib.zig");
 const VkError = base.VkError;
 const Device = base.Device;
 
+const SoftBuffer = @import("SoftBuffer.zig");
 const SoftDevice = @import("SoftDevice.zig");
 
 const Self = @This();
@@ -57,3 +58,49 @@ pub fn clearRange(self: *Self, color: vk.ClearColorValue, range: vk.ImageSubreso
         .r32g32b32a32_sfloat;
     self.clear(.{ .color = color }, clear_format, self.interface.format, range, null);
 }
+
+pub fn copyImage(self: *const Self, self_layout: vk.ImageLayout, dst: *Self, dst_layout: vk.ImageLayout, regions: []const vk.ImageCopy) VkError!void {
+    _ = self;
+    _ = self_layout;
+    _ = dst;
+    _ = dst_layout;
+    _ = regions;
+    std.log.scoped(.commandExecutor).warn("FIXME: implement image to image copy", .{});
+}
+
+pub fn copyImageToBuffer(self: *const Self, self_layout: vk.ImageLayout, dst: *SoftBuffer, regions: []const vk.BufferImageCopy) VkError!void {
+    _ = self_layout;
+    for (regions) |region| {
+        const src_memory = if (self.interface.memory) |memory| memory else return VkError.InvalidDeviceMemoryDrv;
+        const dst_memory = if (dst.interface.memory) |memory| memory else return VkError.InvalidDeviceMemoryDrv;
+
+        const pixel_size: u32 = @intCast(self.interface.getPixelSize());
+        const image_row_pitch: u32 = self.interface.extent.width * pixel_size;
+        const image_size: u32 = @intCast(self.interface.getTotalSize());
+
+        const buffer_row_length: u32 = if (region.buffer_row_length != 0) region.buffer_row_length else region.image_extent.width;
+        const buffer_row_pitch: u32 = buffer_row_length * pixel_size;
+        const buffer_size: u32 = buffer_row_pitch * region.image_extent.height * region.image_extent.depth;
+
+        const src_map: []u8 = @as([*]u8, @ptrCast(try src_memory.map(0, image_size)))[0..image_size];
+        const dst_map: []u8 = @as([*]u8, @ptrCast(try dst_memory.map(region.buffer_offset, buffer_size)))[0..buffer_size];
+
+        const row_size = region.image_extent.width * pixel_size;
+        for (0..self.interface.extent.depth) |z| {
+            for (0..self.interface.extent.height) |y| {
+                const z_as_u32: u32 = @intCast(z);
+                const y_as_u32: u32 = @intCast(y);
+
+                const src_offset = ((@as(u32, @intCast(region.image_offset.z)) + z_as_u32) * self.interface.extent.height + @as(u32, @intCast(region.image_offset.y)) + y_as_u32) * image_row_pitch + @as(u32, @intCast(region.image_offset.x)) * pixel_size;
+                const dst_offset = (z_as_u32 * buffer_row_length * region.image_extent.height + y_as_u32 * buffer_row_length) * pixel_size;
+
+                const src_slice = src_map[src_offset..(src_offset + row_size)];
+                const dst_slice = dst_map[dst_offset..(dst_offset + row_size)];
+                @memcpy(dst_slice, src_slice);
+            }
+        }
+
+        src_memory.unmap();
+        dst_memory.unmap();
+    }
+}

src/soft/SoftPipeline.zig

@@ -15,9 +15,23 @@ const SoftShaderModule = @import("SoftShaderModule.zig");
 const Self = @This();
 pub const Interface = base.Pipeline;
 
-interface: Interface,
+const Shader = struct {
+    module: *SoftShaderModule,
+    runtimes: []spv.Runtime,
+    entry: []const u8,
+};
 
-runtimes: []spv.Runtime,
+const Stages = enum {
+    vertex,
+    tessellation_control,
+    tessellation_evaluation,
+    geometry,
+    fragment,
+    compute,
+};
+
+interface: Interface,
+stages: std.EnumMap(Stages, Shader),
 
 pub fn createCompute(device: *base.Device, allocator: std.mem.Allocator, cache: ?*base.PipelineCache, info: *const vk.ComputePipelineCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -33,19 +47,31 @@ pub fn createCompute(device: *base.Device, allocator: std.mem.Allocator, cache:
     const module = try NonDispatchable(ShaderModule).fromHandleObject(info.stage.module);
     const soft_module: *SoftShaderModule = @alignCast(@fieldParentPtr("interface", module));
 
-    const runtimes = allocator.alloc(spv.Runtime, soft_device.workers.getIdCount()) catch return VkError.OutOfHostMemory;
-    errdefer allocator.free(runtimes);
-
-    for (runtimes) |*runtime| {
-        runtime.* = spv.Runtime.init(allocator, &soft_module.module) catch |err| {
-            std.log.scoped(.SpvRuntimeInit).err("SPIR-V Runtime failed to initialize, {s}", .{@errorName(err)});
-            return VkError.Unknown;
-        };
-    }
+    const device_allocator = soft_device.device_allocator.allocator();
 
     self.* = .{
         .interface = interface,
-        .runtimes = runtimes,
+        .stages = std.EnumMap(Stages, Shader).init(.{
+            .compute = .{
+                .module = blk: {
+                    soft_module.ref();
+                    break :blk soft_module;
+                },
+                .runtimes = blk: {
+                    const runtimes = device_allocator.alloc(spv.Runtime, soft_device.workers.getIdCount()) catch return VkError.OutOfHostMemory;
+                    errdefer device_allocator.free(runtimes);
+
+                    for (runtimes) |*runtime| {
+                        runtime.* = spv.Runtime.init(device_allocator, &soft_module.module) catch |err| {
+                            std.log.scoped(.SpvRuntimeInit).err("SPIR-V Runtime failed to initialize, {s}", .{@errorName(err)});
+                            return VkError.Unknown;
+                        };
+                    }
+                    break :blk runtimes;
+                },
+                .entry = allocator.dupe(u8, std.mem.span(info.stage.p_name)) catch return VkError.OutOfHostMemory,
+            },
+        }),
     };
     return self;
 }
@@ -74,16 +100,24 @@ pub fn createGraphics(device: *base.Device, allocator: std.mem.Allocator, cache:
 
     self.* = .{
         .interface = interface,
-        .runtimes = runtimes,
+        .stages = std.enums.EnumMap(Stages, Shader).init(.{}),
     };
     return self;
 }
 
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    for (self.runtimes) |*runtime| {
-        runtime.deinit(allocator);
+    const soft_device: *SoftDevice = @alignCast(@fieldParentPtr("interface", interface.owner));
+    const device_allocator = soft_device.device_allocator.allocator();
+
+    var it = self.stages.iterator();
+    while (it.next()) |stage| {
+        stage.value.module.unref(allocator);
+        for (stage.value.runtimes) |*runtime| {
+            runtime.deinit(device_allocator);
+        }
+        device_allocator.free(stage.value.runtimes);
+        allocator.free(stage.value.entry);
     }
-    allocator.free(self.runtimes);
     allocator.destroy(self);
 }

src/soft/SoftQueue.zig

@@ -4,11 +4,12 @@ const base = @import("base");
 
 const RefCounter = base.RefCounter;
 
-const Device = @import("device/Device.zig");
+const ExecutionDevice = @import("device/Device.zig");
 const Dispatchable = base.Dispatchable;
 
 const CommandBuffer = base.CommandBuffer;
 const SoftDevice = @import("SoftDevice.zig");
+const SoftCommandBuffer = @import("SoftCommandBuffer.zig");
 
 const VkError = base.VkError;
 
@@ -97,14 +98,13 @@ fn taskRunner(self: *Self, info: Interface.SubmitInfo, p_fence: ?*base.Fence, ru
         command_buffers.deinit(soft_device.device_allocator.allocator());
     }
 
-    var device = Device.init(soft_device);
-    defer device.deinit();
+    var execution_device: ExecutionDevice = .init;
+    execution_device.setup(soft_device);
+    defer execution_device.deinit();
 
     for (info.command_buffers.items) |command_buffer| {
-        command_buffer.submit() catch continue;
-        for (command_buffer.commands.items) |command| {
-            device.execute(&command) catch |err| base.errors.errorLoggerContext(err, "the software command dispatcher");
-        }
+        const soft_command_buffer: *SoftCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+        soft_command_buffer.execute(&execution_device) catch |err| base.errors.errorLoggerContext(err, "the software execution device");
     }
 
     if (p_fence) |fence| {

src/soft/SoftShaderModule.zig

@@ -13,6 +13,10 @@ pub const Interface = base.ShaderModule;
 interface: Interface,
 module: spv.Module,
 
+/// Pipelines need SPIR-V module reference so shader module may not
+/// be destroy on call to `vkDestroyShaderModule`
+ref_count: std.atomic.Value(usize),
+
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.ShaderModuleCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
     errdefer allocator.destroy(self);
@@ -33,12 +37,27 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
             spv.Module.ModuleError.OutOfMemory => return VkError.OutOfHostMemory,
             else => return VkError.ValidationFailed,
         },
+        .ref_count = std.atomic.Value(usize).init(1),
     };
     return self;
 }
 
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    self.unref(allocator);
+}
+
+pub inline fn drop(self: *Self, allocator: std.mem.Allocator) void {
     self.module.deinit(allocator);
     allocator.destroy(self);
 }
+
+pub inline fn ref(self: *Self) void {
+    _ = self.ref_count.fetchAdd(1, .monotonic);
+}
+
+pub inline fn unref(self: *Self, allocator: std.mem.Allocator) void {
+    if (self.ref_count.fetchSub(1, .release) == 1) {
+        self.drop(allocator);
+    }
+}

src/soft/device/ComputeRoutines.zig

@@ -12,16 +12,210 @@ const VkError = base.VkError;
 
 const Self = @This();
 
+const RunData = struct {
+    self: *Self,
+    batch_id: usize,
+    group_count: usize,
+    group_count_x: usize,
+    group_count_y: usize,
+    group_count_z: usize,
+    subgroups_per_workgroup: usize,
+    pipeline: *SoftPipeline,
+};
+
 device: *SoftDevice,
 state: *PipelineState,
+batch_size: usize,
 
 pub fn init(device: *SoftDevice, state: *PipelineState) Self {
     return .{
         .device = device,
         .state = state,
+        .batch_size = 0,
     };
 }
 
 pub fn destroy(self: *Self) void {
     _ = self;
 }
+
+pub fn dispatch(self: *Self, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
+    const group_count: usize = @intCast(group_count_x * group_count_y * group_count_z);
+
+    const pipeline = self.state.pipeline orelse return VkError.InvalidPipelineDrv;
+    const shader = pipeline.stages.getPtr(.compute) orelse return VkError.InvalidPipelineDrv;
+    const spv_module = &shader.module.module;
+    self.batch_size = shader.runtimes.len;
+
+    const invocations_per_subgroup = 4;
+    const invocations_per_workgroup = spv_module.local_size_x * spv_module.local_size_y * spv_module.local_size_z;
+    const subgroups_per_workgroup = @divTrunc(invocations_per_workgroup + invocations_per_subgroup - 1, invocations_per_subgroup);
+
+    var wg: std.Thread.WaitGroup = .{};
+    for (0..@min(self.batch_size, group_count)) |batch_id| {
+        self.device.workers.spawnWg(&wg, runWrapper, .{
+            RunData{
+                .self = self,
+                .batch_id = batch_id,
+                .group_count = group_count,
+                .group_count_x = @as(usize, @intCast(group_count_x)),
+                .group_count_y = @as(usize, @intCast(group_count_y)),
+                .group_count_z = @as(usize, @intCast(group_count_z)),
+                .subgroups_per_workgroup = subgroups_per_workgroup,
+                .pipeline = pipeline,
+            },
+        });
+    }
+    self.device.workers.waitAndWork(&wg);
+}
+
+fn runWrapper(data: RunData) void {
+    @call(.always_inline, run, .{data}) catch |err| {
+        std.log.scoped(.@"SPIR-V runtime").err("SPIR-V runtime catched a '{s}'", .{@errorName(err)});
+        if (@errorReturnTrace()) |trace| {
+            std.debug.dumpStackTrace(trace.*);
+        }
+    };
+}
+
+inline fn run(data: RunData) !void {
+    const allocator = data.self.device.device_allocator.allocator();
+
+    const shader = data.pipeline.stages.getPtrAssertContains(.compute);
+    const rt = &shader.runtimes[data.batch_id];
+
+    const entry = try rt.getEntryPointByName(shader.entry);
+
+    var group_index: usize = data.batch_id;
+    while (group_index < data.group_count) : (group_index += data.self.batch_size) {
+        var modulo: usize = group_index;
+
+        const group_z = @divTrunc(modulo, data.group_count_x * data.group_count_y);
+
+        modulo -= group_z * data.group_count_x * data.group_count_y;
+        const group_y = @divTrunc(modulo, data.group_count_x);
+
+        modulo -= group_y * data.group_count_x;
+        const group_x = modulo;
+
+        try setupWorkgroupBuiltins(
+            data.self,
+            rt,
+            .{
+                @as(u32, @intCast(data.group_count_x)),
+                @as(u32, @intCast(data.group_count_y)),
+                @as(u32, @intCast(data.group_count_z)),
+            },
+            .{
+                @as(u32, @intCast(group_x)),
+                @as(u32, @intCast(group_y)),
+                @as(u32, @intCast(group_z)),
+            },
+        );
+
+        for (0..data.subgroups_per_workgroup) |i| {
+            try setupSubgroupBuiltins(
+                data.self,
+                rt,
+                .{
+                    @as(u32, @intCast(group_x)),
+                    @as(u32, @intCast(group_y)),
+                    @as(u32, @intCast(group_z)),
+                },
+                i,
+            );
+            try data.self.syncDescriptorSets(allocator, rt, true);
+
+            rt.callEntryPoint(allocator, entry) catch |err| switch (err) {
+                spv.Runtime.RuntimeError.OutOfBounds => {},
+                else => return err,
+            };
+
+            try data.self.syncDescriptorSets(allocator, rt, false);
+        }
+    }
+}
+
+fn syncDescriptorSets(self: *Self, allocator: std.mem.Allocator, rt: *spv.Runtime, write: bool) !void {
+    sets: for (self.state.sets[0..], 0..) |set, set_index| {
+        if (set == null)
+            continue :sets;
+
+        bindings: for (set.?.descriptors[0..], 0..) |binding, binding_index| {
+            switch (binding) {
+                .buffer => |buffer_data| if (buffer_data.object) |buffer| {
+                    const memory = if (buffer.interface.memory) |memory| memory else continue :bindings;
+                    const map: []u8 = @as([*]u8, @ptrCast(try memory.map(buffer_data.offset, buffer_data.size)))[0..buffer_data.size];
+                    if (write) {
+                        try rt.writeDescriptorSet(
+                            allocator,
+                            map,
+                            @as(u32, @intCast(set_index)),
+                            @as(u32, @intCast(binding_index)),
+                        );
+                    } else {
+                        try rt.readDescriptorSet(
+                            map,
+                            @as(u32, @intCast(set_index)),
+                            @as(u32, @intCast(binding_index)),
+                        );
+                    }
+                },
+                else => {},
+            }
+        }
+    }
+}
+
+fn setupWorkgroupBuiltins(
+    self: *Self,
+    rt: *spv.Runtime,
+    group_count: [3]u32,
+    group_id: [3]u32,
+) spv.Runtime.RuntimeError!void {
+    const spv_module = &self.state.pipeline.?.stages.getPtrAssertContains(.compute).module.module;
+    const workgroup_size = [3]u32{
+        spv_module.local_size_x,
+        spv_module.local_size_y,
+        spv_module.local_size_z,
+    };
+
+    rt.writeBuiltIn(std.mem.asBytes(&workgroup_size), .WorkgroupSize) catch {};
+    rt.writeBuiltIn(std.mem.asBytes(&group_count), .NumWorkgroups) catch {};
+    rt.writeBuiltIn(std.mem.asBytes(&group_id), .WorkgroupId) catch {};
+}
+
+fn setupSubgroupBuiltins(
+    self: *Self,
+    rt: *spv.Runtime,
+    group_id: [3]u32,
+    local_invocation_index: usize,
+) spv.Runtime.RuntimeError!void {
+    const spv_module = &self.state.pipeline.?.stages.getPtrAssertContains(.compute).module.module;
+    const workgroup_size = [3]u32{
+        spv_module.local_size_x,
+        spv_module.local_size_y,
+        spv_module.local_size_z,
+    };
+    const local_base = [3]u32{
+        workgroup_size[0] * group_id[0],
+        workgroup_size[1] * group_id[1],
+        workgroup_size[2] * group_id[2],
+    };
+    var local_invocation = [3]u32{ 0, 0, 0 };
+
+    var idx: u32 = @intCast(local_invocation_index);
+    local_invocation[2] = @divTrunc(idx, workgroup_size[0] * workgroup_size[1]);
+    idx -= local_invocation[2] * workgroup_size[0] * workgroup_size[1];
+    local_invocation[1] = @divTrunc(idx, workgroup_size[0]);
+    idx -= local_invocation[1] * workgroup_size[0];
+    local_invocation[0] = idx;
+
+    const global_invocation_index = [3]u32{
+        local_base[0] + local_invocation[0],
+        local_base[1] + local_invocation[1],
+        local_base[2] + local_invocation[2],
+    };
+
+    rt.writeBuiltIn(std.mem.asBytes(&global_invocation_index), .GlobalInvocationId) catch {};
+}

src/soft/device/Device.zig

@@ -2,8 +2,6 @@ const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
 
-const copy_routines = @import("copy_routines.zig");
-
 const SoftDescriptorSet = @import("../SoftDescriptorSet.zig");
 const SoftDevice = @import("../SoftDevice.zig");
 const SoftPipeline = @import("../SoftPipeline.zig");
@@ -11,50 +9,31 @@ const SoftPipeline = @import("../SoftPipeline.zig");
 const ComputeRoutines = @import("ComputeRoutines.zig");
 const PipelineState = @import("PipelineState.zig");
 
-const cmd = base.commands;
 const VkError = base.VkError;
 
 const Self = @This();
 
-compute_routine: ComputeRoutines,
+compute_routines: ComputeRoutines,
 
 /// .graphics = 0
 /// .compute = 1
 pipeline_states: [2]PipelineState,
 
-pub fn init(device: *SoftDevice) Self {
-    var self: Self = undefined;
+pub const init: Self = .{
+    .compute_routines = undefined,
+    .pipeline_states = undefined,
+};
 
+pub fn setup(self: *Self, device: *SoftDevice) void {
     for (self.pipeline_states[0..]) |*state| {
         state.* = .{
             .pipeline = null,
             .sets = [_]?*SoftDescriptorSet{null} ** base.VULKAN_MAX_DESCRIPTOR_SETS,
         };
     }
-
-    self.compute_routine = .init(device, &self.pipeline_states[@intFromEnum(vk.PipelineBindPoint.compute)]);
-
-    return self;
+    self.compute_routines = .init(device, &self.pipeline_states[@intFromEnum(vk.PipelineBindPoint.compute)]);
 }
 
 pub fn deinit(self: *Self) void {
-    self.compute_routine.destroy();
-}
-
-pub fn execute(self: *Self, command: *const cmd.Command) VkError!void {
-    switch (command.*) {
-        .BindDescriptorSets => |data| {
-            for (data.first_set.., data.sets[0..]) |i, set| {
-                if (set == null) break;
-                self.pipeline_states[@intCast(@intFromEnum(data.bind_point))].sets[i] = @alignCast(@fieldParentPtr("interface", set.?));
-            }
-        },
-        .BindPipeline => |data| self.pipeline_states[@intCast(@intFromEnum(data.bind_point))].pipeline = @alignCast(@fieldParentPtr("interface", data.pipeline)),
-        .ClearColorImage => |data| try copy_routines.clearColorImage(&data),
-        .CopyBuffer => |data| try copy_routines.copyBuffer(&data),
-        .CopyImage => |data| try copy_routines.copyImage(&data),
-        .CopyImageToBuffer => |data| try copy_routines.copyImageToBuffer(&data),
-        .FillBuffer => |data| try copy_routines.fillBuffer(&data),
-        else => {},
-    }
+    self.compute_routines.destroy();
 }

src/soft/device/copy_routines.zig

@@ -1,86 +0,0 @@
-const std = @import("std");
-const vk = @import("vulkan");
-const base = @import("base");
-
-const cmd = base.commands;
-const VkError = base.VkError;
-
-const SoftImage = @import("../SoftImage.zig");
-
-pub fn clearColorImage(data: *const cmd.CommandClearColorImage) VkError!void {
-    const soft_image: *SoftImage = @alignCast(@fieldParentPtr("interface", data.image));
-    soft_image.clearRange(data.clear_color, data.range);
-}
-
-pub fn copyBuffer(data: *const cmd.CommandCopyBuffer) VkError!void {
-    for (data.regions) |region| {
-        const src_memory = if (data.src.memory) |memory| memory else return VkError.ValidationFailed;
-        const dst_memory = if (data.dst.memory) |memory| memory else return VkError.ValidationFailed;
-
-        const src_map: []u8 = @as([*]u8, @ptrCast(try src_memory.map(region.src_offset, region.size)))[0..region.size];
-        const dst_map: []u8 = @as([*]u8, @ptrCast(try dst_memory.map(region.dst_offset, region.size)))[0..region.size];
-
-        @memcpy(dst_map, src_map);
-
-        src_memory.unmap();
-        dst_memory.unmap();
-    }
-}
-
-pub fn copyImage(data: *const cmd.CommandCopyImage) VkError!void {
-    _ = data;
-    std.log.scoped(.commandExecutor).warn("FIXME: implement image to image copy", .{});
-}
-
-pub fn copyImageToBuffer(data: *const cmd.CommandCopyImageToBuffer) VkError!void {
-    for (data.regions) |region| {
-        const src_memory = if (data.src.memory) |memory| memory else return VkError.ValidationFailed;
-        const dst_memory = if (data.dst.memory) |memory| memory else return VkError.ValidationFailed;
-
-        const pixel_size: u32 = @intCast(data.src.getPixelSize());
-        const image_row_pitch: u32 = data.src.extent.width * pixel_size;
-        const image_size: u32 = @intCast(data.src.getTotalSize());
-
-        const buffer_row_length: u32 = if (region.buffer_row_length != 0) region.buffer_row_length else region.image_extent.width;
-        const buffer_row_pitch: u32 = buffer_row_length * pixel_size;
-        const buffer_size: u32 = buffer_row_pitch * region.image_extent.height * region.image_extent.depth;
-
-        const src_map: []u8 = @as([*]u8, @ptrCast(try src_memory.map(0, image_size)))[0..image_size];
-        const dst_map: []u8 = @as([*]u8, @ptrCast(try dst_memory.map(region.buffer_offset, buffer_size)))[0..buffer_size];
-
-        const row_size = region.image_extent.width * pixel_size;
-        for (0..data.src.extent.depth) |z| {
-            for (0..data.src.extent.height) |y| {
-                const z_as_u32: u32 = @intCast(z);
-                const y_as_u32: u32 = @intCast(y);
-
-                const src_offset = ((@as(u32, @intCast(region.image_offset.z)) + z_as_u32) * data.src.extent.height + @as(u32, @intCast(region.image_offset.y)) + y_as_u32) * image_row_pitch + @as(u32, @intCast(region.image_offset.x)) * pixel_size;
-                const dst_offset = (z_as_u32 * buffer_row_length * region.image_extent.height + y_as_u32 * buffer_row_length) * pixel_size;
-
-                const src_slice = src_map[src_offset..(src_offset + row_size)];
-                const dst_slice = dst_map[dst_offset..(dst_offset + row_size)];
-                @memcpy(dst_slice, src_slice);
-            }
-        }
-
-        src_memory.unmap();
-        dst_memory.unmap();
-    }
-}
-
-pub fn fillBuffer(data: *const cmd.CommandFillBuffer) VkError!void {
-    const memory = if (data.buffer.memory) |memory| memory else return VkError.ValidationFailed;
-    var memory_map: []u32 = @as([*]u32, @ptrCast(@alignCast(try memory.map(data.offset, data.size))))[0..data.size];
-
-    var bytes = if (data.size == vk.WHOLE_SIZE) memory.size - data.offset else data.size;
-
-    var i: usize = 0;
-    while (bytes >= 4) : ({
-        bytes -= 4;
-        i += 1;
-    }) {
-        memory_map[i] = data.data;
-    }
-
-    memory.unmap();
-}