[IR] adding external resources managements
[Soft] adding descriptor sets management
This commit is contained in:
@@ -393,7 +393,9 @@ pub fn write(interface: *Interface, write_data: vk.WriteDescriptorSet) VkError!v
|
||||
const buffer = try NonDispatchable(Buffer).fromHandleObject(buffer_info.buffer);
|
||||
desc.object = @as(*SoftBuffer, @alignCast(@fieldParentPtr("interface", buffer)));
|
||||
if (desc.size == vk.WHOLE_SIZE) {
|
||||
desc.size = if (buffer.memory) |memory| memory.size - desc.offset else return VkError.InvalidDeviceMemoryDrv;
|
||||
if (desc.offset > buffer.size)
|
||||
return VkError.ValidationFailed;
|
||||
desc.size = buffer.size - desc.offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
const std = @import("std");
|
||||
const base = @import("base");
|
||||
const spv = @import("spv");
|
||||
|
||||
const ExecutionDevice = @import("Device.zig");
|
||||
const PipelineState = ExecutionDevice.PipelineState;
|
||||
|
||||
const SoftDevice = @import("../SoftDevice.zig");
|
||||
const SoftPipeline = @import("../SoftPipeline.zig");
|
||||
const ir_compute = @import("../interpreter/compute.zig");
|
||||
|
||||
const VkError = base.VkError;
|
||||
const SpvRuntimeError = spv.Runtime.RuntimeError;
|
||||
|
||||
const Self = @This();
|
||||
|
||||
const RunData = struct {
|
||||
self: *Self,
|
||||
batch_id: usize,
|
||||
group_count: usize,
|
||||
base_group_x: usize,
|
||||
base_group_y: usize,
|
||||
base_group_z: usize,
|
||||
group_count_x: usize,
|
||||
group_count_y: usize,
|
||||
group_count_z: usize,
|
||||
invocations_per_workgroup: usize,
|
||||
local_size: @Vector(3, u32),
|
||||
pipeline: *SoftPipeline,
|
||||
};
|
||||
|
||||
device: *SoftDevice,
|
||||
state: *PipelineState,
|
||||
batch_size: usize,
|
||||
|
||||
invocation_index: std.atomic.Value(usize),
|
||||
|
||||
early_dump: ?u32,
|
||||
final_dump: ?u32,
|
||||
|
||||
pub fn init(device: *SoftDevice, state: *PipelineState) Self {
|
||||
return .{
|
||||
.device = device,
|
||||
.state = state,
|
||||
.batch_size = 0,
|
||||
.invocation_index = .init(0),
|
||||
.early_dump = base.config.soft_compute_dump_early_results_table,
|
||||
.final_dump = base.config.soft_compute_dump_final_results_table,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn dispatch(self: *Self, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
|
||||
try self.dispatchBase(0, 0, 0, group_count_x, group_count_y, group_count_z);
|
||||
}
|
||||
|
||||
fn getLocalSize(rt: *spv.Runtime, allocator: std.mem.Allocator, spv_module: *const spv.Module) VkError!@Vector(3, u32) {
|
||||
if (rt.getWorkgroupSize(allocator) catch return VkError.ValidationFailed) |workgroup_size| {
|
||||
return workgroup_size;
|
||||
}
|
||||
|
||||
return .{
|
||||
spv_module.reflection_infos.local_size_x,
|
||||
spv_module.reflection_infos.local_size_y,
|
||||
spv_module.reflection_infos.local_size_z,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn dispatchBase(self: *Self, base_group_x: u32, base_group_y: u32, base_group_z: u32, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
|
||||
const group_count_xy = std.math.mul(usize, group_count_x, group_count_y) catch return VkError.ValidationFailed;
|
||||
const group_count = std.math.mul(usize, group_count_xy, group_count_z) catch return VkError.ValidationFailed;
|
||||
|
||||
const pipeline = self.state.pipeline orelse return VkError.InvalidPipelineDrv;
|
||||
const shader = pipeline.stages.getPtr(.compute) orelse return VkError.InvalidPipelineDrv;
|
||||
|
||||
const io = self.device.interface.io();
|
||||
const timer = std.Io.Timestamp.now(io, .real);
|
||||
defer if (comptime base.config.logs != .none) {
|
||||
const duration = timer.untilNow(io, .real);
|
||||
const ms: f32 = @floatFromInt(duration.toMicroseconds());
|
||||
std.log.scoped(.ComputeDispatcher).debug("Compute dispatch took {}ms using {s} interpreter", .{ ms / 1000, if (comptime base.config.soft_ir_interpreter) "IR" else "SPIR-V" });
|
||||
};
|
||||
|
||||
if (comptime base.config.soft_ir_interpreter) {
|
||||
return ir_compute.dispatch(shader, base_group_x, base_group_y, base_group_z, group_count_x, group_count_y, group_count_z);
|
||||
} else {
|
||||
const spv_module = &shader.module.module;
|
||||
self.batch_size = if (spv_module.reflection_infos.has_atomics) 1 else shader.runtimes.len;
|
||||
|
||||
const allocator = self.device.interface.device_allocator.allocator();
|
||||
const local_size = try getLocalSize(&shader.runtimes[0].rt, allocator, spv_module);
|
||||
const local_size_xy = std.math.mul(usize, local_size[0], local_size[1]) catch return VkError.ValidationFailed;
|
||||
const invocations_per_workgroup = std.math.mul(usize, local_size_xy, local_size[2]) catch return VkError.ValidationFailed;
|
||||
|
||||
self.invocation_index.store(0, .monotonic);
|
||||
|
||||
var wg: std.Io.Group = .init;
|
||||
for (0..@min(self.batch_size, group_count)) |batch_id| {
|
||||
const run_data: RunData = .{
|
||||
.self = self,
|
||||
.batch_id = batch_id,
|
||||
.group_count = group_count,
|
||||
.base_group_x = @as(usize, @intCast(base_group_x)),
|
||||
.base_group_y = @as(usize, @intCast(base_group_y)),
|
||||
.base_group_z = @as(usize, @intCast(base_group_z)),
|
||||
.group_count_x = @as(usize, @intCast(group_count_x)),
|
||||
.group_count_y = @as(usize, @intCast(group_count_y)),
|
||||
.group_count_z = @as(usize, @intCast(group_count_z)),
|
||||
.invocations_per_workgroup = invocations_per_workgroup,
|
||||
.local_size = local_size,
|
||||
.pipeline = pipeline,
|
||||
};
|
||||
|
||||
wg.async(self.device.interface.io(), runWrapper, .{run_data});
|
||||
}
|
||||
wg.await(self.device.interface.io()) catch return VkError.DeviceLost;
|
||||
}
|
||||
}
|
||||
|
||||
fn runWrapper(data: RunData) void {
|
||||
@call(.always_inline, run, .{data}) catch |err| {
|
||||
std.log.scoped(.@"SPIR-V runtime").err("SPIR-V runtime catched a '{s}'", .{@errorName(err)});
|
||||
if (comptime base.config.logs == .verbose) {
|
||||
if (@errorReturnTrace()) |trace| {
|
||||
std.debug.dumpErrorReturnTrace(trace);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
inline fn run(data: RunData) !void {
|
||||
const allocator = data.self.device.interface.device_allocator.allocator();
|
||||
const io = data.self.device.interface.io();
|
||||
|
||||
const shader = data.pipeline.stages.getPtrAssertContains(.compute);
|
||||
const rt = &shader.runtimes[data.batch_id].rt;
|
||||
|
||||
const entry = try rt.getEntryPointByName(shader.entry);
|
||||
const uses_control_barrier = rt.mod.reflection_infos.has_control_barriers or rt.mod.reflection_infos.has_atomics;
|
||||
|
||||
var barrier_runtimes: []spv.Runtime = &.{};
|
||||
var barrier_statuses: []spv.Runtime.EntryPointStatus = &.{};
|
||||
var initialized_barrier_runtimes: usize = 0;
|
||||
defer {
|
||||
for (barrier_runtimes[0..initialized_barrier_runtimes]) |*barrier_rt| {
|
||||
barrier_rt.resetInvocation(allocator);
|
||||
barrier_rt.deinit(allocator);
|
||||
}
|
||||
allocator.free(barrier_runtimes);
|
||||
allocator.free(barrier_statuses);
|
||||
}
|
||||
|
||||
if (uses_control_barrier) {
|
||||
barrier_runtimes = try allocator.alloc(spv.Runtime, data.invocations_per_workgroup);
|
||||
barrier_statuses = try allocator.alloc(spv.Runtime.EntryPointStatus, data.invocations_per_workgroup);
|
||||
for (barrier_runtimes) |*barrier_rt| {
|
||||
barrier_rt.* = try spv.Runtime.init(allocator, rt.mod, rt.image_api);
|
||||
initialized_barrier_runtimes += 1;
|
||||
try barrier_rt.copySpecializationConstantsFrom(allocator, rt);
|
||||
try prepareRuntime(data.self, barrier_rt);
|
||||
}
|
||||
} else {
|
||||
try prepareRuntime(data.self, rt);
|
||||
}
|
||||
|
||||
var group_index: usize = data.batch_id;
|
||||
while (group_index < data.group_count) : (group_index += data.self.batch_size) {
|
||||
var modulo: usize = group_index;
|
||||
|
||||
const group_z = @divTrunc(modulo, data.group_count_x * data.group_count_y);
|
||||
|
||||
modulo -= group_z * data.group_count_x * data.group_count_y;
|
||||
const group_y = @divTrunc(modulo, data.group_count_x);
|
||||
|
||||
modulo -= group_y * data.group_count_x;
|
||||
const group_x = modulo;
|
||||
|
||||
const group_count_vec = @Vector(3, u32){
|
||||
@as(u32, @intCast(data.group_count_x)),
|
||||
@as(u32, @intCast(data.group_count_y)),
|
||||
@as(u32, @intCast(data.group_count_z)),
|
||||
};
|
||||
const group_id_vec = @Vector(3, u32){
|
||||
@as(u32, @intCast(data.base_group_x + group_x)),
|
||||
@as(u32, @intCast(data.base_group_y + group_y)),
|
||||
@as(u32, @intCast(data.base_group_z + group_z)),
|
||||
};
|
||||
|
||||
if (uses_control_barrier) {
|
||||
try runBarrierWorkgroup(data, barrier_runtimes, barrier_statuses, entry, group_count_vec, group_id_vec);
|
||||
continue;
|
||||
}
|
||||
|
||||
const workgroup_memory = try rt.createWorkgroupMemory(allocator);
|
||||
defer rt.destroyWorkgroupMemory(allocator, workgroup_memory);
|
||||
|
||||
rt.resetInvocation(allocator);
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
try setupWorkgroupBuiltins(data.self, rt, data.local_size, group_count_vec, group_id_vec);
|
||||
|
||||
for (0..data.invocations_per_workgroup) |i| {
|
||||
rt.resetInvocation(allocator);
|
||||
|
||||
const invocation_index = data.self.invocation_index.fetchAdd(1, .monotonic);
|
||||
|
||||
try setupSubgroupBuiltins(data.self, rt, data.local_size, .{
|
||||
@as(u32, @intCast(data.base_group_x + group_x)),
|
||||
@as(u32, @intCast(data.base_group_y + group_y)),
|
||||
@as(u32, @intCast(data.base_group_z + group_z)),
|
||||
}, i);
|
||||
|
||||
if (data.self.early_dump != null and data.self.early_dump.? == invocation_index) {
|
||||
@branchHint(.cold);
|
||||
try dumpResultsTable(allocator, io, rt, true);
|
||||
}
|
||||
|
||||
rt.callEntryPoint(allocator, entry) catch |err| switch (err) {
|
||||
// Some errors can be ignored
|
||||
SpvRuntimeError.OutOfBounds => {},
|
||||
SpvRuntimeError.Killed => continue,
|
||||
else => return err,
|
||||
};
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
|
||||
if (data.self.final_dump != null and data.self.final_dump.? == invocation_index) {
|
||||
@branchHint(.cold);
|
||||
try dumpResultsTable(allocator, io, rt, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepareRuntime(self: *Self, rt: *spv.Runtime) !void {
|
||||
const allocator = self.device.interface.device_allocator.allocator();
|
||||
|
||||
rt.resetInvocation(allocator);
|
||||
if (rt.specialization_constants.count() != 0)
|
||||
try rt.applySpecializationInvocationLayout(allocator);
|
||||
try ExecutionDevice.writeDescriptorSets(self.state, rt);
|
||||
try rt.populatePushConstants(self.state.push_constant_blob[0..]);
|
||||
}
|
||||
|
||||
fn runBarrierWorkgroup(
|
||||
data: RunData,
|
||||
runtimes: []spv.Runtime,
|
||||
statuses: []spv.Runtime.EntryPointStatus,
|
||||
entry: spv.SpvWord,
|
||||
group_count: @Vector(3, u32),
|
||||
group_id: @Vector(3, u32),
|
||||
) !void {
|
||||
const allocator = data.self.device.interface.device_allocator.allocator();
|
||||
|
||||
const workgroup_memory = try runtimes[0].createWorkgroupMemory(allocator);
|
||||
defer runtimes[0].destroyWorkgroupMemory(allocator, workgroup_memory);
|
||||
for (runtimes, 0..) |*rt, i| {
|
||||
rt.resetInvocation(allocator);
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
try setupWorkgroupBuiltins(data.self, rt, data.local_size, group_count, group_id);
|
||||
try setupSubgroupBuiltins(data.self, rt, data.local_size, group_id, i);
|
||||
statuses[i] = try rt.beginEntryPoint(allocator, entry);
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
var pending = false;
|
||||
for (statuses) |status| {
|
||||
if (status == .barrier) {
|
||||
pending = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!pending)
|
||||
break;
|
||||
|
||||
for (runtimes, 0..) |*rt, i| {
|
||||
if (statuses[i] == .completed)
|
||||
continue;
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
statuses[i] = try rt.continueEntryPoint(allocator);
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flushWorkgroupMemory(rt: *spv.Runtime, workgroup_memory: []const spv.Runtime.WorkgroupMemory) spv.Runtime.RuntimeError!void {
|
||||
for (workgroup_memory) |memory| {
|
||||
_ = try (try rt.results[memory.result].getValue()).read(memory.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
fn dumpResultsTable(allocator: std.mem.Allocator, io: std.Io, rt: *spv.Runtime, comptime is_early: bool) !void {
|
||||
@branchHint(.cold);
|
||||
const file = try std.Io.Dir.cwd().createFile(
|
||||
io,
|
||||
std.fmt.comptimePrint("{s}_compute_result_table_dump.txt", .{if (is_early) "early" else "final"}),
|
||||
.{ .truncate = true },
|
||||
);
|
||||
defer file.close(io);
|
||||
var buffer = [_]u8{0} ** 1024;
|
||||
var writer = file.writer(io, buffer[0..]);
|
||||
try rt.dumpResultsTable(allocator, &writer.interface);
|
||||
}
|
||||
|
||||
fn setupWorkgroupBuiltins(self: *Self, rt: *spv.Runtime, local_size: @Vector(3, u32), group_count: @Vector(3, u32), group_id: @Vector(3, u32)) spv.Runtime.RuntimeError!void {
|
||||
const allocator = self.device.interface.device_allocator.allocator();
|
||||
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_size), .WorkgroupSize) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&group_count), .NumWorkgroups) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&group_id), .WorkgroupId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
|
||||
fn setupSubgroupBuiltins(self: *Self, rt: *spv.Runtime, local_size: @Vector(3, u32), group_id: @Vector(3, u32), local_invocation_index: usize) spv.Runtime.RuntimeError!void {
|
||||
const allocator = self.device.interface.device_allocator.allocator();
|
||||
|
||||
const local_base = local_size * group_id;
|
||||
var local_invocation = @Vector(3, u32){ 0, 0, 0 };
|
||||
|
||||
var idx: u32 = @intCast(local_invocation_index);
|
||||
local_invocation[2] = @divTrunc(idx, local_size[0] * local_size[1]);
|
||||
idx -= local_invocation[2] * local_size[0] * local_size[1];
|
||||
local_invocation[1] = @divTrunc(idx, local_size[0]);
|
||||
idx -= local_invocation[1] * local_size[0];
|
||||
local_invocation[0] = idx;
|
||||
|
||||
const global_invocation_index = local_base + local_invocation;
|
||||
const local_invocation_index_u32: u32 = @intCast(local_invocation_index);
|
||||
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_invocation), .LocalInvocationId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_invocation_index_u32), .LocalInvocationIndex) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&global_invocation_index), .GlobalInvocationId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
@@ -4,11 +4,13 @@ const base = @import("base");
|
||||
const lib = @import("../lib.zig");
|
||||
const spv = @import("spv");
|
||||
|
||||
const VkError = base.VkError;
|
||||
|
||||
const SoftDescriptorSet = @import("../SoftDescriptorSet.zig");
|
||||
const SoftDevice = @import("../SoftDevice.zig");
|
||||
const SoftPipeline = @import("../SoftPipeline.zig");
|
||||
|
||||
const ComputeDispatcher = @import("ComputeDispatcher.zig");
|
||||
const ComputeDispatcher = @import("compute/ComputeDispatcher.zig");
|
||||
const Renderer = @import("Renderer.zig");
|
||||
|
||||
const Self = @This();
|
||||
@@ -36,6 +38,46 @@ pub const PipelineState = struct {
|
||||
},
|
||||
};
|
||||
|
||||
pub fn mapStorageBuffer(state: *const PipelineState, set: u32, binding: u32) VkError!?[]u8 {
|
||||
const set_index: usize = set;
|
||||
if (set_index >= state.sets.len)
|
||||
return null;
|
||||
const descriptor_set = state.sets[set_index] orelse return null;
|
||||
|
||||
const binding_index: usize = binding;
|
||||
if (binding_index >= descriptor_set.descriptors.len or binding_index >= descriptor_set.interface.layout.bindings.len)
|
||||
return null;
|
||||
|
||||
const binding_layout = descriptor_set.interface.layout.bindings[binding_index];
|
||||
const dynamic_offset: vk.DeviceSize = switch (binding_layout.descriptor_type) {
|
||||
.storage_buffer_dynamic => blk: {
|
||||
if (binding_layout.dynamic_index >= state.dynamic_offsets[set_index].len)
|
||||
return VkError.ValidationFailed;
|
||||
break :blk state.dynamic_offsets[set_index][binding_layout.dynamic_index];
|
||||
},
|
||||
.storage_buffer => 0,
|
||||
else => return null,
|
||||
};
|
||||
|
||||
const descriptors = switch (descriptor_set.descriptors[binding_index]) {
|
||||
.buffer => |descriptors| descriptors,
|
||||
else => return null,
|
||||
};
|
||||
if (descriptors.len == 0)
|
||||
return null;
|
||||
|
||||
const descriptor = descriptors[0];
|
||||
const buffer = descriptor.object orelse return null;
|
||||
const effective_offset = std.math.add(vk.DeviceSize, descriptor.offset, dynamic_offset) catch return VkError.ValidationFailed;
|
||||
if (effective_offset > buffer.interface.size)
|
||||
return VkError.ValidationFailed;
|
||||
|
||||
const logical_remaining = buffer.interface.size - effective_offset;
|
||||
if (descriptor.size > logical_remaining)
|
||||
return VkError.ValidationFailed;
|
||||
return try buffer.mapAsSliceWithAddedOffset(u8, effective_offset, descriptor.size);
|
||||
}
|
||||
|
||||
compute: ComputeDispatcher,
|
||||
renderer: Renderer,
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
const std = @import("std");
|
||||
const base = @import("base");
|
||||
const spv = @import("spv");
|
||||
|
||||
const ExecutionDevice = @import("../Device.zig");
|
||||
const PipelineState = ExecutionDevice.PipelineState;
|
||||
|
||||
const SoftDevice = @import("../../SoftDevice.zig");
|
||||
const ir_interpreter = @import("ir_interpreter.zig");
|
||||
const spirv_interpreter = @import("spirv_interpreter.zig");
|
||||
|
||||
const VkError = base.VkError;
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub const Batch = struct {
|
||||
worker_index: usize,
|
||||
worker_count: usize,
|
||||
total_groups: usize,
|
||||
base_group: [3]usize,
|
||||
group_count: [3]usize,
|
||||
|
||||
pub fn groupId(self: Batch, linear_index: usize) [3]usize {
|
||||
const groups_xy = self.group_count[0] * self.group_count[1];
|
||||
const group_z = linear_index / groups_xy;
|
||||
const remainder = linear_index - group_z * groups_xy;
|
||||
const group_y = remainder / self.group_count[0];
|
||||
const group_x = remainder - group_y * self.group_count[0];
|
||||
return .{
|
||||
self.base_group[0] + group_x,
|
||||
self.base_group[1] + group_y,
|
||||
self.base_group[2] + group_z,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const BackendContext = if (base.config.soft_ir_interpreter) ir_interpreter.Context else spirv_interpreter.SpvContext;
|
||||
|
||||
device: *SoftDevice,
|
||||
state: *PipelineState,
|
||||
|
||||
invocation_index: std.atomic.Value(usize),
|
||||
|
||||
early_dump: ?u32,
|
||||
final_dump: ?u32,
|
||||
|
||||
pub fn init(device: *SoftDevice, state: *PipelineState) Self {
|
||||
return .{
|
||||
.device = device,
|
||||
.state = state,
|
||||
.invocation_index = .init(0),
|
||||
.early_dump = base.config.soft_compute_dump_early_results_table,
|
||||
.final_dump = base.config.soft_compute_dump_final_results_table,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn dispatch(self: *Self, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
|
||||
try self.dispatchBase(0, 0, 0, group_count_x, group_count_y, group_count_z);
|
||||
}
|
||||
|
||||
fn dispatchBatches(
|
||||
io: std.Io,
|
||||
context: anytype,
|
||||
worker_count: usize,
|
||||
total_groups: usize,
|
||||
base_group: [3]usize,
|
||||
group_count: [3]usize,
|
||||
comptime worker: anytype,
|
||||
) !void {
|
||||
if (total_groups == 0)
|
||||
return;
|
||||
if (worker_count == 0)
|
||||
return error.NoWorkers;
|
||||
|
||||
const active_workers = @min(worker_count, total_groups);
|
||||
var group: std.Io.Group = .init;
|
||||
for (0..active_workers) |worker_index| {
|
||||
group.async(io, worker, .{ context, Batch{
|
||||
.worker_index = worker_index,
|
||||
.worker_count = active_workers,
|
||||
.total_groups = total_groups,
|
||||
.base_group = base_group,
|
||||
.group_count = group_count,
|
||||
} });
|
||||
}
|
||||
try group.await(io);
|
||||
}
|
||||
|
||||
fn getLocalSize(rt: *spv.Runtime, allocator: std.mem.Allocator, spv_module: *const spv.Module) VkError!@Vector(3, u32) {
|
||||
if (rt.getWorkgroupSize(allocator) catch return VkError.ValidationFailed) |workgroup_size| {
|
||||
return workgroup_size;
|
||||
}
|
||||
|
||||
return .{
|
||||
spv_module.reflection_infos.local_size_x,
|
||||
spv_module.reflection_infos.local_size_y,
|
||||
spv_module.reflection_infos.local_size_z,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn dispatchBase(self: *Self, base_group_x: u32, base_group_y: u32, base_group_z: u32, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
|
||||
const group_count_xy = std.math.mul(usize, group_count_x, group_count_y) catch return VkError.ValidationFailed;
|
||||
const group_count = std.math.mul(usize, group_count_xy, group_count_z) catch return VkError.ValidationFailed;
|
||||
|
||||
const pipeline = self.state.pipeline orelse return VkError.InvalidPipelineDrv;
|
||||
const shader = pipeline.stages.getPtr(.compute) orelse return VkError.InvalidPipelineDrv;
|
||||
|
||||
const io = self.device.interface.io();
|
||||
const allocator = self.device.interface.device_allocator.allocator();
|
||||
const timer = std.Io.Timestamp.now(io, .real);
|
||||
defer if (comptime base.config.logs != .none) {
|
||||
const duration = timer.untilNow(io, .real);
|
||||
const ms: f32 = @floatFromInt(duration.toMicroseconds());
|
||||
std.log.scoped(.ComputeDispatcher).debug("Compute dispatch took {}ms using {s} interpreter", .{ ms / 1000, if (comptime base.config.soft_ir_interpreter) "IR" else "SPIR-V" });
|
||||
};
|
||||
|
||||
var context: BackendContext = if (comptime base.config.soft_ir_interpreter)
|
||||
try ir_interpreter.prepare(allocator, shader, self.state, io)
|
||||
else blk: {
|
||||
if (shader.runtimes.len == 0)
|
||||
return VkError.InvalidPipelineDrv;
|
||||
const spv_module = &shader.module.module;
|
||||
const local_size = try getLocalSize(&shader.runtimes[0].rt, allocator, spv_module);
|
||||
const local_size_xy = std.math.mul(usize, local_size[0], local_size[1]) catch return VkError.ValidationFailed;
|
||||
const invocations_per_workgroup = std.math.mul(usize, local_size_xy, local_size[2]) catch return VkError.ValidationFailed;
|
||||
self.invocation_index.store(0, .monotonic);
|
||||
break :blk .{
|
||||
.dispatcher = self,
|
||||
.pipeline = pipeline,
|
||||
.invocations_per_workgroup = invocations_per_workgroup,
|
||||
.local_size = local_size,
|
||||
};
|
||||
};
|
||||
defer if (comptime base.config.soft_ir_interpreter)
|
||||
context.deinit(allocator);
|
||||
|
||||
const worker_count = if (comptime base.config.soft_ir_interpreter)
|
||||
shader.runtimes.len
|
||||
else if (shader.module.module.reflection_infos.has_atomics)
|
||||
1
|
||||
else
|
||||
shader.runtimes.len;
|
||||
|
||||
dispatchBatches(
|
||||
io,
|
||||
context,
|
||||
worker_count,
|
||||
group_count,
|
||||
.{ base_group_x, base_group_y, base_group_z },
|
||||
.{ group_count_x, group_count_y, group_count_z },
|
||||
runWrapper,
|
||||
) catch |err| switch (err) {
|
||||
error.NoWorkers => return VkError.InvalidPipelineDrv,
|
||||
else => return VkError.DeviceLost,
|
||||
};
|
||||
}
|
||||
|
||||
fn runWrapper(context: BackendContext, batch: Batch) void {
|
||||
@call(.always_inline, run, .{ context, batch }) catch |err| {
|
||||
std.log.scoped(.ComputeDispatcher).err("{s} interpreter runtime caught a '{s}'", .{
|
||||
if (comptime base.config.soft_ir_interpreter) "IR" else "SPIR-V",
|
||||
@errorName(err),
|
||||
});
|
||||
if (comptime base.config.logs == .verbose) {
|
||||
if (@errorReturnTrace()) |trace|
|
||||
std.debug.dumpErrorReturnTrace(trace);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
inline fn run(context: BackendContext, batch: Batch) !void {
|
||||
if (comptime base.config.soft_ir_interpreter) {
|
||||
return ir_interpreter.runBatch(context, batch);
|
||||
} else {
|
||||
return spirv_interpreter.runBatch(context, batch);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
const std = @import("std");
|
||||
const base = @import("base");
|
||||
const shader_ir = @import("shader_ir");
|
||||
|
||||
const ExecutionDevice = @import("../Device.zig");
|
||||
const PipelineState = ExecutionDevice.PipelineState;
|
||||
const Batch = @import("ComputeDispatcher.zig").Batch;
|
||||
const Shader = @import("../../interpreter/Shader.zig");
|
||||
|
||||
const VkError = base.VkError;
|
||||
const ir = shader_ir.ir;
|
||||
|
||||
pub const Context = struct {
|
||||
shader: *Shader,
|
||||
io: std.Io,
|
||||
local_size: [3]u32,
|
||||
local_xy: usize,
|
||||
local_count: usize,
|
||||
global_id: ?ir.id.InterfaceVariableId,
|
||||
resource_buffers: []?[]u8,
|
||||
|
||||
pub fn deinit(self: *Context, allocator: std.mem.Allocator) void {
|
||||
allocator.free(self.resource_buffers);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
pub fn prepare(allocator: std.mem.Allocator, shader: *Shader, state: *const PipelineState, io: std.Io) VkError!Context {
|
||||
const local_size = shader.workgroup_size orelse return VkError.ValidationFailed;
|
||||
const local_xy = std.math.mul(usize, local_size[0], local_size[1]) catch return VkError.ValidationFailed;
|
||||
const local_count = std.math.mul(usize, local_xy, local_size[2]) catch return VkError.ValidationFailed;
|
||||
if (shader.runtimes.len == 0)
|
||||
return VkError.InvalidPipelineDrv;
|
||||
|
||||
const resource_buffers = allocator.alloc(?[]u8, shader.program.resources.len) catch return VkError.OutOfDeviceMemory;
|
||||
errdefer allocator.free(resource_buffers);
|
||||
@memset(resource_buffers, null);
|
||||
for (shader.program.resources, resource_buffers) |optional_resource, *buffer| {
|
||||
const resource = optional_resource orelse continue;
|
||||
if (resource.kind == .storage_buffer)
|
||||
buffer.* = try ExecutionDevice.mapStorageBuffer(state, resource.set, resource.binding);
|
||||
}
|
||||
|
||||
return .{
|
||||
.shader = shader,
|
||||
.io = io,
|
||||
.local_size = local_size,
|
||||
.local_xy = local_xy,
|
||||
.local_count = local_count,
|
||||
.global_id = findGlobalInvocationId(&shader.program),
|
||||
.resource_buffers = resource_buffers,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn runBatch(context: Context, batch: Batch) !void {
|
||||
const shader = context.shader;
|
||||
if (batch.worker_index >= shader.runtimes.len)
|
||||
return VkError.InvalidPipelineDrv;
|
||||
|
||||
const slot = &shader.runtimes[batch.worker_index];
|
||||
slot.mutex.lock(context.io) catch return VkError.DeviceLost;
|
||||
defer slot.mutex.unlock(context.io);
|
||||
|
||||
const runtime = &slot.runtime;
|
||||
var group_index = batch.worker_index;
|
||||
while (group_index < batch.total_groups) : (group_index += batch.worker_count) {
|
||||
const group_id = batch.groupId(group_index);
|
||||
const group_x = std.math.cast(u32, group_id[0]) orelse return VkError.ValidationFailed;
|
||||
const group_y = std.math.cast(u32, group_id[1]) orelse return VkError.ValidationFailed;
|
||||
const group_z = std.math.cast(u32, group_id[2]) orelse return VkError.ValidationFailed;
|
||||
|
||||
for (0..context.local_count) |local_index| {
|
||||
if (context.global_id) |variable| {
|
||||
const local_z = local_index / context.local_xy;
|
||||
const local_remainder = local_index - local_z * context.local_xy;
|
||||
const local_y = local_remainder / context.local_size[0];
|
||||
const local_x = local_remainder - local_y * context.local_size[0];
|
||||
try runtime.writeInput(&shader.program, variable, &.{
|
||||
group_x * context.local_size[0] + @as(u32, @intCast(local_x)),
|
||||
group_y * context.local_size[1] + @as(u32, @intCast(local_y)),
|
||||
group_z * context.local_size[2] + @as(u32, @intCast(local_z)),
|
||||
});
|
||||
}
|
||||
_ = try runtime.run(&shader.program, .{ .resource_buffers = context.resource_buffers });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn findGlobalInvocationId(program: *const @import("../../interpreter/Program.zig")) ?ir.id.InterfaceVariableId {
|
||||
for (program.interfaces, 0..) |optional_binding, index| {
|
||||
const binding = optional_binding orelse continue;
|
||||
if (binding.direction == .input and binding.semantic == .builtin and binding.semantic.builtin == .global_invocation_id)
|
||||
return ir.id.InterfaceVariableId.fromIndex(index);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
const std = @import("std");
|
||||
const spv = @import("spv");
|
||||
|
||||
const SpvRuntimeError = spv.Runtime.RuntimeError;
|
||||
|
||||
const ExecutionDevice = @import("../Device.zig");
|
||||
const SoftPipeline = @import("../../SoftPipeline.zig");
|
||||
const Dispatcher = @import("ComputeDispatcher.zig");
|
||||
const Batch = Dispatcher.Batch;
|
||||
|
||||
pub const SpvContext = struct {
|
||||
dispatcher: *Dispatcher,
|
||||
pipeline: *SoftPipeline,
|
||||
invocations_per_workgroup: usize,
|
||||
local_size: @Vector(3, u32),
|
||||
};
|
||||
|
||||
pub fn runBatch(context: SpvContext, batch: Batch) !void {
|
||||
const dispatcher = context.dispatcher;
|
||||
const allocator = dispatcher.device.interface.device_allocator.allocator();
|
||||
const io = dispatcher.device.interface.io();
|
||||
|
||||
const shader = context.pipeline.stages.getPtrAssertContains(.compute);
|
||||
const rt = &shader.runtimes[batch.worker_index].rt;
|
||||
|
||||
const entry = try rt.getEntryPointByName(shader.entry);
|
||||
const uses_control_barrier = rt.mod.reflection_infos.has_control_barriers or rt.mod.reflection_infos.has_atomics;
|
||||
|
||||
var barrier_runtimes: []spv.Runtime = &.{};
|
||||
var barrier_statuses: []spv.Runtime.EntryPointStatus = &.{};
|
||||
var initialized_barrier_runtimes: usize = 0;
|
||||
defer {
|
||||
for (barrier_runtimes[0..initialized_barrier_runtimes]) |*barrier_rt| {
|
||||
barrier_rt.resetInvocation(allocator);
|
||||
barrier_rt.deinit(allocator);
|
||||
}
|
||||
allocator.free(barrier_runtimes);
|
||||
allocator.free(barrier_statuses);
|
||||
}
|
||||
|
||||
if (uses_control_barrier) {
|
||||
barrier_runtimes = try allocator.alloc(spv.Runtime, context.invocations_per_workgroup);
|
||||
barrier_statuses = try allocator.alloc(spv.Runtime.EntryPointStatus, context.invocations_per_workgroup);
|
||||
for (barrier_runtimes) |*barrier_rt| {
|
||||
barrier_rt.* = try spv.Runtime.init(allocator, rt.mod, rt.image_api);
|
||||
initialized_barrier_runtimes += 1;
|
||||
try barrier_rt.copySpecializationConstantsFrom(allocator, rt);
|
||||
try prepareRuntime(dispatcher, barrier_rt);
|
||||
}
|
||||
} else {
|
||||
try prepareRuntime(dispatcher, rt);
|
||||
}
|
||||
|
||||
const group_count_vec = @Vector(3, u32){
|
||||
@intCast(batch.group_count[0]),
|
||||
@intCast(batch.group_count[1]),
|
||||
@intCast(batch.group_count[2]),
|
||||
};
|
||||
var group_index = batch.worker_index;
|
||||
while (group_index < batch.total_groups) : (group_index += batch.worker_count) {
|
||||
const group_id = batch.groupId(group_index);
|
||||
const group_id_vec = @Vector(3, u32){
|
||||
@intCast(group_id[0]),
|
||||
@intCast(group_id[1]),
|
||||
@intCast(group_id[2]),
|
||||
};
|
||||
|
||||
if (uses_control_barrier) {
|
||||
try runBarrierWorkgroup(context, barrier_runtimes, barrier_statuses, entry, group_count_vec, group_id_vec);
|
||||
continue;
|
||||
}
|
||||
|
||||
const workgroup_memory = try rt.createWorkgroupMemory(allocator);
|
||||
defer rt.destroyWorkgroupMemory(allocator, workgroup_memory);
|
||||
|
||||
rt.resetInvocation(allocator);
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
try setupWorkgroupBuiltins(dispatcher, rt, context.local_size, group_count_vec, group_id_vec);
|
||||
|
||||
for (0..context.invocations_per_workgroup) |i| {
|
||||
rt.resetInvocation(allocator);
|
||||
|
||||
const invocation_index = dispatcher.invocation_index.fetchAdd(1, .monotonic);
|
||||
|
||||
try setupSubgroupBuiltins(dispatcher, rt, context.local_size, group_id_vec, i);
|
||||
|
||||
if (dispatcher.early_dump != null and dispatcher.early_dump.? == invocation_index) {
|
||||
@branchHint(.cold);
|
||||
try dumpResultsTable(allocator, io, rt, true);
|
||||
}
|
||||
|
||||
rt.callEntryPoint(allocator, entry) catch |err| switch (err) {
|
||||
SpvRuntimeError.OutOfBounds => {},
|
||||
SpvRuntimeError.Killed => continue,
|
||||
else => return err,
|
||||
};
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
|
||||
if (dispatcher.final_dump != null and dispatcher.final_dump.? == invocation_index) {
|
||||
@branchHint(.cold);
|
||||
try dumpResultsTable(allocator, io, rt, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepareRuntime(dispatcher: *Dispatcher, rt: *spv.Runtime) !void {
|
||||
const allocator = dispatcher.device.interface.device_allocator.allocator();
|
||||
|
||||
rt.resetInvocation(allocator);
|
||||
if (rt.specialization_constants.count() != 0)
|
||||
try rt.applySpecializationInvocationLayout(allocator);
|
||||
try ExecutionDevice.writeDescriptorSets(dispatcher.state, rt);
|
||||
try rt.populatePushConstants(dispatcher.state.push_constant_blob[0..]);
|
||||
}
|
||||
|
||||
fn runBarrierWorkgroup(
|
||||
context: SpvContext,
|
||||
runtimes: []spv.Runtime,
|
||||
statuses: []spv.Runtime.EntryPointStatus,
|
||||
entry: spv.SpvWord,
|
||||
group_count: @Vector(3, u32),
|
||||
group_id: @Vector(3, u32),
|
||||
) !void {
|
||||
const dispatcher = context.dispatcher;
|
||||
const allocator = dispatcher.device.interface.device_allocator.allocator();
|
||||
|
||||
const workgroup_memory = try runtimes[0].createWorkgroupMemory(allocator);
|
||||
defer runtimes[0].destroyWorkgroupMemory(allocator, workgroup_memory);
|
||||
for (runtimes, 0..) |*rt, i| {
|
||||
rt.resetInvocation(allocator);
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
try setupWorkgroupBuiltins(dispatcher, rt, context.local_size, group_count, group_id);
|
||||
try setupSubgroupBuiltins(dispatcher, rt, context.local_size, group_id, i);
|
||||
statuses[i] = try rt.beginEntryPoint(allocator, entry);
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
var pending = false;
|
||||
for (statuses) |status| {
|
||||
if (status == .barrier) {
|
||||
pending = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!pending)
|
||||
break;
|
||||
|
||||
for (runtimes, 0..) |*rt, i| {
|
||||
if (statuses[i] == .completed)
|
||||
continue;
|
||||
try rt.bindWorkgroupMemory(workgroup_memory);
|
||||
statuses[i] = try rt.continueEntryPoint(allocator);
|
||||
try flushWorkgroupMemory(rt, workgroup_memory);
|
||||
try rt.flushDescriptorSets(allocator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flushWorkgroupMemory(rt: *spv.Runtime, workgroup_memory: []const spv.Runtime.WorkgroupMemory) spv.Runtime.RuntimeError!void {
|
||||
for (workgroup_memory) |memory| {
|
||||
_ = try (try rt.results[memory.result].getValue()).read(memory.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
fn dumpResultsTable(allocator: std.mem.Allocator, io: std.Io, rt: *spv.Runtime, comptime is_early: bool) !void {
|
||||
@branchHint(.cold);
|
||||
const file = try std.Io.Dir.cwd().createFile(
|
||||
io,
|
||||
std.fmt.comptimePrint("{s}_compute_result_table_dump.txt", .{if (is_early) "early" else "final"}),
|
||||
.{ .truncate = true },
|
||||
);
|
||||
defer file.close(io);
|
||||
var buffer = [_]u8{0} ** 1024;
|
||||
var writer = file.writer(io, buffer[0..]);
|
||||
try rt.dumpResultsTable(allocator, &writer.interface);
|
||||
}
|
||||
|
||||
fn setupWorkgroupBuiltins(dispatcher: *Dispatcher, rt: *spv.Runtime, local_size: @Vector(3, u32), group_count: @Vector(3, u32), group_id: @Vector(3, u32)) spv.Runtime.RuntimeError!void {
|
||||
const allocator = dispatcher.device.interface.device_allocator.allocator();
|
||||
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_size), .WorkgroupSize) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&group_count), .NumWorkgroups) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&group_id), .WorkgroupId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
|
||||
fn setupSubgroupBuiltins(dispatcher: *Dispatcher, rt: *spv.Runtime, local_size: @Vector(3, u32), group_id: @Vector(3, u32), local_invocation_index: usize) spv.Runtime.RuntimeError!void {
|
||||
const allocator = dispatcher.device.interface.device_allocator.allocator();
|
||||
|
||||
const local_base = local_size * group_id;
|
||||
var local_invocation = @Vector(3, u32){ 0, 0, 0 };
|
||||
|
||||
var idx: u32 = @intCast(local_invocation_index);
|
||||
local_invocation[2] = @divTrunc(idx, local_size[0] * local_size[1]);
|
||||
idx -= local_invocation[2] * local_size[0] * local_size[1];
|
||||
local_invocation[1] = @divTrunc(idx, local_size[0]);
|
||||
idx -= local_invocation[1] * local_size[0];
|
||||
local_invocation[0] = idx;
|
||||
|
||||
const global_invocation_index = local_base + local_invocation;
|
||||
const local_invocation_index_u32: u32 = @intCast(local_invocation_index);
|
||||
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_invocation), .LocalInvocationId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&local_invocation_index_u32), .LocalInvocationIndex) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
rt.writeBuiltIn(allocator, std.mem.asBytes(&global_invocation_index), .GlobalInvocationId) catch |err| switch (err) {
|
||||
SpvRuntimeError.NotFound => {},
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
@@ -25,6 +25,12 @@ pub const InterfaceBinding = struct {
|
||||
span: bc.Span,
|
||||
};
|
||||
|
||||
pub const ResourceBinding = struct {
|
||||
kind: ir.types.ResourceKind,
|
||||
set: u32,
|
||||
binding: u32,
|
||||
};
|
||||
|
||||
pub const RegisterInit = struct {
|
||||
register: bc.Register,
|
||||
value: u32,
|
||||
@@ -43,6 +49,7 @@ copies: []const bc.Copy,
|
||||
branches: []const bc.Branch,
|
||||
initializers: []const RegisterInit,
|
||||
interfaces: []const ?InterfaceBinding,
|
||||
resources: []const ?ResourceBinding,
|
||||
|
||||
pub fn compile(backing_allocator: std.mem.Allocator, module: *const module_ir.Module) !Self {
|
||||
try ir.validator.validate(module);
|
||||
@@ -65,6 +72,7 @@ pub fn compile(backing_allocator: std.mem.Allocator, module: *const module_ir.Mo
|
||||
.branches = lowerer.branches.items,
|
||||
.initializers = lowerer.initializers.items,
|
||||
.interfaces = lowerer.interfaces,
|
||||
.resources = lowerer.resources,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -80,6 +88,13 @@ pub fn interfaceBinding(self: *const Self, variable: ids.InterfaceVariableId) ?I
|
||||
return self.interfaces[variable.index()];
|
||||
}
|
||||
|
||||
pub fn resourceBinding(self: *const Self, resource: ids.ResourceId) ?ResourceBinding {
|
||||
if (resource.index() >= self.resources.len)
|
||||
return null;
|
||||
|
||||
return self.resources[resource.index()];
|
||||
}
|
||||
|
||||
const Lowerer = struct {
|
||||
allocator: std.mem.Allocator,
|
||||
module: *const module_ir.Module,
|
||||
@@ -87,6 +102,7 @@ const Lowerer = struct {
|
||||
entry_block: ids.BlockId,
|
||||
values: []?bc.Span,
|
||||
interfaces: []?InterfaceBinding,
|
||||
resources: []?ResourceBinding,
|
||||
block_pcs: []?u32,
|
||||
register_count: usize = 0,
|
||||
scratch_count: usize = 0,
|
||||
@@ -109,6 +125,14 @@ const Lowerer = struct {
|
||||
@memset(values, null);
|
||||
const interfaces = try allocator.alloc(?InterfaceBinding, module.interface_variables.entries.items.len);
|
||||
@memset(interfaces, null);
|
||||
const resources = try allocator.alloc(?ResourceBinding, module.resources.entries.items.len);
|
||||
for (module.resources.entries.items, resources) |entry, *binding| {
|
||||
binding.* = if (entry) |resource| .{
|
||||
.kind = resource.kind,
|
||||
.set = resource.set,
|
||||
.binding = resource.binding,
|
||||
} else null;
|
||||
}
|
||||
const block_pcs = try allocator.alloc(?u32, module.blocks.entries.items.len);
|
||||
@memset(block_pcs, null);
|
||||
|
||||
@@ -119,6 +143,7 @@ const Lowerer = struct {
|
||||
.entry_block = entry_block,
|
||||
.values = values,
|
||||
.interfaces = interfaces,
|
||||
.resources = resources,
|
||||
.block_pcs = block_pcs,
|
||||
};
|
||||
}
|
||||
@@ -347,10 +372,40 @@ const Lowerer = struct {
|
||||
|
||||
try self.emitCopy(binding.span, src);
|
||||
},
|
||||
.load_buffer => |op| {
|
||||
const dst = result orelse return CompileError.InvalidOperation;
|
||||
const byte_offset = try self.bufferOffset(op.byte_offset);
|
||||
_ = try self.storageBuffer(op.resource);
|
||||
try self.emit(.load_buffer, dst.components, dst.base, byte_offset, bc.invalid_register, bc.invalid_register, @intFromEnum(op.resource));
|
||||
},
|
||||
.store_buffer => |op| {
|
||||
if (result != null)
|
||||
return CompileError.InvalidOperation;
|
||||
const src = try self.span(op.value);
|
||||
const byte_offset = try self.bufferOffset(op.byte_offset);
|
||||
_ = try self.storageBuffer(op.resource);
|
||||
try self.emit(.store_buffer, src.components, src.base, byte_offset, bc.invalid_register, bc.invalid_register, @intFromEnum(op.resource));
|
||||
},
|
||||
.call => return CompileError.UnsupportedOperation,
|
||||
}
|
||||
}
|
||||
|
||||
fn bufferOffset(self: *const Lowerer, id: ids.ValueId) !bc.Register {
|
||||
const byte_offset = try self.span(id);
|
||||
if (byte_offset.components != 1 or byte_offset.kind != .unsigned_integer)
|
||||
return CompileError.InvalidOperation;
|
||||
return byte_offset.base;
|
||||
}
|
||||
|
||||
fn storageBuffer(self: *const Lowerer, id: ids.ResourceId) !ResourceBinding {
|
||||
if (id.index() >= self.resources.len)
|
||||
return CompileError.InvalidOperation;
|
||||
const resource = self.resources[id.index()] orelse return CompileError.InvalidOperation;
|
||||
if (resource.kind != .storage_buffer)
|
||||
return CompileError.InvalidOperation;
|
||||
return resource;
|
||||
}
|
||||
|
||||
fn lowerTerminator(self: *Lowerer, terminator: module_ir.Terminator) !void {
|
||||
switch (terminator) {
|
||||
.branch => |edge| try self.emit(.jump_edge, 1, bc.invalid_register, bc.invalid_register, bc.invalid_register, bc.invalid_register, try self.addEdge(edge)),
|
||||
|
||||
@@ -6,10 +6,13 @@ const Program = @import("Program.zig");
|
||||
const ids = shader_ir.ir.id;
|
||||
|
||||
pub const RuntimeError = error{
|
||||
BufferOutOfBounds,
|
||||
DivisionByZero,
|
||||
IntegerOverflow,
|
||||
InvalidBytecode,
|
||||
InvalidInterface,
|
||||
InvalidResource,
|
||||
ResourceNotBound,
|
||||
ShiftOutOfRange,
|
||||
StepLimitExceeded,
|
||||
UnreachableExecuted,
|
||||
@@ -24,6 +27,7 @@ pub const Outcome = enum {
|
||||
|
||||
pub const RunOptions = struct {
|
||||
max_steps: usize = 1_000_000,
|
||||
resource_buffers: []const ?[]u8 = &.{},
|
||||
};
|
||||
|
||||
const Self = @This();
|
||||
@@ -126,6 +130,8 @@ pub fn run(self: *Self, program: *const Program, options: RunOptions) RuntimeErr
|
||||
.compare_ordered_float_less => self.compareFloat(instruction, .ordered_less),
|
||||
.compare_unordered_float_less => self.compareFloat(instruction, .unordered_less),
|
||||
.select => self.select(instruction),
|
||||
.load_buffer => try self.loadBuffer(program, options.resource_buffers, instruction),
|
||||
.store_buffer => try self.storeBuffer(program, options.resource_buffers, instruction),
|
||||
.jump_edge => pc = try self.applyEdge(program, instruction.immediate),
|
||||
.branch => {
|
||||
if (instruction.immediate >= program.branches.len)
|
||||
@@ -278,6 +284,52 @@ fn select(self: *Self, instruction: bc.Instruction) void {
|
||||
self.registers[@as(usize, instruction.a) + component] = self.registers[@as(usize, selected) + component];
|
||||
}
|
||||
|
||||
fn loadBuffer(self: *Self, program: *const Program, resource_buffers: []const ?[]u8, instruction: bc.Instruction) RuntimeError!void {
|
||||
try self.validateRegisterSpan(instruction);
|
||||
const buffer = try resourceBuffer(program, resource_buffers, instruction.immediate);
|
||||
const bytes = try self.bufferRange(buffer, instruction);
|
||||
for (0..instruction.components) |component| {
|
||||
const offset = component * @sizeOf(u32);
|
||||
self.registers[@as(usize, instruction.a) + component] = std.mem.readInt(u32, bytes[offset..][0..@sizeOf(u32)], .little);
|
||||
}
|
||||
}
|
||||
|
||||
fn storeBuffer(self: *const Self, program: *const Program, resource_buffers: []const ?[]u8, instruction: bc.Instruction) RuntimeError!void {
|
||||
try self.validateRegisterSpan(instruction);
|
||||
const buffer = try resourceBuffer(program, resource_buffers, instruction.immediate);
|
||||
const bytes = try self.bufferRange(buffer, instruction);
|
||||
for (0..instruction.components) |component| {
|
||||
const offset = component * @sizeOf(u32);
|
||||
std.mem.writeInt(u32, bytes[offset..][0..@sizeOf(u32)], self.registers[@as(usize, instruction.a) + component], .little);
|
||||
}
|
||||
}
|
||||
|
||||
fn validateRegisterSpan(self: *const Self, instruction: bc.Instruction) RuntimeError!void {
|
||||
const register_end = std.math.add(usize, instruction.a, instruction.components) catch return RuntimeError.InvalidBytecode;
|
||||
if (register_end > self.registers.len)
|
||||
return RuntimeError.InvalidBytecode;
|
||||
}
|
||||
|
||||
fn bufferRange(self: *const Self, buffer: []u8, instruction: bc.Instruction) RuntimeError![]u8 {
|
||||
if (instruction.b >= self.registers.len)
|
||||
return RuntimeError.InvalidBytecode;
|
||||
|
||||
const byte_offset: usize = self.registers[instruction.b];
|
||||
const byte_count = std.math.mul(usize, instruction.components, @sizeOf(u32)) catch return RuntimeError.BufferOutOfBounds;
|
||||
const end = std.math.add(usize, byte_offset, byte_count) catch return RuntimeError.BufferOutOfBounds;
|
||||
if (end > buffer.len)
|
||||
return RuntimeError.BufferOutOfBounds;
|
||||
return buffer[byte_offset..end];
|
||||
}
|
||||
|
||||
fn resourceBuffer(program: *const Program, resource_buffers: []const ?[]u8, resource_index: u32) RuntimeError![]u8 {
|
||||
const resource = ids.ResourceId.fromIndex(resource_index);
|
||||
_ = program.resourceBinding(resource) orelse return RuntimeError.InvalidResource;
|
||||
if (resource.index() >= resource_buffers.len)
|
||||
return RuntimeError.ResourceNotBound;
|
||||
return resource_buffers[resource.index()] orelse RuntimeError.ResourceNotBound;
|
||||
}
|
||||
|
||||
fn applyEdge(self: *Self, program: *const Program, edge_index: u32) RuntimeError!u32 {
|
||||
if (edge_index >= program.edges.len)
|
||||
return RuntimeError.InvalidBytecode;
|
||||
|
||||
@@ -72,6 +72,8 @@ pub const Opcode = enum(u16) {
|
||||
compare_ordered_float_less,
|
||||
compare_unordered_float_less,
|
||||
select,
|
||||
load_buffer,
|
||||
store_buffer,
|
||||
jump_edge,
|
||||
branch,
|
||||
return_void,
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
const std = @import("std");
|
||||
const base = @import("base");
|
||||
const shader_ir = @import("shader_ir");
|
||||
|
||||
const Shader = @import("Shader.zig");
|
||||
|
||||
const VkError = base.VkError;
|
||||
const ir = shader_ir.ir;
|
||||
|
||||
pub fn dispatch(shader: *Shader, base_group_x: u32, base_group_y: u32, base_group_z: u32, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
|
||||
const local_size = shader.workgroup_size orelse return VkError.ValidationFailed;
|
||||
const local_xy = std.math.mul(usize, local_size[0], local_size[1]) catch return VkError.ValidationFailed;
|
||||
const local_count = std.math.mul(usize, local_xy, local_size[2]) catch return VkError.ValidationFailed;
|
||||
if (shader.runtimes.len == 0)
|
||||
return VkError.InvalidPipelineDrv;
|
||||
|
||||
const global_id = findGlobalInvocationId(&shader.program);
|
||||
var runtime = &shader.runtimes[0].runtime;
|
||||
for (0..group_count_z) |group_z| {
|
||||
for (0..group_count_y) |group_y| {
|
||||
for (0..group_count_x) |group_x| {
|
||||
for (0..local_count) |local_index| {
|
||||
if (global_id) |variable| {
|
||||
const local_z = local_index / local_xy;
|
||||
const local_remainder = local_index - local_z * local_xy;
|
||||
const local_y = local_remainder / local_size[0];
|
||||
const local_x = local_remainder - local_y * local_size[0];
|
||||
runtime.writeInput(&shader.program, variable, &.{
|
||||
(base_group_x + @as(u32, @intCast(group_x))) * local_size[0] + @as(u32, @intCast(local_x)),
|
||||
(base_group_y + @as(u32, @intCast(group_y))) * local_size[1] + @as(u32, @intCast(local_y)),
|
||||
(base_group_z + @as(u32, @intCast(group_z))) * local_size[2] + @as(u32, @intCast(local_z)),
|
||||
}) catch return VkError.Unknown;
|
||||
}
|
||||
_ = runtime.run(&shader.program, .{}) catch return VkError.Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn findGlobalInvocationId(program: *const @import("Program.zig")) ?ir.id.InterfaceVariableId {
|
||||
for (program.interfaces, 0..) |optional_binding, index| {
|
||||
const binding = optional_binding orelse continue;
|
||||
if (binding.direction == .input and binding.semantic == .builtin and binding.semantic.builtin == .global_invocation_id)
|
||||
return ir.id.InterfaceVariableId.fromIndex(index);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
const std = @import("std");
|
||||
const shader_ir = @import("shader_ir");
|
||||
|
||||
const Program = @import("../Program.zig");
|
||||
const Runtime = @import("../Runtime.zig");
|
||||
|
||||
const ir = shader_ir.ir;
|
||||
|
||||
const copy_shader =
|
||||
\\ shader compute @main
|
||||
\\ {
|
||||
\\ @source: vec4[u32] = storage_buffer[set(2), binding(3)]
|
||||
\\ @destination: vec4[u32] = storage_buffer[set(4), binding(5)]
|
||||
\\
|
||||
\\ %source_offset: constant u32 = 1
|
||||
\\ %destination_offset: constant u32 = 2
|
||||
\\
|
||||
\\ fn @main() -> void
|
||||
\\ {
|
||||
\\ .entry():
|
||||
\\ %value: vec4[u32] = load_buffer @source, %source_offset
|
||||
\\ store_buffer @destination, %destination_offset, %value
|
||||
\\ return
|
||||
\\ }
|
||||
\\ }
|
||||
;
|
||||
|
||||
const scalar_shader =
|
||||
\\ shader compute @main
|
||||
\\ {
|
||||
\\ @source: u32 = storage_buffer[set(0), binding(0)]
|
||||
\\ @destination: u32 = storage_buffer[set(0), binding(1)]
|
||||
\\
|
||||
\\ %zero: constant u32 = 0
|
||||
\\ %one: constant u32 = 1
|
||||
\\
|
||||
\\ fn @main() -> void
|
||||
\\ {
|
||||
\\ .entry():
|
||||
\\ %loaded: u32 = load_buffer @source, %zero
|
||||
\\ %value: u32 = integer_add %loaded, %one
|
||||
\\ store_buffer @destination, %zero, %value
|
||||
\\ return
|
||||
\\ }
|
||||
\\ }
|
||||
;
|
||||
|
||||
const bounds_shader =
|
||||
\\ shader compute @main
|
||||
\\ {
|
||||
\\ @buffer: vec2[u32] = storage_buffer[set(0), binding(0)]
|
||||
\\
|
||||
\\ %offset: constant u32 = 1
|
||||
\\ %first: constant u32 = bits(0x11223344)
|
||||
\\ %second: constant u32 = bits(0x55667788)
|
||||
\\
|
||||
\\ fn @main() -> void
|
||||
\\ {
|
||||
\\ .entry():
|
||||
\\ %value: vec2[u32] = composite_construct %first, %second
|
||||
\\ store_buffer @buffer, %offset, %value
|
||||
\\ return
|
||||
\\ }
|
||||
\\ }
|
||||
;
|
||||
|
||||
test "[interpreter] storage-buffer vector load and store use portable little-endian words" {
|
||||
var module = try ir.parser.parseString(std.testing.allocator, copy_shader);
|
||||
defer module.deinit();
|
||||
|
||||
var program = try Program.compile(std.testing.allocator, &module);
|
||||
defer program.deinit();
|
||||
var runtime = try Runtime.init(std.testing.allocator, &program);
|
||||
defer runtime.deinit();
|
||||
|
||||
const source_id = ir.id.ResourceId.fromIndex(0);
|
||||
const destination_id = ir.id.ResourceId.fromIndex(1);
|
||||
try std.testing.expectEqual(@as(u32, 2), program.resourceBinding(source_id).?.set);
|
||||
try std.testing.expectEqual(@as(u32, 3), program.resourceBinding(source_id).?.binding);
|
||||
try std.testing.expectEqual(@as(u32, 4), program.resourceBinding(destination_id).?.set);
|
||||
try std.testing.expectEqual(@as(u32, 5), program.resourceBinding(destination_id).?.binding);
|
||||
|
||||
var source = [_]u8{ 0xff, 0x78, 0x56, 0x34, 0x12, 0xef, 0xcd, 0xab, 0x90, 0x04, 0x03, 0x02, 0x01, 0xdd, 0xcc, 0xbb, 0xaa };
|
||||
var destination = [_]u8{0xcc} ** 20;
|
||||
const resources = [_]?[]u8{ source[0..], destination[0..] };
|
||||
|
||||
try std.testing.expectEqual(Runtime.Outcome.returned, try runtime.run(&program, .{ .resource_buffers = &resources }));
|
||||
try std.testing.expectEqualSlices(u8, source[1..17], destination[2..18]);
|
||||
try std.testing.expectEqual(@as(u8, 0xcc), destination[0]);
|
||||
try std.testing.expectEqual(@as(u8, 0xcc), destination[1]);
|
||||
try std.testing.expectEqual(@as(u8, 0xcc), destination[18]);
|
||||
try std.testing.expectEqual(@as(u8, 0xcc), destination[19]);
|
||||
}
|
||||
|
||||
test "[interpreter] storage-buffer scalar load and store interpret little-endian words" {
|
||||
var module = try ir.parser.parseString(std.testing.allocator, scalar_shader);
|
||||
defer module.deinit();
|
||||
|
||||
var program = try Program.compile(std.testing.allocator, &module);
|
||||
defer program.deinit();
|
||||
var runtime = try Runtime.init(std.testing.allocator, &program);
|
||||
defer runtime.deinit();
|
||||
|
||||
var source = [_]u8{ 0x78, 0x56, 0x34, 0x12 };
|
||||
var destination = [_]u8{0} ** 4;
|
||||
const resources = [_]?[]u8{ source[0..], destination[0..] };
|
||||
try std.testing.expectEqual(Runtime.Outcome.returned, try runtime.run(&program, .{ .resource_buffers = &resources }));
|
||||
try std.testing.expectEqualSlices(u8, &[_]u8{ 0x79, 0x56, 0x34, 0x12 }, &destination);
|
||||
}
|
||||
|
||||
test "[interpreter] storage-buffer accesses report unbound and out-of-bounds resources" {
|
||||
var module = try ir.parser.parseString(std.testing.allocator, bounds_shader);
|
||||
defer module.deinit();
|
||||
|
||||
var program = try Program.compile(std.testing.allocator, &module);
|
||||
defer program.deinit();
|
||||
var runtime = try Runtime.init(std.testing.allocator, &program);
|
||||
defer runtime.deinit();
|
||||
|
||||
try std.testing.expectError(Runtime.RuntimeError.ResourceNotBound, runtime.run(&program, .{}));
|
||||
|
||||
var buffer = [_]u8{0xa5} ** 8;
|
||||
const resources = [_]?[]u8{buffer[0..]};
|
||||
try std.testing.expectError(Runtime.RuntimeError.BufferOutOfBounds, runtime.run(&program, .{ .resource_buffers = &resources }));
|
||||
const unchanged = [_]u8{0xa5} ** 8;
|
||||
try std.testing.expectEqualSlices(u8, &unchanged, &buffer);
|
||||
}
|
||||
@@ -10,5 +10,6 @@ comptime {
|
||||
_ = @import("arithmetic.zig");
|
||||
_ = @import("branching.zig");
|
||||
_ = @import("loops.zig");
|
||||
_ = @import("storage_buffers.zig");
|
||||
_ = @import("termination.zig");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user