diff --git a/src/Module.zig b/src/Module.zig index 2dcb84c..5d9f25f 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -277,7 +277,15 @@ fn applyDecorations(self: *Self, allocator: std.mem.Allocator) ModuleError!void const helpers = struct { fn applyValueLayout(results: []Result, value: *Value, type_word: SpvWord) ModuleError!void { const type_variant = results[type_word].variant orelse return; - const type_data = switch (type_variant) { + const resolved_type_word = switch (type_variant) { + .Type => |type_data| switch (type_data) { + .Pointer => |ptr| ptr.target, + else => type_word, + }, + else => return, + }; + const resolved_type_variant = results[resolved_type_word].variant orelse return; + const type_data = switch (resolved_type_variant) { .Type => |type_data| type_data, else => return, }; @@ -287,8 +295,13 @@ fn applyDecorations(self: *Self, allocator: std.mem.Allocator) ModuleError!void .Structure => |type_structure| { @memcpy(@constCast(structure.offsets), type_structure.members_offsets); @memcpy(@constCast(structure.matrix_strides), type_structure.members_matrix_strides); + @memcpy(@constCast(structure.row_major), type_structure.members_row_major); - for (structure.values, type_structure.members_type_word) |*member_value, member_type_word| { + for (structure.values, type_structure.members_type_word, 0..) |*member_value, member_type_word, member_index| { + if (member_value.* == .RuntimeArray) { + member_value.RuntimeArray.matrix_stride = type_structure.members_matrix_strides[member_index]; + member_value.RuntimeArray.row_major = type_structure.members_row_major[member_index]; + } try applyValueLayout(results, member_value, member_type_word); } }, @@ -355,6 +368,8 @@ fn applyDecorations(self: *Self, allocator: std.mem.Allocator) ModuleError!void switch (decoration.rtype) { .Offset => s.members_offsets[decoration.index] = decoration.literal_1, .MatrixStride => s.members_matrix_strides[decoration.index] = decoration.literal_1, + .RowMajor => s.members_row_major[decoration.index] = true, + .ColMajor => s.members_row_major[decoration.index] = false, else => {}, } }, diff --git a/src/Result.zig b/src/Result.zig index c477d29..5768411 100644 --- a/src/Result.zig +++ b/src/Result.zig @@ -123,6 +123,7 @@ pub const TypeData = union(Type) { members_type_word: []const SpvWord, members_offsets: []?SpvWord, members_matrix_strides: []?SpvWord, + members_row_major: []bool, member_names: std.ArrayList([]const u8), }, Function: struct { @@ -247,6 +248,7 @@ pub fn deinit(self: *Self, allocator: std.mem.Allocator) void { } allocator.free(data.members_offsets); allocator.free(data.members_matrix_strides); + allocator.free(data.members_row_major); data.member_names.deinit(allocator); }, else => {}, @@ -331,6 +333,7 @@ pub fn dupe(self: *const Self, allocator: std.mem.Allocator) RuntimeError!Self { .members_type_word = allocator.dupe(SpvWord, s.members_type_word) catch return RuntimeError.OutOfMemory, .members_offsets = allocator.dupe(?SpvWord, s.members_offsets) catch return RuntimeError.OutOfMemory, .members_matrix_strides = allocator.dupe(?SpvWord, s.members_matrix_strides) catch return RuntimeError.OutOfMemory, + .members_row_major = allocator.dupe(bool, s.members_row_major) catch return RuntimeError.OutOfMemory, .member_names = blk2: { const member_names = s.member_names.clone(allocator) catch return RuntimeError.OutOfMemory; for (member_names.items, s.member_names.items) |*new_name, name| { diff --git a/src/Value.zig b/src/Value.zig index 47d82b3..4b26f4e 100644 --- a/src/Value.zig +++ b/src/Value.zig @@ -78,6 +78,8 @@ pub const Value = union(Type) { type_word: SpvWord, stride: SpvWord, data: []u8, + matrix_stride: ?SpvWord = null, + row_major: bool = false, pub inline fn createValueFromIndex(self: *const @This(), allocator: std.mem.Allocator, results: []const Result, index: usize) RuntimeError!*Value { const offset = try self.getCheckedOffsetOfIndex(index); @@ -86,7 +88,10 @@ pub const Value = union(Type) { value.* = try Value.init(allocator, results, self.type_word, false); errdefer value.deinit(allocator); - _ = try value.write(self.data[offset .. offset + self.stride]); + _ = if (self.matrix_stride) |matrix_stride| + try value.writeWithMatrixLayout(self.data[offset .. offset + self.stride], matrix_stride, self.row_major) + else + try value.write(self.data[offset .. offset + self.stride]); return value; } @@ -95,7 +100,10 @@ pub const Value = union(Type) { const offset = try self.getCheckedOffsetOfIndex(index); var value = try Value.init(allocator, results, self.type_word, false); errdefer value.deinit(allocator); - _ = try value.write(self.data[offset .. offset + self.stride]); + _ = if (self.matrix_stride) |matrix_stride| + try value.writeWithMatrixLayout(self.data[offset .. offset + self.stride], matrix_stride, self.row_major) + else + try value.write(self.data[offset .. offset + self.stride]); return value; } @@ -119,6 +127,7 @@ pub const Value = union(Type) { external_data: ?[]u8, offsets: []const ?SpvWord, matrix_strides: []const ?SpvWord, + row_major: []const bool, values: []Self, }, Function: noreturn, @@ -160,6 +169,7 @@ pub const Value = union(Type) { uniform_backing_value: ?*Self = null, owns_uniform_backing_value: bool = false, matrix_stride: ?SpvWord = null, + matrix_row_major: bool = false, }, pub inline fn getCompositeDataOrNull(self: *const Self) ?[]Self { @@ -261,6 +271,8 @@ pub const Value = union(Type) { .type_word = a.components_type_word, .stride = a.stride, .data = &.{}, + .matrix_stride = null, + .row_major = false, }, }; } @@ -284,6 +296,7 @@ pub const Value = union(Type) { .external_data = null, .offsets = allocator.dupe(?SpvWord, s.members_offsets) catch return RuntimeError.OutOfMemory, .matrix_strides = allocator.dupe(?SpvWord, s.members_matrix_strides) catch return RuntimeError.OutOfMemory, + .row_major = allocator.dupe(bool, s.members_row_major) catch return RuntimeError.OutOfMemory, .values = allocator.alloc(Self, member_count) catch return RuntimeError.OutOfMemory, }, }; @@ -299,6 +312,8 @@ pub const Value = union(Type) { .type_word = a.components_type_word, .stride = a.stride, .data = &.{}, + .matrix_stride = null, + .row_major = false, }, }, .Image => .{ @@ -369,6 +384,7 @@ pub const Value = union(Type) { .external_data = s.external_data, .offsets = allocator.dupe(?SpvWord, s.offsets) catch return RuntimeError.OutOfMemory, .matrix_strides = allocator.dupe(?SpvWord, s.matrix_strides) catch return RuntimeError.OutOfMemory, + .row_major = allocator.dupe(bool, s.row_major) catch return RuntimeError.OutOfMemory, .values = values, }; }, @@ -472,7 +488,10 @@ pub const Value = union(Type) { for (s.values, 0..) |v, i| { const member_offset: usize = @intCast(s.offsets[i] orelse end_offset); if (member_offset > output.len) return RuntimeError.OutOfBounds; - const read_size = try v.read(output[member_offset..]); + const read_size = if (s.matrix_strides[i]) |matrix_stride| + try v.readWithMatrixLayout(output[member_offset..], matrix_stride, s.row_major[i]) + else + try v.read(output[member_offset..]); end_offset = @max(end_offset, member_offset + read_size); } return end_offset; @@ -483,7 +502,90 @@ pub const Value = union(Type) { return 0; } - pub fn readMatrixWithStride(self: *const Self, output: []u8, matrix_stride: SpvWord) RuntimeError!usize { + fn readVectorLane(self: *const Self, lane_index: usize, output: []u8) RuntimeError!usize { + switch (self.*) { + .Vector => |values| { + if (lane_index >= values.len) return RuntimeError.OutOfBounds; + return values[lane_index].read(output); + }, + .Vector2f32, .Vector3f32, .Vector4f32 => { + const value = try readLane(.Float, 32, self, lane_index); + if (output.len < @sizeOf(f32)) return RuntimeError.OutOfBounds; + @memcpy(output[0..@sizeOf(f32)], std.mem.asBytes(&value)); + return @sizeOf(f32); + }, + .Vector2i32, .Vector3i32, .Vector4i32 => { + const value = try readLane(.SInt, 32, self, lane_index); + if (output.len < @sizeOf(i32)) return RuntimeError.OutOfBounds; + @memcpy(output[0..@sizeOf(i32)], std.mem.asBytes(&value)); + return @sizeOf(i32); + }, + .Vector2u32, .Vector3u32, .Vector4u32 => { + const value = try readLane(.UInt, 32, self, lane_index); + if (output.len < @sizeOf(u32)) return RuntimeError.OutOfBounds; + @memcpy(output[0..@sizeOf(u32)], std.mem.asBytes(&value)); + return @sizeOf(u32); + }, + else => return RuntimeError.InvalidValueType, + } + } + + fn writeVectorLane(self: *Self, lane_index: usize, input: []const u8) RuntimeError!usize { + switch (self.*) { + .Vector => |values| { + if (lane_index >= values.len) return RuntimeError.OutOfBounds; + return values[lane_index].write(input); + }, + .Vector2f32, .Vector3f32, .Vector4f32 => { + if (input.len < @sizeOf(f32)) return RuntimeError.OutOfBounds; + try writeLane(.Float, 32, self, lane_index, std.mem.bytesToValue(f32, input[0..@sizeOf(f32)])); + return @sizeOf(f32); + }, + .Vector2i32, .Vector3i32, .Vector4i32 => { + if (input.len < @sizeOf(i32)) return RuntimeError.OutOfBounds; + try writeLane(.SInt, 32, self, lane_index, std.mem.bytesToValue(i32, input[0..@sizeOf(i32)])); + return @sizeOf(i32); + }, + .Vector2u32, .Vector3u32, .Vector4u32 => { + if (input.len < @sizeOf(u32)) return RuntimeError.OutOfBounds; + try writeLane(.UInt, 32, self, lane_index, std.mem.bytesToValue(u32, input[0..@sizeOf(u32)])); + return @sizeOf(u32); + }, + else => return RuntimeError.InvalidValueType, + } + } + + pub fn readVectorWithStride(self: *const Self, output: []u8, stride: SpvWord) RuntimeError!usize { + const lane_count: usize = @intCast(try self.resolveLaneCount()); + if (lane_count == 0) return 0; + + const stride_usize: usize = @intCast(stride); + const scalar_size = @divExact(try self.getPlainMemorySize(), lane_count); + const vector_size = (lane_count - 1) * stride_usize + scalar_size; + if (output.len < vector_size) return RuntimeError.OutOfBounds; + + for (0..lane_count) |index| { + _ = try self.readVectorLane(index, output[index * stride_usize ..]); + } + return vector_size; + } + + pub fn writeVectorWithStride(self: *Self, input: []const u8, stride: SpvWord) RuntimeError!usize { + const lane_count: usize = @intCast(try self.resolveLaneCount()); + if (lane_count == 0) return 0; + + const stride_usize: usize = @intCast(stride); + const scalar_size = @divExact(try self.getPlainMemorySize(), lane_count); + const vector_size = (lane_count - 1) * stride_usize + scalar_size; + if (input.len < vector_size) return RuntimeError.OutOfBounds; + + for (0..lane_count) |index| { + _ = try self.writeVectorLane(index, input[index * stride_usize ..]); + } + return vector_size; + } + + pub fn readMatrixWithStride(self: *const Self, output: []u8, matrix_stride: SpvWord, row_major: bool) RuntimeError!usize { const columns = switch (self.*) { .Matrix => |columns| columns, else => return RuntimeError.InvalidValueType, @@ -491,17 +593,46 @@ pub const Value = union(Type) { if (columns.len == 0) return 0; const matrix_stride_usize: usize = @intCast(matrix_stride); - const column_size = try columns[0].getPlainMemorySize(); - const matrix_size = (columns.len - 1) * matrix_stride_usize + column_size; + const row_count: usize = @intCast(try columns[0].resolveLaneCount()); + if (row_count == 0) return 0; + + const scalar_size = @divExact(try columns[0].getPlainMemorySize(), row_count); + const matrix_size = if (row_major) + (row_count - 1) * matrix_stride_usize + columns.len * scalar_size + else + (columns.len - 1) * matrix_stride_usize + try columns[0].getPlainMemorySize(); if (output.len < matrix_size) return RuntimeError.OutOfBounds; - for (columns, 0..) |column, column_index| { - _ = try column.read(output[column_index * matrix_stride_usize ..]); + if (row_major) { + for (columns, 0..) |column, column_index| { + for (0..row_count) |row_index| { + _ = try column.readVectorLane(row_index, output[row_index * matrix_stride_usize + column_index * scalar_size ..]); + } + } + } else { + for (columns, 0..) |column, column_index| { + _ = try column.read(output[column_index * matrix_stride_usize ..]); + } } return matrix_size; } - pub fn writeMatrixWithStride(self: *Self, input: []const u8, matrix_stride: SpvWord) RuntimeError!usize { + pub fn readWithMatrixLayout(self: *const Self, output: []u8, matrix_stride: SpvWord, row_major: bool) RuntimeError!usize { + return switch (self.*) { + .Matrix => try self.readMatrixWithStride(output, matrix_stride, row_major), + .Array => |arr| blk: { + var offset: usize = 0; + for (arr.values) |*value| { + _ = try value.readWithMatrixLayout(output[offset..], matrix_stride, row_major); + offset += arr.stride; + } + break :blk offset; + }, + else => try self.read(output), + }; + } + + pub fn writeMatrixWithStride(self: *Self, input: []const u8, matrix_stride: SpvWord, row_major: bool) RuntimeError!usize { const columns = switch (self.*) { .Matrix => |columns| columns, else => return RuntimeError.InvalidValueType, @@ -509,16 +640,45 @@ pub const Value = union(Type) { if (columns.len == 0) return 0; const matrix_stride_usize: usize = @intCast(matrix_stride); - const column_size = try columns[0].getPlainMemorySize(); - const matrix_size = (columns.len - 1) * matrix_stride_usize + column_size; + const row_count: usize = @intCast(try columns[0].resolveLaneCount()); + if (row_count == 0) return 0; + + const scalar_size = @divExact(try columns[0].getPlainMemorySize(), row_count); + const matrix_size = if (row_major) + (row_count - 1) * matrix_stride_usize + columns.len * scalar_size + else + (columns.len - 1) * matrix_stride_usize + try columns[0].getPlainMemorySize(); if (input.len < matrix_size) return RuntimeError.OutOfBounds; - for (columns, 0..) |*column, column_index| { - _ = try column.write(input[column_index * matrix_stride_usize ..]); + if (row_major) { + for (columns, 0..) |*column, column_index| { + for (0..row_count) |row_index| { + _ = try column.writeVectorLane(row_index, input[row_index * matrix_stride_usize + column_index * scalar_size ..]); + } + } + } else { + for (columns, 0..) |*column, column_index| { + _ = try column.write(input[column_index * matrix_stride_usize ..]); + } } return matrix_size; } + pub fn writeWithMatrixLayout(self: *Self, input: []const u8, matrix_stride: SpvWord, row_major: bool) RuntimeError!usize { + return switch (self.*) { + .Matrix => try self.writeMatrixWithStride(input, matrix_stride, row_major), + .Array => |*arr| blk: { + var offset: usize = 0; + for (arr.values) |*value| { + _ = try value.writeWithMatrixLayout(input[offset..], matrix_stride, row_major); + offset += arr.stride; + } + break :blk offset; + }, + else => try self.write(input), + }; + } + pub fn write(self: *Self, input: []const u8) RuntimeError!usize { const vecRoutine = struct { inline fn routine(comptime T: type, vec: *T, in: []const u8) RuntimeError!usize { @@ -620,14 +780,10 @@ pub const Value = union(Type) { for (s.values, 0..) |*v, i| { const member_offset: usize = @intCast(s.offsets[i] orelse end_offset); if (member_offset > input.len) return RuntimeError.OutOfBounds; - const write_size = switch (v.*) { - .Matrix => |*columns| blk: { - const matrix_stride = s.matrix_strides[i] orelse break :blk try v.write(input[member_offset..]); - _ = columns; - break :blk try v.writeMatrixWithStride(input[member_offset..], matrix_stride); - }, - else => try v.write(input[member_offset..]), - }; + const write_size = if (s.matrix_strides[i]) |matrix_stride| + try v.writeWithMatrixLayout(input[member_offset..], matrix_stride, s.row_major[i]) + else + try v.write(input[member_offset..]); end_offset = @max(end_offset, member_offset + write_size); } if (end_offset > input.len) return RuntimeError.OutOfBounds; @@ -736,14 +892,20 @@ pub const Value = union(Type) { pub fn flushPtr(self: *Self, allocator: std.mem.Allocator) RuntimeError!void { _ = allocator; switch (self.*) { + .Structure => |*s| { + if (s.external_data) |data| { + _ = try self.read(data); + } + }, .Pointer => |*p| { if (p.uniform_slice_window) |window| { switch (p.ptr) { .common => |ptr| { - _ = if (p.matrix_stride) |matrix_stride| - try ptr.readMatrixWithStride(window, matrix_stride) - else - try ptr.read(window); + _ = if (p.matrix_stride) |matrix_stride| blk: { + if (p.matrix_row_major and ptr.isVector()) + break :blk try ptr.readVectorWithStride(window, matrix_stride); + break :blk try ptr.readWithMatrixLayout(window, matrix_stride, p.matrix_row_major); + } else try ptr.read(window); }, .f32_ptr => |ptr| { if (window.len < @sizeOf(f32)) return RuntimeError.OutOfBounds; @@ -781,6 +943,7 @@ pub const Value = union(Type) { allocator.free(s.values); allocator.free(s.offsets); allocator.free(s.matrix_strides); + allocator.free(s.row_major); }, .Pointer => |*p| { if (p.owns_uniform_backing_value) { diff --git a/src/opcodes.zig b/src/opcodes.zig index 337a819..a06c565 100644 --- a/src/opcodes.zig +++ b/src/opcodes.zig @@ -211,13 +211,13 @@ pub const SetupDispatcher = block: { .ImageQuerySizeLod = autoSetupConstant, .ImageRead = autoSetupConstant, .ImageSampleExplicitLod = autoSetupConstant, - .ImageSampleImplicitLod = autoSetupConstant, + .ImageSampleImplicitLod = opDerivativeSetup, .ImageSampleDrefExplicitLod = autoSetupConstant, - .ImageSampleDrefImplicitLod = autoSetupConstant, + .ImageSampleDrefImplicitLod = opDerivativeSetup, .ImageSampleProjDrefExplicitLod = autoSetupConstant, - .ImageSampleProjDrefImplicitLod = autoSetupConstant, + .ImageSampleProjDrefImplicitLod = opDerivativeSetup, .ImageSampleProjExplicitLod = autoSetupConstant, - .ImageSampleProjImplicitLod = autoSetupConstant, + .ImageSampleProjImplicitLod = opDerivativeSetup, .ImageTexelPointer = autoSetupConstant, .InBoundsAccessChain = setupAccessChain, .IsFinite = autoSetupConstant, @@ -1463,25 +1463,25 @@ fn ImageEngine(comptime Op: ImageOp) type { } } - fn sampleImageImplicitLod(rt: *Runtime, dst: *Value, driver_image: *anyopaque, driver_sampler: *anyopaque, dim: spv.SpvDim, x: f32, y: f32, z: f32, offset: Runtime.ImageOffset) RuntimeError!void { + fn sampleImageImplicitLod(rt: *Runtime, dst: *Value, driver_image: *anyopaque, driver_sampler: *anyopaque, dim: spv.SpvDim, x: f32, y: f32, z: f32, lod: ?f32, offset: Runtime.ImageOffset) RuntimeError!void { switch (dst.*) { .Vector4f32, .Vector3f32, .Vector2f32, - => try writeFloatTexel(dst, try rt.image_api.sampleImageFloat4(driver_image, driver_sampler, dim, x, y, z, null, offset)), + => try writeFloatTexel(dst, try rt.image_api.sampleImageFloat4(driver_image, driver_sampler, dim, x, y, z, lod, offset)), .Vector4i32, .Vector3i32, .Vector2i32, .Vector4u32, .Vector3u32, .Vector2u32, - => try writeIntTexel(dst, try rt.image_api.sampleImageInt4(driver_image, driver_sampler, dim, x, y, z, null, offset)), + => try writeIntTexel(dst, try rt.image_api.sampleImageInt4(driver_image, driver_sampler, dim, x, y, z, lod, offset)), .Vector => |lanes| { if (lanes.len == 0) return RuntimeError.InvalidSpirV; switch (lanes[0]) { - .Float => try writeFloatTexel(dst, try rt.image_api.sampleImageFloat4(driver_image, driver_sampler, dim, x, y, z, null, offset)), - .Int => try writeIntTexel(dst, try rt.image_api.sampleImageInt4(driver_image, driver_sampler, dim, x, y, z, null, offset)), + .Float => try writeFloatTexel(dst, try rt.image_api.sampleImageFloat4(driver_image, driver_sampler, dim, x, y, z, lod, offset)), + .Int => try writeIntTexel(dst, try rt.image_api.sampleImageInt4(driver_image, driver_sampler, dim, x, y, z, lod, offset)), else => return RuntimeError.InvalidValueType, } }, @@ -1490,6 +1490,81 @@ fn ImageEngine(comptime Op: ImageOp) type { } } + fn cubeFaceCoord(x: f32, y: f32, z: f32) struct { u: f32, v: f32 } { + const ax = @abs(x); + const ay = @abs(y); + const az = @abs(z); + + var sc: f32 = 0.0; + var tc: f32 = 0.0; + var ma: f32 = 1.0; + + if (ax >= ay and ax >= az) { + ma = ax; + if (x >= 0.0) { + sc = -z; + tc = -y; + } else { + sc = z; + tc = -y; + } + } else if (ay >= ax and ay >= az) { + ma = ay; + if (y >= 0.0) { + sc = x; + tc = z; + } else { + sc = x; + tc = -z; + } + } else { + ma = az; + if (z >= 0.0) { + sc = x; + tc = -y; + } else { + sc = -x; + tc = -y; + } + } + + const inv_ma = if (ma == 0.0) 0.0 else 1.0 / ma; + return .{ + .u = (sc * inv_ma + 1.0) * 0.5, + .v = (tc * inv_ma + 1.0) * 0.5, + }; + } + + fn implicitSampleLod(rt: *Runtime, coordinate_id: SpvWord, driver_image: *anyopaque, driver_sampler: *anyopaque, dim: spv.SpvDim, x: f32, y: f32, z: f32) RuntimeError!?f32 { + const coord_derivative = rt.derivatives.get(coordinate_id) orelse return null; + const coord_dx_x = try readSampleCoordLane(&coord_derivative.dx, 0); + const coord_dx_y = readSampleCoordLane(&coord_derivative.dx, 1) catch 0.0; + const coord_dx_z = readSampleCoordLane(&coord_derivative.dx, 2) catch 0.0; + const coord_dy_x = try readSampleCoordLane(&coord_derivative.dy, 0); + const coord_dy_y = readSampleCoordLane(&coord_derivative.dy, 1) catch 0.0; + const coord_dy_z = readSampleCoordLane(&coord_derivative.dy, 2) catch 0.0; + const lod_dim, const derivatives = if (dim == .Cube) blk: { + const center = cubeFaceCoord(x, y, z); + const dx = cubeFaceCoord(x + coord_dx_x, y + coord_dx_y, z + coord_dx_z); + const dy = cubeFaceCoord(x + coord_dy_x, y + coord_dy_y, z + coord_dy_z); + break :blk .{ + spv.SpvDim.@"2D", + Runtime.ImageDerivatives{ + .dx = .{ .x = dx.u - center.u, .y = dx.v - center.v, .z = 0.0, .w = 0.0 }, + .dy = .{ .x = dy.u - center.u, .y = dy.v - center.v, .z = 0.0, .w = 0.0 }, + }, + }; + } else .{ + dim, + Runtime.ImageDerivatives{ + .dx = .{ .x = coord_dx_x, .y = coord_dx_y, .z = coord_dx_z, .w = 0.0 }, + .dy = .{ .x = coord_dy_x, .y = coord_dy_y, .z = coord_dy_z, .w = 0.0 }, + }, + }; + const lod = try rt.image_api.queryImageLod(driver_image, driver_sampler, lod_dim, derivatives); + return lod.y; + } + fn setImplicitSampleDerivative( allocator: std.mem.Allocator, rt: *Runtime, @@ -1521,9 +1596,10 @@ fn ImageEngine(comptime Op: ImageOp) type { const coord_dy_x = try readSampleCoordLane(&coord_derivative.dy, 0); const coord_dy_y = readSampleCoordLane(&coord_derivative.dy, 1) catch 0.0; const coord_dy_z = readSampleCoordLane(&coord_derivative.dy, 2) catch 0.0; + const sample_lod = try implicitSampleLod(rt, coordinate_id, driver_image, driver_sampler, dim, x, y, z); - try sampleImageImplicitLod(rt, &dx_sample, driver_image, driver_sampler, dim, x + coord_dx_x, y + coord_dx_y, z + coord_dx_z, offset); - try sampleImageImplicitLod(rt, &dy_sample, driver_image, driver_sampler, dim, x + coord_dy_x, y + coord_dy_y, z + coord_dy_z, offset); + try sampleImageImplicitLod(rt, &dx_sample, driver_image, driver_sampler, dim, x + coord_dx_x, y + coord_dx_y, z + coord_dx_z, sample_lod, offset); + try sampleImageImplicitLod(rt, &dy_sample, driver_image, driver_sampler, dim, x + coord_dy_x, y + coord_dy_y, z + coord_dy_z, sample_lod, offset); var dx = try Value.init(allocator, rt.results, result_type_word, false); defer dx.deinit(allocator); @@ -1892,6 +1968,16 @@ fn ImageEngine(comptime Op: ImageOp) type { }; const image_operands = if (word_count > 4) try rt.it.next() else 0; const parsed_operands = try parseImageOperands(rt, image_operands); + const lod = try implicitSampleLod( + rt, + coordinate_id, + sampled_image_operand.driver_image, + sampled_image_operand.driver_sampler, + sampled_image_operand.dim, + coords.x, + coords.y, + coords.z, + ); try sampleImageImplicitLod( rt, @@ -1902,6 +1988,7 @@ fn ImageEngine(comptime Op: ImageOp) type { coords.x, coords.y, coords.z, + lod, parsed_operands.offset, ); try setImplicitSampleDerivative( @@ -2968,11 +3055,12 @@ fn copyValue(dst: *Value, src: *const Value) RuntimeError!void { } } - inline fn readWindow(dst_v: *Value, window: []const u8, matrix_stride: ?SpvWord) RuntimeError!void { - _ = if (matrix_stride) |stride| - try dst_v.writeMatrixWithStride(window, stride) - else - try dst_v.write(window); + inline fn readWindow(dst_v: *Value, window: []const u8, matrix_stride: ?SpvWord, matrix_row_major: bool) RuntimeError!void { + _ = if (matrix_stride) |stride| blk: { + if (matrix_row_major and dst_v.isVector()) + break :blk try dst_v.writeVectorWithStride(window, stride); + break :blk try dst_v.writeWithMatrixLayout(window, stride, matrix_row_major); + } else try dst_v.write(window); } }; @@ -2988,10 +3076,11 @@ fn copyValue(dst: *Value, src: *const Value) RuntimeError!void { else => try copyValue(dst_val_ptr, src), } if (dst_ptr.uniform_slice_window) |window| { - _ = if (dst_ptr.matrix_stride) |matrix_stride| - try dst_val_ptr.readMatrixWithStride(window, matrix_stride) - else - try dst_val_ptr.read(window); + _ = if (dst_ptr.matrix_stride) |matrix_stride| blk: { + if (dst_ptr.matrix_row_major and dst_val_ptr.isVector()) + break :blk try dst_val_ptr.readVectorWithStride(window, matrix_stride); + break :blk try dst_val_ptr.readWithMatrixLayout(window, matrix_stride, dst_ptr.matrix_row_major); + } else try dst_val_ptr.read(window); } return; }, @@ -3035,6 +3124,7 @@ fn copyValue(dst: *Value, src: *const Value) RuntimeError!void { if (dst.* == .Structure) { @memcpy(@constCast(dst.Structure.offsets), s.offsets); @memcpy(@constCast(dst.Structure.matrix_strides), s.matrix_strides); + @memcpy(@constCast(dst.Structure.row_major), s.row_major); } const dst_slice = helpers.getDstSlice(dst); try helpers.copySlice(dst_slice.?, s.values); @@ -3043,28 +3133,28 @@ fn copyValue(dst: *Value, src: *const Value) RuntimeError!void { .common => |src_val_ptr| { if (ptr.uniform_slice_window) |window| { try copyValue(dst, src_val_ptr); - try helpers.readWindow(dst, window, ptr.matrix_stride); + try helpers.readWindow(dst, window, ptr.matrix_stride, ptr.matrix_row_major); } else { try copyValue(dst, src_val_ptr); } }, .f32_ptr => |src_f32_ptr| { if (ptr.uniform_slice_window) |window| { - try helpers.readWindow(dst, window, ptr.matrix_stride); + try helpers.readWindow(dst, window, ptr.matrix_stride, ptr.matrix_row_major); } else { try helpers.readF32(dst, src_f32_ptr); } }, .i32_ptr => |src_i32_ptr| { if (ptr.uniform_slice_window) |window| { - try helpers.readWindow(dst, window, ptr.matrix_stride); + try helpers.readWindow(dst, window, ptr.matrix_stride, ptr.matrix_row_major); } else { try helpers.readI32(dst, src_i32_ptr); } }, .u32_ptr => |src_u32_ptr| { if (ptr.uniform_slice_window) |window| { - try helpers.readWindow(dst, window, ptr.matrix_stride); + try helpers.readWindow(dst, window, ptr.matrix_stride, ptr.matrix_row_major); } else { try helpers.readU32(dst, src_u32_ptr); } @@ -3136,12 +3226,22 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime } return null; } + + fn laneOffset(matrix_stride: ?SpvWord, matrix_row_major: bool, lane_index: usize, lane_size: usize) usize { + if (matrix_row_major) { + if (matrix_stride) |stride| { + return lane_index * @as(usize, @intCast(stride)); + } + } + return lane_index * lane_size; + } }; var uniform_slice_window: ?[]u8 = null; var uniform_backing_value: ?*Value = null; var owns_uniform_backing_value = false; var matrix_stride: ?SpvWord = null; + var matrix_row_major = false; if (std.meta.activeTag(value_ptr.*) == .Pointer) { const ptr = value_ptr.Pointer; @@ -3149,6 +3249,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime uniform_backing_value = ptr.uniform_backing_value; owns_uniform_backing_value = false; matrix_stride = ptr.matrix_stride; + matrix_row_major = ptr.matrix_row_major; switch (ptr.ptr) { .common => |common| value_ptr = common, else => return RuntimeError.InvalidSpirV, @@ -3173,6 +3274,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime uniform_backing_value = ptr.uniform_backing_value; owns_uniform_backing_value = false; matrix_stride = ptr.matrix_stride; + matrix_row_major = ptr.matrix_row_major; switch (ptr.ptr) { .common => |common| value_ptr = common, else => return RuntimeError.InvalidSpirV, @@ -3184,7 +3286,15 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime switch (value_ptr.*) { .Vector, .Matrix => |v| { if (component_index >= v.len) return RuntimeError.OutOfBounds; - const offset = if (std.meta.activeTag(value_ptr.*) == .Matrix and matrix_stride != null) + const is_matrix = std.meta.activeTag(value_ptr.*) == .Matrix; + const offset = if (!is_matrix and matrix_stride != null and matrix_row_major) + component_index * @as(usize, @intCast(matrix_stride.?)) + else if (is_matrix and matrix_stride != null and matrix_row_major) row_major_offset_blk: { + const column = &v[component_index]; + const lane_count: usize = @intCast(try column.resolveLaneCount()); + if (lane_count == 0) return RuntimeError.OutOfBounds; + break :row_major_offset_blk component_index * @divExact(try column.getPlainMemorySize(), lane_count); + } else if (is_matrix and matrix_stride != null) component_index * @as(usize, @intCast(matrix_stride.?)) else plain_offset_blk: { var plain_offset: usize = 0; @@ -3195,12 +3305,24 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime }; uniform_slice_window = try helpers.advanceWindow(uniform_slice_window, offset); value_ptr = &v[component_index]; - if (std.meta.activeTag(value_ptr.*) != .Matrix) matrix_stride = null; + if (is_matrix and matrix_row_major) { + // Keep stride for the selected column vector; its lanes are separated by MatrixStride. + } else if (std.meta.activeTag(value_ptr.*) != .Matrix) { + matrix_stride = null; + matrix_row_major = false; + } }, .Array => |a| { if (component_index >= a.values.len) return RuntimeError.OutOfBounds; uniform_slice_window = try helpers.advanceWindow(uniform_slice_window, component_index * a.stride); value_ptr = &a.values[component_index]; + switch (value_ptr.*) { + .Array, .Matrix => {}, + else => { + matrix_stride = null; + matrix_row_major = false; + }, + } }, .Structure => |s| { if (component_index >= s.values.len) return RuntimeError.OutOfBounds; @@ -3219,6 +3341,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime value_ptr = &s.values[component_index]; matrix_stride = s.matrix_strides[component_index]; + matrix_row_major = s.row_major[component_index]; }, .RuntimeArray => |*arr| { if (component_index >= arr.getLen()) return RuntimeError.OutOfBounds; @@ -3242,11 +3365,22 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime uniform_backing_value = backing; owns_uniform_backing_value = true; uniform_slice_window = arr.data[element_offset .. element_offset + arr.stride]; + if (arr.matrix_stride) |stride| { + matrix_stride = stride; + matrix_row_major = arr.row_major; + } + switch (value_ptr.*) { + .Array, .Matrix => {}, + else => { + matrix_stride = null; + matrix_row_major = false; + }, + } }, .Vector4f32 => |*v| switch (component_index) { inline 0...3 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .f32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(f32), @sizeOf(f32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(f32)), @sizeOf(f32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3256,7 +3390,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector3f32 => |*v| switch (component_index) { inline 0...2 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .f32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(f32), @sizeOf(f32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(f32)), @sizeOf(f32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3266,7 +3400,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector2f32 => |*v| switch (component_index) { inline 0...1 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .f32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(f32), @sizeOf(f32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(f32)), @sizeOf(f32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3276,7 +3410,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector4i32 => |*v| switch (component_index) { inline 0...3 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .i32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(i32), @sizeOf(i32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(i32)), @sizeOf(i32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3286,7 +3420,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector3i32 => |*v| switch (component_index) { inline 0...2 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .i32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(i32), @sizeOf(i32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(i32)), @sizeOf(i32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3296,7 +3430,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector2i32 => |*v| switch (component_index) { inline 0...1 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .i32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(i32), @sizeOf(i32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(i32)), @sizeOf(i32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3306,7 +3440,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector4u32 => |*v| switch (component_index) { inline 0...3 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .u32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(u32), @sizeOf(u32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(u32)), @sizeOf(u32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3316,7 +3450,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector3u32 => |*v| switch (component_index) { inline 0...2 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .u32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(u32), @sizeOf(u32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(u32)), @sizeOf(u32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3326,7 +3460,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .Vector2u32 => |*v| switch (component_index) { inline 0...1 => |idx| break :blk .{ .Pointer = .{ .ptr = .{ .u32_ptr = &v[idx] }, - .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, idx * @sizeOf(u32), @sizeOf(u32)), + .uniform_slice_window = try helpers.advanceWindowSized(uniform_slice_window, helpers.laneOffset(matrix_stride, matrix_row_major, idx, @sizeOf(u32)), @sizeOf(u32)), .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = null, @@ -3346,6 +3480,7 @@ fn opAccessChain(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime .uniform_backing_value = uniform_backing_value, .owns_uniform_backing_value = owns_uniform_backing_value, .matrix_stride = matrix_stride, + .matrix_row_major = matrix_row_major, }, }; }, @@ -4721,6 +4856,7 @@ fn opMemberName(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime) .members_type_word = undefined, .members_offsets = undefined, .members_matrix_strides = undefined, + .members_row_major = undefined, .member_names = .empty, }, }, @@ -5146,6 +5282,8 @@ fn opTypeStruct(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime) @memset(members_offsets, null); const members_matrix_strides = allocator.alloc(?SpvWord, word_count - 1) catch return RuntimeError.OutOfMemory; @memset(members_matrix_strides, null); + const members_row_major = allocator.alloc(bool, word_count - 1) catch return RuntimeError.OutOfMemory; + @memset(members_row_major, false); if (rt.results[id].variant) |*variant| { switch (variant.*) { @@ -5154,6 +5292,7 @@ fn opTypeStruct(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime) s.members_type_word = members_type_word; s.members_offsets = members_offsets; s.members_matrix_strides = members_matrix_strides; + s.members_row_major = members_row_major; }, else => unreachable, }, @@ -5166,6 +5305,7 @@ fn opTypeStruct(allocator: std.mem.Allocator, word_count: SpvWord, rt: *Runtime) .members_type_word = members_type_word, .members_offsets = members_offsets, .members_matrix_strides = members_matrix_strides, + .members_row_major = members_row_major, .member_names = .empty, }, },