[Phi] adding full formats support, finishing blitter port from software

blitter

build.zig

@@ -607,6 +607,7 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u
     const sources = [_][]const u8{
         "src/phi/mic/main.c",
         "src/phi/mic/Blitter.c",
+        "src/phi/mic/BlitFormats.c",
         "src/phi/mic/Buffer.c",
         "src/phi/mic/CommandBuffer.c",
         "src/phi/mic/Daemon.c",
@@ -648,7 +649,7 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u
         cmd.addFileArg(avx_object);
     }
 
-    cmd.addArgs(&.{ "-lscif", "-o" });
+    cmd.addArgs(&.{ "-lscif", "-lm", "-o" });
     return cmd.addOutputFileArg("phi_device.mic");
 }
 

src/phi/PhiQueue.zig

@@ -18,6 +18,15 @@ pub const Interface = base.Queue;
 const ring_capacity: usize = @intCast(proto.PHI_QUEUE_RING_CAPACITY);
 const ring_capacity_u64: u64 = @intCast(ring_capacity);
 const shutdown_sequence = std.math.maxInt(u64);
+const shutdown_timeout_ns = 5 * std.time.ns_per_s;
+const shutdown_poll_ns = 10 * std.time.ns_per_ms;
+
+const CompletionShutdown = enum {
+    acknowledged,
+    stopped_without_acknowledgement,
+    timed_out,
+    wait_failed,
+};
 
 const PreparedSubmit = struct {
     wait_semaphores: std.ArrayList(*base.BinarySemaphore),
@@ -63,6 +72,7 @@ pending: [ring_capacity]?PendingCompletion,
 
 error_state: ?VkError,
 shutting_down: bool,
+completion_stopped: bool,
 remote_stopped: bool,
 
 pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, family_index: u32, flags: vk.DeviceQueueCreateFlags) VkError!*Interface {
@@ -129,6 +139,7 @@ pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, fa
         .pending = [_]?PendingCompletion{null} ** ring_capacity,
         .error_state = null,
         .shutting_down = false,
+        .completion_stopped = false,
         .remote_stopped = false,
     };
 
@@ -141,27 +152,63 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) VkError!void
     const io = interface.owner.io();
     const device_allocator = interface.owner.device_allocator.allocator();
 
+    var graceful_shutdown = true;
     waitIdle(interface) catch |err| {
+        graceful_shutdown = false;
         std.log.scoped(.PhiQueue).warn("Queue did not become idle during destruction: {s}", .{@errorName(err)});
     };
-
+    var mutex_locked = true;
-    var graceful_shutdown = true;
     self.mutex.lock(io) catch {
+        mutex_locked = false;
         graceful_shutdown = false;
     };
-    if (graceful_shutdown) {
+    var shutdown_next_sequence: u64 = 0;
+    var shutdown_completed_sequence: u64 = 0;
+    if (mutex_locked) {
         self.shutting_down = true;
+        shutdown_next_sequence = self.next_remote_sequence;
+        shutdown_completed_sequence = self.completed_sequence;
         self.condition.broadcast(io);
         self.mutex.unlock(io);
+    }
 
+    if (graceful_shutdown) {
+        std.log.scoped(.PhiQueue).info(
+            "Sending shutdown doorbell (next remote sequence {d}, completed {d})",
+            .{ shutdown_next_sequence, shutdown_completed_sequence },
+        );
         self.transport.sendQueueDoorbell(shutdown_sequence) catch |err| {
             graceful_shutdown = false;
             std.log.scoped(.PhiQueue).warn("Failed to send queue shutdown doorbell: {s}", .{@errorName(err)});
-            self.transport.close();
         };
-    } else {
+
-        // Wake the blocking completion receiver before releasing queue storage
+        if (graceful_shutdown) {
-        self.transport.close();
+            switch (self.waitForCompletionShutdown(io)) {
+                .acknowledged => {},
+                .stopped_without_acknowledgement => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Remote queue stopped without acknowledging shutdown", .{});
+                },
+                .timed_out => {
+                    graceful_shutdown = false;
+                    const progress = self.completionProgress(io);
+                    std.log.scoped(.PhiQueue).warn(
+                        "Timed out waiting for remote queue shutdown (next remote sequence {d}, completed {d}); closing SCIF endpoint",
+                        .{ progress.next, progress.completed },
+                    );
+                },
+                .wait_failed => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Failed while waiting for remote queue shutdown; closing SCIF endpoint", .{});
+                },
+            }
+        }
+    }
+
+    if (!graceful_shutdown) {
+        // Wake the blocking completion receiver before releasing queue storage.
+        // Keep libscif loaded until the receiver has returned from scif_recv.
+        self.transport.interrupt();
     }
 
     self.completion_group.await(io) catch |err| {
@@ -521,9 +568,16 @@ fn publish(self: *Self, prepared: *PreparedSubmit, fence: ?*base.Fence) VkError!
 }
 
 fn completionRunner(self: *Self) void {
+    defer self.markCompletionStopped();
+
     while (true) {
-        const completion = self.transport.receiveQueueCompletion() catch {
+        const completion = self.transport.receiveQueueCompletion() catch |err| {
-            if (!self.isShuttingDown()) self.markLost(VkError.DeviceLost);
+            if (!self.isShuttingDown()) {
+                std.log.scoped(.PhiQueue).err("Queue completion receive failed: {s}", .{@errorName(err)});
+                self.markLost(VkError.DeviceLost);
+            } else {
+                std.log.scoped(.PhiQueue).warn("Queue completion receiver stopped without shutdown acknowledgement: {s}", .{@errorName(err)});
+            }
             return;
         };
 
@@ -531,6 +585,7 @@ fn completionRunner(self: *Self) void {
             const io = self.interface.owner.io();
             self.mutex.lock(io) catch return;
             self.remote_stopped = completion.status == proto.PHI_STATUS_OK;
+            std.log.scoped(.PhiQueue).info("Received remote queue shutdown acknowledgement (status {d})", .{completion.status});
             self.condition.broadcast(io);
             self.mutex.unlock(io);
             return;
@@ -574,7 +629,14 @@ fn completeOne(self: *Self, completion: proto.PhiQueueCompletion) void {
     if (pending.command_backing) |backing| allocator.free(backing);
 
     if (completion.status != proto.PHI_STATUS_OK or cleanup_failed) {
-        self.markLost(VkError.DeviceLost);
+        std.log.scoped(.PhiQueue).err(
+            "Queue completion {d} failed with remote status {d} (cleanup failed: {})",
+            .{ completion.sequence, completion.status, cleanup_failed },
+        );
+        // A remote command error is confined to this submission. The protocol
+        // stream and ring remain synchronized, so poisoning every later CTS
+        // submission would only hide the command that actually failed.
+        if (cleanup_failed) self.markLost(VkError.DeviceLost);
         failPending(&pending);
     } else if (self.hasError()) {
         failPending(&pending);
@@ -649,6 +711,43 @@ fn hasError(self: *Self) bool {
     return self.error_state != null;
 }
 
+fn completionProgress(self: *Self, io: std.Io) struct { next: u64, completed: u64 } {
+    self.mutex.lock(io) catch return .{ .next = 0, .completed = 0 };
+    defer self.mutex.unlock(io);
+    return .{ .next = self.next_remote_sequence, .completed = self.completed_sequence };
+}
+
+fn waitForCompletionShutdown(self: *Self, io: std.Io) CompletionShutdown {
+    const deadline = std.Io.Clock.Timestamp.fromNow(io, .{
+        .raw = .fromNanoseconds(shutdown_timeout_ns),
+        .clock = .awake,
+    });
+
+    while (true) {
+        self.mutex.lock(io) catch return .wait_failed;
+        const stopped = self.completion_stopped;
+        const acknowledged = self.remote_stopped;
+        self.mutex.unlock(io);
+
+        if (stopped) return if (acknowledged) .acknowledged else .stopped_without_acknowledgement;
+
+        const remaining = deadline.durationFromNow(io);
+        if (remaining.raw.nanoseconds <= 0) return .timed_out;
+        (std.Io.Clock.Duration{
+            .raw = .fromNanoseconds(@min(remaining.raw.nanoseconds, shutdown_poll_ns)),
+            .clock = .awake,
+        }).sleep(io) catch return .wait_failed;
+    }
+}
+
+fn markCompletionStopped(self: *Self) void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return;
+    self.completion_stopped = true;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
 fn isShuttingDown(self: *Self) bool {
     const io = self.interface.owner.io();
     self.mutex.lock(io) catch return true;

src/phi/PhiTransport.zig

@@ -12,6 +12,8 @@ const Self = @This();
 epd: Endpoint,
 sequence: u64 = 1,
 mutex: std.Io.Mutex = .init,
+endpoint_mutex: base.SpinMutex = .{},
+library_loaded: bool = true,
 instance: *base.Instance,
 node_id: u16,
 
@@ -68,13 +70,25 @@ pub fn deinit(self: *Self) void {
     std.log.scoped(.PhiTransport).info("Closed connection", .{});
 }
 
-/// Close a transport without issuing an RPC shutdown. Queue transports switch
+/// Close the endpoint so a thread blocked in SCIF receive wakes up. The SCIF
-/// to a raw full-duplex doorbell protocol after setup and must use this path
+/// library stays loaded until `close`, because that thread may still be
-pub fn close(self: *Self) void {
+/// returning through a dynamically loaded function.
-    if (self.epd < 0) return;
+pub fn interrupt(self: *Self) void {
-
+    self.endpoint_mutex.lock();
-    closeEndpoint(self.epd);
+    const endpoint = self.epd;
     self.epd = -1;
+    self.endpoint_mutex.unlock();
+
+    if (endpoint >= 0) closeEndpoint(endpoint);
+}
+
+/// Close a transport without issuing an RPC shutdown. Queue transports switch
+/// to a raw full-duplex doorbell protocol after setup and must use this path.
+pub fn close(self: *Self) void {
+    self.interrupt();
+    if (!self.library_loaded) return;
+
+    self.library_loaded = false;
     scif.unload();
 }
 
@@ -137,9 +151,10 @@ pub fn statusToErr(status: c_int) VkError {
 }
 
 fn writeAll(self: *Self, bytes: []const u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const written = scif.send(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
+        const written = scif.send(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
         if (written <= 0) {
             return VkError.DeviceLost;
         }
@@ -148,9 +163,10 @@ fn writeAll(self: *Self, bytes: []const u8) VkError!void {
 }
 
 fn readAll(self: *Self, bytes: []u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const read = scif.recv(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
+        const read = scif.recv(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
         if (read <= 0) {
             return VkError.DeviceLost;
         }
@@ -158,6 +174,12 @@ fn readAll(self: *Self, bytes: []u8) VkError!void {
     }
 }
 
+fn getEndpoint(self: *Self) ?Endpoint {
+    self.endpoint_mutex.lock();
+    defer self.endpoint_mutex.unlock();
+    return if (self.epd >= 0) self.epd else null;
+}
+
 fn closeEndpoint(endpoint: Endpoint) void {
     _ = scif.close(endpoint);
 }
@@ -181,8 +203,9 @@ fn handshake(self: *Self) VkError!void {
 }
 
 pub fn registerHostMemory(self: *Self, memory: []u8) VkError!u64 {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     const offset = scif.register(
-        self.epd,
+        endpoint,
         memory.ptr,
         memory.len,
         0,
@@ -196,7 +219,8 @@ pub fn registerHostMemory(self: *Self, memory: []u8) VkError!u64 {
 }
 
 pub fn unregisterHostMemory(self: *Self, offset: u64, size: usize) VkError!void {
-    if (scif.unregister(self.epd, @intCast(offset), size) != 0) {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
+    if (scif.unregister(endpoint, @intCast(offset), size) != 0) {
         return VkError.Unknown;
     }
 }

src/phi/mic/BlitFormats.h

@@ -0,0 +1,43 @@
+#ifndef APE_PHI_BLIT_FORMATS_H
+#define APE_PHI_BLIT_FORMATS_H
+
+#include <Protocol.h>
+#include <stdint.h>
+
+typedef struct PhiBlitFormatInfo
+{
+	uint32_t texel_size;
+	uint8_t is_integer;
+	uint8_t is_signed;
+	uint8_t is_float;
+	uint8_t is_srgb;
+	uint8_t is_unsigned;
+	uint8_t can_read_float;
+	uint8_t can_write_float;
+	uint8_t can_read_int;
+	uint8_t can_write_int;
+	uint8_t vector_unorm8x4;
+} PhiBlitFormatInfo;
+
+typedef struct PhiBlitFloat4
+{
+	float values[4];
+} PhiBlitFloat4;
+
+typedef struct PhiBlitInt4
+{
+	uint32_t values[4];
+} PhiBlitInt4;
+
+int PhiGetBlitFormatInfo(PhiFormat format, PhiBlitFormatInfo* info);
+PhiBlitFloat4 PhiReadBlitFloat4(const uint8_t* map, PhiFormat format);
+void PhiWriteBlitFloat4(PhiBlitFloat4 color, uint8_t* map, PhiFormat format);
+PhiBlitInt4 PhiReadBlitInt4(const uint8_t* map, PhiFormat format);
+void PhiWriteBlitInt4(PhiBlitInt4 color, uint8_t* map, PhiFormat format);
+PhiBlitFloat4 PhiConvertBlitFloat4(PhiBlitFloat4 color,
+                                   PhiFormat src_format,
+                                   PhiFormat dst_format,
+                                   int allow_srgb_conversion,
+                                   int apply_srgb_conversion);
+
+#endif

src/phi/mic/Blitter.c

@@ -1,3 +1,4 @@
+#include <BlitFormats.h>
 #include <Blitter.h>
 #include <Logger.h>
 #include <Memory.h>
@@ -8,7 +9,6 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#define UNORM8_SCALE (1.0f / 255.0f)
 #define BLIT_WEIGHT_SCALE 1024.0f
 #define BLIT_PARALLEL_MIN_PIXELS (256u * 1024u)
 #define BLIT_TASK_TARGET_BYTES (64u * 1024u)
@@ -20,18 +20,8 @@ enum
 	PHI_FILTER_LINEAR = 1,
 };
 
-typedef struct Color
+typedef PhiBlitFloat4 Color;
-{
+typedef PhiBlitFormatInfo FormatInfo;
-	float r;
-	float g;
-	float b;
-	float a;
-} Color;
-
-typedef struct FormatInfo
-{
-	uint32_t texel_size;
-} FormatInfo;
 
 typedef struct SampleCoordinate
 {
@@ -51,149 +41,11 @@ typedef struct BlitWork
 	uint64_t rows_per_layer;
 } BlitWork;
 
-static int GetFormatInfo(PhiFormat format, FormatInfo* info)
-{
-	switch(format)
-	{
-		case PHI_FORMAT_R8_UNORM:
-			info->texel_size = 1;
-			return 1;
-
-		case PHI_FORMAT_R8G8_UNORM:
-			info->texel_size = 2;
-			return 1;
-
-		case PHI_FORMAT_R8G8B8_UNORM:
-		case PHI_FORMAT_B8G8R8_UNORM:
-			info->texel_size = 3;
-			return 1;
-
-		case PHI_FORMAT_R8G8B8A8_UNORM:
-		case PHI_FORMAT_B8G8R8A8_UNORM:
-		case PHI_FORMAT_A8B8G8R8_UNORM_PACK32:
-			info->texel_size = 4;
-			return 1;
-
-		default:
-			return 0;
-	}
-}
-
-static Color ReadUnorm8(const uint8_t* texel, PhiFormat format)
-{
-	Color color = { 0.0f, 0.0f, 0.0f, 1.0f };
-
-	switch(format)
-	{
-		case PHI_FORMAT_R8_UNORM:
-			color.r = (float)texel[0] * UNORM8_SCALE;
-			break;
-
-		case PHI_FORMAT_R8G8_UNORM:
-			color.r = (float)texel[0] * UNORM8_SCALE;
-			color.g = (float)texel[1] * UNORM8_SCALE;
-			break;
-
-		case PHI_FORMAT_R8G8B8_UNORM:
-			color.r = (float)texel[0] * UNORM8_SCALE;
-			color.g = (float)texel[1] * UNORM8_SCALE;
-			color.b = (float)texel[2] * UNORM8_SCALE;
-			break;
-
-		case PHI_FORMAT_B8G8R8_UNORM:
-			color.r = (float)texel[2] * UNORM8_SCALE;
-			color.g = (float)texel[1] * UNORM8_SCALE;
-			color.b = (float)texel[0] * UNORM8_SCALE;
-			break;
-
-		case PHI_FORMAT_R8G8B8A8_UNORM:
-		case PHI_FORMAT_A8B8G8R8_UNORM_PACK32:
-			color.r = (float)texel[0] * UNORM8_SCALE;
-			color.g = (float)texel[1] * UNORM8_SCALE;
-			color.b = (float)texel[2] * UNORM8_SCALE;
-			color.a = (float)texel[3] * UNORM8_SCALE;
-			break;
-
-		case PHI_FORMAT_B8G8R8A8_UNORM:
-			color.r = (float)texel[2] * UNORM8_SCALE;
-			color.g = (float)texel[1] * UNORM8_SCALE;
-			color.b = (float)texel[0] * UNORM8_SCALE;
-			color.a = (float)texel[3] * UNORM8_SCALE;
-			break;
-
-		default:
-			break;
-	}
-
-	return color;
-}
-
-static inline uint8_t QuantizeUnorm8(float value)
-{
-	if(value <= 0.0f)
-		return 0;
-	if(value >= 1.0f)
-		return UINT8_MAX;
-	return (uint8_t)(value * 255.0f + 0.5f);
-}
-
-static void WriteUnorm8(uint8_t* texel, PhiFormat format, Color color)
-{
-	const uint8_t r = QuantizeUnorm8(color.r);
-	const uint8_t g = QuantizeUnorm8(color.g);
-	const uint8_t b = QuantizeUnorm8(color.b);
-	const uint8_t a = QuantizeUnorm8(color.a);
-
-	switch(format)
-	{
-		case PHI_FORMAT_R8_UNORM:
-			texel[0] = r;
-			break;
-
-		case PHI_FORMAT_R8G8_UNORM:
-			texel[0] = r;
-			texel[1] = g;
-			break;
-
-		case PHI_FORMAT_R8G8B8_UNORM:
-			texel[0] = r;
-			texel[1] = g;
-			texel[2] = b;
-			break;
-
-		case PHI_FORMAT_B8G8R8_UNORM:
-			texel[0] = b;
-			texel[1] = g;
-			texel[2] = r;
-			break;
-
-		case PHI_FORMAT_R8G8B8A8_UNORM:
-		case PHI_FORMAT_A8B8G8R8_UNORM_PACK32:
-			texel[0] = r;
-			texel[1] = g;
-			texel[2] = b;
-			texel[3] = a;
-			break;
-
-		case PHI_FORMAT_B8G8R8A8_UNORM:
-			texel[0] = b;
-			texel[1] = g;
-			texel[2] = r;
-			texel[3] = a;
-			break;
-
-		default:
-			break;
-	}
-}
-
 static inline Color LerpColor(Color a, Color b, float factor)
 {
 	Color result;
-	result.r = a.r + (b.r - a.r) * factor;
+	for(uint32_t component = 0; component < 4; ++component)
-	result.g = a.g + (b.g - a.g) * factor;
+		result.values[component] = a.values[component] + (b.values[component] - a.values[component]) * factor;
-	result.b = a.b + (b.b - a.b) * factor;
-	result.a = a.a + (b.a - a.a) * factor;
 	return result;
 }
 
@@ -238,59 +90,133 @@ static Color ReadTexel(const uint8_t* src,
                        uint32_t z)
 {
 	const uint64_t offset = (uint64_t)z * slice_pitch + (uint64_t)y * row_pitch + (uint64_t)x * texel_size;
-	return ReadUnorm8(src + (size_t)offset, format);
+	return PhiReadBlitFloat4(src + (size_t)offset, format);
+}
+
+static inline Color PrepareLinearSample(Color color,
+                                        const PhiCmdBlitImage* command,
+                                        const FormatInfo* src_info,
+                                        int* apply_srgb_conversion)
+{
+	if(command->allow_srgb_conversion && src_info->is_srgb)
+	{
+		*apply_srgb_conversion = 0;
+		return PhiConvertBlitFloat4(color, (PhiFormat)command->src_format, (PhiFormat)command->dst_format, 1, 1);
+	}
+	return color;
 }
 
 static Color
-Sample(const uint8_t* src, const PhiCmdBlitImage* command, uint32_t texel_size, float x, float y, float z, int filter_3d)
+Sample(const uint8_t* src, const PhiCmdBlitImage* command, const FormatInfo* src_info, float x, float y, float z, int filter_3d)
 {
 	const PhiFormat format = (PhiFormat)command->src_format;
+	int apply_srgb_conversion = 1;
 
 	if(command->filter == PHI_FILTER_NEAREST)
 	{
-		return ReadTexel(src,
+		Color color = ReadTexel(src,
-		                 format,
+		                        format,
-		                 texel_size,
+		                        src_info->texel_size,
-		                 command->src_row_pitch,
+		                        command->src_row_pitch,
-		                 command->src_slice_pitch,
+		                        command->src_slice_pitch,
-		                 GetNearestCoordinate(x, command->src_width),
+		                        GetNearestCoordinate(x, command->src_width),
-		                 GetNearestCoordinate(y, command->src_height),
+		                        GetNearestCoordinate(y, command->src_height),
-		                 GetNearestCoordinate(z, command->src_depth));
+		                        GetNearestCoordinate(z, command->src_depth));
+		return PhiConvertBlitFloat4(color, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, 1);
 	}
 
 	const SampleCoordinate sample_x = GetLinearCoordinate(x, command->src_width);
 	const SampleCoordinate sample_y = GetLinearCoordinate(y, command->src_height);
 	const SampleCoordinate sample_z = GetLinearCoordinate(z, command->src_depth);
 
-	const Color color_0_0 = ReadTexel(
+	Color color_0_0 = ReadTexel(src,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.lo, sample_z.lo);
+	                            format,
-	const Color color_0_1 = ReadTexel(
+	                            src_info->texel_size,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.lo, sample_z.lo);
+	                            command->src_row_pitch,
-	const Color color_1_0 = ReadTexel(
+	                            command->src_slice_pitch,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.hi, sample_z.lo);
+	                            sample_x.lo,
-	const Color color_1_1 = ReadTexel(
+	                            sample_y.lo,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.hi, sample_z.lo);
+	                            sample_z.lo);
+	Color color_0_1 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.hi,
+	                            sample_y.lo,
+	                            sample_z.lo);
+	Color color_1_0 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.lo,
+	                            sample_y.hi,
+	                            sample_z.lo);
+	Color color_1_1 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.hi,
+	                            sample_y.hi,
+	                            sample_z.lo);
+	color_0_0 = PrepareLinearSample(color_0_0, command, src_info, &apply_srgb_conversion);
+	color_0_1 = PrepareLinearSample(color_0_1, command, src_info, &apply_srgb_conversion);
+	color_1_0 = PrepareLinearSample(color_1_0, command, src_info, &apply_srgb_conversion);
+	color_1_1 = PrepareLinearSample(color_1_1, command, src_info, &apply_srgb_conversion);
 
 	const Color row_0 = LerpColor(color_0_0, color_0_1, sample_x.factor);
 	const Color row_1 = LerpColor(color_1_0, color_1_1, sample_x.factor);
 	const Color slice_0 = LerpColor(row_0, row_1, sample_y.factor);
 
 	if(!filter_3d)
-		return slice_0;
+		return PhiConvertBlitFloat4(
+		    slice_0, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, apply_srgb_conversion);
 
-	const Color color_0_0_1 = ReadTexel(
+	Color color_0_0_1 = ReadTexel(src,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.lo, sample_z.hi);
+	                              format,
-	const Color color_0_1_1 = ReadTexel(
+	                              src_info->texel_size,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.lo, sample_z.hi);
+	                              command->src_row_pitch,
-	const Color color_1_0_1 = ReadTexel(
+	                              command->src_slice_pitch,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.hi, sample_z.hi);
+	                              sample_x.lo,
-	const Color color_1_1_1 = ReadTexel(
+	                              sample_y.lo,
-	    src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.hi, sample_z.hi);
+	                              sample_z.hi);
+	Color color_0_1_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.hi,
+	                              sample_y.lo,
+	                              sample_z.hi);
+	Color color_1_0_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.lo,
+	                              sample_y.hi,
+	                              sample_z.hi);
+	Color color_1_1_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.hi,
+	                              sample_y.hi,
+	                              sample_z.hi);
+	color_0_0_1 = PrepareLinearSample(color_0_0_1, command, src_info, &apply_srgb_conversion);
+	color_0_1_1 = PrepareLinearSample(color_0_1_1, command, src_info, &apply_srgb_conversion);
+	color_1_0_1 = PrepareLinearSample(color_1_0_1, command, src_info, &apply_srgb_conversion);
+	color_1_1_1 = PrepareLinearSample(color_1_1_1, command, src_info, &apply_srgb_conversion);
 
 	const Color row_0_1 = LerpColor(color_0_0_1, color_0_1_1, sample_x.factor);
 	const Color row_1_1 = LerpColor(color_1_0_1, color_1_1_1, sample_x.factor);
 	const Color slice_1 = LerpColor(row_0_1, row_1_1, sample_y.factor);
-	return LerpColor(slice_0, slice_1, sample_z.factor);
+	Color color = LerpColor(slice_0, slice_1, sample_z.factor);
+	return PhiConvertBlitFloat4(
+	    color, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, apply_srgb_conversion);
 }
 
 static inline int IsMemoryRangeValid(const Memory* memory, uint64_t offset, uint64_t size)
@@ -425,13 +351,26 @@ static inline void BlitScalarPixel(uint8_t* dst,
                                    const uint8_t* src,
                                    const PhiCmdBlitImage* command,
                                    const FormatInfo* src_info,
+                                   const FormatInfo* dst_info,
                                    float source_x,
                                    float source_y,
                                    float source_z,
                                    int filter_3d)
 {
-	const Color color = Sample(src, command, src_info->texel_size, source_x, source_y, source_z, filter_3d);
+	if(src_info->is_integer && dst_info->is_integer)
-	WriteUnorm8(dst, (PhiFormat)command->dst_format, color);
+	{
+		const uint32_t x = GetNearestCoordinate(source_x, command->src_width);
+		const uint32_t y = GetNearestCoordinate(source_y, command->src_height);
+		const uint32_t z = GetNearestCoordinate(source_z, command->src_depth);
+		const uint64_t offset =
+		    (uint64_t)z * command->src_slice_pitch + (uint64_t)y * command->src_row_pitch + (uint64_t)x * src_info->texel_size;
+		const PhiBlitInt4 color = PhiReadBlitInt4(src + (size_t)offset, (PhiFormat)command->src_format);
+		PhiWriteBlitInt4(color, dst, (PhiFormat)command->dst_format);
+		return;
+	}
+
+	const Color color = Sample(src, command, src_info, source_x, source_y, source_z, filter_3d);
+	PhiWriteBlitFloat4(color, dst, (PhiFormat)command->dst_format);
 }
 
 static void BlitRow(uint8_t* dst,
@@ -448,7 +387,8 @@ static void BlitRow(uint8_t* dst,
 
 	// Fast path
 	if(command->filter == PHI_FILTER_NEAREST && command->src_format == command->dst_format && command->step_x == 1.0f &&
-	   first_source_x >= 0.0f && first_source_x + (float)(pixel_count - 1) < (float)command->src_width)
+	   !(command->allow_srgb_conversion && (src_info->is_srgb || dst_info->is_srgb)) && first_source_x >= 0.0f &&
+	   first_source_x + (float)(pixel_count - 1) < (float)command->src_width)
 	{
 		const uint32_t source_texel_x = GetNearestCoordinate(first_source_x, command->src_width);
 		const uint32_t source_texel_y = GetNearestCoordinate(source_y, command->src_height);
@@ -461,13 +401,14 @@ static void BlitRow(uint8_t* dst,
 	}
 
 	uint32_t processed = 0;
-	if(src_info->texel_size == 4 && dst_info->texel_size == 4 && command->src_width <= INT32_MAX)
+	if(src_info->vector_unorm8x4 && dst_info->vector_unorm8x4 && command->src_width <= INT32_MAX)
 	{
 		while(processed < pixel_count && ((uintptr_t)(dst + (size_t)processed * 4) & 63) != 0)
 		{
 			const int32_t destination_x = command->dst_x0 + (int32_t)processed;
 			const float source_x = command->src_x0 + (float)destination_x * command->step_x;
-			BlitScalarPixel(dst + (size_t)processed * 4, src, command, src_info, source_x, source_y, source_z, filter_3d);
+			BlitScalarPixel(
+			    dst + (size_t)processed * 4, src, command, src_info, dst_info, source_x, source_y, source_z, filter_3d);
 			++processed;
 		}
 
@@ -536,8 +477,15 @@ static void BlitRow(uint8_t* dst,
 	{
 		const int32_t destination_x = command->dst_x0 + (int32_t)processed;
 		const float source_x = command->src_x0 + (float)destination_x * command->step_x;
-		BlitScalarPixel(
+		BlitScalarPixel(dst + (size_t)processed * dst_info->texel_size,
-		    dst + (size_t)processed * dst_info->texel_size, src, command, src_info, source_x, source_y, source_z, filter_3d);
+		                src,
+		                command,
+		                src_info,
+		                dst_info,
+		                source_x,
+		                source_y,
+		                source_z,
+		                filter_3d);
 		++processed;
 	}
 }
@@ -581,12 +529,21 @@ PhiStatus BlitImage(const PhiCmdBlitImage* command)
 	FormatInfo src_info;
 	FormatInfo dst_info;
 
-	if(!GetFormatInfo((PhiFormat)command->src_format, &src_info) || !GetFormatInfo((PhiFormat)command->dst_format, &dst_info))
+	if(!PhiGetBlitFormatInfo((PhiFormat)command->src_format, &src_info) ||
+	   !PhiGetBlitFormatInfo((PhiFormat)command->dst_format, &dst_info))
 	{
 		LogErrorFmt("Unsupported blit image formats: src=%u dst=%u", command->src_format, command->dst_format);
 		return PHI_STATUS_INVALID_ARGUMENT;
 	}
 
+	const int integer_path = src_info.is_integer && dst_info.is_integer;
+	if((integer_path && (!src_info.can_read_int || !dst_info.can_write_int)) ||
+	   (!integer_path && (!src_info.can_read_float || !dst_info.can_write_float)))
+	{
+		LogErrorFmt("Unsupported blit image format direction: src=%u dst=%u", command->src_format, command->dst_format);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
 	PhiStatus status = ValidateCommand(command, src_memory, dst_memory, &src_info, &dst_info);
 	if(status != PHI_STATUS_OK)
 		return status;

src/phi/mic/CommandBuffer.c

@@ -7,7 +7,7 @@
 #include <string.h>
 
 static const char* CommandName[] = {
-	"CopyBuffer", "FillBuffer", "CopyBufferToImage", "CopyImageToBuffer", "CopyImage",
+	"CopyBuffer", "FillBuffer", "CopyBufferToImage", "CopyImageToBuffer", "CopyImage", "BlitImage",
 };
 
 PhiStatus ReadCommandData(PhiCommandReader* reader, void* data, uint64_t size)

src/phi/mic/Memory.c

@@ -82,7 +82,9 @@ int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 		if(memory == NULL)
 			LogErrorFmt("Failed to allocate %zu bytes", (size_t)request.size);
 		else
-			LogInfoFmt("Allocated %llu bytes to handle 0x%X", request.size, (uintptr_t)memory);
+			LogInfoFmt("Allocated %llu bytes to handle 0x%llX",
+			           (unsigned long long)request.size,
+			           (unsigned long long)(uintptr_t)memory);
 	}
 	else if(header->type == PHI_PACKET_MAP_HOST_MEMORY)
 	{
@@ -93,7 +95,7 @@ int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 		if(memory == NULL)
 			reply.result.status = PHI_STATUS_MAP_HOST_MEMORY_FAILED;
 		else
-			LogInfoFmt("Mapped host memory to handle 0x%X", (uint64_t)(uintptr_t)memory);
+			LogInfoFmt("Mapped host memory to handle 0x%llX", (unsigned long long)(uintptr_t)memory);
 	}
 
 	if(memory != NULL)
@@ -131,20 +133,36 @@ int HandleDestroyMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 	if(request.remote_handle == 0)
 	{
 		reply.result.status = PHI_STATUS_INVALID_HANDLE;
-		LogErrorFmt("Could not free memory: invalid handle 0x%X", request.remote_handle);
+		LogErrorFmt("Could not free memory: invalid handle 0x%llX", (unsigned long long)request.remote_handle);
 	}
 	else
 	{
-		const Memory* memory = (const Memory*)(uintptr_t)request.remote_handle;
+		Memory* memory = (Memory*)(uintptr_t)request.remote_handle;
+		const MemoryType memory_type = memory->type;
+		const char* memory_type_name;
 
-		if(memory->type == PHI_MEMORY_LOCAL)
+		if(memory_type == PHI_MEMORY_LOCAL)
-			free((void*)memory);
+			memory_type_name = "local";
-		else if(memory->type == PHI_MEMORY_HOST_MAPPED)
+		else if(memory_type == PHI_MEMORY_HOST_MAPPED)
-			scif_munmap((void*)memory->ptr, memory->size);
+			memory_type_name = "host-mapped";
+		else
+		{
+			reply.result.status = PHI_STATUS_INVALID_HANDLE;
+			LogErrorFmt("Could not free memory handle 0x%llX: invalid memory type", (unsigned long long)request.remote_handle);
+			return SendReply(endpoint, header, &reply, sizeof(reply));
+		}
 
-		LogInfoFmt("Destroyed %s memory handle 0x%X",
+		if(memory_type == PHI_MEMORY_HOST_MAPPED && scif_munmap(memory->ptr, (size_t)memory->scif_size) != 0)
-		           memory->type == PHI_MEMORY_LOCAL ? "local" : "host-mapped",
+		{
-		           request.remote_handle);
+			reply.result.status = PHI_STATUS_INVALID_HANDLE;
+			LogErrorFmt("Failed to unmap memory handle 0x%llX: %s", (unsigned long long)request.remote_handle, strerror(errno));
+		}
+		else
+		{
+			LogInfoFmt("Destroyed %s memory handle 0x%llX", memory_type_name, (unsigned long long)request.remote_handle);
+		}
+
+		free(memory);
 	}
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));

src/phi/mic/Queue.c

@@ -49,16 +49,21 @@ static int SendQueueCompletion(PhiEndpoint endpoint, uint64_t sequence, PhiStatu
 static int RunQueue(PhiEndpoint endpoint, volatile PhiQueueShared* shared)
 {
 	uint64_t next_sequence = 1;
-	PhiStatus fatal_status = PHI_STATUS_OK;
 
 	for(;;)
 	{
 		PhiQueueDoorbell doorbell;
 		if(ReadAll(endpoint, &doorbell, sizeof(doorbell)) < 0)
+		{
+			LogWarningFmt("Queue peer disconnected while waiting for sequence %llu", (unsigned long long)next_sequence);
 			return 0;
+		}
 
 		if(doorbell.sequence == PHI_QUEUE_SHUTDOWN_SEQUENCE)
+		{
+			LogInfoFmt("Received queue shutdown doorbell at sequence %llu", (unsigned long long)next_sequence);
 			return 1;
+		}
 
 		if(doorbell.sequence < next_sequence)
 			continue;
@@ -76,20 +81,14 @@ static int RunQueue(PhiEndpoint endpoint, volatile PhiQueueShared* shared)
 				.command_count = remote_submission->command_count,
 			};
 
-			PhiStatus status = fatal_status;
+			PhiStatus status;
-			if(status == PHI_STATUS_OK)
+			if(submission.sequence != next_sequence)
-			{
+				status = PHI_STATUS_BAD_MESSAGE;
-				if(submission.sequence != next_sequence)
+			else
-					status = PHI_STATUS_BAD_MESSAGE;
+				status = ExecuteQueueSubmission(endpoint, &submission);
-				else
-					status = ExecuteQueueSubmission(endpoint, &submission);
-			}
 
-			if(status != PHI_STATUS_OK && fatal_status == PHI_STATUS_OK)
+			if(status != PHI_STATUS_OK)
-			{
 				LogErrorFmt("Queue submission %llu failed: %s", (unsigned long long)next_sequence, StatusName[status]);
-				fatal_status = status;
-			}
 
 			__atomic_store_n(&shared->completed_sequence, next_sequence, __ATOMIC_RELEASE);
 			if(SendQueueCompletion(endpoint, next_sequence, status) < 0)
@@ -143,10 +142,22 @@ int HandleQueueSetup(PhiEndpoint endpoint, const PhiMessageHeader* header)
 
 	const int run_result = RunQueue(endpoint, shared);
 	if(scif_munmap((void*)shared, (size_t)request.scif_size) != 0)
+	{
+		LogErrorFmt("Failed to unmap queue ring during shutdown: %s", strerror(errno));
 		return -1;
+	}
 
 	if(run_result == 1)
-		return SendQueueCompletion(endpoint, PHI_QUEUE_SHUTDOWN_SEQUENCE, PHI_STATUS_OK);
+	{
+		LogInfo("Queue ring unmapped; sending shutdown acknowledgement");
+		if(SendQueueCompletion(endpoint, PHI_QUEUE_SHUTDOWN_SEQUENCE, PHI_STATUS_OK) < 0)
+		{
+			LogErrorFmt("Failed to send queue shutdown acknowledgement: %s", strerror(errno));
+			return -1;
+		}
+		LogInfo("Queue shutdown acknowledgement sent");
+		return 0;
+	}
 
 	return run_result;
 }

src/phi/shared/Protocol.h

@@ -157,6 +157,7 @@ typedef enum PhiFormat
 	PHI_FORMAT_R32G32B32A32_SINT = 108,
 	PHI_FORMAT_R32G32B32A32_SFLOAT = 109,
 	PHI_FORMAT_B10G11R11_UFLOAT_PACK32 = 122,
+	PHI_FORMAT_E5B9G9R9_UFLOAT_PACK32 = 123,
 	PHI_FORMAT_D16_UNORM = 124,
 	PHI_FORMAT_X8_D24_UNORM_PACK32 = 125,
 	PHI_FORMAT_D32_SFLOAT = 126,
@@ -164,6 +165,9 @@ typedef enum PhiFormat
 	PHI_FORMAT_D16_UNORM_S8_UINT = 128,
 	PHI_FORMAT_D24_UNORM_S8_UINT = 129,
 	PHI_FORMAT_D32_SFLOAT_S8_UINT = 130,
+
+	PHI_FORMAT_A4R4G4B4_UNORM_PACK16 = 1000340000,
+	PHI_FORMAT_A4B4G4R4_UNORM_PACK16 = 1000340001,
 } PhiFormat;
 
 typedef struct PhiMessageHeader