adding daemon upload from phi device

build.zig

@@ -542,11 +542,28 @@ fn customPhi(
     const daemon = try addPhiCardDaemon(b, optimize, cc, sysroot);
     const install_daemon = b.addInstallFile(daemon, "lib/phi_device.mic");
     lib.step.dependOn(&install_daemon.step);
+
+    const embedded_daemon = addEmbeddedPhiDaemon(b, daemon);
+    lib_mod.addAnonymousImport("phi_daemon", .{
+        .root_source_file = embedded_daemon,
+    });
 }
 
 fn optionsPhi(b: *std.Build, options: *Step.Options) !void {
-    _ = b;
+    const daemon_remote_path = b.option(
-    _ = options;
+        []const u8,
+        "phi-daemon-remote-path",
+        "Path where the Xeon Phi daemon is copied on the card",
+    ) orelse "/tmp/phi_device.mic";
+
+    const daemon_host_prefix = b.option(
+        []const u8,
+        "phi-daemon-host-prefix",
+        "Host prefix used to reach cards over ssh/scp; card N uses <prefix>N",
+    ) orelse "mic";
+
+    options.addOption([]const u8, "phi_daemon_remote_path", daemon_remote_path);
+    options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix);
 }
 
 fn addPhiCardDaemon(
@@ -601,3 +618,11 @@ fn addPhiCardDaemon(
 
     return cmd.addOutputFileArg("phi_device.mic");
 }
+
+fn addEmbeddedPhiDaemon(b: *std.Build, daemon: std.Build.LazyPath) std.Build.LazyPath {
+    const wf = b.addWriteFiles();
+    _ = wf.addCopyFile(daemon, "phi_device.mic");
+    return wf.add("phi_daemon.zig",
+        \\pub const data = @embedFile("phi_device.mic");
+    );
+}

src/phi/PhiDevice.zig

@@ -1,10 +1,12 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
+const lib = @import("lib.zig");
 
 const PhiQueue = @import("PhiQueue.zig");
 const PhiPhysicalDevice = @import("PhiPhysicalDevice.zig");
 const PhiTransport = @import("PhiTransport.zig");
+const phi_daemon = @import("phi_daemon");
 
 pub const PhiBinarySemaphore = @import("PhiBinarySemaphore.zig");
 pub const PhiBuffer = @import("PhiBuffer.zig");
@@ -28,6 +30,8 @@ pub const PhiRenderPass = @import("PhiRenderPass.zig");
 pub const PhiSampler = @import("PhiSampler.zig");
 pub const PhiShaderModule = @import("PhiShaderModule.zig");
 
+const config = lib.config;
+const daemon_binary = phi_daemon.data;
 const VkError = base.VkError;
 
 const Self = @This();
@@ -75,7 +79,9 @@ pub fn create(instance: *base.Instance, physical_device: *base.PhysicalDevice, a
     };
 
     const phi_physical_device: *PhiPhysicalDevice = @alignCast(@fieldParentPtr("interface", physical_device));
-    const transport = try PhiTransport.init(phi_physical_device.scif_node_id);
+    try uploadAndLaunchDaemon(instance, allocator, phi_physical_device.mic_device_num);
+
+    const transport = try PhiTransport.init(instance, phi_physical_device.scif_node_id);
 
     self.* = .{
         .interface = interface,
@@ -214,3 +220,65 @@ pub fn getDeviceGroupPresentCapabilitiesKHR(_: *Interface, capabilities: *vk.Dev
 pub fn getDeviceGroupSurfacePresentModesKHR(_: *Interface, _: *base.SurfaceKHR) VkError!vk.DeviceGroupPresentModeFlagsKHR {
     return .{ .local_bit_khr = true };
 }
+
+fn uploadAndLaunchDaemon(instance: *base.Instance, allocator: std.mem.Allocator, mic_device_num: u32) VkError!void {
+    const io = instance.io();
+
+    const local_path = std.fmt.allocPrint(allocator, "/tmp/ape_phi_device_{d}_{d}.mic", .{ std.os.linux.getpid(), mic_device_num }) catch return VkError.OutOfHostMemory;
+    defer allocator.free(local_path);
+    defer std.Io.Dir.deleteFileAbsolute(io, local_path) catch {};
+
+    std.Io.Dir.writeFile(.cwd(), io, .{
+        .sub_path = local_path,
+        .data = daemon_binary,
+    }) catch |err| {
+        std.log.scoped(.PhiDevice).err("Failed to write embedded Phi daemon: {s}", .{@errorName(err)});
+        return VkError.InitializationFailed;
+    };
+
+    const host = std.fmt.allocPrint(allocator, "{s}{d}", .{ config.phi_daemon_host_prefix, mic_device_num }) catch return VkError.OutOfHostMemory;
+    defer allocator.free(host);
+
+    const remote_target = std.fmt.allocPrint(allocator, "{s}:{s}", .{ host, config.phi_daemon_remote_path }) catch return VkError.OutOfHostMemory;
+    defer allocator.free(remote_target);
+
+    try runHostCommand(instance, allocator, &.{
+        "scp",
+        local_path,
+        remote_target,
+    });
+
+    const launch_command = std.fmt.allocPrint(
+        allocator,
+        "chmod +x {s} && nohup {s} >/tmp/phi_device.log 2>&1 </dev/null &",
+        .{ config.phi_daemon_remote_path, config.phi_daemon_remote_path },
+    ) catch return VkError.OutOfHostMemory;
+    defer allocator.free(launch_command);
+
+    try runHostCommand(instance, allocator, &.{
+        "ssh",
+        host,
+        launch_command,
+    });
+}
+
+fn runHostCommand(instance: *base.Instance, allocator: std.mem.Allocator, argv: []const []const u8) VkError!void {
+    const result = std.process.run(allocator, instance.io(), .{
+        .argv = argv,
+        .stdout_limit = .limited(4096),
+        .stderr_limit = .limited(4096),
+    }) catch |err| {
+        std.log.scoped(.PhiDevice).err("Failed to run {s}: {s}", .{ argv[0], @errorName(err) });
+        return VkError.InitializationFailed;
+    };
+    defer allocator.free(result.stdout);
+    defer allocator.free(result.stderr);
+
+    switch (result.term) {
+        .exited => |code| if (code == 0) return,
+        else => {},
+    }
+
+    std.log.scoped(.PhiDevice).err("{s} failed: stdout=\"{s}\" stderr=\"{s}\"", .{ argv[0], result.stdout, result.stderr });
+    return VkError.InitializationFailed;
+}

src/phi/PhiDeviceMemory.zig

@@ -2,8 +2,10 @@ const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
 const lib = @import("lib.zig");
+const proto = lib.proto;
 
 const PhiDevice = @import("PhiDevice.zig");
+const PhiTransport = @import("PhiTransport.zig");
 
 const VkError = base.VkError;
 
@@ -38,10 +40,24 @@ pub fn create(device: *PhiDevice, allocator: std.mem.Allocator, size: vk.DeviceS
     const allocation_size = std.math.cast(usize, size) orelse return VkError.OutOfDeviceMemory;
 
     const remote_handle = if (device_local) blk: {
-        const remote = try device.transport.allocMemory(size, memory_type_index);
+        const alloc_request: proto.PhiAllocMemoryRequest = .{
-        break :blk remote.remote_handle;
+            .size = size,
+            .memory_type_index = memory_type_index,
+            .flags = 0,
+        };
+
+        var reply: proto.PhiAllocMemoryReply = undefined;
+        try device.transport.request(proto.PHI_COMMAND_ALLOC_MEMORY, std.mem.asBytes(&alloc_request), std.mem.asBytes(&reply));
+
+        if (reply.result.status != proto.PHI_STATUS_OK) {
+            return PhiTransport.statusToErr(reply.result.status);
+        }
+
+        std.log.scoped(.PhiDeviceMemory).info("Recieved remote handle 0x{X}", .{reply.remote_handle});
+
+        break :blk reply.remote_handle;
     } else 0;
-    errdefer if (remote_handle != 0) device.transport.freeMemory(remote_handle);
+    errdefer if (remote_handle != 0) self.interface.destroy(allocator);
 
     const data = if (host_visible)
         device.interface.device_allocator.allocator().alloc(u8, allocation_size) catch return VkError.OutOfDeviceMemory
@@ -64,7 +80,17 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
         interface.owner.device_allocator.allocator().free(data);
     }
     if (self.remote_handle != 0) {
-        device.transport.freeMemory(self.remote_handle);
+        const request_payload: proto.PhiFreeMemoryRequest = .{
+            .remote_handle = self.remote_handle,
+        };
+        var reply: proto.PhiFreeMemoryReply = undefined;
+        device.transport.request(proto.PHI_COMMAND_FREE_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| {
+            std.log.scoped(.PhiTransport).err("Remote free failed: {s}", .{@errorName(err)});
+            return;
+        };
+        if (reply.result.status != proto.PHI_STATUS_OK) {
+            std.log.scoped(.PhiTransport).err("Remote free returned status {d}", .{reply.result.status});
+        }
     }
     allocator.destroy(self);
 }

src/phi/PhiPhysicalDevice.zig

@@ -32,6 +32,7 @@ pub const EXTENSIONS = [_]vk.ExtensionProperties{
 
 interface: Interface,
 scif_node_id: u16,
+mic_device_num: u32,
 
 pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device: mic.Device, mic_device_num: u32) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -262,6 +263,7 @@ pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device
     self.* = .{
         .interface = interface,
         .scif_node_id = deviceNumToScifNode(mic_device_num),
+        .mic_device_num = mic_device_num,
     };
 
     return self;

src/phi/PhiTransport.zig

@@ -10,9 +10,10 @@ const Self = @This();
 
 epd: scif.epd_t,
 sequence: u64 = 1,
-mutex: base.SpinMutex = .{},
+mutex: std.Io.Mutex = .init,
+instance: *base.Instance,
 
-pub fn init(node_id: u16) VkError!Self {
+pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
     try scif.load();
     errdefer scif.unload();
 
@@ -33,7 +34,10 @@ pub fn init(node_id: u16) VkError!Self {
         return VkError.InitializationFailed;
     }
 
-    var self: Self = .{ .epd = epd };
+    var self: Self = .{
+        .epd = epd,
+        .instance = instance,
+    };
     try self.handshake();
     return self;
 }
@@ -43,57 +47,10 @@ pub fn deinit(self: *Self) void {
     scif.unload();
 }
 
-pub fn allocMemory(self: *Self, size: u64, memory_type_index: u32) VkError!proto.PhiAllocMemoryReply {
+pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void {
-    self.mutex.lock();
+    self.mutex.lock(self.instance.io()) catch return VkError.DeviceLost;
-    defer self.mutex.unlock();
+    defer self.mutex.unlock(self.instance.io());
 
-    const alloc_request: proto.PhiAllocMemoryRequest = .{
-        .size = size,
-        .memory_type_index = memory_type_index,
-        .flags = 0,
-    };
-    var reply: proto.PhiAllocMemoryReply = undefined;
-    try self.request(proto.PHI_COMMAND_ALLOC_MEMORY, std.mem.asBytes(&alloc_request), std.mem.asBytes(&reply));
-    if (reply.result.status != proto.PHI_STATUS_OK) {
-        return mapStatus(reply.result.status);
-    }
-    return reply;
-}
-
-pub fn freeMemory(self: *Self, remote_handle: u64) void {
-    self.mutex.lock();
-    defer self.mutex.unlock();
-
-    const request_payload: proto.PhiFreeMemoryRequest = .{
-        .remote_handle = remote_handle,
-    };
-    var reply: proto.PhiFreeMemoryReply = undefined;
-    self.request(proto.PHI_COMMAND_FREE_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| {
-        std.log.scoped(.PhiTransport).err("Remote free failed: {s}", .{@errorName(err)});
-        return;
-    };
-    if (reply.result.status != proto.PHI_STATUS_OK) {
-        std.log.scoped(.PhiTransport).err("Remote free returned status {d}", .{reply.result.status});
-    }
-}
-
-fn handshake(self: *Self) VkError!void {
-    const request_payload: proto.PhiHelloRequest = .{
-        .host_protocol_version = proto.PHI_PROTOCOL_VERSION,
-        .reserved = 0,
-    };
-    var reply: proto.PhiHelloReply = undefined;
-    try self.request(proto.PHI_COMMAND_HELLO, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply));
-    if (reply.result.status != proto.PHI_STATUS_OK) {
-        return mapStatus(reply.result.status);
-    }
-    if (reply.device_protocol_version != proto.PHI_PROTOCOL_VERSION) {
-        std.log.scoped(.PhiTransport).err("Unsupported Phi protocol version {d}", .{reply.device_protocol_version});
-        return VkError.InitializationFailed;
-    }
-}
-
-fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void {
     const sequence = self.sequence;
     self.sequence += 1;
 
@@ -124,6 +81,14 @@ fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u
     try self.readAll(reply_payload);
 }
 
+pub fn statusToErr(status: c_int) VkError {
+    return switch (status) {
+        proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory,
+        proto.PHI_STATUS_UNSUPPORTED_VERSION => VkError.InitializationFailed,
+        else => VkError.Unknown,
+    };
+}
+
 fn writeAll(self: *Self, bytes: []const u8) VkError!void {
     var offset: usize = 0;
     while (offset < bytes.len) {
@@ -146,10 +111,18 @@ fn readAll(self: *Self, bytes: []u8) VkError!void {
     }
 }
 
-fn mapStatus(status: c_int) VkError {
+fn handshake(self: *Self) VkError!void {
-    return switch (status) {
+    const request_payload: proto.PhiHelloRequest = .{
-        proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory,
+        .host_protocol_version = proto.PHI_PROTOCOL_VERSION,
-        proto.PHI_STATUS_UNSUPPORTED_VERSION => VkError.InitializationFailed,
+        .reserved = 0,
-        else => VkError.Unknown,
     };
+    var reply: proto.PhiHelloReply = undefined;
+    try self.request(proto.PHI_COMMAND_HELLO, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply));
+    if (reply.result.status != proto.PHI_STATUS_OK) {
+        return statusToErr(reply.result.status);
+    }
+    if (reply.device_protocol_version != proto.PHI_PROTOCOL_VERSION) {
+        std.log.scoped(.PhiTransport).err("Unsupported Phi protocol version {d}", .{reply.device_protocol_version});
+        return VkError.InitializationFailed;
+    }
 }

src/phi/mic/Logger.c

@@ -42,7 +42,7 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu
 	time_t now = time(0);
 	struct tm tstruct = *localtime(&now);
 	char buffer[128];
-	strftime(buffer, sizeof(buffer), "[%X] ", &tstruct);
+	strftime(buffer, sizeof(buffer), "%X", &tstruct);
 
 	FILE* out = stdout;
 
@@ -51,7 +51,7 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu
 
 	// Way too much printf calls
 	SetConsoleColor(out, MAGENTA);
-	fprintf(out, "[Phi device ");
+	fprintf(out, "[ApeDriver ");
 	SetConsoleColor(out, GREEN);
 	fprintf(out, "Phi ");
 	SetConsoleColor(out, YELLOW);
@@ -76,11 +76,13 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu
 			break;
 	}
 
+	SetConsoleColor(out, RESET);
+
 	va_list argptr;
 	va_start(argptr, line);
+	vfprintf(out, fmt, argptr);
+	va_end(argptr);
 
-	SetConsoleColor(out, RESET);
-	fprintf(out, fmt, argptr);
 	fputc('\n', out);
 
 	if(level == PHI_LOG_LEVEL_FATAL)

src/phi/mic/Memory.c

@@ -14,7 +14,6 @@ int HandleAllocMemory(scif_epd_t endpoint, const PhiMessageHeader* header)
 		.remote_handle = 0,
 		.size = 0,
 	};
-	void* memory;
 
 	if(header->payload_size != sizeof(request))
 	{
@@ -26,15 +25,18 @@ int HandleAllocMemory(scif_epd_t endpoint, const PhiMessageHeader* header)
 	if(ReadAll(endpoint, &request, sizeof(request)) < 0)
 		return -1;
 
-	memory = calloc(1, (size_t)request.size);
+	void* memory = malloc((size_t)request.size);
 
 	if(memory == NULL)
+	{
 		reply.result.status = PHI_STATUS_OUT_OF_MEMORY;
+		PhiLogInfoFmt("Failed to allocate %zu bytes", (size_t)request.size);
+	}
 	else
 	{
 		reply.remote_handle = (uint64_t)(uintptr_t)memory;
 		reply.size = request.size;
-		PhiLogInfoFmt("Allocated %zu bytes", (size_t)request.size);
+		PhiLogInfoFmt("Allocated %llu bytes to handle 0x%X", reply.size, reply.remote_handle);
 	}
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));
@@ -57,16 +59,19 @@ int HandleFreeMemory(scif_epd_t endpoint, const PhiMessageHeader* header)
 		reply.result.status = PHI_STATUS_BAD_MESSAGE;
 		return SendReply(endpoint, header, &reply, sizeof(reply));
 	}
+
 	if(ReadAll(endpoint, &request, sizeof(request)) < 0)
 		return -1;
 
 	if(request.remote_handle == 0)
 	{
 		reply.result.status = PHI_STATUS_INVALID_HANDLE;
+		PhiLogErrorFmt("Could not free memory: invalid handle 0x%X", request.remote_handle);
 	}
 	else
 	{
 		free((void*)(uintptr_t)request.remote_handle);
+		PhiLogInfoFmt("Freed memory handle 0x%X", request.remote_handle);
 	}
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));