From 1f6b0b1f37b86a7944426111b10526995ea0fff7 Mon Sep 17 00:00:00 2001 From: Kbz-8 Date: Tue, 7 Jul 2026 01:02:45 +0200 Subject: [PATCH] adding daemon upload from phi device --- build.zig | 29 +++++++++++- src/phi/PhiDevice.zig | 70 ++++++++++++++++++++++++++- src/phi/PhiDeviceMemory.zig | 34 +++++++++++-- src/phi/PhiPhysicalDevice.zig | 2 + src/phi/PhiTransport.zig | 89 ++++++++++++----------------------- src/phi/mic/Logger.c | 12 +++-- src/phi/mic/Memory.c | 11 +++-- 7 files changed, 174 insertions(+), 73 deletions(-) diff --git a/build.zig b/build.zig index b9563ad..b4b5a6a 100644 --- a/build.zig +++ b/build.zig @@ -542,11 +542,28 @@ fn customPhi( const daemon = try addPhiCardDaemon(b, optimize, cc, sysroot); const install_daemon = b.addInstallFile(daemon, "lib/phi_device.mic"); lib.step.dependOn(&install_daemon.step); + + const embedded_daemon = addEmbeddedPhiDaemon(b, daemon); + lib_mod.addAnonymousImport("phi_daemon", .{ + .root_source_file = embedded_daemon, + }); } fn optionsPhi(b: *std.Build, options: *Step.Options) !void { - _ = b; - _ = options; + const daemon_remote_path = b.option( + []const u8, + "phi-daemon-remote-path", + "Path where the Xeon Phi daemon is copied on the card", + ) orelse "/tmp/phi_device.mic"; + + const daemon_host_prefix = b.option( + []const u8, + "phi-daemon-host-prefix", + "Host prefix used to reach cards over ssh/scp; card N uses N", + ) orelse "mic"; + + options.addOption([]const u8, "phi_daemon_remote_path", daemon_remote_path); + options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix); } fn addPhiCardDaemon( @@ -601,3 +618,11 @@ fn addPhiCardDaemon( return cmd.addOutputFileArg("phi_device.mic"); } + +fn addEmbeddedPhiDaemon(b: *std.Build, daemon: std.Build.LazyPath) std.Build.LazyPath { + const wf = b.addWriteFiles(); + _ = wf.addCopyFile(daemon, "phi_device.mic"); + return wf.add("phi_daemon.zig", + \\pub const data = @embedFile("phi_device.mic"); + ); +} diff --git a/src/phi/PhiDevice.zig b/src/phi/PhiDevice.zig index f98957d..0b4c160 100644 --- a/src/phi/PhiDevice.zig +++ b/src/phi/PhiDevice.zig @@ -1,10 +1,12 @@ const std = @import("std"); const vk = @import("vulkan"); const base = @import("base"); +const lib = @import("lib.zig"); const PhiQueue = @import("PhiQueue.zig"); const PhiPhysicalDevice = @import("PhiPhysicalDevice.zig"); const PhiTransport = @import("PhiTransport.zig"); +const phi_daemon = @import("phi_daemon"); pub const PhiBinarySemaphore = @import("PhiBinarySemaphore.zig"); pub const PhiBuffer = @import("PhiBuffer.zig"); @@ -28,6 +30,8 @@ pub const PhiRenderPass = @import("PhiRenderPass.zig"); pub const PhiSampler = @import("PhiSampler.zig"); pub const PhiShaderModule = @import("PhiShaderModule.zig"); +const config = lib.config; +const daemon_binary = phi_daemon.data; const VkError = base.VkError; const Self = @This(); @@ -75,7 +79,9 @@ pub fn create(instance: *base.Instance, physical_device: *base.PhysicalDevice, a }; const phi_physical_device: *PhiPhysicalDevice = @alignCast(@fieldParentPtr("interface", physical_device)); - const transport = try PhiTransport.init(phi_physical_device.scif_node_id); + try uploadAndLaunchDaemon(instance, allocator, phi_physical_device.mic_device_num); + + const transport = try PhiTransport.init(instance, phi_physical_device.scif_node_id); self.* = .{ .interface = interface, @@ -214,3 +220,65 @@ pub fn getDeviceGroupPresentCapabilitiesKHR(_: *Interface, capabilities: *vk.Dev pub fn getDeviceGroupSurfacePresentModesKHR(_: *Interface, _: *base.SurfaceKHR) VkError!vk.DeviceGroupPresentModeFlagsKHR { return .{ .local_bit_khr = true }; } + +fn uploadAndLaunchDaemon(instance: *base.Instance, allocator: std.mem.Allocator, mic_device_num: u32) VkError!void { + const io = instance.io(); + + const local_path = std.fmt.allocPrint(allocator, "/tmp/ape_phi_device_{d}_{d}.mic", .{ std.os.linux.getpid(), mic_device_num }) catch return VkError.OutOfHostMemory; + defer allocator.free(local_path); + defer std.Io.Dir.deleteFileAbsolute(io, local_path) catch {}; + + std.Io.Dir.writeFile(.cwd(), io, .{ + .sub_path = local_path, + .data = daemon_binary, + }) catch |err| { + std.log.scoped(.PhiDevice).err("Failed to write embedded Phi daemon: {s}", .{@errorName(err)}); + return VkError.InitializationFailed; + }; + + const host = std.fmt.allocPrint(allocator, "{s}{d}", .{ config.phi_daemon_host_prefix, mic_device_num }) catch return VkError.OutOfHostMemory; + defer allocator.free(host); + + const remote_target = std.fmt.allocPrint(allocator, "{s}:{s}", .{ host, config.phi_daemon_remote_path }) catch return VkError.OutOfHostMemory; + defer allocator.free(remote_target); + + try runHostCommand(instance, allocator, &.{ + "scp", + local_path, + remote_target, + }); + + const launch_command = std.fmt.allocPrint( + allocator, + "chmod +x {s} && nohup {s} >/tmp/phi_device.log 2>&1 |code| if (code == 0) return, + else => {}, + } + + std.log.scoped(.PhiDevice).err("{s} failed: stdout=\"{s}\" stderr=\"{s}\"", .{ argv[0], result.stdout, result.stderr }); + return VkError.InitializationFailed; +} diff --git a/src/phi/PhiDeviceMemory.zig b/src/phi/PhiDeviceMemory.zig index 14664a0..e40a36c 100644 --- a/src/phi/PhiDeviceMemory.zig +++ b/src/phi/PhiDeviceMemory.zig @@ -2,8 +2,10 @@ const std = @import("std"); const vk = @import("vulkan"); const base = @import("base"); const lib = @import("lib.zig"); +const proto = lib.proto; const PhiDevice = @import("PhiDevice.zig"); +const PhiTransport = @import("PhiTransport.zig"); const VkError = base.VkError; @@ -38,10 +40,24 @@ pub fn create(device: *PhiDevice, allocator: std.mem.Allocator, size: vk.DeviceS const allocation_size = std.math.cast(usize, size) orelse return VkError.OutOfDeviceMemory; const remote_handle = if (device_local) blk: { - const remote = try device.transport.allocMemory(size, memory_type_index); - break :blk remote.remote_handle; + const alloc_request: proto.PhiAllocMemoryRequest = .{ + .size = size, + .memory_type_index = memory_type_index, + .flags = 0, + }; + + var reply: proto.PhiAllocMemoryReply = undefined; + try device.transport.request(proto.PHI_COMMAND_ALLOC_MEMORY, std.mem.asBytes(&alloc_request), std.mem.asBytes(&reply)); + + if (reply.result.status != proto.PHI_STATUS_OK) { + return PhiTransport.statusToErr(reply.result.status); + } + + std.log.scoped(.PhiDeviceMemory).info("Recieved remote handle 0x{X}", .{reply.remote_handle}); + + break :blk reply.remote_handle; } else 0; - errdefer if (remote_handle != 0) device.transport.freeMemory(remote_handle); + errdefer if (remote_handle != 0) self.interface.destroy(allocator); const data = if (host_visible) device.interface.device_allocator.allocator().alloc(u8, allocation_size) catch return VkError.OutOfDeviceMemory @@ -64,7 +80,17 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void { interface.owner.device_allocator.allocator().free(data); } if (self.remote_handle != 0) { - device.transport.freeMemory(self.remote_handle); + const request_payload: proto.PhiFreeMemoryRequest = .{ + .remote_handle = self.remote_handle, + }; + var reply: proto.PhiFreeMemoryReply = undefined; + device.transport.request(proto.PHI_COMMAND_FREE_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| { + std.log.scoped(.PhiTransport).err("Remote free failed: {s}", .{@errorName(err)}); + return; + }; + if (reply.result.status != proto.PHI_STATUS_OK) { + std.log.scoped(.PhiTransport).err("Remote free returned status {d}", .{reply.result.status}); + } } allocator.destroy(self); } diff --git a/src/phi/PhiPhysicalDevice.zig b/src/phi/PhiPhysicalDevice.zig index 33456e7..d0bc429 100644 --- a/src/phi/PhiPhysicalDevice.zig +++ b/src/phi/PhiPhysicalDevice.zig @@ -32,6 +32,7 @@ pub const EXTENSIONS = [_]vk.ExtensionProperties{ interface: Interface, scif_node_id: u16, +mic_device_num: u32, pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device: mic.Device, mic_device_num: u32) VkError!*Self { const self = allocator.create(Self) catch return VkError.OutOfHostMemory; @@ -262,6 +263,7 @@ pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device self.* = .{ .interface = interface, .scif_node_id = deviceNumToScifNode(mic_device_num), + .mic_device_num = mic_device_num, }; return self; diff --git a/src/phi/PhiTransport.zig b/src/phi/PhiTransport.zig index 6c3413c..57c825c 100644 --- a/src/phi/PhiTransport.zig +++ b/src/phi/PhiTransport.zig @@ -10,9 +10,10 @@ const Self = @This(); epd: scif.epd_t, sequence: u64 = 1, -mutex: base.SpinMutex = .{}, +mutex: std.Io.Mutex = .init, +instance: *base.Instance, -pub fn init(node_id: u16) VkError!Self { +pub fn init(instance: *base.Instance, node_id: u16) VkError!Self { try scif.load(); errdefer scif.unload(); @@ -33,7 +34,10 @@ pub fn init(node_id: u16) VkError!Self { return VkError.InitializationFailed; } - var self: Self = .{ .epd = epd }; + var self: Self = .{ + .epd = epd, + .instance = instance, + }; try self.handshake(); return self; } @@ -43,57 +47,10 @@ pub fn deinit(self: *Self) void { scif.unload(); } -pub fn allocMemory(self: *Self, size: u64, memory_type_index: u32) VkError!proto.PhiAllocMemoryReply { - self.mutex.lock(); - defer self.mutex.unlock(); +pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void { + self.mutex.lock(self.instance.io()) catch return VkError.DeviceLost; + defer self.mutex.unlock(self.instance.io()); - const alloc_request: proto.PhiAllocMemoryRequest = .{ - .size = size, - .memory_type_index = memory_type_index, - .flags = 0, - }; - var reply: proto.PhiAllocMemoryReply = undefined; - try self.request(proto.PHI_COMMAND_ALLOC_MEMORY, std.mem.asBytes(&alloc_request), std.mem.asBytes(&reply)); - if (reply.result.status != proto.PHI_STATUS_OK) { - return mapStatus(reply.result.status); - } - return reply; -} - -pub fn freeMemory(self: *Self, remote_handle: u64) void { - self.mutex.lock(); - defer self.mutex.unlock(); - - const request_payload: proto.PhiFreeMemoryRequest = .{ - .remote_handle = remote_handle, - }; - var reply: proto.PhiFreeMemoryReply = undefined; - self.request(proto.PHI_COMMAND_FREE_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| { - std.log.scoped(.PhiTransport).err("Remote free failed: {s}", .{@errorName(err)}); - return; - }; - if (reply.result.status != proto.PHI_STATUS_OK) { - std.log.scoped(.PhiTransport).err("Remote free returned status {d}", .{reply.result.status}); - } -} - -fn handshake(self: *Self) VkError!void { - const request_payload: proto.PhiHelloRequest = .{ - .host_protocol_version = proto.PHI_PROTOCOL_VERSION, - .reserved = 0, - }; - var reply: proto.PhiHelloReply = undefined; - try self.request(proto.PHI_COMMAND_HELLO, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)); - if (reply.result.status != proto.PHI_STATUS_OK) { - return mapStatus(reply.result.status); - } - if (reply.device_protocol_version != proto.PHI_PROTOCOL_VERSION) { - std.log.scoped(.PhiTransport).err("Unsupported Phi protocol version {d}", .{reply.device_protocol_version}); - return VkError.InitializationFailed; - } -} - -fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void { const sequence = self.sequence; self.sequence += 1; @@ -124,6 +81,14 @@ fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u try self.readAll(reply_payload); } +pub fn statusToErr(status: c_int) VkError { + return switch (status) { + proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory, + proto.PHI_STATUS_UNSUPPORTED_VERSION => VkError.InitializationFailed, + else => VkError.Unknown, + }; +} + fn writeAll(self: *Self, bytes: []const u8) VkError!void { var offset: usize = 0; while (offset < bytes.len) { @@ -146,10 +111,18 @@ fn readAll(self: *Self, bytes: []u8) VkError!void { } } -fn mapStatus(status: c_int) VkError { - return switch (status) { - proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory, - proto.PHI_STATUS_UNSUPPORTED_VERSION => VkError.InitializationFailed, - else => VkError.Unknown, +fn handshake(self: *Self) VkError!void { + const request_payload: proto.PhiHelloRequest = .{ + .host_protocol_version = proto.PHI_PROTOCOL_VERSION, + .reserved = 0, }; + var reply: proto.PhiHelloReply = undefined; + try self.request(proto.PHI_COMMAND_HELLO, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)); + if (reply.result.status != proto.PHI_STATUS_OK) { + return statusToErr(reply.result.status); + } + if (reply.device_protocol_version != proto.PHI_PROTOCOL_VERSION) { + std.log.scoped(.PhiTransport).err("Unsupported Phi protocol version {d}", .{reply.device_protocol_version}); + return VkError.InitializationFailed; + } } diff --git a/src/phi/mic/Logger.c b/src/phi/mic/Logger.c index 3f2ad60..9dd3b5e 100644 --- a/src/phi/mic/Logger.c +++ b/src/phi/mic/Logger.c @@ -42,7 +42,7 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu time_t now = time(0); struct tm tstruct = *localtime(&now); char buffer[128]; - strftime(buffer, sizeof(buffer), "[%X] ", &tstruct); + strftime(buffer, sizeof(buffer), "%X", &tstruct); FILE* out = stdout; @@ -51,9 +51,9 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu // Way too much printf calls SetConsoleColor(out, MAGENTA); - fprintf(out, "[Phi device "); + fprintf(out, "[ApeDriver "); SetConsoleColor(out, GREEN); - fprintf(out, "Phi"); + fprintf(out, "Phi "); SetConsoleColor(out, YELLOW); fprintf(out, "%s", buffer); SetConsoleColor(out, MAGENTA); @@ -76,11 +76,13 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu break; } + SetConsoleColor(out, RESET); + va_list argptr; va_start(argptr, line); + vfprintf(out, fmt, argptr); + va_end(argptr); - SetConsoleColor(out, RESET); - fprintf(out, fmt, argptr); fputc('\n', out); if(level == PHI_LOG_LEVEL_FATAL) diff --git a/src/phi/mic/Memory.c b/src/phi/mic/Memory.c index cd17892..44dfb08 100644 --- a/src/phi/mic/Memory.c +++ b/src/phi/mic/Memory.c @@ -14,7 +14,6 @@ int HandleAllocMemory(scif_epd_t endpoint, const PhiMessageHeader* header) .remote_handle = 0, .size = 0, }; - void* memory; if(header->payload_size != sizeof(request)) { @@ -26,15 +25,18 @@ int HandleAllocMemory(scif_epd_t endpoint, const PhiMessageHeader* header) if(ReadAll(endpoint, &request, sizeof(request)) < 0) return -1; - memory = calloc(1, (size_t)request.size); + void* memory = malloc((size_t)request.size); if(memory == NULL) + { reply.result.status = PHI_STATUS_OUT_OF_MEMORY; + PhiLogInfoFmt("Failed to allocate %zu bytes", (size_t)request.size); + } else { reply.remote_handle = (uint64_t)(uintptr_t)memory; reply.size = request.size; - PhiLogInfoFmt("Allocated %zu bytes", (size_t)request.size); + PhiLogInfoFmt("Allocated %llu bytes to handle 0x%X", reply.size, reply.remote_handle); } return SendReply(endpoint, header, &reply, sizeof(reply)); @@ -57,16 +59,19 @@ int HandleFreeMemory(scif_epd_t endpoint, const PhiMessageHeader* header) reply.result.status = PHI_STATUS_BAD_MESSAGE; return SendReply(endpoint, header, &reply, sizeof(reply)); } + if(ReadAll(endpoint, &request, sizeof(request)) < 0) return -1; if(request.remote_handle == 0) { reply.result.status = PHI_STATUS_INVALID_HANDLE; + PhiLogErrorFmt("Could not free memory: invalid handle 0x%X", request.remote_handle); } else { free((void*)(uintptr_t)request.remote_handle); + PhiLogInfoFmt("Freed memory handle 0x%X", request.remote_handle); } return SendReply(endpoint, header, &reply, sizeof(reply));