[Phi] fixing device memory creation, implemented AVX copy and fill for

vkCmdCopyBuffers and vkCmdFillBuffer

src/phi/mic/Buffer.c

@@ -1,6 +1,7 @@
 #include <Buffer.h>
+#include <Memory.h>
 
-#include <string.h>
+#include <avx/Avx.h>
 
 int PhiIsBufferCommand(const PhiCmdHeader* header)
 {
@@ -25,10 +26,13 @@ static PhiStatus CopyBuffer(PhiCommandReader* reader)
 	if(command.src_memory == 0 || command.dst_memory == 0)
 		return PHI_STATUS_INVALID_HANDLE;
 
-	void* dst = (void*)((uintptr_t)command.dst_memory + (uintptr_t)command.dst_offset);
-	const void* src = (const void*)((uintptr_t)command.src_memory + (uintptr_t)command.src_offset);
+	Memory* dst_memory = (Memory*)command.dst_memory;
+	const Memory* src_memory = (const Memory*)command.src_memory;
 
-	memcpy(dst, src, (size_t)command.size);
+	uint8_t* dst = (uint8_t*)dst_memory->ptr + (size_t)command.dst_offset;
+	const uint8_t* src = (const uint8_t*)src_memory->ptr + (size_t)command.src_offset;
+
+	AvxCopy(dst, src, (size_t)command.size);
 
 	return PHI_STATUS_OK;
 }
@@ -36,17 +40,57 @@ static PhiStatus CopyBuffer(PhiCommandReader* reader)
 static PhiStatus FillBuffer(PhiCommandReader* reader)
 {
 	PhiCmdFillBuffer command;
+
 	PhiStatus status = PhiReadCommandData(reader, &command, sizeof(command));
+
 	if(status != PHI_STATUS_OK)
 		return status;
 
 	if(command.memory == 0)
 		return PHI_STATUS_INVALID_HANDLE;
 
-	uint32_t* dst = (uint32_t*)((uintptr_t)command.memory + (uintptr_t)command.offset);
+	Memory* memory = (Memory*)command.memory;
 
-	for(; command.size >= 4; command.size -= 4, dst++)
-		*dst = command.data;
+	uint8_t* dst = (uint8_t*)memory->ptr + (size_t)command.offset;
+
+	size_t size = (size_t)command.size;
+	const uint32_t value = command.data;
+
+	// Check if dst and size are 4-byte aligned
+	if((((uintptr_t)dst | size) & 3) != 0)
+		return PHI_STATUS_INVALID_ARGUMENT;
+
+	// Bring dst to a 64-byte cache-line boundary.
+	while(size >= 4 && ((uintptr_t)dst & 63) != 0)
+	{
+		*(uint32_t*)dst = value;
+
+		dst += 4;
+		size -= 4;
+	}
+
+	while(size >= 256)
+	{
+		AvxFill256(dst, value);
+
+		dst += 256;
+		size -= 256;
+	}
+
+	while(size >= 64)
+	{
+		AvxFill64(dst, value);
+
+		dst += 64;
+		size -= 64;
+	}
+
+	uint32_t* tail = (uint32_t*)dst;
+	while(size >= 4)
+	{
+		*tail++ = value;
+		size -= 4;
+	}
 
 	return PHI_STATUS_OK;
 }

src/phi/mic/CommandBuffer.c

@@ -47,7 +47,7 @@ static PhiStatus ExecuteCommand(PhiCommandReader* reader, const PhiCmdHeader* co
 int HandleWorkExecution(PhiEndpoint endpoint, const PhiMessageHeader* header)
 {
 	PhiWorkExecutionRequest request;
-	PhiWorkExecutionReply reply = {
+	PhiResultReply reply = {
 		.result = {
 			.status = PHI_STATUS_OK,
 			.reserved = 0,

src/phi/mic/Daemon.c

@@ -1,3 +1,4 @@
+#include "Protocol.h"
 #include <CommandBuffer.h>
 #include <Daemon.h>
 #include <Logger.h>
@@ -75,13 +76,14 @@ int HandlePacket(PhiEndpoint endpoint)
 					return -1;
 				break;
 
+			case PHI_PACKET_MAP_HOST_MEMORY:
 			case PHI_PACKET_ALLOC_MEMORY:
-				if(HandleAllocMemory(endpoint, &header) < 0)
+				if(HandleNewMemory(endpoint, &header) < 0)
 					return -1;
 				break;
 
-			case PHI_PACKET_FREE_MEMORY:
-				if(HandleFreeMemory(endpoint, &header) < 0)
+			case PHI_PACKET_DESTROY_MEMORY:
+				if(HandleDestroyMemory(endpoint, &header) < 0)
 					return -1;
 				break;
 
@@ -157,7 +159,7 @@ int SendReply(PhiEndpoint endpoint, const PhiMessageHeader* request, const void*
 
 int SendStatus(PhiEndpoint endpoint, const PhiMessageHeader* request, PhiStatus status)
 {
-	PhiFreeMemoryReply reply = {
+	PhiResultReply reply = {
 		.result = {
 			.status = status,
 			.reserved = 0,

src/phi/mic/Memory.c

@@ -1,12 +1,60 @@
+#include <errno.h>
 #include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
 
 #include <Logger.h>
 #include <Memory.h>
 
-int HandleAllocMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
+static size_t AlignUp(size_t value, size_t alignment)
 {
-	PhiAllocMemoryRequest request;
-	PhiAllocMemoryReply reply = {
+	return (value + alignment - 1) & ~(alignment - 1);
+}
+
+static Memory* MapHostMemory(PhiEndpoint epd, const PhiMapHostMemoryRequest* request)
+{
+	void* ptr = scif_mmap(NULL, request->scif_size, PROT_READ | PROT_WRITE, 0, epd, request->scif_offset);
+
+	if(ptr == MAP_FAILED)
+	{
+		PhiLogErrorFmt("Failed to map host memory: %s", strerror(errno));
+		return NULL;
+	}
+
+	Memory* memory = malloc(sizeof(*memory));
+	if(!memory)
+	{
+		scif_munmap(ptr, request->scif_size);
+		PhiLogError("Failed to allocate memory");
+		return NULL;
+	}
+
+	memory->type = PHI_MEMORY_HOST_MAPPED;
+	memory->ptr = ptr;
+	memory->size = request->size;
+	memory->scif_size = request->scif_size;
+
+	return memory;
+}
+
+static Memory* AllocMemory(PhiEndpoint epd, const PhiAllocMemoryRequest* request)
+{
+	Memory* memory = (Memory*)malloc(sizeof(Memory) + PHI_MEMORY_ALIGNMENT + request->size);
+
+	if(!memory)
+		return NULL;
+
+	memory->type = PHI_MEMORY_LOCAL;
+	memory->ptr = (void*)AlignUp((uintptr_t)memory + sizeof(Memory), PHI_MEMORY_ALIGNMENT);
+	memory->size = request->size;
+	memory->scif_size = 0;
+
+	return memory;
+}
+
+int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
+{
+	PhiNewMemoryReply reply = {
 		.result = {
 			.status = PHI_STATUS_OK,
 			.reserved = 0,
@@ -15,37 +63,54 @@ int HandleAllocMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 		.size = 0,
 	};
 
-	if(header->payload_size != sizeof(request))
+	if(header->payload_size != sizeof(PhiAllocMemoryRequest) && header->payload_size != sizeof(PhiMapHostMemoryRequest))
 	{
 		if(DrainPayload(endpoint, header->payload_size) < 0)
 			return -1;
 		reply.result.status = PHI_STATUS_BAD_MESSAGE;
 		return SendReply(endpoint, header, &reply, sizeof(reply));
 	}
-	if(ReadAll(endpoint, &request, sizeof(request)) < 0)
-		return -1;
 
-	const void* memory = malloc((size_t)request.size);
+	Memory* memory;
 
-	if(memory == NULL)
+	if(header->type == PHI_PACKET_ALLOC_MEMORY)
 	{
-		reply.result.status = PHI_STATUS_OUT_OF_MEMORY;
-		PhiLogInfoFmt("Failed to allocate %zu bytes", (size_t)request.size);
+		PhiAllocMemoryRequest request;
+		if(ReadAll(endpoint, &request, sizeof(request)) < 0)
+			return -1;
+		memory = AllocMemory(endpoint, &request);
+		if(memory == NULL)
+			PhiLogErrorFmt("Failed to allocate %zu bytes", (size_t)request.size);
+		else
+			PhiLogInfoFmt("Allocated %llu bytes to handle 0x%X", request.size, (uintptr_t)memory);
 	}
-	else
+	else if(header->type == PHI_PACKET_MAP_HOST_MEMORY)
+	{
+		PhiMapHostMemoryRequest request;
+		if(ReadAll(endpoint, &request, sizeof(request)) < 0)
+			return -1;
+		memory = MapHostMemory(endpoint, &request);
+		if(memory == NULL)
+			reply.result.status = PHI_STATUS_MAP_HOST_MEMORY_FAILED;
+		else
+			PhiLogInfoFmt("Mapped host memory to handle 0x%X", (uint64_t)(uintptr_t)memory);
+	}
+
+	if(memory != NULL)
 	{
 		reply.remote_handle = (uint64_t)(uintptr_t)memory;
-		reply.size = request.size;
-		PhiLogInfoFmt("Allocated %llu bytes to handle 0x%X", reply.size, reply.remote_handle);
+		reply.size = memory->size;
 	}
+	else if(reply.result.status == PHI_STATUS_OK)
+		reply.result.status = PHI_STATUS_OUT_OF_MEMORY;
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));
 }
 
-int HandleFreeMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
+int HandleDestroyMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 {
-	PhiFreeMemoryRequest request;
-	PhiFreeMemoryReply reply = {
+	PhiDestroyMemoryRequest request;
+	PhiResultReply reply = {
 		.result = {
 			.status = PHI_STATUS_OK,
 			.reserved = 0,
@@ -70,8 +135,16 @@ int HandleFreeMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 	}
 	else
 	{
-		free((void*)(uintptr_t)request.remote_handle);
-		PhiLogInfoFmt("Freed memory handle 0x%X", request.remote_handle);
+		const Memory* memory = (const Memory*)(uintptr_t)request.remote_handle;
+
+		if(memory->type == PHI_MEMORY_LOCAL)
+			free((void*)memory);
+		else if(memory->type == PHI_MEMORY_HOST_MAPPED)
+			scif_munmap((void*)memory->ptr, memory->size);
+
+		PhiLogInfoFmt("Destroyed %s memory handle 0x%X",
+		              memory->type == PHI_MEMORY_LOCAL ? "local" : "host-mapped",
+		              request.remote_handle);
 	}
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));

src/phi/mic/Memory.h

@@ -3,7 +3,25 @@
 
 #include <Daemon.h>
 
-int HandleAllocMemory(PhiEndpoint endpoint, const PhiMessageHeader* header);
-int HandleFreeMemory(PhiEndpoint endpoint, const PhiMessageHeader* header);
+typedef enum MemoryType
+{
+	PHI_MEMORY_LOCAL,
+	PHI_MEMORY_HOST_MAPPED,
+} MemoryType;
+
+typedef struct Memory
+{
+	MemoryType type;
+
+	void* ptr;
+
+	uint64_t size;
+	uint64_t scif_size;
+
+	off_t scif_offset;
+} Memory;
+
+int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header);
+int HandleDestroyMemory(PhiEndpoint endpoint, const PhiMessageHeader* header);
 
 #endif

src/phi/mic/Transport.c

@@ -1,73 +1,5 @@
 #include <Transport.h>
 
-#ifdef PHI_HOST_EMULATION
-
-#include <arpa/inet.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-static struct sockaddr_in PhiLoopbackAddress(uint16_t port)
-{
-	struct sockaddr_in address = {
-		.sin_family = AF_INET,
-		.sin_port = htons(port),
-		.sin_addr = {
-			.s_addr = htonl(INADDR_LOOPBACK),
-		},
-	};
-	return address;
-}
-
-PhiEndpoint PhiTransportAccept(PhiEndpoint endpoint)
-{
-	return accept(endpoint, NULL, NULL);
-}
-
-int PhiTransportClose(PhiEndpoint endpoint)
-{
-	return close(endpoint);
-}
-
-PhiEndpoint PhiTransportListen(uint16_t port)
-{
-	PhiEndpoint endpoint = socket(AF_INET, SOCK_STREAM, 0);
-	if(endpoint == PHI_ENDPOINT_INVALID)
-		return PHI_ENDPOINT_INVALID;
-
-	const int reuse_address = 1;
-	if(setsockopt(endpoint, SOL_SOCKET, SO_REUSEADDR, &reuse_address, sizeof(reuse_address)) < 0)
-	{
-		PhiTransportClose(endpoint);
-		return PHI_ENDPOINT_INVALID;
-	}
-
-	const struct sockaddr_in address = PhiLoopbackAddress(port);
-	if(bind(endpoint, (const struct sockaddr*)&address, sizeof(address)) < 0 || listen(endpoint, 16) < 0)
-	{
-		PhiTransportClose(endpoint);
-		return PHI_ENDPOINT_INVALID;
-	}
-
-	return endpoint;
-}
-
-ssize_t PhiTransportReceive(PhiEndpoint endpoint, void* data, size_t size)
-{
-	return recv(endpoint, data, size, 0);
-}
-
-ssize_t PhiTransportSend(PhiEndpoint endpoint, const void* data, size_t size)
-{
-#ifdef MSG_NOSIGNAL
-	return send(endpoint, data, size, MSG_NOSIGNAL);
-#else
-	return send(endpoint, data, size, 0);
-#endif
-}
-
-#else
-
 PhiEndpoint PhiTransportAccept(PhiEndpoint endpoint)
 {
 	struct scif_portID peer;
@@ -105,5 +37,3 @@ ssize_t PhiTransportSend(PhiEndpoint endpoint, const void* data, size_t size)
 {
 	return scif_send(endpoint, (void*)data, size, SCIF_SEND_BLOCK);
 }
-
-#endif

src/phi/mic/Transport.h

@@ -5,12 +5,8 @@
 #include <stdint.h>
 #include <sys/types.h>
 
-#ifdef PHI_HOST_EMULATION
-typedef int PhiEndpoint;
-#else
 #include <scif.h>
 typedef scif_epd_t PhiEndpoint;
-#endif
 
 #define PHI_ENDPOINT_INVALID ((PhiEndpoint) - 1)
 

src/phi/mic/avx/Avx.h

@@ -0,0 +1,12 @@
+#ifndef APE_PHI_AVX_H
+#define APE_PHI_AVX_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size);
+
+void AvxFill64(void* dst, uint32_t value);
+void AvxFill256(void* dst, uint32_t value);
+
+#endif

src/phi/mic/avx/Copy.c

@@ -0,0 +1,107 @@
+#include <immintrin.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#define PHI_CACHE_LINE_SIZE 64
+
+static inline __m512i Load512KNC(const uint8_t* src)
+{
+	return _mm512_load_epi32((const void*)src);
+}
+
+void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size)
+{
+	if(size == 0)
+		return;
+
+	// Generic fallback if not 4-byte aligned
+	if((((uintptr_t)dst | (uintptr_t)src | size) & 3) != 0)
+	{
+		memcpy(dst, src, size);
+		return;
+	}
+
+	// Align the destination to 64 bytes.
+	size_t prefix = (-(uintptr_t)dst) & (PHI_CACHE_LINE_SIZE - 1);
+
+	if(prefix > size)
+		prefix = size;
+
+	if(prefix != 0)
+	{
+		memcpy(dst, src, prefix);
+
+		dst += prefix;
+		src += prefix;
+		size -= prefix;
+	}
+
+	if(((uintptr_t)src & (PHI_CACHE_LINE_SIZE - 1)) == 0)
+	{
+		// Unroll four cache lines at a time
+		while(size >= 256)
+		{
+			const __m512i v0 = _mm512_load_epi32((const void*)(src + 0));
+			const __m512i v1 = _mm512_load_epi32((const void*)(src + 64));
+			const __m512i v2 = _mm512_load_epi32((const void*)(src + 128));
+			const __m512i v3 = _mm512_load_epi32((const void*)(src + 192));
+
+			_mm512_store_epi32((void*)(dst + 0), v0);
+			_mm512_store_epi32((void*)(dst + 64), v1);
+			_mm512_store_epi32((void*)(dst + 128), v2);
+			_mm512_store_epi32((void*)(dst + 192), v3);
+
+			src += 256;
+			dst += 256;
+			size -= 256;
+		}
+
+		while(size >= 64)
+		{
+			const __m512i value = _mm512_load_epi32((const void*)src);
+
+			_mm512_store_epi32((void*)dst, value);
+
+			src += 64;
+			dst += 64;
+			size -= 64;
+		}
+	}
+	else
+	{
+		// Source is only 4-byte aligned.
+		// KNC's loadunpack pair implements the conceptual unaligned 64-byte load.
+		while(size >= 256)
+		{
+			const __m512i v0 = Load512KNC(src + 0);
+			const __m512i v1 = Load512KNC(src + 64);
+			const __m512i v2 = Load512KNC(src + 128);
+			const __m512i v3 = Load512KNC(src + 192);
+
+			_mm512_store_epi32((void*)(dst + 0), v0);
+			_mm512_store_epi32((void*)(dst + 64), v1);
+			_mm512_store_epi32((void*)(dst + 128), v2);
+			_mm512_store_epi32((void*)(dst + 192), v3);
+
+			src += 256;
+			dst += 256;
+			size -= 256;
+		}
+
+		while(size >= 64)
+		{
+			const __m512i value = Load512KNC(src);
+
+			_mm512_store_epi32((void*)dst, value);
+
+			src += 64;
+			dst += 64;
+			size -= 64;
+		}
+	}
+
+	// At most 63 bytes remain
+	if(size != 0)
+		memcpy(dst, src, size);
+}

src/phi/mic/avx/Fill.c

@@ -0,0 +1,23 @@
+#include <immintrin.h>
+#include <stdint.h>
+
+// Dst must be 64-byte aligned
+void PhiFill256KNC(void* dst, uint32_t value)
+{
+	__m512i v = _mm512_set1_epi32((int)value);
+
+	uint8_t* d = (uint8_t*)dst;
+
+	_mm512_store_epi32((void*)(d + 0), v);
+	_mm512_store_epi32((void*)(d + 64), v);
+	_mm512_store_epi32((void*)(d + 128), v);
+	_mm512_store_epi32((void*)(d + 192), v);
+}
+
+// Dst must be 64-byte aligned
+void PhiFill64KNC(void* dst, uint32_t value)
+{
+	__m512i v = _mm512_set1_epi32((int)value);
+
+	_mm512_store_epi32(dst, v);
+}

src/phi/mic/main.c

@@ -2,11 +2,6 @@
 #include <pthread.h>
 #include <stdint.h>
 
-#ifdef PHI_HOST_EMULATION
-#include <string.h>
-#include <unistd.h>
-#endif
-
 #include <Daemon.h>
 #include <Logger.h>
 
@@ -21,13 +16,8 @@ static void* HandleClient(void* const argument)
 
 int main(int argc, char** argv)
 {
-#ifdef PHI_HOST_EMULATION
-	if(argc == 2 && strcmp(argv[1], "--unlink-on-start") == 0)
-		(void)unlink(argv[0]);
-#else
 	(void)argc;
 	(void)argv;
-#endif
 
 	PhiEndpoint endpoint = StartDaemon();
 	pthread_attr_t client_thread_attributes;