[Phi] adding image management and image/buffer copy commands

src/phi/mic/Buffer.c

@@ -4,19 +4,6 @@
 
 #include <avx/Avx.h>
 
-int IsBufferCommand(const PhiCmdHeader* header)
-{
-	switch((PhiCmdType)header->type)
-	{
-		case PHI_CMD_COPY_BUFFER:
-		case PHI_CMD_FILL_BUFFER:
-			return 1;
-
-		default:
-			return 0;
-	}
-}
-
 static PhiStatus CopyBuffer(PhiCommandReader* reader)
 {
 	PhiCmdCopyBuffer command;
@@ -24,8 +11,16 @@ static PhiStatus CopyBuffer(PhiCommandReader* reader)
 	if(status != PHI_STATUS_OK)
 		return status;
 
-	if(command.src_memory == 0 || command.dst_memory == 0)
+	if(command.src_memory == 0)
+	{
+		LogError("Invalid src memory handle");
 		return PHI_STATUS_INVALID_HANDLE;
+	}
+	if(command.dst_memory == 0)
+	{
+		LogError("Invalid dst memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
 
 	Memory* dst_memory = (Memory*)command.dst_memory;
 	const Memory* src_memory = (const Memory*)command.src_memory;
@@ -103,6 +98,19 @@ static PhiStatus FillBuffer(PhiCommandReader* reader)
 	return PHI_STATUS_OK;
 }
 
+int IsBufferCommand(const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER:
+		case PHI_CMD_FILL_BUFFER:
+			return 1;
+
+		default:
+			return 0;
+	}
+}
+
 PhiStatus ExecuteBufferCommand(PhiCommandReader* reader, const PhiCmdHeader* header)
 {
 	switch((PhiCmdType)header->type)

src/phi/mic/CommandBuffer.c

@@ -2,6 +2,11 @@
 #include <Logger.h>
 
 #include <Buffer.h>
+#include <Image.h>
+
+static const char* CommandName[] = {
+	"CopyBuffer", "FillBuffer", "CopyBufferToImage", "CopyImageToBuffer", "CopyImage",
+};
 
 PhiStatus ReadCommandData(PhiCommandReader* reader, void* data, uint64_t size)
 {
@@ -42,14 +47,12 @@ static PhiStatus ExecuteCommand(PhiCommandReader* reader, const PhiCmdHeader* co
 	if(IsBufferCommand(command_header))
 		return ExecuteBufferCommand(reader, command_header);
 
+	if(IsImageCommand(command_header))
+		return ExecuteImageCommand(reader, command_header);
+
 	return PHI_STATUS_BAD_MESSAGE;
 }
 
-static const char* CommandName[] = {
-	"CopyBuffer",
-	"FillBuffer",
-};
-
 int HandleWorkExecution(PhiEndpoint endpoint, const PhiMessageHeader* header)
 {
 	PhiWorkExecutionRequest request;

src/phi/mic/Image.c

@@ -0,0 +1,372 @@
+#include <stddef.h>
+#include <stdint.h>
+
+#include <Image.h>
+#include <Logger.h>
+#include <Memory.h>
+
+#include <avx/Avx.h>
+
+static int GetRegionSpan(uint64_t row_pitch,
+                         uint64_t slice_pitch,
+                         uint64_t layer_pitch,
+                         uint64_t row_size,
+                         uint32_t row_count,
+                         uint32_t slice_count,
+                         uint32_t layer_count,
+                         uint64_t* span)
+{
+	uint64_t result = 0;
+	uint64_t term = 0;
+
+	if(row_size == 0 || row_count == 0 || slice_count == 0 || layer_count == 0)
+		return 0;
+
+	if(row_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)row_count - 1, row_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(slice_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)slice_count - 1, slice_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(layer_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)layer_count - 1, layer_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(__builtin_add_overflow(result, row_size, &result))
+		return 0;
+
+	*span = result;
+	return 1;
+}
+
+static inline int IsMemoryRangeValid(const Memory* memory, uint64_t offset, uint64_t size)
+{
+	if(memory == NULL)
+		return 0;
+	if(offset > memory->size)
+		return 0;
+	return size <= memory->size - offset;
+}
+
+static PhiStatus ValidateCopyCommand(const PhiCmdCopyImage* command, const Memory* src_memory, const Memory* dst_memory)
+{
+	uint64_t src_span;
+	uint64_t dst_span;
+
+	if(command->row_size == 0 || command->row_count == 0 || command->slice_count == 0 || command->layer_count == 0)
+	{
+		LogErrorFmt("Invalid image copy command: one of this arguments is zero: row_size=%lu row_count=%u slice_count=%u"
+		            "layer_count=%u",
+		            command->row_size,
+		            command->row_count,
+		            command->slice_count,
+		            command->layer_count);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(command->row_count > 1)
+	{
+		if(command->src_row_pitch < command->row_size || command->dst_row_pitch < command->row_size)
+		{
+			LogErrorFmt("Invalid image copy command: row_size=%lu is larger than src_row_pitch=%lu or dst_row_pitch=%lu",
+			            command->row_size,
+			            command->src_row_pitch,
+			            command->dst_row_pitch);
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+	}
+
+	if(!GetRegionSpan(command->src_row_pitch,
+	                  command->src_slice_pitch,
+	                  command->src_layer_pitch,
+	                  command->row_size,
+	                  command->row_count,
+	                  command->slice_count,
+	                  command->layer_count,
+	                  &src_span))
+	{
+		LogErrorFmt("Invalid image copy command: computed src region span is zero: src_row_pitch=%lu src_slice_pitch=%lu "
+		            "src_layer_pitch=%lu",
+		            command->src_row_pitch,
+		            command->src_slice_pitch,
+		            command->src_layer_pitch);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!GetRegionSpan(command->dst_row_pitch,
+	                  command->dst_slice_pitch,
+	                  command->dst_layer_pitch,
+	                  command->row_size,
+	                  command->row_count,
+	                  command->slice_count,
+	                  command->layer_count,
+	                  &dst_span))
+	{
+		LogErrorFmt("Invalid image copy command: computed dst region span is zero: dst_row_pitch=%lu dst_slice_pitch=%lu "
+		            "dst_layer_pitch=%lu",
+		            command->dst_row_pitch,
+		            command->dst_slice_pitch,
+		            command->dst_layer_pitch);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!IsMemoryRangeValid(src_memory, command->src_offset, src_span))
+	{
+		LogErrorFmt("Invalid image copy command: src memory range is invalid: src_offset=%lu src_span=%lu",
+		            command->src_offset,
+		            src_span);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!IsMemoryRangeValid(dst_memory, command->dst_offset, dst_span))
+	{
+		LogErrorFmt("Invalid image copy command: dst memory range is invalid: dst_offset=%lu dst_span=%lu",
+		            command->dst_offset,
+		            dst_span);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(command->src_offset > SIZE_MAX || command->dst_offset > SIZE_MAX || command->src_row_pitch > SIZE_MAX ||
+	   command->dst_row_pitch > SIZE_MAX || command->src_slice_pitch > SIZE_MAX || command->dst_slice_pitch > SIZE_MAX ||
+	   command->src_layer_pitch > SIZE_MAX || command->dst_layer_pitch > SIZE_MAX || command->row_size > SIZE_MAX)
+	{
+		LogErrorFmt(
+		    "Invalid image copy command: size_t overflow: src_offset=%lu dst_offset=%lu src_row_pitch=%lu dst_row_pitch=%lu "
+		    "src_slice_pitch=%lu dst_slice_pitch=%lu src_layer_pitch=%lu dst_layer_pitch=%lu row_size=%lu",
+		    command->src_offset,
+		    command->dst_offset,
+		    command->src_row_pitch,
+		    command->dst_row_pitch,
+		    command->src_slice_pitch,
+		    command->dst_slice_pitch,
+		    command->src_layer_pitch,
+		    command->dst_layer_pitch,
+		    command->row_size);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	return PHI_STATUS_OK;
+}
+
+static inline int GetTightSliceSize(const PhiCmdCopyImage* command, uint64_t* slice_size)
+{
+	return !__builtin_mul_overflow(command->row_size, command->row_count, slice_size);
+}
+
+static inline int GetTightLayerSize(const PhiCmdCopyImage* command, uint64_t* layer_size)
+{
+	uint64_t slice_size;
+	if(!GetTightSliceSize(command, &slice_size))
+		return 0;
+	return !__builtin_mul_overflow(slice_size, command->slice_count, layer_size);
+}
+
+static inline int GetTightCopySize(const PhiCmdCopyImage* command, uint64_t* copy_size)
+{
+	uint64_t layer_size;
+	if(!GetTightLayerSize(command, &layer_size))
+		return 0;
+	return !__builtin_mul_overflow(layer_size, command->layer_count, copy_size);
+}
+
+static inline int RowsAreContiguous(const PhiCmdCopyImage* command)
+{
+	if(command->row_count <= 1)
+		return 1;
+	return command->src_row_pitch == command->row_size && command->dst_row_pitch == command->row_size;
+}
+
+static int SlicesAreContiguous(const PhiCmdCopyImage* command)
+{
+	uint64_t slice_size;
+
+	if(!RowsAreContiguous(command))
+		return 0;
+	if(command->slice_count <= 1)
+		return 1;
+	if(!GetTightSliceSize(command, &slice_size))
+		return 0;
+
+	return command->src_slice_pitch == slice_size && command->dst_slice_pitch == slice_size;
+}
+
+static int LayersAreContiguous(const PhiCmdCopyImage* command)
+{
+	uint64_t layer_size;
+
+	if(!SlicesAreContiguous(command))
+		return 0;
+	if(command->layer_count <= 1)
+		return 1;
+	if(!GetTightLayerSize(command, &layer_size))
+		return 0;
+
+	return command->src_layer_pitch == layer_size && command->dst_layer_pitch == layer_size;
+}
+
+static PhiStatus CopyImageRegion(const PhiCmdCopyImage* command)
+{
+	if(command->src_memory == 0)
+	{
+		LogError("Invalid src memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
+	if(command->dst_memory == 0)
+	{
+		LogError("Invalid dst memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
+
+	const Memory* src_memory = (const Memory*)(uintptr_t)command->src_memory;
+	Memory* dst_memory = (Memory*)(uintptr_t)command->dst_memory;
+
+	PhiStatus status = ValidateCopyCommand(command, src_memory, dst_memory);
+	if(status != PHI_STATUS_OK)
+		return status;
+
+	const uint8_t* src = (const uint8_t*)src_memory->ptr + (size_t)command->src_offset;
+	uint8_t* dst = (uint8_t*)dst_memory->ptr + (size_t)command->dst_offset;
+
+	// Fast path: the entire region is tightly packed on both sides
+	if(LayersAreContiguous(command))
+	{
+		uint64_t copy_size;
+
+		if(!GetTightCopySize(command, &copy_size) || copy_size > SIZE_MAX)
+		{
+			LogErrorFmt("Invalid copy size: %lu", copy_size);
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		AvxCopy(dst, src, (size_t)copy_size);
+
+		return PHI_STATUS_OK;
+	}
+
+	// Second fast path: all slices inside each layer are contiguous, but layers themselves have padding
+	if(SlicesAreContiguous(command))
+	{
+		uint64_t layer_size;
+
+		if(!GetTightLayerSize(command, &layer_size) || layer_size > SIZE_MAX)
+		{
+			LogErrorFmt("Invalid layer size: %lu", layer_size);
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+		{
+			const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+			const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+			AvxCopy(dst + (size_t)dst_layer_offset, src + (size_t)src_layer_offset, (size_t)layer_size);
+		}
+
+		return PHI_STATUS_OK;
+	}
+
+	// Third fast path: rows are tightly packed, so each slice is a single AVX copy
+	if(RowsAreContiguous(command))
+	{
+		uint64_t slice_size;
+
+		if(!GetTightSliceSize(command, &slice_size) || slice_size > SIZE_MAX)
+		{
+			LogErrorFmt("Invalid slice size: %lu", slice_size);
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+		{
+			const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+			const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+			for(uint32_t slice = 0; slice < command->slice_count; ++slice)
+			{
+				const uint64_t src_slice_offset = src_layer_offset + (uint64_t)slice * command->src_slice_pitch;
+				const uint64_t dst_slice_offset = dst_layer_offset + (uint64_t)slice * command->dst_slice_pitch;
+
+				AvxCopy(dst + (size_t)dst_slice_offset, src + (size_t)src_slice_offset, (size_t)slice_size);
+			}
+		}
+
+		return PHI_STATUS_OK;
+	}
+
+	// General path: only performs a pitched byte copy
+	for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+	{
+		const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+		const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+		for(uint32_t slice = 0; slice < command->slice_count; ++slice)
+		{
+			const uint64_t src_slice_offset = src_layer_offset + (uint64_t)slice * command->src_slice_pitch;
+			const uint64_t dst_slice_offset = dst_layer_offset + (uint64_t)slice * command->dst_slice_pitch;
+
+			for(uint32_t row = 0; row < command->row_count; ++row)
+			{
+				const uint64_t src_row_offset = src_slice_offset + (uint64_t)row * command->src_row_pitch;
+				const uint64_t dst_row_offset = dst_slice_offset + (uint64_t)row * command->dst_row_pitch;
+
+				AvxCopy(dst + (size_t)dst_row_offset, src + (size_t)src_row_offset, (size_t)command->row_size);
+			}
+		}
+	}
+
+	return PHI_STATUS_OK;
+}
+
+static PhiStatus ExecuteCopyImage(PhiCommandReader* reader)
+{
+	PhiCmdCopyImage command;
+
+	PhiStatus status = ReadCommandData(reader, &command, sizeof(command));
+
+	if(status != PHI_STATUS_OK)
+		return status;
+
+	return CopyImageRegion(&command);
+}
+
+int IsImageCommand(const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER_TO_IMAGE:
+		case PHI_CMD_COPY_IMAGE_TO_BUFFER:
+		case PHI_CMD_COPY_IMAGE:
+			return 1;
+
+		default:
+			return 0;
+	}
+}
+
+PhiStatus ExecuteImageCommand(PhiCommandReader* reader, const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER_TO_IMAGE:
+		case PHI_CMD_COPY_IMAGE_TO_BUFFER:
+		case PHI_CMD_COPY_IMAGE:
+			return ExecuteCopyImage(reader);
+
+		default:
+			return PHI_STATUS_BAD_MESSAGE;
+	}
+}

src/phi/mic/Image.h

@@ -0,0 +1,9 @@
+#ifndef APE_PHI_IMAGE_H
+#define APE_PHI_IMAGE_H
+
+#include <CommandBuffer.h>
+
+int IsImageCommand(const PhiCmdHeader* header);
+PhiStatus ExecuteImageCommand(PhiCommandReader* reader, const PhiCmdHeader* header);
+
+#endif

src/phi/mic/avx/Copy.c

@@ -3,7 +3,7 @@
 #include <stdint.h>
 #include <string.h>
 
-#define PHI_CACHE_LINE_SIZE 64
+#include <avx/Utils.h>
 
 void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size)
 {
@@ -63,16 +63,14 @@ void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size)
 			size -= 64;
 		}
 	}
-	else
+	else // Unaligned
 	{
-		// Source is only 4-byte aligned.
-		// KNC's loadunpack pair implements the conceptual unaligned 64-byte load.
 		while(size >= 256)
 		{
-			const __m512i v0 = _mm512_load_epi32((const void*)(src + 0));
-			const __m512i v1 = _mm512_load_epi32((const void*)(src + 64));
-			const __m512i v2 = _mm512_load_epi32((const void*)(src + 128));
-			const __m512i v3 = _mm512_load_epi32((const void*)(src + 192));
+			const __m512i v0 = Load512Unaligned(src + 0);
+			const __m512i v1 = Load512Unaligned(src + 64);
+			const __m512i v2 = Load512Unaligned(src + 128);
+			const __m512i v3 = Load512Unaligned(src + 192);
 
 			_mm512_store_epi32((void*)(dst + 0), v0);
 			_mm512_store_epi32((void*)(dst + 64), v1);
@@ -86,7 +84,7 @@ void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size)
 
 		while(size >= 64)
 		{
-			const __m512i value = _mm512_load_epi32((const void*)src);
+			const __m512i value = Load512Unaligned(src);
 
 			_mm512_store_epi32((void*)dst, value);
 

src/phi/mic/avx/Intrinsic.h

@@ -4,11 +4,23 @@
 #include <immintrin.h>
 #include <stdint.h>
 
-inline __attribute__((always_inline)) __m512i _mm512_set1_epi32_knc(uint32_t value)
+inline __attribute__((always_inline, __artificial__)) __m512i _mm512_set1_epi32_knc(uint32_t value)
 {
 	__m512i result;
-	__asm__("vpbroadcastd %1, %0" : "=x"(result) : "m"(value));
+	__asm__ volatile("vpbroadcastd %1, %0" : "=x"(result) : "m"(value));
 	return result;
 }
 
+inline __m512i __attribute__((always_inline, __artificial__)) _mm512_loadunpacklo_epi32(__m512i src, const void* ptr)
+{
+	__asm__ volatile("vloadunpackld (%1), %0" : "+v"(src) : "r"(ptr) : "memory");
+	return src;
+}
+
+inline __m512i __attribute__((always_inline, __artificial__)) _mm512_loadunpackhi_epi32(__m512i src, const void* ptr)
+{
+	__asm__ volatile("vloadunpackhd (%1), %0" : "+v"(src) : "r"(ptr) : "memory");
+	return src;
+}
+
 #endif

src/phi/mic/avx/Utils.h

@@ -0,0 +1,19 @@
+#ifndef APE_PHI_AVX_UTILS_H
+#define APE_PHI_AVX_UTILS_H
+
+#include <immintrin.h>
+#include <stdint.h>
+
+#include <avx/Intrinsic.h>
+
+#define PHI_CACHE_LINE_SIZE 64
+
+static inline __m512i Load512Unaligned(const uint8_t* src)
+{
+	__m512i value = _mm512_setzero_epi32();
+	value = _mm512_loadunpacklo_epi32(value, (const void*)src);
+	value = _mm512_loadunpackhi_epi32(value, (const void*)(src + PHI_CACHE_LINE_SIZE));
+	return value;
+}
+
+#endif