diff --git a/build.zig b/build.zig index 98c1a88..54763f0 100644 --- a/build.zig +++ b/build.zig @@ -615,6 +615,7 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u "src/phi/mic/Memory.c", "src/phi/mic/Queue.c", "src/phi/mic/Transport.c", + "src/phi/mic/WorkerPool.c", // Add non-AVX files here }; @@ -625,6 +626,7 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u // Keep KNC AVX-512/IMCI code in separate translation units. The GCC port // in use must not compile the daemon's scalar/control code with -mavx512f const avx_sources = [_][]const u8{ + "src/phi/mic/avx/Blit.c", "src/phi/mic/avx/Copy.c", "src/phi/mic/avx/Fill.c", // Add AVX files here diff --git a/src/phi/mic/Blitter.c b/src/phi/mic/Blitter.c index 94171e2..c9f127b 100644 --- a/src/phi/mic/Blitter.c +++ b/src/phi/mic/Blitter.c @@ -1,14 +1,638 @@ #include +#include #include +#include -PhiStatus BlitImage(const PhiCmdBlitImage* command) +#include + +#include +#include + +#define UNORM8_SCALE (1.0f / 255.0f) +#define BLIT_WEIGHT_SCALE 1024.0f +#define BLIT_PARALLEL_MIN_PIXELS (256u * 1024u) +#define BLIT_TASK_TARGET_BYTES (64u * 1024u) +#define BLIT_MAX_GRAIN_ROWS 64u + +enum { - const Memory* src_memory = (const Memory*)(uintptr_t)command->src_memory; - Memory* dst_memory = (Memory*)(uintptr_t)command->dst_memory; + PHI_FILTER_NEAREST = 0, + PHI_FILTER_LINEAR = 1, +}; - for(uint32_t layer = 0; layer < command->layer_count; ++layer) +typedef struct Color +{ + float r; + float g; + float b; + float a; +} Color; + +typedef struct FormatInfo +{ + uint32_t texel_size; +} FormatInfo; + +typedef struct SampleCoordinate +{ + uint32_t lo; + uint32_t hi; + float factor; +} SampleCoordinate; + +typedef struct BlitWork +{ + const PhiCmdBlitImage* command; + const FormatInfo* src_info; + const FormatInfo* dst_info; + const uint8_t* src; + uint8_t* dst; + uint32_t row_count; + uint64_t rows_per_layer; +} BlitWork; + +static int GetFormatInfo(PhiFormat format, FormatInfo* info) +{ + switch(format) { + case PHI_FORMAT_R8_UNORM: + info->texel_size = 1; + return 1; + + case PHI_FORMAT_R8G8_UNORM: + info->texel_size = 2; + return 1; + + case PHI_FORMAT_R8G8B8_UNORM: + case PHI_FORMAT_B8G8R8_UNORM: + info->texel_size = 3; + return 1; + + case PHI_FORMAT_R8G8B8A8_UNORM: + case PHI_FORMAT_B8G8R8A8_UNORM: + case PHI_FORMAT_A8B8G8R8_UNORM_PACK32: + info->texel_size = 4; + return 1; + + default: + return 0; + } +} + +static Color ReadUnorm8(const uint8_t* texel, PhiFormat format) +{ + Color color = { 0.0f, 0.0f, 0.0f, 1.0f }; + + switch(format) + { + case PHI_FORMAT_R8_UNORM: + color.r = (float)texel[0] * UNORM8_SCALE; + break; + + case PHI_FORMAT_R8G8_UNORM: + color.r = (float)texel[0] * UNORM8_SCALE; + color.g = (float)texel[1] * UNORM8_SCALE; + break; + + case PHI_FORMAT_R8G8B8_UNORM: + color.r = (float)texel[0] * UNORM8_SCALE; + color.g = (float)texel[1] * UNORM8_SCALE; + color.b = (float)texel[2] * UNORM8_SCALE; + break; + + case PHI_FORMAT_B8G8R8_UNORM: + color.r = (float)texel[2] * UNORM8_SCALE; + color.g = (float)texel[1] * UNORM8_SCALE; + color.b = (float)texel[0] * UNORM8_SCALE; + break; + + case PHI_FORMAT_R8G8B8A8_UNORM: + case PHI_FORMAT_A8B8G8R8_UNORM_PACK32: + color.r = (float)texel[0] * UNORM8_SCALE; + color.g = (float)texel[1] * UNORM8_SCALE; + color.b = (float)texel[2] * UNORM8_SCALE; + color.a = (float)texel[3] * UNORM8_SCALE; + break; + + case PHI_FORMAT_B8G8R8A8_UNORM: + color.r = (float)texel[2] * UNORM8_SCALE; + color.g = (float)texel[1] * UNORM8_SCALE; + color.b = (float)texel[0] * UNORM8_SCALE; + color.a = (float)texel[3] * UNORM8_SCALE; + break; + + default: + break; + } + + return color; +} + +static inline uint8_t QuantizeUnorm8(float value) +{ + if(value <= 0.0f) + return 0; + if(value >= 1.0f) + return UINT8_MAX; + return (uint8_t)(value * 255.0f + 0.5f); +} + +static void WriteUnorm8(uint8_t* texel, PhiFormat format, Color color) +{ + const uint8_t r = QuantizeUnorm8(color.r); + const uint8_t g = QuantizeUnorm8(color.g); + const uint8_t b = QuantizeUnorm8(color.b); + const uint8_t a = QuantizeUnorm8(color.a); + + switch(format) + { + case PHI_FORMAT_R8_UNORM: + texel[0] = r; + break; + + case PHI_FORMAT_R8G8_UNORM: + texel[0] = r; + texel[1] = g; + break; + + case PHI_FORMAT_R8G8B8_UNORM: + texel[0] = r; + texel[1] = g; + texel[2] = b; + break; + + case PHI_FORMAT_B8G8R8_UNORM: + texel[0] = b; + texel[1] = g; + texel[2] = r; + break; + + case PHI_FORMAT_R8G8B8A8_UNORM: + case PHI_FORMAT_A8B8G8R8_UNORM_PACK32: + texel[0] = r; + texel[1] = g; + texel[2] = b; + texel[3] = a; + break; + + case PHI_FORMAT_B8G8R8A8_UNORM: + texel[0] = b; + texel[1] = g; + texel[2] = r; + texel[3] = a; + break; + + default: + break; + } +} + +static inline Color LerpColor(Color a, Color b, float factor) +{ + Color result; + result.r = a.r + (b.r - a.r) * factor; + result.g = a.g + (b.g - a.g) * factor; + result.b = a.b + (b.b - a.b) * factor; + result.a = a.a + (b.a - a.a) * factor; + return result; +} + +static inline double ClampCoordinate(float coordinate, uint32_t dimension) +{ + const double upper = (double)dimension - 0.5; + + if(coordinate < 0.5f) + return 0.5; + if((double)coordinate > upper) + return upper; + return (double)coordinate; +} + +static inline uint32_t GetNearestCoordinate(float coordinate, uint32_t dimension) +{ + return (uint32_t)ClampCoordinate(coordinate, dimension); +} + +static SampleCoordinate GetLinearCoordinate(float coordinate, uint32_t dimension) +{ + const double source = ClampCoordinate(coordinate, dimension) - 0.5; + SampleCoordinate result; + result.lo = (uint32_t)source; + result.hi = result.lo + 1 < dimension ? result.lo + 1 : result.lo; + result.factor = (float)(source - (double)result.lo); + return result; +} + +static inline uint32_t QuantizeBlitWeight(float factor) +{ + return (uint32_t)(factor * BLIT_WEIGHT_SCALE + 0.5f); +} + +static Color ReadTexel(const uint8_t* src, + PhiFormat format, + uint32_t texel_size, + uint64_t row_pitch, + uint64_t slice_pitch, + uint32_t x, + uint32_t y, + uint32_t z) +{ + const uint64_t offset = (uint64_t)z * slice_pitch + (uint64_t)y * row_pitch + (uint64_t)x * texel_size; + return ReadUnorm8(src + (size_t)offset, format); +} + +static Color +Sample(const uint8_t* src, const PhiCmdBlitImage* command, uint32_t texel_size, float x, float y, float z, int filter_3d) +{ + const PhiFormat format = (PhiFormat)command->src_format; + + if(command->filter == PHI_FILTER_NEAREST) + { + return ReadTexel(src, + format, + texel_size, + command->src_row_pitch, + command->src_slice_pitch, + GetNearestCoordinate(x, command->src_width), + GetNearestCoordinate(y, command->src_height), + GetNearestCoordinate(z, command->src_depth)); + } + + const SampleCoordinate sample_x = GetLinearCoordinate(x, command->src_width); + const SampleCoordinate sample_y = GetLinearCoordinate(y, command->src_height); + const SampleCoordinate sample_z = GetLinearCoordinate(z, command->src_depth); + + const Color color_0_0 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.lo, sample_z.lo); + const Color color_0_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.lo, sample_z.lo); + const Color color_1_0 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.hi, sample_z.lo); + const Color color_1_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.hi, sample_z.lo); + + const Color row_0 = LerpColor(color_0_0, color_0_1, sample_x.factor); + const Color row_1 = LerpColor(color_1_0, color_1_1, sample_x.factor); + const Color slice_0 = LerpColor(row_0, row_1, sample_y.factor); + + if(!filter_3d) + return slice_0; + + const Color color_0_0_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.lo, sample_z.hi); + const Color color_0_1_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.lo, sample_z.hi); + const Color color_1_0_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.lo, sample_y.hi, sample_z.hi); + const Color color_1_1_1 = ReadTexel( + src, format, texel_size, command->src_row_pitch, command->src_slice_pitch, sample_x.hi, sample_y.hi, sample_z.hi); + + const Color row_0_1 = LerpColor(color_0_0_1, color_0_1_1, sample_x.factor); + const Color row_1_1 = LerpColor(color_1_0_1, color_1_1_1, sample_x.factor); + const Color slice_1 = LerpColor(row_0_1, row_1_1, sample_y.factor); + return LerpColor(slice_0, slice_1, sample_z.factor); +} + +static inline int IsMemoryRangeValid(const Memory* memory, uint64_t offset, uint64_t size) +{ + if(memory == NULL || memory->ptr == NULL || offset > memory->size) + return 0; + return size <= memory->size - offset; +} + +static int ComputeRegionSpan(uint64_t row_pitch, + uint64_t slice_pitch, + uint64_t layer_pitch, + uint64_t row_size, + uint32_t row_count, + uint32_t slice_count, + uint32_t layer_count, + uint64_t* span) +{ + uint64_t result = 0; + uint64_t term; + + if(row_size == 0 || row_count == 0 || slice_count == 0 || layer_count == 0) + return 0; + if(__builtin_mul_overflow((uint64_t)row_count - 1, row_pitch, &term) || __builtin_add_overflow(result, term, &result)) + return 0; + if(__builtin_mul_overflow((uint64_t)slice_count - 1, slice_pitch, &term) || __builtin_add_overflow(result, term, &result)) + return 0; + if(__builtin_mul_overflow((uint64_t)layer_count - 1, layer_pitch, &term) || __builtin_add_overflow(result, term, &result)) + return 0; + if(__builtin_add_overflow(result, row_size, &result)) + return 0; + + *span = result; + return 1; +} + +static PhiStatus ValidateCommand(const PhiCmdBlitImage* command, + const Memory* src_memory, + const Memory* dst_memory, + const FormatInfo* src_info, + const FormatInfo* dst_info) +{ + if(command->src_width == 0 || command->src_height == 0 || command->src_depth == 0 || command->layer_count == 0 || + command->dst_x0 < 0 || command->dst_y0 < 0 || command->dst_z0 < 0 || command->dst_x1 <= command->dst_x0 || + command->dst_y1 <= command->dst_y0 || command->dst_z1 <= command->dst_z0 || command->filter > PHI_FILTER_LINEAR || + !__builtin_isfinite(command->src_x0) || !__builtin_isfinite(command->src_y0) || !__builtin_isfinite(command->src_z0) || + !__builtin_isfinite(command->step_x) || !__builtin_isfinite(command->step_y) || !__builtin_isfinite(command->step_z)) + { + LogError("Invalid blit image dimensions, coordinates, or filter"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + if(command->src_offset > SIZE_MAX || command->dst_offset > SIZE_MAX || command->src_row_pitch > SIZE_MAX || + command->src_slice_pitch > SIZE_MAX || command->src_layer_pitch > SIZE_MAX || command->dst_row_pitch > SIZE_MAX || + command->dst_slice_pitch > SIZE_MAX || command->dst_layer_pitch > SIZE_MAX) + { + LogError("Blit image address does not fit in size_t"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + uint64_t src_row_size; + uint64_t src_slice_span; + uint64_t src_volume_span; + uint64_t src_span; + if(__builtin_mul_overflow((uint64_t)command->src_width, src_info->texel_size, &src_row_size) || + command->src_row_pitch < src_row_size || + !ComputeRegionSpan(command->src_row_pitch, 0, 0, src_row_size, command->src_height, 1, 1, &src_slice_span) || + command->src_slice_pitch < src_slice_span || + !ComputeRegionSpan(command->src_row_pitch, + command->src_slice_pitch, + 0, + src_row_size, + command->src_height, + command->src_depth, + 1, + &src_volume_span) || + command->src_layer_pitch < src_volume_span || + !ComputeRegionSpan(command->src_row_pitch, + command->src_slice_pitch, + command->src_layer_pitch, + src_row_size, + command->src_height, + command->src_depth, + command->layer_count, + &src_span)) + { + LogError("Invalid blit image source pitches"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + uint64_t dst_row_size; + uint64_t dst_slice_span; + uint64_t dst_volume_span; + uint64_t dst_span; + if(__builtin_mul_overflow((uint64_t)(uint32_t)command->dst_x1, dst_info->texel_size, &dst_row_size) || + command->dst_row_pitch < dst_row_size || + !ComputeRegionSpan(command->dst_row_pitch, 0, 0, dst_row_size, (uint32_t)command->dst_y1, 1, 1, &dst_slice_span) || + command->dst_slice_pitch < dst_slice_span || + !ComputeRegionSpan(command->dst_row_pitch, + command->dst_slice_pitch, + 0, + dst_row_size, + (uint32_t)command->dst_y1, + (uint32_t)command->dst_z1, + 1, + &dst_volume_span) || + command->dst_layer_pitch < dst_volume_span || + !ComputeRegionSpan(command->dst_row_pitch, + command->dst_slice_pitch, + command->dst_layer_pitch, + dst_row_size, + (uint32_t)command->dst_y1, + (uint32_t)command->dst_z1, + command->layer_count, + &dst_span)) + { + LogError("Invalid blit image destination pitches"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + if(src_span > SIZE_MAX || dst_span > SIZE_MAX || !IsMemoryRangeValid(src_memory, command->src_offset, src_span) || + !IsMemoryRangeValid(dst_memory, command->dst_offset, dst_span)) + { + LogError("Blit image memory range is invalid"); + return PHI_STATUS_INVALID_ARGUMENT; } return PHI_STATUS_OK; } + +static inline void BlitScalarPixel(uint8_t* dst, + const uint8_t* src, + const PhiCmdBlitImage* command, + const FormatInfo* src_info, + float source_x, + float source_y, + float source_z, + int filter_3d) +{ + const Color color = Sample(src, command, src_info->texel_size, source_x, source_y, source_z, filter_3d); + WriteUnorm8(dst, (PhiFormat)command->dst_format, color); +} + +static void BlitRow(uint8_t* dst, + const uint8_t* src, + const PhiCmdBlitImage* command, + const FormatInfo* src_info, + const FormatInfo* dst_info, + float source_y, + float source_z) +{ + const uint32_t pixel_count = (uint32_t)(command->dst_x1 - command->dst_x0); + const float first_source_x = command->src_x0 + (float)command->dst_x0 * command->step_x; + const int filter_3d = command->step_z != 1.0f; + + // Fast path + if(command->filter == PHI_FILTER_NEAREST && command->src_format == command->dst_format && command->step_x == 1.0f && + first_source_x >= 0.0f && first_source_x + (float)(pixel_count - 1) < (float)command->src_width) + { + const uint32_t source_texel_x = GetNearestCoordinate(first_source_x, command->src_width); + const uint32_t source_texel_y = GetNearestCoordinate(source_y, command->src_height); + const uint32_t source_texel_z = GetNearestCoordinate(source_z, command->src_depth); + const uint64_t source_offset = (uint64_t)source_texel_z * command->src_slice_pitch + + (uint64_t)source_texel_y * command->src_row_pitch + + (uint64_t)source_texel_x * src_info->texel_size; + AvxCopy(dst, src + (size_t)source_offset, (size_t)pixel_count * src_info->texel_size); + return; + } + + uint32_t processed = 0; + if(src_info->texel_size == 4 && dst_info->texel_size == 4 && command->src_width <= INT32_MAX) + { + while(processed < pixel_count && ((uintptr_t)(dst + (size_t)processed * 4) & 63) != 0) + { + const int32_t destination_x = command->dst_x0 + (int32_t)processed; + const float source_x = command->src_x0 + (float)destination_x * command->step_x; + BlitScalarPixel(dst + (size_t)processed * 4, src, command, src_info, source_x, source_y, source_z, filter_3d); + ++processed; + } + + _Alignas(64) uint32_t source_x0[16]; + _Alignas(64) uint32_t source_x1[16]; + _Alignas(64) uint32_t weights_x[16]; + + if(command->filter == PHI_FILTER_NEAREST) + { + const uint32_t source_row = GetNearestCoordinate(source_y, command->src_height); + const uint32_t source_slice = GetNearestCoordinate(source_z, command->src_depth); + const uint64_t source_offset = + (uint64_t)source_slice * command->src_slice_pitch + (uint64_t)source_row * command->src_row_pitch; + const uint8_t* src_row = src + (size_t)source_offset; + + while(pixel_count - processed >= 16) + { + for(uint32_t lane = 0; lane < 16; ++lane) + { + const int32_t destination_x = command->dst_x0 + (int32_t)(processed + lane); + const float source_x = command->src_x0 + (float)destination_x * command->step_x; + source_x0[lane] = GetNearestCoordinate(source_x, command->src_width); + } + + AvxBlitNearestUnorm8x4( + dst + (size_t)processed * 4, src_row, source_x0, command->src_format, command->dst_format); + processed += 16; + } + } + else if(!filter_3d) + { + const SampleCoordinate sample_y = GetLinearCoordinate(source_y, command->src_height); + const SampleCoordinate sample_z = GetLinearCoordinate(source_z, command->src_depth); + const uint64_t slice_offset = (uint64_t)sample_z.lo * command->src_slice_pitch; + const uint8_t* src_row_0 = src + (size_t)(slice_offset + (uint64_t)sample_y.lo * command->src_row_pitch); + const uint8_t* src_row_1 = src + (size_t)(slice_offset + (uint64_t)sample_y.hi * command->src_row_pitch); + const uint32_t weight_y = QuantizeBlitWeight(sample_y.factor); + + while(pixel_count - processed >= 16) + { + for(uint32_t lane = 0; lane < 16; ++lane) + { + const int32_t destination_x = command->dst_x0 + (int32_t)(processed + lane); + const float source_x = command->src_x0 + (float)destination_x * command->step_x; + const SampleCoordinate sample_x = GetLinearCoordinate(source_x, command->src_width); + source_x0[lane] = sample_x.lo; + source_x1[lane] = sample_x.hi; + weights_x[lane] = QuantizeBlitWeight(sample_x.factor); + } + + AvxBlitLinearUnorm8x4(dst + (size_t)processed * 4, + src_row_0, + src_row_1, + source_x0, + source_x1, + weights_x, + weight_y, + command->src_format, + command->dst_format); + processed += 16; + } + } + } + + while(processed < pixel_count) + { + const int32_t destination_x = command->dst_x0 + (int32_t)processed; + const float source_x = command->src_x0 + (float)destination_x * command->step_x; + BlitScalarPixel( + dst + (size_t)processed * dst_info->texel_size, src, command, src_info, source_x, source_y, source_z, filter_3d); + ++processed; + } +} + +static void BlitRows(void* context, uint64_t begin, uint64_t end) +{ + const BlitWork* work = context; + const PhiCmdBlitImage* command = work->command; + + for(uint64_t row = begin; row < end; ++row) + { + const uint32_t layer = (uint32_t)(row / work->rows_per_layer); + const uint64_t row_in_layer = row - (uint64_t)layer * work->rows_per_layer; + const uint32_t slice = (uint32_t)(row_in_layer / work->row_count); + const uint32_t row_in_slice = (uint32_t)(row_in_layer - (uint64_t)slice * work->row_count); + const int32_t z = command->dst_z0 + (int32_t)slice; + const int32_t y = command->dst_y0 + (int32_t)row_in_slice; + const float source_z = command->src_z0 + (float)z * command->step_z; + const float source_y = command->src_y0 + (float)y * command->step_y; + + const uint8_t* src_layer = work->src + (size_t)((uint64_t)layer * command->src_layer_pitch); + uint8_t* dst_layer = work->dst + (size_t)((uint64_t)layer * command->dst_layer_pitch); + uint8_t* dst_slice = dst_layer + (size_t)((uint64_t)(uint32_t)z * command->dst_slice_pitch); + const uint64_t row_offset = + (uint64_t)(uint32_t)y * command->dst_row_pitch + (uint64_t)(uint32_t)command->dst_x0 * work->dst_info->texel_size; + + BlitRow(dst_slice + (size_t)row_offset, src_layer, command, work->src_info, work->dst_info, source_y, source_z); + } +} + +PhiStatus BlitImage(const PhiCmdBlitImage* command) +{ + if(command->src_memory == 0 || command->dst_memory == 0) + { + LogError("Invalid blit image memory handle"); + return PHI_STATUS_INVALID_HANDLE; + } + + const Memory* src_memory = (const Memory*)(uintptr_t)command->src_memory; + Memory* dst_memory = (Memory*)(uintptr_t)command->dst_memory; + FormatInfo src_info; + FormatInfo dst_info; + + if(!GetFormatInfo((PhiFormat)command->src_format, &src_info) || !GetFormatInfo((PhiFormat)command->dst_format, &dst_info)) + { + LogErrorFmt("Unsupported blit image formats: src=%u dst=%u", command->src_format, command->dst_format); + return PHI_STATUS_INVALID_ARGUMENT; + } + + PhiStatus status = ValidateCommand(command, src_memory, dst_memory, &src_info, &dst_info); + if(status != PHI_STATUS_OK) + return status; + + const uint8_t* src = (const uint8_t*)src_memory->ptr + (size_t)command->src_offset; + uint8_t* dst = (uint8_t*)dst_memory->ptr + (size_t)command->dst_offset; + const uint32_t row_count = (uint32_t)(command->dst_y1 - command->dst_y0); + const uint32_t slice_count = (uint32_t)(command->dst_z1 - command->dst_z0); + const uint32_t width = (uint32_t)(command->dst_x1 - command->dst_x0); + uint64_t rows_per_layer; + uint64_t total_rows; + uint64_t total_pixels; + + if(__builtin_mul_overflow((uint64_t)row_count, slice_count, &rows_per_layer) || + __builtin_mul_overflow(rows_per_layer, command->layer_count, &total_rows) || + __builtin_mul_overflow(total_rows, width, &total_pixels)) + { + LogError("Blit image work size overflow"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + const BlitWork work = { + .command = command, + .src_info = &src_info, + .dst_info = &dst_info, + .src = src, + .dst = dst, + .row_count = row_count, + .rows_per_layer = rows_per_layer, + }; + + uint64_t row_bytes; + if(__builtin_mul_overflow((uint64_t)width, dst_info.texel_size, &row_bytes)) + { + LogError("Blit image row size overflow"); + return PHI_STATUS_INVALID_ARGUMENT; + } + + uint64_t grain_rows = row_bytes < BLIT_TASK_TARGET_BYTES ? BLIT_TASK_TARGET_BYTES / row_bytes : 1; + if(grain_rows > BLIT_MAX_GRAIN_ROWS) + grain_rows = BLIT_MAX_GRAIN_ROWS; + + if(src_memory != dst_memory && total_pixels >= BLIT_PARALLEL_MIN_PIXELS && WorkerPoolGetWorkerCount() != 0) + WorkerPoolParallelFor(total_rows, grain_rows, BlitRows, (void*)&work); + else + BlitRows((void*)&work, 0, total_rows); + + return PHI_STATUS_OK; +} diff --git a/src/phi/mic/WorkerPool.c b/src/phi/mic/WorkerPool.c new file mode 100644 index 0000000..ad7805a --- /dev/null +++ b/src/phi/mic/WorkerPool.c @@ -0,0 +1,184 @@ +#include + +#include + +#include +#include +#include +#include + +#define PHI_WORKER_STACK_SIZE (128u * 1024u) +#define PHI_WORKER_MAX_COUNT 255u + +typedef struct WorkerPool +{ + pthread_mutex_t submission_mutex; + pthread_mutex_t work_mutex; + pthread_cond_t work_available; + pthread_cond_t work_complete; + + uint64_t generation; + uint64_t item_count; + uint64_t grain_size; + uint64_t next_item; + + WorkerPoolTask task; + void* context; + + uint32_t worker_count; + uint32_t workers_pending; +} WorkerPool; + +static WorkerPool Pool = { + .submission_mutex = PTHREAD_MUTEX_INITIALIZER, + .work_mutex = PTHREAD_MUTEX_INITIALIZER, + .work_available = PTHREAD_COND_INITIALIZER, + .work_complete = PTHREAD_COND_INITIALIZER, +}; +static pthread_once_t PoolInitialization = PTHREAD_ONCE_INIT; + +static int TakeWork(WorkerPoolTask* task, void** context, uint64_t* begin, uint64_t* end) +{ + int has_work = 0; + pthread_mutex_lock(&Pool.work_mutex); + + if(Pool.next_item < Pool.item_count) + { + *begin = Pool.next_item; + uint64_t remaining = Pool.item_count - Pool.next_item; + uint64_t count = remaining < Pool.grain_size ? remaining : Pool.grain_size; + Pool.next_item += count; + *end = Pool.next_item; + *task = Pool.task; + *context = Pool.context; + has_work = 1; + } + + pthread_mutex_unlock(&Pool.work_mutex); + return has_work; +} + +static void RunAvailableWork(void) +{ + WorkerPoolTask task; + void* context; + uint64_t begin; + uint64_t end; + + while(TakeWork(&task, &context, &begin, &end)) + task(context, begin, end); +} + +static void* WorkerMain(void* argument) +{ + (void)argument; + uint64_t generation = 0; + + for(;;) + { + pthread_mutex_lock(&Pool.work_mutex); + while(Pool.generation == generation) + pthread_cond_wait(&Pool.work_available, &Pool.work_mutex); + generation = Pool.generation; + pthread_mutex_unlock(&Pool.work_mutex); + + RunAvailableWork(); + + pthread_mutex_lock(&Pool.work_mutex); + --Pool.workers_pending; + if(Pool.workers_pending == 0) + pthread_cond_signal(&Pool.work_complete); + pthread_mutex_unlock(&Pool.work_mutex); + } + + return NULL; +} + +static uint32_t GetConfiguredWorkerCount(void) +{ + long online_cpu_count = sysconf(_SC_NPROCESSORS_ONLN); + uint32_t worker_count = online_cpu_count > 1 ? (uint32_t)(online_cpu_count - 1) : 0; + if(worker_count > PHI_WORKER_MAX_COUNT) + worker_count = PHI_WORKER_MAX_COUNT; + + const char* configured_count = getenv("PHI_BLIT_THREADS"); + if(configured_count != NULL && configured_count[0] != '\0') + { + char* end; + errno = 0; + unsigned long value = strtoul(configured_count, &end, 10); + if(errno == 0 && *end == '\0' && value <= PHI_WORKER_MAX_COUNT) + worker_count = (uint32_t)value; + } + + return worker_count; +} + +static void InitializePool(void) +{ + const uint32_t requested_count = GetConfiguredWorkerCount(); + if(requested_count == 0) + return; + + pthread_attr_t attributes; + if(pthread_attr_init(&attributes) != 0) + return; + + (void)pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + size_t stack_size = PHI_WORKER_STACK_SIZE; + const long minimum_stack_size = sysconf(_SC_THREAD_STACK_MIN); + if(minimum_stack_size > 0 && stack_size < (size_t)minimum_stack_size) + stack_size = (size_t)minimum_stack_size; + (void)pthread_attr_setstacksize(&attributes, stack_size); + + for(uint32_t worker = 0; worker < requested_count; ++worker) + { + pthread_t thread; + if(pthread_create(&thread, &attributes, WorkerMain, NULL) != 0) + break; + ++Pool.worker_count; + } + + pthread_attr_destroy(&attributes); +} + +uint32_t WorkerPoolGetWorkerCount(void) +{ + pthread_once(&PoolInitialization, InitializePool); + return Pool.worker_count; +} + +void WorkerPoolParallelFor(uint64_t item_count, uint64_t grain_size, WorkerPoolTask task, void* context) +{ + if(item_count == 0 || task == NULL) + return; + if(grain_size == 0) + grain_size = 1; + + pthread_once(&PoolInitialization, InitializePool); + if(Pool.worker_count == 0 || item_count <= grain_size) + { + task(context, 0, item_count); + return; + } + + pthread_mutex_lock(&Pool.submission_mutex); + pthread_mutex_lock(&Pool.work_mutex); + Pool.item_count = item_count; + Pool.grain_size = grain_size; + Pool.next_item = 0; + Pool.task = task; + Pool.context = context; + Pool.workers_pending = Pool.worker_count; + ++Pool.generation; + pthread_cond_broadcast(&Pool.work_available); + pthread_mutex_unlock(&Pool.work_mutex); + + RunAvailableWork(); + + pthread_mutex_lock(&Pool.work_mutex); + while(Pool.workers_pending != 0) + pthread_cond_wait(&Pool.work_complete, &Pool.work_mutex); + pthread_mutex_unlock(&Pool.work_mutex); + pthread_mutex_unlock(&Pool.submission_mutex); +} diff --git a/src/phi/mic/WorkerPool.h b/src/phi/mic/WorkerPool.h new file mode 100644 index 0000000..c8a77a4 --- /dev/null +++ b/src/phi/mic/WorkerPool.h @@ -0,0 +1,11 @@ +#ifndef APE_PHI_WORKER_POOL_H +#define APE_PHI_WORKER_POOL_H + +#include + +typedef void (*WorkerPoolTask)(void* context, uint64_t begin, uint64_t end); + +uint32_t WorkerPoolGetWorkerCount(void); +void WorkerPoolParallelFor(uint64_t item_count, uint64_t grain_size, WorkerPoolTask task, void* context); + +#endif diff --git a/src/phi/mic/avx/Avx.h b/src/phi/mic/avx/Avx.h index 63813a9..57981ff 100644 --- a/src/phi/mic/avx/Avx.h +++ b/src/phi/mic/avx/Avx.h @@ -6,6 +6,21 @@ void AvxCopy(uint8_t* dst, const uint8_t* src, size_t size); +void AvxBlitNearestUnorm8x4(uint8_t* dst, + const uint8_t* src_row, + const uint32_t* source_indices, + uint32_t src_format, + uint32_t dst_format); +void AvxBlitLinearUnorm8x4(uint8_t* dst, + const uint8_t* src_row_0, + const uint8_t* src_row_1, + const uint32_t* source_x0, + const uint32_t* source_x1, + const uint32_t* weights_x, + uint32_t weight_y, + uint32_t src_format, + uint32_t dst_format); + void AvxFill64(void* dst, uint32_t value); void AvxFill256(void* dst, uint32_t value); diff --git a/src/phi/mic/avx/Blit.c b/src/phi/mic/avx/Blit.c new file mode 100644 index 0000000..d07b99f --- /dev/null +++ b/src/phi/mic/avx/Blit.c @@ -0,0 +1,138 @@ +#include +#include + +#include +#include + +#define BLIT_WEIGHT_SCALE 1024u +#define BLIT_WEIGHT_SHIFT 20u +#define BLIT_WEIGHT_ROUND (1u << (BLIT_WEIGHT_SHIFT - 1)) + +static inline int IsBgra(uint32_t format) +{ + return format == PHI_FORMAT_B8G8R8A8_UNORM; +} + +static inline __m512i SwapRedBlue(__m512i packed) +{ + const __m512i keep_mask = _mm512_set1_epi32_knc(0xff00ff00u); + const __m512i red_mask = _mm512_set1_epi32_knc(0x000000ffu); + const __m512i blue_mask = _mm512_set1_epi32_knc(0x00ff0000u); + const __m512i keep = _mm512_and_epi32(packed, keep_mask); + const __m512i red = _mm512_slli_epi32(_mm512_and_epi32(packed, red_mask), 16); + const __m512i blue = _mm512_srli_epi32(_mm512_and_epi32(packed, blue_mask), 16); + return _mm512_or_epi32(keep, _mm512_or_epi32(red, blue)); +} + +static inline __m512i ToCanonicalRgba(__m512i packed, uint32_t format) +{ + return IsBgra(format) ? SwapRedBlue(packed) : packed; +} + +static inline __m512i FromCanonicalRgba(__m512i packed, uint32_t format) +{ + return IsBgra(format) ? SwapRedBlue(packed) : packed; +} + +static inline __m512i GatherCanonical(const uint8_t* source_row, __m512i indices, uint32_t format) +{ + const __m512i packed = _mm512_i32gather_epi32_knc(indices, (const uint32_t*)source_row); + return ToCanonicalRgba(packed, format); +} + +#define EXTRACT_CHANNEL(packed, shift) _mm512_and_epi32(_mm512_srli_epi32((packed), (shift)), _mm512_set1_epi32_knc(0xffu)) + +static inline __m512i LerpHorizontal(__m512i a, __m512i b, __m512i weight) +{ + const __m512i scale = _mm512_set1_epi32_knc(BLIT_WEIGHT_SCALE); + const __m512i inverse_weight = _mm512_sub_epi32(scale, weight); + return _mm512_add_epi32(_mm512_mullo_epi32(a, inverse_weight), _mm512_mullo_epi32(b, weight)); +} + +static inline __m512i LerpVertical(__m512i a, __m512i b, __m512i weight) +{ + const __m512i scale = _mm512_set1_epi32_knc(BLIT_WEIGHT_SCALE); + const __m512i round = _mm512_set1_epi32_knc(BLIT_WEIGHT_ROUND); + const __m512i inverse_weight = _mm512_sub_epi32(scale, weight); + const __m512i sum = _mm512_add_epi32(_mm512_mullo_epi32(a, inverse_weight), _mm512_mullo_epi32(b, weight)); + return _mm512_srli_epi32(_mm512_add_epi32(sum, round), BLIT_WEIGHT_SHIFT); +} + +static inline __m512i InterpolateChannel(__m512i color_0_0, + __m512i color_0_1, + __m512i color_1_0, + __m512i color_1_1, + __m512i weight_x, + __m512i weight_y) +{ + const __m512i row_0 = LerpHorizontal(color_0_0, color_0_1, weight_x); + const __m512i row_1 = LerpHorizontal(color_1_0, color_1_1, weight_x); + return LerpVertical(row_0, row_1, weight_y); +} + +static inline __m512i PackChannels(__m512i red, __m512i green, __m512i blue, __m512i alpha) +{ + green = _mm512_slli_epi32(green, 8); + blue = _mm512_slli_epi32(blue, 16); + alpha = _mm512_slli_epi32(alpha, 24); + return _mm512_or_epi32(_mm512_or_epi32(red, green), _mm512_or_epi32(blue, alpha)); +} + +void AvxBlitNearestUnorm8x4(uint8_t* dst, + const uint8_t* src_row, + const uint32_t* source_indices, + uint32_t src_format, + uint32_t dst_format) +{ + const __m512i indices = _mm512_load_epi32(source_indices); + __m512i packed = GatherCanonical(src_row, indices, src_format); + packed = FromCanonicalRgba(packed, dst_format); + _mm512_store_epi32(dst, packed); +} + +void AvxBlitLinearUnorm8x4(uint8_t* dst, + const uint8_t* src_row_0, + const uint8_t* src_row_1, + const uint32_t* source_x0, + const uint32_t* source_x1, + const uint32_t* weights_x, + uint32_t weight_y, + uint32_t src_format, + uint32_t dst_format) +{ + const __m512i x0 = _mm512_load_epi32(source_x0); + const __m512i x1 = _mm512_load_epi32(source_x1); + const __m512i weight_x = _mm512_load_epi32(weights_x); + const __m512i vector_weight_y = _mm512_set1_epi32_knc(weight_y); + const __m512i color_0_0 = GatherCanonical(src_row_0, x0, src_format); + const __m512i color_0_1 = GatherCanonical(src_row_0, x1, src_format); + const __m512i color_1_0 = GatherCanonical(src_row_1, x0, src_format); + const __m512i color_1_1 = GatherCanonical(src_row_1, x1, src_format); + + __m512i packed = PackChannels(InterpolateChannel(EXTRACT_CHANNEL(color_0_0, 0), + EXTRACT_CHANNEL(color_0_1, 0), + EXTRACT_CHANNEL(color_1_0, 0), + EXTRACT_CHANNEL(color_1_1, 0), + weight_x, + vector_weight_y), + InterpolateChannel(EXTRACT_CHANNEL(color_0_0, 8), + EXTRACT_CHANNEL(color_0_1, 8), + EXTRACT_CHANNEL(color_1_0, 8), + EXTRACT_CHANNEL(color_1_1, 8), + weight_x, + vector_weight_y), + InterpolateChannel(EXTRACT_CHANNEL(color_0_0, 16), + EXTRACT_CHANNEL(color_0_1, 16), + EXTRACT_CHANNEL(color_1_0, 16), + EXTRACT_CHANNEL(color_1_1, 16), + weight_x, + vector_weight_y), + InterpolateChannel(EXTRACT_CHANNEL(color_0_0, 24), + EXTRACT_CHANNEL(color_0_1, 24), + EXTRACT_CHANNEL(color_1_0, 24), + EXTRACT_CHANNEL(color_1_1, 24), + weight_x, + vector_weight_y)); + packed = FromCanonicalRgba(packed, dst_format); + _mm512_store_epi32(dst, packed); +} diff --git a/src/phi/mic/avx/Intrinsic.h b/src/phi/mic/avx/Intrinsic.h index 2d5dcd8..e40bbb0 100644 --- a/src/phi/mic/avx/Intrinsic.h +++ b/src/phi/mic/avx/Intrinsic.h @@ -11,6 +11,24 @@ inline __attribute__((always_inline, __artificial__)) __m512i _mm512_set1_epi32_ return result; } +inline __attribute__((always_inline, __artificial__)) __m512i _mm512_i32gather_epi32_knc(__m512i indices, const uint32_t* base) +{ + __m512i result = _mm512_setzero_epi32(); + uint32_t pending = UINT16_MAX; + + do + { + __asm__ volatile("kmov %1, %%k1\n\t" + "vpgatherdd (%2,%3,4), %0%{%%k1%}\n\t" + "kmov %%k1, %1" + : "+v"(result), "+r"(pending) + : "r"(base), "v"(indices) + : "k1", "cc", "memory"); + } while(pending != 0); + + return result; +} + inline __m512i __attribute__((always_inline, __artificial__)) _mm512_loadunpacklo_epi32(__m512i src, const void* ptr) { __asm__ volatile("vloadunpackld (%1), %0" : "+v"(src) : "r"(ptr) : "memory");