[CI] fixing ci, removing soft cts ci

allocation

.gitea/workflows/Build.yml

@@ -22,8 +22,8 @@ jobs:
         run: |
           set -euxo pipefail
 
-          apt-get update
-          apt-get install -y unzip curl
+          apt update
+          apt install -y unzip curl
 
           MPSS_URL="https://drive.kbz8.me/dav/public-files/KSKeiOKxnFPyfws/mpss.zip"
           MPSS_ZIP="/tmp/mpss.zip"

.gitea/workflows/CTS_Soft.yml

@@ -1,64 +0,0 @@
-name: CTS Soft
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_and_test:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - uses: https://codeberg.org/mlugg/setup-zig@v2
-    - name: Install system dependencies
-      run: |
-        apt update
-        apt install -y libgl1 libwayland-egl1 libwayland-cursor0 clang libwayland-server0
-
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
-      with:
-        toolchain: stable
-        targets: x86_64-unknown-linux-gnu
-
-    - uses: Swatinem/rust-cache@v2
-      with:
-        cache-all-crates: "true"
-        cache-on-failure: "false"
-
-    - name: Install deqp-runner
-      run: cargo install deqp-runner
-
-    - name: Verify installations
-      shell: bash
-      run: |
-        source $HOME/.cargo/env
-        echo "Verifying tool installations..."
-        which deqp-runner && deqp-runner --version || echo "deqp-runner not found"
-
-    - name: Run Vulkan CTS
-      run: zig build cts-soft --release=fast -- -j4
-      continue-on-error: true
-
-    - name: Verify tests
-      run: ls cts | grep "results.csv";
-
-    - name: Archiving results
-      run: |
-        mkdir -p assets/
-        zip ./assets/results.zip ./cts/results.csv
-
-    - name: Deploying CTS results
-      uses: milanmk/actions-file-deployer@master
-      with:
-        remote-protocol: sftp
-        remote-host: ${{ secrets.SFTP_HOST_CTS_SOFT }}
-        remote-user: ${{ secrets.SFTP_USER_CTS_SOFT }}
-        remote-password: ${{ secrets.SFTP_PASSWORD_CTS_SOFT }}
-        remote-port: 6969
-        local-path: "./assets"
-        remote-path: "/www/assets"
-        sync: full

.gitea/workflows/Test.yml

@@ -19,8 +19,8 @@ jobs:
         run: |
           set -euxo pipefail
 
-          apt-get update
-          apt-get install -y unzip curl
+          apt update
+          apt install -y unzip curl
 
           MPSS_URL="https://drive.kbz8.me/dav/public-files/KSKeiOKxnFPyfws/mpss.zip"
           MPSS_ZIP="/tmp/mpss.zip"

README.md

@@ -129,8 +129,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateSwapchainKHR                             | ✅ Implemented |
 | vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
 | vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ✅ Implemented |
 | vkDestroyCommandPool                             | ✅ Implemented |
@@ -194,8 +194,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetPhysicalDeviceSurfaceSupportKHR             | ✅ Implemented |
 | vkGetPhysicalDeviceWaylandPresentationSupportKHR | ✅ Implemented |
 | vkGetPhysicalDeviceWin32PresentationSupportKHR   | ⚙️ WIP         |
-| vkGetPhysicalDeviceXcbPresentationSupportKHR     | ⚙️ WIP         |
-| vkGetPhysicalDeviceXlibPresentationSupportKHR    | ⚙️ WIP         |
+| vkGetPhysicalDeviceXcbPresentationSupportKHR     | ✅ Implemented |
+| vkGetPhysicalDeviceXlibPresentationSupportKHR    | ✅ Implemented |
 | vkGetPipelineCacheData                           | ✅ Implemented |
 | vkGetQueryPoolResults                            | ✅ Implemented |
 | vkGetRenderAreaGranularity                       | ✅ Implemented |
@@ -264,7 +264,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkAllocateDescriptorSets                         | ⚙️ WIP         |
 | vkAllocateMemory                                 | ✅ Implemented |
 | vkBeginCommandBuffer                             | ✅ Implemented |
-| vkBindBufferMemory                               | ⚙️ WIP         |
+| vkBindBufferMemory                               | ✅ Implemented |
 | vkBindImageMemory                                | ✅ Implemented |
 | vkCmdBeginQuery                                  | ⚙️ WIP         |
 | vkCmdBeginRenderPass                             | ⚙️ WIP         |
@@ -272,13 +272,13 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdBindIndexBuffer                             | ⚙️ WIP         |
 | vkCmdBindPipeline                                | ⚙️ WIP         |
 | vkCmdBindVertexBuffers                           | ⚙️ WIP         |
-| vkCmdBlitImage                                   | ✅ Implemented |
+| vkCmdBlitImage                                   | ⚙️ WIP         |
 | vkCmdClearAttachments                            | ⚙️ WIP         |
 | vkCmdClearColorImage                             | ⚙️ WIP         |
 | vkCmdClearDepthStencilImage                      | ⚙️ WIP         |
 | vkCmdCopyBuffer                                  | ✅ Implemented |
 | vkCmdCopyBufferToImage                           | ✅ Implemented |
-| vkCmdCopyImage                                   | ⚙️ WIP         |
+| vkCmdCopyImage                                   | ✅ Implemented |
 | vkCmdCopyImageToBuffer                           | ✅ Implemented |
 | vkCmdCopyQueryPoolResults                        | ⚙️ WIP         |
 | vkCmdDispatch                                    | ⚙️ WIP         |
@@ -312,7 +312,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdUpdateBuffer                                | ⚙️ WIP         |
 | vkCmdWaitEvents                                  | ⚙️ WIP         |
 | vkCmdWriteTimestamp                              | ⚙️ WIP         |
-| vkCreateBuffer                                   | ⚙️ WIP         |
+| vkCreateBuffer                                   | ✅ Implemented |
 | vkCreateBufferView                               | ⚙️ WIP         |
 | vkCreateCommandPool                              | ✅ Implemented |
 | vkCreateComputePipelines                         | ⚙️ WIP         |
@@ -320,7 +320,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateDescriptorSetLayout                      | ⚙️ WIP         |
 | vkCreateDevice                                   | ✅ Implemented |
 | vkCreateEvent                                    | ⚙️ WIP         |
-| vkCreateFence                                    | ⚙️ WIP         |
+| vkCreateFence                                    | ✅ Implemented |
 | vkCreateFramebuffer                              | ⚙️ WIP         |
 | vkCreateGraphicsPipelines                        | ⚙️ WIP         |
 | vkCreateImage                                    | ✅ Implemented |
@@ -332,20 +332,20 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateRenderPass                               | ⚙️ WIP         |
 | vkCreateSampler                                  | ⚙️ WIP         |
 | vkCreateSemaphore                                | ⚙️ WIP         |
-| vkCreateShaderModule                             | ⚙️ WIP         |
-| vkCreateSwapchainKHR                             | ⚙️ WIP         |
-| vkCreateWaylandSurfaceKHR                        | ⚙️ WIP         |
+| vkCreateShaderModule                             | ✅ Implemented |
+| vkCreateSwapchainKHR                             | ✅ Implemented |
+| vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
-| vkDestroyBuffer                                  | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
+| vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ⚙️ WIP         |
 | vkDestroyCommandPool                             | ✅ Implemented |
 | vkDestroyDescriptorPool                          | ⚙️ WIP         |
 | vkDestroyDescriptorSetLayout                     | ⚙️ WIP         |
 | vkDestroyDevice                                  | ✅ Implemented |
 | vkDestroyEvent                                   | ⚙️ WIP         |
-| vkDestroyFence                                   | ⚙️ WIP         |
+| vkDestroyFence                                   | ✅ Implemented |
 | vkDestroyFramebuffer                             | ⚙️ WIP         |
 | vkDestroyImage                                   | ✅ Implemented |
 | vkDestroyImageView                               | ⚙️ WIP         |
@@ -356,11 +356,11 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkDestroyQueryPool                               | ⚙️ WIP         |
 | vkDestroyRenderPass                              | ⚙️ WIP         |
 | vkDestroySampler                                 | ⚙️ WIP         |
-| vkDestroySemaphore                               | ⚙️ WIP         |
+| vkDestroySemaphore                               | ✅ Implemented |
 | vkDestroyShaderModule                            | ⚙️ WIP         |
 | vkDestroySurfaceKHR                              | ⚙️ WIP         |
-| vkDestroySwapchainKHR                            | ⚙️ WIP         |
-| vkDeviceWaitIdle                                 | ⚙️ WIP         |
+| vkDestroySwapchainKHR                            | ✅ Implemented |
+| vkDeviceWaitIdle                                 | ✅ Implemented |
 | vkEndCommandBuffer                               | ✅ Implemented |
 | vkEnumerateDeviceExtensionProperties             | ⚙️ WIP         |
 | vkEnumerateDeviceLayerProperties                 | ⚙️ WIP         |
@@ -369,7 +369,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkEnumeratePhysicalDeviceGroupsKHR               | ⚙️ WIP         |
 | vkEnumeratePhysicalDevices                       | ✅ Implemented |
 | vkFlushMappedMemoryRanges                        | ⚙️ WIP         |
-| vkFreeCommandBuffers                             | ⚙️ WIP         |
+| vkFreeCommandBuffers                             | ✅ Implemented |
 | vkFreeDescriptorSets                             | ⚙️ WIP         |
 | vkFreeMemory                                     | ✅ Implemented |
 | vkGetBufferDeviceAddress                         | ⚙️ WIP         |
@@ -380,14 +380,14 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetDeviceGroupPresentCapabilitiesKHR           | ⚙️ WIP         |
 | vkGetDeviceGroupSurfacePresentModesKHR           | ⚙️ WIP         |
 | vkGetDeviceMemoryCommitment                      | ⚙️ WIP         |
-| vkGetDeviceProcAddr                              | ⚙️ WIP         |
-| vkGetDeviceQueue                                 | ⚙️ WIP         |
-| vkGetEventStatus                                 | ⚙️ WIP         |
-| vkGetFenceStatus                                 | ⚙️ WIP         |
+| vkGetDeviceProcAddr                              | ✅ Implemented |
+| vkGetDeviceQueue                                 | ✅ Implemented |
+| vkGetEventStatus                                 | ✅ Implemented |
+| vkGetFenceStatus                                 | ✅ Implemented |
 | vkGetImageMemoryRequirements                     | ✅ Implemented |
 | vkGetImageSparseMemoryRequirements               | ⚙️ WIP         |
 | vkGetImageSubresourceLayout                      | ✅ Implemented |
-| vkGetInstanceProcAddr                            | ⚙️ WIP         |
+| vkGetInstanceProcAddr                            | ✅ Implemented |
 | vkGetPhysicalDeviceFeatures                      | ✅ Implemented |
 | vkGetPhysicalDeviceFormatProperties              | ⚙️ WIP         |
 | vkGetPhysicalDeviceImageFormatProperties         | ✅ Implemented |
@@ -408,22 +408,22 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetRenderAreaGranularity                       | ⚙️ WIP         |
 | vkGetSwapchainImagesKHR                          | ⚙️ WIP         |
 | vkInvalidateMappedMemoryRanges                   | ⚙️ WIP         |
-| vkMapMemory                                      | ⚙️ WIP         |
+| vkMapMemory                                      | ✅ Implemented |
 | vkMergePipelineCaches                            | ⚙️ WIP         |
 | vkQueueBindSparse                                | ⚙️ WIP         |
 | vkQueuePresentKHR                                | ⚙️ WIP         |
 | vkQueueSubmit                                    | ✅ Implemented |
-| vkQueueWaitIdle                                  | ⚙️ WIP         |
-| vkResetCommandBuffer                             | ⚙️ WIP         |
+| vkQueueWaitIdle                                  | ✅ Implemented |
+| vkResetCommandBuffer                             | ✅ Implemented |
 | vkResetCommandPool                               | ✅ Implemented |
 | vkResetDescriptorPool                            | ⚙️ WIP         |
 | vkResetEvent                                     | ⚙️ WIP         |
-| vkResetFences                                    | ⚙️ WIP         |
+| vkResetFences                                    | ✅ Implemented |
 | vkResetQueryPool                                 | ⚙️ WIP         |
 | vkSetEvent                                       | ⚙️ WIP         |
-| vkUnmapMemory                                    | ⚙️ WIP         |
+| vkUnmapMemory                                    | ✅ Implemented |
 | vkUpdateDescriptorSets                           | ⚙️ WIP         |
-| vkWaitForFences                                  | ⚙️ WIP         |
+| vkWaitForFences                                  | ✅ Implemented |
 
 </details>
 
@@ -473,9 +473,9 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkBindImageMemory                                | ✅ Implemented |
 | vkCmdBeginQuery                                  | ⚙️ WIP         |
 | vkCmdBeginRenderPass                             | ⚙️ WIP         |
-| vkCmdBindDescriptorSets                          | ⚙️ WIP         |
+| vkCmdBindDescriptorSets                          | ✅ Implemented |
 | vkCmdBindIndexBuffer                             | ⚙️ WIP         |
-| vkCmdBindPipeline                                | ⚙️ WIP         |
+| vkCmdBindPipeline                                | ✅ Implemented |
 | vkCmdBindVertexBuffers                           | ⚙️ WIP         |
 | vkCmdBlitImage                                   | ⚙️ WIP         |
 | vkCmdClearAttachments                            | ⚙️ WIP         |
@@ -486,8 +486,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdCopyImage                                   | ✅ Implemented |
 | vkCmdCopyImageToBuffer                           | ✅ Implemented |
 | vkCmdCopyQueryPoolResults                        | ⚙️ WIP         |
-| vkCmdDispatch                                    | ⚙️ WIP         |
-| vkCmdDispatchBaseKHR                             | ⚙️ WIP         |
+| vkCmdDispatch                                    | ✅ Implemented |
+| vkCmdDispatchBaseKHR                             | ✅ Implemented |
 | vkCmdDispatchIndirect                            | ⚙️ WIP         |
 | vkCmdDraw                                        | ⚙️ WIP         |
 | vkCmdDrawIndexed                                 | ⚙️ WIP         |
@@ -520,9 +520,9 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateBuffer                                   | ✅ Implemented |
 | vkCreateBufferView                               | ⚙️ WIP         |
 | vkCreateCommandPool                              | ✅ Implemented |
-| vkCreateComputePipelines                         | ⚙️ WIP         |
-| vkCreateDescriptorPool                           | ⚙️ WIP         |
-| vkCreateDescriptorSetLayout                      | ⚙️ WIP         |
+| vkCreateComputePipelines                         | ✅ Implemented |
+| vkCreateDescriptorPool                           | ✅ Implemented |
+| vkCreateDescriptorSetLayout                      | ✅ Implemented |
 | vkCreateDevice                                   | ✅ Implemented |
 | vkCreateEvent                                    | ⚙️ WIP         |
 | vkCreateFence                                    | ✅ Implemented |
@@ -537,17 +537,17 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateRenderPass                               | ⚙️ WIP         |
 | vkCreateSampler                                  | ⚙️ WIP         |
 | vkCreateSemaphore                                | ✅ Implemented |
-| vkCreateShaderModule                             | ⚙️ WIP         |
-| vkCreateSwapchainKHR                             | ⚙️ WIP         |
-| vkCreateWaylandSurfaceKHR                        | ⚙️ WIP         |
+| vkCreateShaderModule                             | ✅ Implemented |
+| vkCreateSwapchainKHR                             | ✅ Implemented |
+| vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
 | vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ⚙️ WIP         |
 | vkDestroyCommandPool                             | ✅ Implemented |
-| vkDestroyDescriptorPool                          | ⚙️ WIP         |
-| vkDestroyDescriptorSetLayout                     | ⚙️ WIP         |
+| vkDestroyDescriptorPool                          | ✅ Implemented |
+| vkDestroyDescriptorSetLayout                     | ✅ Implemented |
 | vkDestroyDevice                                  | ✅ Implemented |
 | vkDestroyEvent                                   | ⚙️ WIP         |
 | vkDestroyFence                                   | ✅ Implemented |
@@ -555,16 +555,16 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkDestroyImage                                   | ✅ Implemented |
 | vkDestroyImageView                               | ⚙️ WIP         |
 | vkDestroyInstance                                | ✅ Implemented |
-| vkDestroyPipeline                                | ⚙️ WIP         |
+| vkDestroyPipeline                                | ✅ Implemented |
 | vkDestroyPipelineCache                           | ⚙️ WIP         |
 | vkDestroyPipelineLayout                          | ⚙️ WIP         |
 | vkDestroyQueryPool                               | ⚙️ WIP         |
 | vkDestroyRenderPass                              | ⚙️ WIP         |
 | vkDestroySampler                                 | ⚙️ WIP         |
 | vkDestroySemaphore                               | ✅ Implemented |
-| vkDestroyShaderModule                            | ⚙️ WIP         |
-| vkDestroySurfaceKHR                              | ⚙️ WIP         |
-| vkDestroySwapchainKHR                            | ⚙️ WIP         |
+| vkDestroyShaderModule                            | ✅ Implemented |
+| vkDestroySurfaceKHR                              | ✅ Implemented |
+| vkDestroySwapchainKHR                            | ✅ Implemented |
 | vkDeviceWaitIdle                                 | ✅ Implemented |
 | vkEndCommandBuffer                               | ✅ Implemented |
 | vkEnumerateDeviceExtensionProperties             | ✅ Implemented |
@@ -575,7 +575,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkEnumeratePhysicalDevices                       | ✅ Implemented |
 | vkFlushMappedMemoryRanges                        | ✅ Implemented |
 | vkFreeCommandBuffers                             | ✅ Implemented |
-| vkFreeDescriptorSets                             | ⚙️ WIP         |
+| vkFreeDescriptorSets                             | ✅ Implemented |
 | vkFreeMemory                                     | ✅ Implemented |
 | vkGetBufferDeviceAddress                         | ⚙️ WIP         |
 | vkGetBufferDeviceAddressEXT                      | ⚙️ WIP         |
@@ -592,7 +592,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetImageMemoryRequirements                     | ⚙️ WIP         |
 | vkGetImageSparseMemoryRequirements               | ⚙️ WIP         |
 | vkGetImageSubresourceLayout                      | ⚙️ WIP         |
-| vkGetInstanceProcAddr                            | ⚙️ WIP         |
+| vkGetInstanceProcAddr                            | ✅ Implemented |
 | vkGetPhysicalDeviceFeatures                      | ✅ Implemented |
 | vkGetPhysicalDeviceFormatProperties              | ⚙️ WIP         |
 | vkGetPhysicalDeviceImageFormatProperties         | ⚙️ WIP         |
@@ -621,7 +621,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkQueueWaitIdle                                  | ✅ Implemented |
 | vkResetCommandBuffer                             | ✅ Implemented |
 | vkResetCommandPool                               | ✅ Implemented |
-| vkResetDescriptorPool                            | ⚙️ WIP         |
+| vkResetDescriptorPool                            | ✅ Implemented |
 | vkResetEvent                                     | ⚙️ WIP         |
 | vkResetFences                                    | ✅ Implemented |
 | vkResetQueryPool                                 | ⚙️ WIP         |

build.zig

@@ -257,12 +257,17 @@ pub fn build(b: *std.Build) !void {
 }
 
 fn addCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const ImplementationDesc, impl_lib: *Step.Compile, comptime mode: RunningMode) !*Step {
+    const arch = if (target.query.cpu_arch) |arch| arch else builtin.cpu.arch;
+    if (!arch.isX86())
+        return error.NoCTSForPlatform;
+
     const cts = b.dependency("cts_bin", .{});
 
     const cts_exe_name = cts.path(b.fmt("deqp-vk-{s}", .{
         switch (if (target.query.os_tag) |tag| tag else builtin.target.os.tag) {
             .linux => "linux.x86_64",
             .windows => "windows.exe",
+            .macos => "macos.x86_64",
             else => return error.NoCTSForPlatform,
         },
     }));
@@ -334,12 +339,17 @@ fn addCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const Implemen
 }
 
 fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const ImplementationDesc, impl_lib: *Step.Compile, comptime mode: RunningMode) !*Step {
+    const arch = if (target.query.cpu_arch) |arch| arch else builtin.cpu.arch;
+    if (!arch.isX86())
+        return error.NoCTSForPlatform;
+
     const cts = b.dependency("cts_bin", .{});
 
     const cts_exe_name = cts.path(b.fmt("deqp-vk-{s}", .{
         switch (if (target.query.os_tag) |tag| tag else builtin.target.os.tag) {
             .linux => "linux.x86_64",
             .windows => "windows.exe",
+            .macos => "macos.x86_64",
             else => return error.NoCTSForPlatform,
         },
     }));
@@ -470,8 +480,8 @@ fn customSoft(
 // Flint specialized functions
 
 fn customFlint(
-    _: *std.Build,
-    _: *Step.Options,
+    b: *std.Build,
+    options: *Step.Options,
     _: *Step.Compile,
     lib_mod: *std.Build.Module,
     _: *std.Build.Module,
@@ -484,6 +494,12 @@ fn customFlint(
 ) !void {
     lib_mod.addImport("intel_c", base_c_mod);
     lib_mod.addImport("shader_ir", shader_ir_mod);
+
+    const dump_common_ir = b.option(bool, "flint-dump-common-ir", "Print backend-agnostic shader IR after translation") orelse false;
+    const dump_ir = b.option(bool, "flint-dump-ir", "Print final Flint IR after backend lowering") orelse false;
+
+    options.addOption(bool, "flint_dump_common_ir", dump_common_ir);
+    options.addOption(bool, "flint_dump_ir", dump_ir);
 }
 
 // Phi specialized functions
@@ -496,7 +512,7 @@ fn customPhi(
     _: *std.Build.Module,
     _: *std.Build.Module,
     base_c_mod: *std.Build.Module,
-    _: *std.Build.Module,
+    shader_ir_mod: *std.Build.Module,
     target: std.Build.ResolvedTarget,
     optimize: std.builtin.OptimizeMode,
     use_llvm: bool,
@@ -517,6 +533,7 @@ fn customPhi(
     options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix);
 
     lib_mod.addImport("phi_c", base_c_mod);
+    lib_mod.addImport("shader_ir", shader_ir_mod);
 
     const miclib = b.lazyDependency("miclib", .{
         .target = target,
@@ -576,7 +593,9 @@ fn addPhiDaemonCompilerArgs(
         "-std=c11",
         "-Wall",
         "-Wextra",
+        "-Werror",
         "-Wno-unused-parameter",
+        "-Wno-unused-variable",
         "-pthread",
     });
 
@@ -593,7 +612,7 @@ fn addPhiDaemonCompilerArgs(
     switch (optimize) {
         .Debug => cmd.addArgs(&.{ "-O0", "-g3" }),
         .ReleaseSafe => cmd.addArgs(&.{ "-O2", "-g", "-DNDEBUG" }),
-        .ReleaseFast => cmd.addArgs(&.{ "-O3", "-DNDEBUG" }),
+        .ReleaseFast => cmd.addArgs(&.{ "-O3", "-DNDEBUG", "-DNOLOGS" }),
         .ReleaseSmall => cmd.addArgs(&.{ "-Os", "-DNDEBUG" }),
     }
 }
@@ -604,12 +623,17 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u
 
     const sources = [_][]const u8{
         "src/phi/mic/main.c",
+        "src/phi/mic/Blitter.c",
+        "src/phi/mic/BlitFormats.c",
         "src/phi/mic/Buffer.c",
         "src/phi/mic/CommandBuffer.c",
         "src/phi/mic/Daemon.c",
+        "src/phi/mic/Image.c",
         "src/phi/mic/Logger.c",
         "src/phi/mic/Memory.c",
+        "src/phi/mic/Queue.c",
         "src/phi/mic/Transport.c",
+        "src/phi/mic/WorkerPool.c",
         // Add non-AVX files here
     };
 
@@ -617,9 +641,10 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u
         cmd.addFileArg(b.path(source));
     }
 
-    // Keep KNC AVX-512/IMCI code in separate translation units. This GCC
-    // port must not compile the daemon's scalar/control code with -mavx512f.
+    // Keep KNC AVX-512/IMCI code in separate translation units. The GCC port
+    // in use must not compile the daemon's scalar/control code with -mavx512f
     const avx_sources = [_][]const u8{
+        "src/phi/mic/avx/Blit.c",
         "src/phi/mic/avx/Copy.c",
         "src/phi/mic/avx/Fill.c",
         // Add AVX files here
@@ -641,7 +666,7 @@ fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u
         cmd.addFileArg(avx_object);
     }
 
-    cmd.addArgs(&.{ "-lscif", "-o" });
+    cmd.addArgs(&.{ "-lscif", "-lm", "-o" });
     return cmd.addOutputFileArg("phi_device.mic");
 }
 

build.zig.zon

@@ -23,8 +23,8 @@
             .hash = "zmath-0.11.0-dev-wjwivdMsAwD-xaLj76YHUq3t9JDH-X16xuMTmnDzqbu2",
         },
         .cts_bin = .{
-            .url = "git+https://git.kbz8.me/kbz_8/Vulkan-CTS-bin.git#b316a134bc0aa7ac21d9c57a1df588809824dcdc",
-            .hash = "N-V-__8AAF9uOh0I4P_99za7N822J3JwsDaqONrFVrcEQo59",
+            .url = "git+https://git.kbz8.me/kbz_8/Vulkan-CTS-bin.git#190a251847dd8d41b1cd05493d5e2b5fd3d4cd32",
+            .hash = "N-V-__8AAF_AmSKsM1ucsQiHOIC71T6pFAneEsOu2hzlc2IA",
         },
         .drm = .{
             .url = "git+https://github.com/Kbz-8/zig-drm#409f58daa8f5174b2fcb8897f1c30f0b0729b611",

compile_flags.txt

@@ -1,5 +1,6 @@
 -xc
 -std=c11
+-mavx512f
 -Isrc/phi/shared
 -Isrc/phi/mic
 -isystem/opt/mpss/3.8.6/sysroots/k1om-mpss-linux/usr/include/

src/compiler/ir/instruction.zig

@@ -115,6 +115,12 @@ pub const Call = struct {
     arguments: []const ValueId,
 };
 
+pub const ArrayLength = struct {
+    resource: ResourceId,
+    byte_offset: ValueId,
+    stride: u32,
+};
+
 pub const Operation = union(enum) {
     unary: Unary,
     binary: Binary,
@@ -128,6 +134,7 @@ pub const Operation = union(enum) {
     load_buffer: LoadBuffer,
     store_buffer: StoreBuffer,
     call: Call,
+    array_length: ArrayLength,
 
     pub fn visitValueUses(self: Operation, context: anytype, comptime visitor: anytype) void {
         switch (self) {
@@ -163,6 +170,7 @@ pub const Operation = union(enum) {
                 for (op.arguments) |argument|
                     visitor(context, argument);
             },
+            .array_length => |op| visitor(context, op.byte_offset),
         }
     }
 
@@ -201,6 +209,7 @@ pub const Operation = union(enum) {
                 replaceOne(&op.value, old, replacement, &count);
             },
             .call => |*op| op.arguments = try replaceSlice(allocator, op.arguments, old, replacement, &count),
+            .array_length => |*op| replaceOne(&op.byte_offset, old, replacement, &count),
         }
         return count;
     }

src/compiler/ir/module.zig

@@ -94,6 +94,8 @@ pub const Builtin = enum {
     frag_coord,
     frag_depth,
     global_invocation_id,
+    num_workgroups,
+    workgroup_size,
 };
 
 pub const InterfaceSemantic = union(enum) {

src/compiler/ir/parser/ast.zig

@@ -101,5 +101,6 @@ pub const ParsedOperation = union(enum) {
     store_interface: struct { interface_name: []const u8, value: ValueRef },
     load_buffer: struct { resource_name: []const u8, byte_offset: ValueRef },
     store_buffer: struct { resource_name: []const u8, byte_offset: ValueRef, value: ValueRef },
+    array_length: struct { resource_name: []const u8, byte_offset: ValueRef, stride: u32 },
     call: struct { function_name: []const u8, arguments: []const ValueRef },
 };

src/compiler/ir/parser/lower.zig

@@ -285,6 +285,19 @@ fn lowerOperation(
                 .inferred_type = null,
             };
         },
+        .array_length => |op| blk: {
+            const resource_id = resources.get(op.resource_name) orelse return error.UnknownResource;
+            const byte_offset = resolveValue(values, op.byte_offset) orelse return error.UnknownValue;
+
+            break :blk .{
+                .operation = .{ .array_length = .{
+                    .resource = resource_id,
+                    .byte_offset = byte_offset,
+                    .stride = op.stride,
+                } },
+                .inferred_type = null,
+            };
+        },
         .call => |op| blk: {
             const function_id = functions.get(op.function_name) orelse return error.UnknownFunction;
             var arguments: std.ArrayList(ids.ValueId) = .empty;

src/compiler/ir/parser/parser.zig

@@ -412,6 +412,19 @@ const Parser = struct {
             } };
         }
 
+        if (std.mem.eql(u8, name, "array_length")) {
+            const resource_name = (try self.expect(.at_name)).text;
+            try self.expectDiscard(.comma);
+            const byte_offset = try self.parseValueRef();
+            try self.expectDiscard(.comma);
+            try self.expectIdentifier("stride");
+            return .{ .array_length = .{
+                .resource_name = resource_name,
+                .byte_offset = byte_offset,
+                .stride = try self.parseUnsigned(u32, .number),
+            } };
+        }
+
         if (std.mem.eql(u8, name, "call")) {
             const function_name = (try self.expect(.at_name)).text;
             try self.expectDiscard(.left_paren);
@@ -522,6 +535,13 @@ const Parser = struct {
             });
         }
 
+        if (std.mem.eql(u8, token.text, "runtime_array")) {
+            try self.expectDiscard(.left_square);
+            const element_type = try self.parseType();
+            try self.expectDiscard(.right_square);
+            return module.internType(.{ .runtime_array = .{ .element_type = element_type } });
+        }
+
         if (std.mem.eql(u8, token.text, "struct")) {
             try self.expectDiscard(.left_square);
             var members: std.ArrayList(ids.TypeId) = .empty;

src/compiler/ir/printer.zig

@@ -170,6 +170,11 @@ fn writeType(module: *const module_ir.Module, writer: *std.Io.Writer, type_id: i
             try writer.writeByte(']');
         },
         .resource_handle => |handle| try writer.print("resourceHandle[{t}]", .{handle.kind}),
+        .runtime_array => |array| {
+            try writer.writeAll("runtime_array[");
+            try writeType(module, writer, array.element_type);
+            try writer.writeByte(']');
+        },
     }
 }
 
@@ -247,6 +252,14 @@ fn writeOperation(module: *const module_ir.Module, writer: *std.Io.Writer, opera
             try writeValueList(module, writer, op.arguments);
             try writer.writeByte(')');
         },
+        .array_length => |op| {
+            try writer.writeAll("array_length ");
+            const resource = module.resources.get(op.resource);
+            try writeNamedRef(writer, if (resource) |r| r.name else null, "resource", op.resource.index());
+            try writer.writeAll(", ");
+            try writeValueRef(module, writer, op.byte_offset);
+            try writer.print(", stride {}", .{op.stride});
+        },
     }
 }
 

src/compiler/ir/transformers/inline_all_functions.zig

@@ -391,6 +391,11 @@ fn remapOperation(
             .value = try mappedValue(module, value_map, op.value),
         } },
         .call => Error.InvalidModule,
+        .array_length => |op| .{ .array_length = .{
+            .resource = op.resource,
+            .byte_offset = try mappedValue(module, value_map, op.byte_offset),
+            .stride = op.stride,
+        } },
     };
 }
 

src/compiler/ir/type.zig

@@ -24,6 +24,10 @@ pub const ArrayType = struct {
     length: u32,
 };
 
+pub const RuntimeArrayType = struct {
+    element_type: TypeId,
+};
+
 pub const StructureType = struct {
     members: []const TypeId,
 };
@@ -68,6 +72,7 @@ pub const Type = union(enum) {
     structure: StructureType,
     pointer: PointerType,
     resource_handle: ResourceHandleType,
+    runtime_array: RuntimeArrayType,
 
     pub fn eql(a: Type, b: Type) bool {
         return switch (a) {
@@ -101,6 +106,10 @@ pub const Type = union(enum) {
                 .resource_handle => |other| std.meta.eql(value, other),
                 else => false,
             },
+            .runtime_array => |value| switch (b) {
+                .runtime_array => |other| std.meta.eql(value, other),
+                else => false,
+            },
         };
     }
 };

src/compiler/ir/validator/validator.zig

@@ -34,6 +34,11 @@ pub const ValidationError = error{
     WrongReturnType,
 };
 
+const IntegerShape = struct {
+    bits: u16,
+    components: u8,
+};
+
 pub const Error = ValidationError || std.mem.Allocator.Error;
 
 pub fn validate(module: *const module_ir.Module) Error!void {
@@ -227,13 +232,37 @@ fn validateOperation(module: *const module_ir.Module, function_id: ids.FunctionI
 
     switch (instruction.operation) {
         .unary => |op| {
-            const operand_type = try operandType(module, function_id, op.operand);
+            const operand_type =
+                try operandType(module, function_id, op.operand);
 
-            if (result_type == null)
-                return ValidationError.WrongResultPresence;
+            const result =
+                result_type orelse return ValidationError.WrongResultPresence;
 
-            if (result_type.? != operand_type)
-                return ValidationError.WrongResultType;
+            switch (op.opcode) {
+                .bitwise_not => {
+                    if (!isIntegerScalarOrVector(module, operand_type))
+                        return ValidationError.WrongOperandType;
+
+                    if (!isIntegerScalarOrVector(module, result))
+                        return ValidationError.WrongResultType;
+
+                    if (!haveSameIntegerShape(module, operand_type, result))
+                        return ValidationError.WrongResultType;
+                },
+
+                .logical_not => {
+                    if (!isBoolean(module, operand_type))
+                        return ValidationError.WrongOperandType;
+
+                    if (!isBoolean(module, result))
+                        return ValidationError.WrongResultType;
+                },
+
+                .negate => {
+                    if (result != operand_type)
+                        return ValidationError.WrongResultType;
+                },
+            }
         },
         .binary => |op| {
             const lhs_type = try operandType(module, function_id, op.lhs);
@@ -375,6 +404,23 @@ fn validateOperation(module: *const module_ir.Module, function_id: ids.FunctionI
             } else if (result_type == null or result_type.? != callee.return_type)
                 return ValidationError.WrongResultType;
         },
+        .array_length => |op| {
+            const resource = module.resources.get(op.resource) orelse return ValidationError.InvalidValue;
+
+            if (resource.kind != .storage_buffer)
+                return ValidationError.WrongResourceKind;
+
+            if (!isUnsignedInteger(module, try operandType(module, function_id, op.byte_offset)))
+                return ValidationError.WrongOperandType;
+
+            if (op.stride == 0)
+                return ValidationError.InvalidInstruction;
+
+            const result = result_type orelse return ValidationError.WrongResultPresence;
+
+            if (!isArrayLengthResultType(module, result))
+                return ValidationError.WrongResultType;
+        },
     }
 }
 
@@ -506,6 +552,56 @@ fn isBufferAccessibleType(module: *const module_ir.Module, type_id: ids.TypeId)
     };
 }
 
+fn isArrayLengthResultType(module: *const module_ir.Module, type_id: ids.TypeId) bool {
+    const ty = module.types.get(type_id) orelse return false;
+
+    return switch (ty.*) {
+        .integer => |integer| integer.signedness == .unsigned and (integer.bits == 32 or integer.bits == 64),
+
+        else => false,
+    };
+}
+
+fn integerShape(module: *const module_ir.Module, type_id: ids.TypeId) ?IntegerShape {
+    const ty = module.types.get(type_id) orelse return null;
+
+    return switch (ty.*) {
+        .integer => |integer| .{
+            .bits = integer.bits,
+            .components = 1,
+        },
+
+        .vector => |vector| blk: {
+            const element =
+                module.types.get(vector.element_type) orelse return null;
+
+            const integer = switch (element.*) {
+                .integer => |integer| integer,
+                else => return null,
+            };
+
+            break :blk .{
+                .bits = integer.bits,
+                .components = vector.length,
+            };
+        },
+
+        else => null,
+    };
+}
+
+fn isIntegerScalarOrVector(module: *const module_ir.Module, type_id: ids.TypeId) bool {
+    return integerShape(module, type_id) != null;
+}
+
+fn haveSameIntegerShape(module: *const module_ir.Module, lhs: ids.TypeId, rhs: ids.TypeId) bool {
+    const lhs_shape = integerShape(module, lhs) orelse return false;
+    const rhs_shape = integerShape(module, rhs) orelse return false;
+
+    return lhs_shape.bits == rhs_shape.bits and
+        lhs_shape.components == rhs_shape.components;
+}
+
 fn targetsBlock(terminator: module_ir.Terminator, target: ids.BlockId) bool {
     return switch (terminator) {
         .branch => |edge| edge.target == target,

src/compiler/spirv/spirv.zig

@@ -1030,3 +1030,135 @@ pub const Decoration = enum(u32) {
     offset = 35,
     _,
 };
+
+pub const Builtin = enum(u32) {
+    position = 0,
+    point_size = 1,
+    clip_distance = 3,
+    cull_distance = 4,
+    vertex_id = 5,
+    instance_id = 6,
+    primitive_id = 7,
+    invocation_id = 8,
+    layer = 9,
+    viewport_index = 10,
+    tess_level_outer = 11,
+    tess_level_inner = 12,
+    tess_coord = 13,
+    patch_vertices = 14,
+    frag_coord = 15,
+    point_coord = 16,
+    front_facing = 17,
+    sample_id = 18,
+    sample_position = 19,
+    sample_mask = 20,
+    frag_depth = 22,
+    helper_invocation = 23,
+    num_workgroups = 24,
+    workgroup_size = 25,
+    workgroup_id = 26,
+    local_invocation_id = 27,
+    global_invocation_id = 28,
+    local_invocation_index = 29,
+    work_dim = 30,
+    global_size = 31,
+    enqueued_workgroup_size = 32,
+    global_offset = 33,
+    global_linear_id = 34,
+    subgroup_size = 36,
+    subgroup_max_size = 37,
+    num_subgroups = 38,
+    num_enqueued_subgroups = 39,
+    subgroup_id = 40,
+    subgroup_local_invocation_id = 41,
+    vertex_index = 42,
+    instance_index = 43,
+    core_id_arm = 4160,
+    core_count_arm = 4161,
+    core_max_id_arm = 4162,
+    warp_id_arm = 4163,
+    warp_max_id_arm = 4164,
+    subgroup_eq_mask = 4416,
+    subgroup_ge_mask = 4417,
+    subgroup_gt_mask = 4418,
+    subgroup_le_mask = 4419,
+    subgroup_lt_mask = 4420,
+    base_vertex = 4424,
+    base_instance = 4425,
+    draw_index = 4426,
+    primitive_shading_rate_khr = 4432,
+    device_index = 4438,
+    view_index = 4440,
+    shading_rate_khr = 4444,
+    tile_offset_qcom = 4492,
+    tile_dimension_qcom = 4493,
+    tile_apron_size_qcom = 4494,
+    bary_coord_no_persp_amd = 4992,
+    bary_coord_no_persp_centroid_amd = 4993,
+    bary_coord_no_persp_sample_amd = 4994,
+    bary_coord_smooth_amd = 4995,
+    bary_coord_smooth_centroid_amd = 4996,
+    bary_coord_smooth_sample_amd = 4997,
+    bary_coord_pull_model_amd = 4998,
+    frag_stencil_ref_ext = 5014,
+    remaining_recursion_levels_amdx = 5021,
+    shader_index_amdx = 5073,
+    sampler_heap_ext = 5122,
+    resource_heap_ext = 5123,
+    viewport_mask_nv = 5253,
+    secondary_position_nv = 5257,
+    secondary_viewport_mask_nv = 5258,
+    position_per_view_nv = 5261,
+    viewport_mask_per_view_nv = 5262,
+    fully_covered_ext = 5264,
+    task_count_nv = 5274,
+    primitive_count_nv = 5275,
+    primitive_indices_nv = 5276,
+    clip_distance_per_view_nv = 5277,
+    cull_distance_per_view_nv = 5278,
+    layer_per_view_nv = 5279,
+    mesh_view_count_nv = 5280,
+    mesh_view_indices_nv = 5281,
+    bary_coord_khr = 5286,
+    bary_coord_no_persp_khr = 5287,
+    frag_size_ext = 5292,
+    frag_invocation_count_ext = 5293,
+    primitive_point_indices_ext = 5294,
+    primitive_line_indices_ext = 5295,
+    primitive_triangle_indices_ext = 5296,
+    cull_primitive_ext = 5299,
+    launch_id_khr = 5319,
+    launch_size_khr = 5320,
+    world_ray_origin_khr = 5321,
+    world_ray_direction_khr = 5322,
+    object_ray_origin_khr = 5323,
+    object_ray_direction_khr = 5324,
+    ray_tmin_khr = 5325,
+    ray_tmax_khr = 5326,
+    instance_custom_index_khr = 5327,
+    object_to_world_khr = 5330,
+    world_to_object_khr = 5331,
+    hit_t_nv = 5332,
+    hit_kind_khr = 5333,
+    current_ray_time_nv = 5334,
+    hit_triangle_vertex_positions_khr = 5335,
+    hit_micro_triangle_vertex_positions_nv = 5337,
+    hit_micro_triangle_vertex_barycentrics_nv = 5344,
+    incoming_ray_flags_khr = 5351,
+    ray_geometry_index_khr = 5352,
+    hit_is_sphere_nv = 5359,
+    hit_is_lss_nv = 5360,
+    hit_sphere_position_nv = 5361,
+    warps_per_sm_nv = 5374,
+    sm_count_nv = 5375,
+    warp_id_nv = 5376,
+    sm_id_nv = 5377,
+    hit_lss_positions_nv = 5396,
+    hit_kind_front_facing_micro_triangle_nv = 5405,
+    hit_kind_back_facing_micro_triangle_nv = 5406,
+    hit_sphere_radius_nv = 5420,
+    hit_lss_radii_nv = 5421,
+    cluster_id_nv = 5436,
+    cull_mask_khr = 6021,
+    _,
+};

src/intel/FlintCommandBuffer.zig

@@ -12,6 +12,7 @@ const MemoryRange = @import("MemoryRange.zig");
 
 const copy = @import("copy_commands.zig");
 const blitter = @import("blitter.zig");
+const gen9_dispatch = @import("compiler/targets/gen9/compute/dispatch.zig");
 
 const Self = @This();
 pub const Interface = base.CommandBuffer;
@@ -19,6 +20,8 @@ pub const Interface = base.CommandBuffer;
 interface: Interface,
 batch: std.ArrayList(u32),
 relocations: std.ArrayList(kmd.Relocation),
+gpu_allocations: std.ArrayList(kmd.Memory),
+engine: ?kmd.Engine,
 bound_compute_pipeline: ?*FlintPipeline,
 bound_compute_descriptor_sets: [base.vulkan_max_descriptor_sets]?*FlintDescriptorSet,
 
@@ -84,6 +87,8 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
         .interface = interface,
         .batch = .empty,
         .relocations = .empty,
+        .gpu_allocations = .empty,
+        .engine = null,
         .bound_compute_pipeline = null,
         .bound_compute_descriptor_sets = @splat(null),
     };
@@ -93,8 +98,10 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     const command_allocator = self.interface.host_allocator.allocator();
+    self.releaseGpuAllocations();
     self.batch.deinit(command_allocator);
     self.relocations.deinit(command_allocator);
+    self.gpu_allocations.deinit(command_allocator);
     allocator.destroy(self);
 }
 
@@ -105,7 +112,7 @@ pub fn submitGpuBatch(self: *Self, syncs: []const kmd.SyncDependency) VkError!vo
     // Empty command buffers still need a no-op submission to carry queue synchronization.
     const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
     const allocator = self.interface.host_allocator.allocator();
-    try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.batch.items, self.relocations.items, syncs);
+    try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.engine orelse .blitter, self.batch.items, self.relocations.items, syncs);
 }
 
 pub fn begin(interface: *Interface, info: *const vk.CommandBufferBeginInfo) VkError!void {
@@ -119,22 +126,46 @@ pub fn end(interface: *Interface) VkError!void {
 
 pub fn reset(interface: *Interface, flags: vk.CommandBufferResetFlags) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    self.releaseGpuAllocations();
     if (flags.release_resources_bit) {
         const command_allocator = self.interface.host_allocator.allocator();
         self.batch.clearAndFree(command_allocator);
         self.relocations.clearAndFree(command_allocator);
+        self.gpu_allocations.clearAndFree(command_allocator);
     } else {
         self.batch.clearRetainingCapacity();
         self.relocations.clearRetainingCapacity();
+        self.gpu_allocations.clearRetainingCapacity();
     }
+    self.engine = null;
     self.bound_compute_pipeline = null;
     self.bound_compute_descriptor_sets = @splat(null);
 }
 
+fn releaseGpuAllocations(self: *Self) void {
+    const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+    for (self.gpu_allocations.items) |*allocation|
+        allocation.deinit(&device.kmd, self.interface.owner.io());
+    self.gpu_allocations.clearRetainingCapacity();
+}
+
+pub fn requireEngine(self: *Self, engine: kmd.Engine) VkError!void {
+    if (self.engine) |current| {
+        if (current != engine)
+            return VkError.FeatureNotPresent;
+    } else {
+        self.engine = engine;
+    }
+}
+
 pub fn emit(self: *Self, dword: u32) VkError!void {
     self.batch.append(self.interface.host_allocator.allocator(), dword) catch return VkError.OutOfHostMemory;
 }
 
+fn emitSlice(self: *Self, words: []const u32) VkError!void {
+    self.batch.appendSlice(self.interface.host_allocator.allocator(), words) catch return VkError.OutOfHostMemory;
+}
+
 pub fn emitRelocatedAddress(self: *Self, range: MemoryRange, read: bool, write: bool) VkError!void {
     const address_offset = self.batch.items.len * @sizeOf(u32);
     try self.emit(@intCast(range.offset));
@@ -145,6 +176,7 @@ pub fn emitRelocatedAddress(self: *Self, range: MemoryRange, read: bool, write:
         .delta = @intCast(range.offset),
         .read = read,
         .write = write,
+        .domain = if ((self.engine orelse .blitter) == .render) .render else .none,
     }) catch return VkError.OutOfHostMemory;
 }
 
@@ -300,35 +332,143 @@ pub fn dispatchBase(interface: *Interface, base_group_x: u32, base_group_y: u32,
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     if (group_count_x == 0 or group_count_y == 0 or group_count_z == 0)
         return;
-
-    inline for ([_]struct { u32, u32 }{
-        .{ base_group_x, group_count_x },
-        .{ base_group_y, group_count_y },
-        .{ base_group_z, group_count_z },
-    }) |dimension| {
-        const group_end = std.math.add(u32, dimension[0], dimension[1]) catch return VkError.ValidationFailed;
-        if (group_end > 65535)
-            return VkError.ValidationFailed;
-    }
+    if (base_group_x != 0 or base_group_y != 0 or base_group_z != 0 or
+        group_count_x != 1 or group_count_y != 1 or group_count_z != 1)
+        return VkError.FeatureNotPresent;
 
     const pipeline = self.bound_compute_pipeline orelse return VkError.ValidationFailed;
     const artifact = pipeline.computeArtifact() orelse return VkError.FeatureNotPresent;
+    const kernel = artifact.kernel orelse return VkError.FeatureNotPresent;
+    if (!std.mem.eql(u32, &artifact.program.workgroup_size, &.{ 1, 1, 1 }) or
+        artifact.program.program_data.scratch_size_bytes != 0)
+        return VkError.FeatureNotPresent;
+
+    var ranges: [gen9_dispatch.max_surfaces]?MemoryRange = @splat(null);
+    var sizes: [gen9_dispatch.max_surfaces]u64 = @splat(0);
     for (artifact.resources.bindings) |resource| {
-        if (resource.set >= base.vulkan_max_descriptor_sets)
+        if (resource.set >= base.vulkan_max_descriptor_sets or @as(usize, resource.binding_table_index) >= gen9_dispatch.max_storage_surfaces)
             return VkError.ValidationFailed;
 
         const descriptor_set = self.bound_compute_descriptor_sets[resource.set] orelse return VkError.ValidationFailed;
         const expected_layout = pipeline.interface.layout.set_layouts[resource.set] orelse return VkError.ValidationFailed;
-
         if (descriptor_set.interface.layout != expected_layout)
             return VkError.ValidationFailed;
 
         const descriptor = try descriptor_set.getBuffer(resource.binding, 0);
         const buffer = descriptor.buffer orelse return VkError.ValidationFailed;
-
         if (!buffer.usage.storage_buffer_bit or buffer.memory == null)
             return VkError.ValidationFailed;
+
+        const range = try MemoryRange.fromBuffer(buffer, descriptor.offset, descriptor.size);
+        ranges[resource.binding_table_index] = range;
+        sizes[resource.binding_table_index] = range.size;
     }
+
+    const old_engine = self.engine;
+    try self.requireEngine(.render);
+    const old_batch_len = self.batch.items.len;
+    const old_relocation_len = self.relocations.items.len;
+    const old_allocation_len = self.gpu_allocations.items.len;
+    errdefer {
+        self.engine = old_engine;
+        self.batch.items.len = old_batch_len;
+        self.relocations.items.len = old_relocation_len;
+        while (self.gpu_allocations.items.len > old_allocation_len) {
+            const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+            self.gpu_allocations.items[self.gpu_allocations.items.len - 1].deinit(&device.kmd, self.interface.owner.io());
+            self.gpu_allocations.items.len -= 1;
+        }
+    }
+
+    const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+    var state = try device.kmd.allocateMemory(self.interface.owner.io(), gen9_dispatch.page_size);
+    var state_owned = true;
+    errdefer if (state_owned) state.deinit(&device.kmd, self.interface.owner.io());
+
+    const mapped = try state.map(&device.kmd, self.interface.owner.io(), 0, gen9_dispatch.page_size);
+    const state_layout = gen9_dispatch.writeState(mapped, kernel, sizes[0..artifact.resources.bindings.len]) catch |err| switch (err) {
+        error.StateTooLarge,
+        error.UnsupportedBufferSize,
+        error.EmptyBuffer,
+        error.TooManySurfaces,
+        => return VkError.FeatureNotPresent,
+    };
+    state.unmap();
+    try state.flushRange(&device.kmd, self.interface.owner.io(), 0, state_layout.size);
+    const state_handle = try state.handle();
+
+    self.gpu_allocations.append(self.interface.host_allocator.allocator(), state) catch return VkError.OutOfHostMemory;
+    state_owned = false;
+
+    for (0..@as(usize, state_layout.storage_surface_count)) |index| {
+        const range = ranges[index] orelse return VkError.ValidationFailed;
+        if (range.offset > std.math.maxInt(u32))
+            return VkError.FeatureNotPresent;
+        self.relocations.append(self.interface.host_allocator.allocator(), .{
+            .source_handle = state_handle,
+            .target_handle = try range.memory.allocation.handle(),
+            .offset = state_layout.surface_address_offsets[index],
+            .delta = @intCast(range.offset),
+            .read = true,
+            .write = true,
+            .domain = .render,
+        }) catch return VkError.OutOfHostMemory;
+    }
+
+    const size_table_surface: usize = state_layout.storage_surface_count;
+    self.relocations.append(self.interface.host_allocator.allocator(), .{
+        .source_handle = state_handle,
+        .target_handle = state_handle,
+        .offset = state_layout.surface_address_offsets[size_table_surface],
+        .delta = state_layout.size_table_offset,
+        .read = true,
+        .write = false,
+        .domain = .render,
+    }) catch return VkError.OutOfHostMemory;
+
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush |
+        gen9_dispatch.pipe_control.render_target_flush |
+        gen9_dispatch.pipe_control.depth_flush));
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.texture_invalidate |
+        gen9_dispatch.pipe_control.constant_invalidate |
+        gen9_dispatch.pipe_control.state_invalidate |
+        gen9_dispatch.pipe_control.instruction_invalidate));
+    try self.emitSlice(&gen9_dispatch.ccStatePointers);
+    try self.emitSlice(&gen9_dispatch.pipelineSelectGpgpu);
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush |
+        gen9_dispatch.pipe_control.render_target_flush));
+
+    const sba_start = self.batch.items.len * @sizeOf(u32);
+    try self.emitSlice(&gen9_dispatch.stateBaseAddress());
+    inline for (.{
+        .{ 4, kmd.Domain.render },
+        .{ 6, kmd.Domain.render },
+        .{ 10, kmd.Domain.instruction },
+    }) |base_address| {
+        self.relocations.append(self.interface.host_allocator.allocator(), .{
+            .target_handle = state_handle,
+            .offset = sba_start + base_address[0] * @sizeOf(u32),
+            .delta = gen9_dispatch.base_address_delta,
+            .read = true,
+            .domain = base_address[1],
+        }) catch return VkError.OutOfHostMemory;
+    }
+
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.texture_invalidate |
+        gen9_dispatch.pipe_control.constant_invalidate |
+        gen9_dispatch.pipe_control.state_invalidate |
+        gen9_dispatch.pipe_control.instruction_invalidate));
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall));
+    try self.emitSlice(&gen9_dispatch.mediaVfeState());
+    try self.emitSlice(&gen9_dispatch.interfaceDescriptorLoad(state_layout.interface_descriptor_offset));
+    try self.emitSlice(&gen9_dispatch.gpgpuWalker(.{ 1, 1, 1 }, 1));
+    try self.emitSlice(&gen9_dispatch.mediaStateFlush);
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush));
 }
 
 pub fn setDeviceMask(interface: *Interface, device_mask: u32) VkError!void {
@@ -382,23 +522,30 @@ pub fn endRenderPass(interface: *Interface) VkError!void {
 pub fn executeCommands(interface: *Interface, commands: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     const secondary: *Self = @alignCast(@fieldParentPtr("interface", commands));
+    if (secondary.gpu_allocations.items.len != 0)
+        return VkError.FeatureNotPresent;
+    if (secondary.engine) |engine|
+        try self.requireEngine(engine);
+
     const allocator = self.interface.host_allocator.allocator();
     const relocation_offset = self.batch.items.len * @sizeOf(u32);
-
     self.batch.appendSlice(allocator, secondary.batch.items) catch return VkError.OutOfHostMemory;
     for (secondary.relocations.items) |relocation| {
         self.relocations.append(allocator, .{
+            .source_handle = relocation.source_handle,
             .target_handle = relocation.target_handle,
-            .offset = relocation.offset + relocation_offset,
+            .offset = relocation.offset + if (relocation.source_handle == null) relocation_offset else 0,
             .delta = relocation.delta,
             .read = relocation.read,
             .write = relocation.write,
+            .domain = relocation.domain,
         }) catch return VkError.OutOfHostMemory;
     }
 }
 
 pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.DeviceSize, size: vk.DeviceSize, data: u32) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    try self.requireEngine(.blitter);
     const dst_range = try copy.fillRange(buffer, offset, size);
 
     var filled: vk.DeviceSize = 0;

src/intel/FlintInstance.zig

@@ -31,6 +31,8 @@ pub const extensions = [_]vk.ExtensionProperties{
     castExtension(vk.extensions.khr_get_physical_device_properties_2),
     castExtension(vk.extensions.khr_surface),
     castExtension(vk.extensions.khr_wayland_surface),
+    castExtension(vk.extensions.khr_xlib_surface),
+    castExtension(vk.extensions.khr_xcb_surface),
 };
 
 pub fn create(allocator: std.mem.Allocator, infos: *const vk.InstanceCreateInfo) VkError!*Interface {

src/intel/FlintPipeline.zig

@@ -15,16 +15,7 @@ const PipelineKind = enum {
     compute,
 };
 
-pub const ComputeArtifact = struct {
-    program: compiler.Program,
-    resources: compiler.targets.ComputeResourceLayout,
-
-    fn deinit(self: *ComputeArtifact, allocator: std.mem.Allocator) void {
-        self.resources.deinit(allocator);
-        self.program.deinit();
-        self.* = undefined;
-    }
-};
+pub const ComputeArtifact = compiler.targets.ComputeArtifact;
 
 const CommonStage = struct {
     stage: shader_ir.ir.module.Stage,
@@ -58,7 +49,7 @@ pub fn createCompute(device: *base.Device, allocator: std.mem.Allocator, cache:
     };
     initialized = true;
 
-    self.stages = try compileStages(self.artifact_allocator.allocator(), &.{info.stage}, .compute, compilerDeviceInfo(device));
+    self.stages = try compileStages(self.artifact_allocator.allocator(), device.io(), &.{info.stage}, .compute, compilerDeviceInfo(device));
     if (self.computeArtifact()) |artifact|
         try validateComputePipelineLayout(self.interface.layout, &artifact.resources);
     return self;
@@ -83,11 +74,11 @@ pub fn createGraphics(device: *base.Device, allocator: std.mem.Allocator, cache:
         stages[0..info.stage_count]
     else
         return VkError.ValidationFailed;
-    self.stages = try compileStages(self.artifact_allocator.allocator(), stage_infos, .graphics, compilerDeviceInfo(device));
+    self.stages = try compileStages(self.artifact_allocator.allocator(), device.io(), stage_infos, .graphics, compilerDeviceInfo(device));
     return self;
 }
 
-fn compileStages(allocator: std.mem.Allocator, infos: []const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError![]CommonStage {
+fn compileStages(allocator: std.mem.Allocator, io: std.Io, infos: []const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError![]CommonStage {
     if (infos.len == 0)
         return VkError.ValidationFailed;
 
@@ -100,13 +91,13 @@ fn compileStages(allocator: std.mem.Allocator, infos: []const vk.PipelineShaderS
     }
 
     for (infos, stages) |*info, *stage| {
-        stage.* = try compileStage(allocator, info, pipeline_kind, device_info);
+        stage.* = try compileStage(allocator, io, info, pipeline_kind, device_info);
         initialized += 1;
     }
     return stages;
 }
 
-fn compileStage(allocator: std.mem.Allocator, info: *const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError!CommonStage {
+fn compileStage(allocator: std.mem.Allocator, io: std.Io, info: *const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError!CommonStage {
     const specializations = try specializationValues(allocator, info.p_specialization_info);
     defer if (specializations.len != 0) allocator.free(specializations);
 
@@ -131,9 +122,15 @@ fn compileStage(allocator: std.mem.Allocator, info: *const vk.PipelineShaderStag
     errdefer module.deinit();
 
     std.debug.assert(module.stage == expected_stage);
+    if (base.config.flint_dump_common_ir)
+        dumpCommonIr(allocator, io, std.mem.span(info.p_name), &module);
 
     var artifact = try lowerToFlint(allocator, &module, device_info);
     errdefer if (artifact) |*value| value.deinit(allocator);
+    if (base.config.flint_dump_ir) {
+        if (artifact) |*value|
+            dumpFlintIr(allocator, io, std.mem.span(info.p_name), &value.program);
+    }
 
     return .{
         .stage = expected_stage,
@@ -142,58 +139,43 @@ fn compileStage(allocator: std.mem.Allocator, info: *const vk.PipelineShaderStag
     };
 }
 
+fn dumpCommonIr(allocator: std.mem.Allocator, io: std.Io, entry_point: []const u8, module: *const base.ShaderModule.IrModule) void {
+    const text = shader_ir.ir.printer.allocPrint(allocator, module) catch |err| {
+        std.log.scoped(.FlintPipeline).err("could not print backend-agnostic IR: {s}", .{@errorName(err)});
+        return;
+    };
+    defer allocator.free(text);
+
+    var stdout_buffer: [1024]u8 = undefined;
+    var stdout_file_writer: std.Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
+    const stdout_writer = &stdout_file_writer.interface;
+    stdout_writer.print("\n=== backend-agnostic IR: {s} ===\n{s}\n", .{ entry_point, text }) catch @panic("Debug printing failed");
+    stdout_writer.flush() catch @panic("Debug printing failed");
+}
+
+fn dumpFlintIr(allocator: std.mem.Allocator, io: std.Io, entry_point: []const u8, program: *const compiler.program.Program) void {
+    const text = compiler.printer.allocPrint(allocator, program) catch |err| {
+        std.log.scoped(.FlintPipeline).err("could not print Flint IR: {s}", .{@errorName(err)});
+        return;
+    };
+    defer allocator.free(text);
+
+    var stdout_buffer: [1024]u8 = undefined;
+    var stdout_file_writer: std.Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
+    const stdout_writer = &stdout_file_writer.interface;
+    stdout_writer.print("\n=== Flint IR: {s} ===\n{s}\n", .{ entry_point, text }) catch @panic("Debug printing failed");
+    stdout_writer.flush() catch @panic("Debug printing failed");
+}
+
 fn lowerToFlint(allocator: std.mem.Allocator, module: *base.ShaderModule.IrModule, device_info: ?compiler.device.DeviceInfo) VkError!?ComputeArtifact {
     const target = device_info orelse return null;
-    var program = compiler.targets.lower(allocator, module, target, .{}) catch |err| switch (err) {
+    return compiler.targets.compileCompute(allocator, module, target, .{}) catch |err| switch (err) {
         error.OutOfMemory => return VkError.OutOfHostMemory,
-
-        error.UnsupportedGeneration,
-        error.UnsupportedStage,
-        error.UnsupportedDispatchWidth,
-        error.UnsupportedGrfSize,
-        error.UnsupportedWorkgroupSize,
-        error.MissingWorkgroupSize,
-        error.UnsupportedType,
-        error.UnsupportedOperation,
-        error.UnsupportedTerminator,
-        => return null,
-
         else => {
-            std.log.scoped(.FlintPipeline).err("shader lowering failed: {s}", .{@errorName(err)});
+            std.log.scoped(.FlintPipeline).err("compute compilation failed: {s}", .{@errorName(err)});
             return VkError.ValidationFailed;
         },
     };
-    errdefer program.deinit();
-
-    var resources = compiler.targets.layoutComputeResources(allocator, &program) catch |err| switch (err) {
-        error.OutOfMemory => return VkError.OutOfHostMemory,
-        error.UnsupportedGeneration,
-        error.TooManyStorageBuffers,
-        => {
-            program.deinit();
-            return null;
-        },
-    };
-    errdefer resources.deinit(allocator);
-
-    compiler.targets.lowerComputeResources(&program, &resources) catch |err| switch (err) {
-        error.UnsupportedGeneration => {
-            resources.deinit(allocator);
-            program.deinit();
-            return null;
-        },
-        error.InvalidProgram,
-        error.InvalidResourceLayout,
-        => {
-            std.log.scoped(.FlintPipeline).err("Flint compute resource lowering failed: {s}", .{@errorName(err)});
-            return VkError.ValidationFailed;
-        },
-    };
-
-    return .{
-        .program = program,
-        .resources = resources,
-    };
 }
 
 fn validateComputePipelineLayout(layout: *const base.PipelineLayout, resources: *const compiler.targets.ComputeResourceLayout) VkError!void {
@@ -318,9 +300,16 @@ test "Flint pipeline: lower common compute IR" {
     const program = &artifact.program;
 
     try std.testing.expect(program.properties.common_ir_lowered);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expectEqual(@as(u16, 0), program.payload.header_grf.?.number);
     try std.testing.expect(program.properties.block_parameters_lowered);
-    try std.testing.expect(!program.properties.system_values_lowered);
+    try std.testing.expect(program.properties.system_values_lowered);
     try std.testing.expect(program.properties.resources_lowered);
+    try std.testing.expect(program.properties.messages_lowered);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+    try std.testing.expect(program.properties.message_payloads_lowered);
+    try std.testing.expect(program.properties.registers_allocated);
     try std.testing.expect(!program.properties.instructions_selected);
     try std.testing.expectEqual([3]u32{ 1, 1, 1 }, program.workgroup_size);
     try std.testing.expectEqual(@as(usize, 1), program.storage_buffers.entries.items.len);
@@ -330,6 +319,7 @@ test "Flint pipeline: lower common compute IR" {
 
     const text = try compiler.printer.allocPrint(std.testing.allocator, program);
     defer std.testing.allocator.free(text);
-    try std.testing.expect(std.mem.indexOf(u8, text, "load_global_invocation_id %id_x:u32, component(0)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "store_buffer bti(0), 0:u32, %id_x:u32") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "load_global_invocation_id") == null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "mov r1:u32, 0:u32") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "surface_message write bti(0)") != null);
 }

src/intel/FlintQueue.zig

@@ -97,6 +97,7 @@ pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*bas
                 try device.kmd.submitBatch(
                     interface.owner.io(),
                     allocator,
+                    .blitter,
                     &.{},
                     &.{},
                     syncs.items,
@@ -129,6 +130,7 @@ pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*bas
         try device.kmd.submitBatch(
             interface.owner.io(),
             allocator,
+            .blitter,
             &.{},
             &.{},
             syncs[0..sync_count],

src/intel/compiler/device.zig

@@ -47,18 +47,39 @@ pub const DeviceInfo = struct {
         const platform: Platform = switch (pci_device_id & 0xff00) {
             0x1900 => .skylake,
             0x5900 => .kabylake,
+
             0x3e00 => switch (pci_device_id) {
-                0x3ea0, 0x3ea1, 0x3ea2, 0x3ea3, 0x3ea4 => .whiskey_lake,
+                0x3ea0,
+                0x3ea1,
+                0x3ea2,
+                0x3ea3,
+                0x3ea4,
+                => .whiskey_lake,
+
                 else => .coffee_lake,
             },
+
             0x9b00 => .comet_lake,
             0x8a00 => .ice_lake,
             0x4500 => .elkhart_lake,
             0x4e00 => .jasper_lake,
+
             else => switch (pci_device_id) {
-                0x0a84, 0x1a84, 0x1a85, 0x5a84, 0x5a85 => .broxton,
-                0x3184, 0x3185 => .gemini_lake,
-                0x87c0, 0x87ca => .kabylake,
+                0x0a84,
+                0x1a84,
+                0x1a85,
+                0x5a84,
+                0x5a85,
+                => .broxton,
+
+                0x3184,
+                0x3185,
+                => .gemini_lake,
+
+                0x87c0,
+                0x87ca,
+                => .kabylake,
+
                 else => return null,
             },
         };
@@ -72,7 +93,11 @@ pub const DeviceInfo = struct {
             .whiskey_lake,
             .comet_lake,
             => .gen9,
-            .ice_lake, .elkhart_lake, .jasper_lake => .gen11,
+
+            .ice_lake,
+            .elkhart_lake,
+            .jasper_lake,
+            => .gen11,
         };
 
         return .{

src/intel/compiler/ir/instruction.zig

@@ -28,6 +28,40 @@ pub const StoreBuffer = struct {
     source: operand.Source,
 };
 
+pub const ArrayLength = struct {
+    destination: operand.Destination,
+    buffer: BufferReference,
+    byte_offset: operand.Source,
+    stride: u32,
+};
+
+pub const SurfaceRead = struct {
+    destination: operand.Destination,
+    binding_table: u8,
+    address: operand.Source,
+    immediate_offset: u32 = 0,
+};
+
+pub const SurfaceWrite = struct {
+    binding_table: u8,
+    address: operand.Source,
+    immediate_offset: u32 = 0,
+    data: operand.Source,
+};
+
+pub const SurfaceMessageKind = enum {
+    read,
+    write,
+};
+
+pub const SurfaceMessage = struct {
+    kind: SurfaceMessageKind,
+    binding_table: u8,
+    payload: operand.RegisterSpan,
+    response: ?operand.RegisterSpan,
+    data_type: operand.DataType,
+};
+
 pub const Move = struct {
     destination: operand.Destination,
     source: operand.Source,
@@ -66,12 +100,28 @@ pub const Compare = struct {
     rhs: operand.Source,
 };
 
+pub const MathOpcode = enum {
+    integer_quotient,
+};
+
+pub const Math = struct {
+    opcode: MathOpcode,
+    destination: operand.Destination,
+    lhs: operand.Source,
+    rhs: operand.Source,
+};
+
 pub const Operation = union(enum) {
     load_global_invocation_id: LoadGlobalInvocationId,
     load_buffer: LoadBuffer,
     store_buffer: StoreBuffer,
+    array_length: ArrayLength,
+    surface_read: SurfaceRead,
+    surface_write: SurfaceWrite,
+    surface_message: SurfaceMessage,
     move: Move,
     binary: Binary,
+    math: Math,
     compare: Compare,
     parallel_copy: pseudo.ParallelCopy,
 };

src/intel/compiler/ir/printer.zig

@@ -136,6 +136,42 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex
             try writer.writeAll(", ");
             try writeSource(program, writer, execution_size, op.source);
         },
+        .array_length => |op| {
+            try writer.writeAll("array_length ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeBufferReference(program, writer, op.buffer);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.byte_offset);
+            try writer.print(", stride({d})", .{op.stride});
+        },
+        .surface_read => |op| {
+            try writer.writeAll("surface_read ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.print(", bti({d}), ", .{op.binding_table});
+            try writeSource(program, writer, execution_size, op.address);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+        },
+        .surface_write => |op| {
+            try writer.print("surface_write bti({d}), ", .{op.binding_table});
+            try writeSource(program, writer, execution_size, op.address);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.data);
+        },
+        .surface_message => |op| {
+            try writer.print("surface_message {t} bti({d}), payload(", .{ op.kind, op.binding_table });
+            try writeRegister(program, writer, op.payload.base);
+            try writer.print(", {d})", .{op.payload.register_count});
+            if (op.response) |response| {
+                try writer.writeAll(", response(");
+                try writeRegister(program, writer, response.base);
+                try writer.print(", {d})", .{response.register_count});
+            }
+            try writer.print(", type({t})", .{op.data_type});
+        },
         .move => |op| {
             try writer.writeAll("mov ");
             try writeDestination(program, writer, execution_size, op.destination);
@@ -158,6 +194,14 @@ fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, ex
             try writer.writeAll(", ");
             try writeSource(program, writer, execution_size, op.rhs);
         },
+        .math => |op| {
+            try writer.print("{t} ", .{op.opcode});
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.lhs);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.rhs);
+        },
         .parallel_copy => |op| try writeParallelCopy(program, writer, execution_size, op),
     }
 }

src/intel/compiler/ir/program.zig

@@ -11,9 +11,12 @@ pub const Properties = packed struct {
     block_parameters_lowered: bool = false,
     parallel_copies_lowered: bool = false,
 
+    compute_abi_lowered: bool = false,
     system_values_lowered: bool = false,
     resources_lowered: bool = false,
     messages_lowered: bool = false,
+    message_addresses_lowered: bool = false,
+    message_payloads_lowered: bool = false,
     control_flow_lowered: bool = false,
 
     regions_legalized: bool = false,
@@ -23,7 +26,7 @@ pub const Properties = packed struct {
     flags_allocated: bool = false,
     branches_resolved: bool = false,
 
-    _padding: u19 = 0,
+    _padding: u16 = 0,
 };
 
 pub const StorageBuffer = struct {

src/intel/compiler/ir/validator.zig

@@ -5,40 +5,43 @@ const program_ir = @import("program.zig");
 const pseudo = @import("pseudo.zig");
 
 pub const Error = error{
-    MissingEntryBlock,
-    InvalidBlock,
-    MissingTerminator,
-    InvalidInstruction,
-    InvalidVirtualRegister,
-    InvalidVirtualFlag,
-    UnallocatedVirtualFlag,
-    InvalidPhysicalRegister,
-    InvalidRegisterSize,
-    InvalidRegisterAlignment,
-    InvalidLaneCount,
-    InvalidRegion,
-    InvalidDestination,
-    InvalidImmediateType,
-    InvalidStorageBuffer,
-    InvalidBufferReference,
-    InvalidGlobalInvocationId,
-    InvalidBufferAccess,
-    InvalidWorkgroupSize,
-    EmptyParallelCopy,
-    InvalidParallelCopyDestination,
-    ParallelCopyTypeMismatch,
-    DuplicateParallelCopyDestination,
-    PredicatedParallelCopy,
-    UnloweredParallelCopy,
-    UnloweredSystemValue,
-    UnloweredResource,
-    InvalidPayloadLayout,
-    EntryBlockHasParameters,
     DuplicateBlockParameter,
+    DuplicateParallelCopyDestination,
     EdgeArgumentCountMismatch,
     EdgeArgumentKindMismatch,
     EdgeArgumentTypeMismatch,
+    EmptyParallelCopy,
+    EntryBlockHasParameters,
+    InvalidBlock,
+    InvalidBufferAccess,
+    InvalidBufferReference,
+    InvalidDestination,
+    InvalidGlobalInvocationId,
+    InvalidImmediateType,
+    InvalidInstruction,
+    InvalidLaneCount,
+    InvalidMath,
+    InvalidMessage,
+    InvalidParallelCopyDestination,
+    InvalidPayloadLayout,
+    InvalidPhysicalRegister,
+    InvalidRegion,
+    InvalidRegisterAlignment,
+    InvalidRegisterSize,
+    InvalidStorageBuffer,
+    InvalidVirtualFlag,
+    InvalidVirtualRegister,
+    InvalidWorkgroupSize,
+    MissingEntryBlock,
+    MissingTerminator,
+    ParallelCopyTypeMismatch,
+    PredicatedParallelCopy,
+    UnallocatedVirtualFlag,
     UnloweredBlockParameter,
+    UnloweredMessage,
+    UnloweredParallelCopy,
+    UnloweredResource,
+    UnloweredSystemValue,
 };
 
 pub fn validate(program: *const program_ir.Program) Error!void {
@@ -138,6 +141,8 @@ fn validateInstruction(program: *const program_ir.Program, inst: instruction.Ins
                 return Error.InvalidGlobalInvocationId;
         },
         .load_buffer => |op| {
+            if (program.properties.messages_lowered)
+                return Error.UnloweredMessage;
             try validateBufferReference(program, op.buffer);
             try validateDestination(program, op.destination);
             try validateBufferOffset(program, op.byte_offset);
@@ -145,12 +150,49 @@ fn validateInstruction(program: *const program_ir.Program, inst: instruction.Ins
                 return Error.InvalidBufferAccess;
         },
         .store_buffer => |op| {
+            if (program.properties.messages_lowered)
+                return Error.UnloweredMessage;
             try validateBufferReference(program, op.buffer);
             try validateBufferOffset(program, op.byte_offset);
             try validateSource(program, op.source);
             if (!op.source.type.isInitialTargetType())
                 return Error.InvalidBufferAccess;
         },
+        .array_length => |op| {
+            try validateBufferReference(program, op.buffer);
+            try validateDestination(program, op.destination);
+            try validateBufferOffset(program, op.byte_offset);
+            if (op.destination.type != .u32 or op.stride == 0)
+                return Error.InvalidBufferAccess;
+        },
+        .surface_read => |op| {
+            try validateDestination(program, op.destination);
+            try validateBufferOffset(program, op.address);
+            if (!op.destination.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .surface_write => |op| {
+            try validateBufferOffset(program, op.address);
+            try validateSource(program, op.data);
+            if (!op.data.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .surface_message => |op| {
+            try validateRegisterSpan(program, op.payload);
+            if (!op.data_type.isInitialTargetType())
+                return Error.InvalidMessage;
+            switch (op.kind) {
+                .read => {
+                    if (op.payload.register_count != 1 or op.response == null)
+                        return Error.InvalidMessage;
+                    try validateRegisterSpan(program, op.response.?);
+                    if (op.response.?.register_count != 1)
+                        return Error.InvalidMessage;
+                },
+                .write => if (op.payload.register_count != 2 or op.response != null)
+                    return Error.InvalidMessage,
+            }
+        },
         .move => |op| {
             try validateDestination(program, op.destination);
             try validateSource(program, op.source);
@@ -165,6 +207,22 @@ fn validateInstruction(program: *const program_ir.Program, inst: instruction.Ins
             try validateSource(program, op.lhs);
             try validateSource(program, op.rhs);
         },
+        .math => |op| {
+            try validateDestination(program, op.destination);
+            try validateSource(program, op.lhs);
+            try validateSource(program, op.rhs);
+
+            switch (op.opcode) {
+                .integer_quotient => {
+                    if (inst.execution_size != .simd8)
+                        return Error.InvalidMath;
+                    if (op.destination.type != .u32 and op.destination.type != .i32)
+                        return Error.InvalidMath;
+                    if (op.lhs.type != op.destination.type or op.rhs.type != op.destination.type)
+                        return Error.InvalidMath;
+                },
+            }
+        },
 
         .parallel_copy => |op| {
             if (program.properties.parallel_copies_lowered)
@@ -286,6 +344,15 @@ fn validateDestination(program: *const program_ir.Program, destination: operand.
     }
 }
 
+fn validateRegisterSpan(program: *const program_ir.Program, span: operand.RegisterSpan) Error!void {
+    if (span.register_count == 0)
+        return Error.InvalidMessage;
+    switch (span.base) {
+        .virtual, .physical_grf => try validateRegisterRef(program, span.base),
+        else => return Error.InvalidMessage,
+    }
+}
+
 fn validateRegisterRef(program: *const program_ir.Program, register: operand.RegisterRef) Error!void {
     switch (register) {
         .virtual => |id| if (!program.virtual_registers.isLive(id))

src/intel/compiler/lower/block_arguments.zig

@@ -6,13 +6,14 @@ const operand = @import("../ir/operand.zig");
 const program_ir = @import("../ir/program.zig");
 const pseudo = @import("../ir/pseudo.zig");
 const validator = @import("../ir/validator.zig");
+const device = @import("../device.zig");
 
 pub const Error = std.mem.Allocator.Error || error{
     InvalidProgram,
 };
 
 pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
     if (program.properties.block_parameters_lowered)
         return;
 
@@ -27,8 +28,8 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
 
     var emitted_parallel_copy = false;
     for (original_blocks.items) |block_id| {
-        const block = program.blocks.get(block_id) orelse return error.InvalidProgram;
-        const terminator = block.terminator orelse return error.InvalidProgram;
+        const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+        const terminator = block.terminator orelse return Error.InvalidProgram;
         const rewritten: instruction.Terminator = switch (terminator) {
             .jump => |edge| .{ .jump = try rewriteEdge(
                 allocator,
@@ -73,9 +74,9 @@ fn rewriteEdge(
     if (edge.arguments.len == 0)
         return .{ .target = edge.target, .arguments = &.{} };
 
-    const target = builder.program.blocks.get(edge.target) orelse return error.InvalidProgram;
+    const target = builder.program.blocks.get(edge.target) orelse return Error.InvalidProgram;
     if (target.parameters.items.len != edge.arguments.len)
-        return error.InvalidProgram;
+        return Error.InvalidProgram;
 
     var register_copies: std.ArrayList(pseudo.RegisterCopy) = .empty;
     defer register_copies.deinit(allocator);
@@ -87,10 +88,10 @@ fn rewriteEdge(
             .register => |destination_id| {
                 const source = switch (argument) {
                     .source => |value| value,
-                    .predicate => return error.InvalidProgram,
+                    .predicate => return Error.InvalidProgram,
                 };
                 const destination = builder.program.virtual_registers.get(destination_id) orelse
-                    return error.InvalidProgram;
+                    return Error.InvalidProgram;
                 try register_copies.append(allocator, .{
                     .destination = .{
                         .register = .{ .virtual = destination_id },
@@ -101,7 +102,7 @@ fn rewriteEdge(
             },
             .flag => |destination_id| {
                 const source = switch (argument) {
-                    .source => return error.InvalidProgram,
+                    .source => return Error.InvalidProgram,
                     .predicate => |value| value,
                 };
                 try flag_copies.append(allocator, .{
@@ -128,19 +129,18 @@ fn rewriteEdge(
     return .{ .target = edge_block, .arguments = &.{} };
 }
 
-fn executionSize(dispatch_width: @import("../device.zig").DispatchWidth) @import("../device.zig").ExecutionSize {
+fn executionSize(dispatch_width: device.DispatchWidth) device.ExecutionSize {
     return @enumFromInt(@intFromEnum(dispatch_width));
 }
 
 fn mapBuilderError(err: anyerror) Error {
     return switch (err) {
-        error.OutOfMemory => error.OutOfMemory,
-        else => error.InvalidProgram,
+        Error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
     };
 }
 
 test "[ir] block arguments: lower register and flag parameters" {
-    const device = @import("../device.zig");
     const printer = @import("../ir/printer.zig");
 
     const device_info: device.DeviceInfo = .{
@@ -215,8 +215,6 @@ test "[ir] block arguments: lower register and flag parameters" {
 }
 
 test "[ir] block arguments: split same-target conditional edges" {
-    const device = @import("../device.zig");
-
     const device_info: device.DeviceInfo = .{
         .generation = .gen9,
         .platform = .skylake,

src/intel/compiler/lower/common_ir.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const base = @import("base");
 const shader_compiler = @import("shader_ir");
 const shader_ir = shader_compiler.ir;
 const device = @import("../device.zig");
@@ -431,6 +432,7 @@ const LoweringState = struct {
             .load_buffer => |operation| try self.lowerLoadBuffer(block_id, source_instruction.result, operation),
             .store_buffer => |operation| try self.lowerStoreBuffer(block_id, source_instruction.result, operation),
             .call => return Error.UnsanitizedModule,
+            .array_length => |operation| try self.lowerArrayLength(block_id, source_instruction.result, operation),
         }
     }
 
@@ -506,6 +508,32 @@ const LoweringState = struct {
         if (lhs_components.len == 0 or lhs_components.len != rhs_components.len or lhs_components.len != result_components.len)
             return Error.InvalidModule;
 
+        switch (operation.opcode) {
+            .unsigned_divide,
+            .signed_divide,
+            => {
+                const signed = operation.opcode == .signed_divide;
+                const target_type: operand.DataType = if (signed) .i32 else .u32;
+
+                for (lhs_components, rhs_components, result_components) |lhs_value, rhs_value, result_component| {
+                    var dst = try destinationFromSource(result_component);
+                    dst.type = target_type;
+
+                    try self.appendInstruction(block_id, null, .{
+                        .math = .{
+                            .opcode = .integer_quotient,
+                            .destination = dst,
+                            .lhs = try retypeIntegerSource(lhs_value, target_type),
+                            .rhs = try retypeIntegerSource(rhs_value, target_type),
+                        },
+                    });
+                }
+                return;
+            },
+
+            else => {},
+        }
+
         const data_type = lhs_components[0].type;
         const opcode: instruction.BinaryOpcode = switch (operation.opcode) {
             .integer_add => if (data_type == .u32 or data_type == .i32) .add else return Error.UnsupportedOperation,
@@ -520,8 +548,11 @@ const LoweringState = struct {
             .bitwise_and => if (data_type == .u32 or data_type == .i32) .bitwise_and else return Error.UnsupportedOperation,
             .bitwise_or => if (data_type == .u32 or data_type == .i32) .bitwise_or else return Error.UnsupportedOperation,
             .bitwise_xor => if (data_type == .u32 or data_type == .i32) .bitwise_xor else return Error.UnsupportedOperation,
+
             .unsigned_divide,
             .signed_divide,
+            => unreachable,
+
             .unsigned_modulo,
             .signed_modulo,
             .float_divide,
@@ -548,6 +579,47 @@ const LoweringState = struct {
         }
     }
 
+    fn retypeIntegerSource(source_value: operand.Source, target_type: operand.DataType) Error!operand.Source {
+        if (target_type != .u32 and target_type != .i32)
+            return Error.UnsupportedType;
+
+        var result = source_value;
+
+        switch (source_value.type) {
+            .u32, .i32 => {},
+            else => return Error.UnsupportedType,
+        }
+
+        result.register = switch (source_value.register) {
+            .immediate => |immediate| .{
+                .immediate = switch (target_type) {
+                    .u32 => .{
+                        .u32 = switch (immediate) {
+                            .u32 => |v| v,
+                            .i32 => |v| @bitCast(v),
+                            else => return Error.UnsupportedType,
+                        },
+                    },
+
+                    .i32 => .{
+                        .i32 = switch (immediate) {
+                            .u32 => |v| @bitCast(v),
+                            .i32 => |v| v,
+                            else => return Error.UnsupportedType,
+                        },
+                    },
+
+                    else => unreachable,
+                },
+            },
+
+            else => source_value.register,
+        };
+
+        result.type = target_type;
+        return result;
+    }
+
     fn lowerCompare(self: *LoweringState, block_id: ids.BlockId, result: ?shader_ir.id.ValueId, operation: shader_ir.instruction.Compare) Error!void {
         const result_id = try requireResult(result);
         const result_value = self.lowerer.module.values.get(result_id) orelse return Error.InvalidModule;
@@ -688,21 +760,36 @@ const LoweringState = struct {
         const variable = self.lowerer.module.interface_variables.get(operation.variable) orelse return Error.InvalidModule;
         if (variable.direction != .input)
             return Error.InvalidModule;
+
         switch (variable.semantic) {
-            .builtin => |builtin| if (builtin != .global_invocation_id)
-                return Error.UnsupportedOperation,
+            .builtin => |builtin| switch (builtin) {
+                .global_invocation_id => try self.lowerGlobalInvocationId(block_id, result_id, variable),
+                .num_workgroups => try self.lowerNumWorkgroups(result_id),
+                .workgroup_size => base.unsupported("workgroup size builtin is not yet supported in Flint", .{}),
+                else => return Error.UnsupportedOperation,
+            },
             .location => return Error.UnsupportedOperation,
         }
+    }
 
+    fn lowerGlobalInvocationId(
+        self: *LoweringState,
+        block_id: ids.BlockId,
+        result_id: shader_ir.id.ValueId,
+        variable: *const shader_ir.module.InterfaceVariable,
+    ) Error!void {
         const result_value = self.lowerer.module.values.get(result_id) orelse return Error.InvalidModule;
         if (result_value.type != variable.type)
             return Error.InvalidModule;
+
         const result_components = try self.addRegisterLocation(result_id, .temporary);
         if (result_components.len != 3)
             return Error.UnsupportedOperation;
+
         for (result_components, 0..) |result_component, component_index| {
             if (result_component.type != .u32)
                 return Error.UnsupportedOperation;
+
             try self.appendInstruction(block_id, null, .{
                 .load_global_invocation_id = .{
                     .destination = try destinationFromSource(result_component),
@@ -712,6 +799,28 @@ const LoweringState = struct {
         }
     }
 
+    fn lowerNumWorkgroups(self: *LoweringState, result_id: shader_ir.id.ValueId) Error!void {
+        const result_value = self.lowerer.module.values.get(result_id) orelse return Error.InvalidModule;
+        const lowered_type = try self.lowerType(result_value.type);
+
+        if (lowered_type.element_type != .u32 or lowered_type.component_count != 3)
+            return Error.UnsupportedType;
+
+        const vec = try self.storage.alloc(operand.Source, 3);
+
+        for (self.lowerer.module.execution_modes.workgroup_size.?, vec) |value, *component| {
+            component.* = .{
+                .register = .{
+                    .immediate = .{ .u32 = value },
+                },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            };
+        }
+
+        try self.putLocation(result_id, .{ .components = vec });
+    }
+
     fn lowerStoreInterface(self: *LoweringState, block_id: ids.BlockId, result: ?shader_ir.id.ValueId, operation: shader_ir.instruction.StoreInterface) Error!void {
         _ = self;
         _ = block_id;
@@ -758,6 +867,26 @@ const LoweringState = struct {
         }
     }
 
+    fn lowerArrayLength(self: *LoweringState, block_id: ids.BlockId, result: ?shader_ir.id.ValueId, operation: shader_ir.instruction.ArrayLength) Error!void {
+        const result_id = try requireResult(result);
+        const byte_offset = try self.source(operation.byte_offset);
+        if (byte_offset.type != .u32)
+            return Error.UnsupportedType;
+
+        const result_components = try self.addRegisterLocation(result_id, .temporary);
+        if (result_components.len != 1 or result_components[0].type != .u32)
+            return Error.UnsupportedType;
+
+        try self.appendInstruction(block_id, null, .{
+            .array_length = .{
+                .destination = try destinationFromSource(result_components[0]),
+                .buffer = .{ .logical = try self.storageBuffer(operation.resource) },
+                .byte_offset = byte_offset,
+                .stride = operation.stride,
+            },
+        });
+    }
+
     fn lowerControlAndTerminators(self: *LoweringState, allocator: std.mem.Allocator) Error!void {
         const source_entry = try self.sourceEntryFunction();
         const function = source_entry[1];
@@ -870,7 +999,7 @@ pub const Lowerer = struct {
 
         var transformer_context: shader_ir.transformer_manager.Context = .{ .allocator = allocator };
         _ = transformer_manager.run(self.module, &transformer_context) catch |err| return switch (err) {
-            error.OutOfMemory => Error.OutOfMemory,
+            Error.OutOfMemory => Error.OutOfMemory,
             else => Error.SanitizationFailed,
         };
         if (!self.module.properties.no_function_calls)
@@ -913,12 +1042,6 @@ pub const Lowerer = struct {
 
         program.properties.common_ir_lowered = true;
         validator.validate(&program) catch return Error.InvalidLoweredProgram;
-
-        block_arguments.run(allocator, &program) catch |err| return switch (err) {
-            error.OutOfMemory => Error.OutOfMemory,
-            else => Error.InvalidLoweredProgram,
-        };
-        validator.validate(&program) catch return Error.InvalidLoweredProgram;
         return program;
     }
 };
@@ -964,6 +1087,7 @@ fn expectLowered(source: []const u8, expected: []const u8) !void {
 
     var program = try lower(std.testing.allocator, &module, test_device, .{});
     defer program.deinit();
+    try block_arguments.run(std.testing.allocator, &program);
     try std.testing.expect(program.properties.common_ir_lowered);
     try std.testing.expect(!program.properties.instructions_selected);
 
@@ -979,6 +1103,7 @@ fn expectLoweredFragments(source: []const u8, expected: []const []const u8, unex
 
     var program = try lower(std.testing.allocator, &module, test_device, .{});
     defer program.deinit();
+    try block_arguments.run(std.testing.allocator, &program);
     try std.testing.expect(program.properties.common_ir_lowered);
     try std.testing.expect(!program.properties.instructions_selected);
 
@@ -1300,6 +1425,28 @@ test "[ir] Lower: vector storage-buffer operations" {
     }, &.{});
 }
 
+test "[ir] Lower: runtime array length" {
+    const source =
+        \\shader compute @main
+        \\{
+        \\    @storage: runtime_array[u32] = storage_buffer[set(0), binding(3)]
+        \\    %offset: constant u32 = 16
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %length: u32 = array_length @storage, %offset, stride 4
+        \\            return
+        \\    }
+        \\}
+    ;
+
+    try expectLoweredFragments(source, &.{
+        "@storage = storage_buffer[set(0), binding(3)]",
+        "%length: vgrf u32[8], class(temporary)",
+        "[simd8] array_length %length:u32, @storage, 16:u32, stride(4)",
+    }, &.{});
+}
+
 test "[ir] Lower: vector block parameter" {
     const source =
         \\shader compute @main
@@ -1405,20 +1552,6 @@ test "[ir] Lower: boolean block parameter" {
 }
 
 test "[ir] Lower: unsupported operations" {
-    try expectLoweringError(
-        \\shader compute @main
-        \\{
-        \\    %one: constant u32 = bits(0x1)
-        \\    %two: constant u32 = bits(0x2)
-        \\    fn @main() -> void
-        \\    {
-        \\        .entry():
-        \\            %quotient: u32 = unsigned_divide %one, %two
-        \\            return
-        \\    }
-        \\}
-    , Error.UnsupportedOperation);
-
     try expectLoweringError(
         \\shader compute @main
         \\{

src/intel/compiler/lower/parallel_copies.zig

@@ -28,7 +28,7 @@ const FlagWrite = struct {
 };
 
 pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
     if (program.properties.parallel_copies_lowered)
         return;
 
@@ -39,12 +39,12 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
         var instruction_index: usize = 0;
 
         while (true) {
-            const block = program.blocks.get(block_id) orelse return error.InvalidProgram;
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
             if (instruction_index >= block.instructions.items.len)
                 break;
 
             const instruction_id = block.instructions.items[instruction_index];
-            const inst = program.instructions.get(instruction_id) orelse return error.InvalidProgram;
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
             const parallel_copy = switch (inst.operation) {
                 .parallel_copy => |copy| copy,
                 else => {
@@ -53,7 +53,7 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
                 },
             };
             if (inst.predicate != null)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
             const execution_size = inst.execution_size;
 
             var emitted: std.ArrayList(EmittedInstruction) = .empty;
@@ -61,16 +61,16 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
             try lowerParallelCopy(allocator, &builder, execution_size, parallel_copy, &emitted);
 
             if (emitted.items.len == 0) {
-                const mutable_block = program.blocks.getMut(block_id) orelse return error.InvalidProgram;
+                const mutable_block = program.blocks.getMut(block_id) orelse return Error.InvalidProgram;
                 const removed_id = mutable_block.instructions.orderedRemove(instruction_index);
                 if (removed_id != instruction_id or !program.instructions.remove(instruction_id))
-                    return error.InvalidProgram;
+                    return Error.InvalidProgram;
                 continue;
             }
 
             builder.replaceOperation(instruction_id, emitted.items[0].operation) catch |err|
                 return mapBuilderError(err);
-            const replacement = program.instructions.getMut(instruction_id) orelse return error.InvalidProgram;
+            const replacement = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
             replacement.predicate = emitted.items[0].predicate;
 
             for (emitted.items[1..], 1..) |item, offset| {
@@ -87,7 +87,7 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
     }
 
     program.properties.parallel_copies_lowered = true;
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
 }
 
 fn lowerParallelCopy(
@@ -130,9 +130,9 @@ fn scheduleRegisterCopies(
 
         const cycle_copy = &pending.items[0];
         const destination_id = destinationVirtualRegister(cycle_copy.destination) orelse
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
         const destination_register = builder.program.virtual_registers.get(destination_id) orelse
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
         const temporary = builder.addVirtualRegister(.{
             .size_bytes = destination_register.size_bytes,
             .alignment_bytes = destination_register.alignment_bytes,
@@ -295,8 +295,8 @@ fn immediateU32(value: u32) operand.Source {
 
 fn mapBuilderError(err: anyerror) Error {
     return switch (err) {
-        error.OutOfMemory => error.OutOfMemory,
-        else => error.InvalidProgram,
+        Error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
     };
 }
 

src/intel/compiler/targets/gen9/compute/abi.zig

@@ -0,0 +1,60 @@
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = error{
+    InvalidPayloadLayout,
+};
+
+const thread_header: operand.PhysicalGrf = .{
+    .number = 0,
+    .byte_offset = 0,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (program.properties.compute_abi_lowered)
+        return;
+
+    if (program.payload.header_grf) |header| {
+        if (header.number != thread_header.number or header.byte_offset != thread_header.byte_offset)
+            return Error.InvalidPayloadLayout;
+    }
+    if (program.program_data.payload_grf_count > 1)
+        return Error.InvalidPayloadLayout;
+
+    program.payload.header_grf = thread_header;
+    program.program_data.payload_grf_count = 1;
+    program.properties.compute_abi_lowered = true;
+}
+
+const std = @import("std");
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+test "[gen9] compute ABI: reserve thread header" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    try run(&program);
+
+    try std.testing.expectEqual(thread_header, program.payload.header_grf.?);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+
+    try run(&program);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+}
+
+test "[gen9] compute ABI: reject conflicting payload" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+    program.payload.header_grf = .{ .number = 1 };
+
+    try std.testing.expectError(Error.InvalidPayloadLayout, run(&program));
+    try std.testing.expect(!program.properties.compute_abi_lowered);
+}

src/intel/compiler/targets/gen9/compute/array_length_lowering.zig

@@ -0,0 +1,91 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const ids = @import("../../../ir/id.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const resource_layout = @import("resource_layout.zig");
+
+pub const Error = std.mem.Allocator.Error || error{InvalidProgram};
+
+pub fn run(program: *program_ir.Program, layout: *const resource_layout.Layout) Error!void {
+    if (!program.properties.resources_lowered)
+        return Error.InvalidProgram;
+    if (layout.bindings.len >= std.math.maxInt(u8))
+        return Error.InvalidProgram;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const op = switch (inst.operation) {
+                .array_length => |value| value,
+                else => {
+                    instruction_index += 1;
+                    continue;
+                },
+            };
+            const resource_index = switch (op.buffer) {
+                .binding_table => |value| value,
+                .logical => return Error.InvalidProgram,
+            };
+            if (resource_index >= layout.bindings.len or op.stride == 0)
+                return Error.InvalidProgram;
+
+            const execution_size = inst.execution_size;
+            const predicate = inst.predicate;
+            const result_source: operand.Source = .{
+                .register = op.destination.register,
+                .type = .u32,
+                .region = operand.Region.contiguous(execution_size),
+            };
+            var negated_offset = op.byte_offset;
+            negated_offset.negate = !negated_offset.negate;
+
+            const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+            mutable.operation = .{ .load_buffer = .{
+                .destination = op.destination,
+                .buffer = .{ .binding_table = @intCast(layout.bindings.len) },
+                .byte_offset = immediate(@as(u32, resource_index) * @sizeOf(u32)),
+            } };
+
+            _ = builder.insertInstruction(block_id, instruction_index + 1, execution_size, predicate, .{ .binary = .{
+                .opcode = .add,
+                .destination = op.destination,
+                .lhs = result_source,
+                .rhs = negated_offset,
+            } }) catch |err| return mapBuilderError(err);
+            _ = builder.insertInstruction(block_id, instruction_index + 2, execution_size, predicate, .{ .math = .{
+                .opcode = .integer_quotient,
+                .destination = op.destination,
+                .lhs = result_source,
+                .rhs = immediate(op.stride),
+            } }) catch |err| return mapBuilderError(err);
+            instruction_index += 3;
+        }
+    }
+}
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}

src/intel/compiler/targets/gen9/compute/compute.zig

@@ -1,7 +1,17 @@
 const std = @import("std");
 
+pub const abi = @import("abi.zig");
+pub const dispatch = @import("dispatch.zig");
+pub const eu_encoder = @import("eu_encoder.zig");
+pub const kernel_encoder = @import("kernel_encoder.zig");
+pub const message_addresses = @import("message_addresses.zig");
+pub const message_descriptor = @import("message_descriptor.zig");
+pub const message_lowering = @import("message_lowering.zig");
+pub const message_payloads = @import("message_payloads.zig");
 pub const resource_layout = @import("resource_layout.zig");
 pub const resource_lowering = @import("resource_lowering.zig");
+pub const regions = @import("regions.zig");
+pub const system_values = @import("system_values.zig");
 pub const ResourceLayout = resource_layout.Layout;
 
 pub const Error = error{UnsupportedWorkgroupSize};

src/intel/compiler/targets/gen9/compute/dispatch.zig

@@ -0,0 +1,209 @@
+const std = @import("std");
+
+pub const max_storage_surfaces: usize = 4;
+pub const max_surfaces: usize = max_storage_surfaces + 1;
+pub const page_size: usize = 4096;
+pub const surface_state_size: usize = 64;
+pub const interface_descriptor_size: usize = 32;
+
+const mocs: u32 = 0x78;
+pub const base_address_delta: u32 = 1 | (mocs << 4);
+const raw_surface_format: u32 = 0x1ff;
+
+pub const Error = error{
+    EmptyBuffer,
+    StateTooLarge,
+    UnsupportedBufferSize,
+    TooManySurfaces,
+};
+
+pub const StateLayout = struct {
+    size: usize,
+    kernel_offset: u32,
+    surface_offsets: [max_surfaces]u32,
+    surface_address_offsets: [max_surfaces]u32,
+    surface_count: u8,
+    storage_surface_count: u8,
+    size_table_offset: u32,
+    binding_table_offset: u32,
+    interface_descriptor_offset: u32,
+};
+
+pub fn writeState(destination: []u8, kernel: []const u8, buffer_sizes: []const u64) Error!StateLayout {
+    if (buffer_sizes.len > max_storage_surfaces)
+        return Error.TooManySurfaces;
+
+    var layout: StateLayout = .{
+        .size = 0,
+        .kernel_offset = 0,
+        .surface_offsets = @splat(0),
+        .surface_address_offsets = @splat(0),
+        .surface_count = @intCast(buffer_sizes.len + 1),
+        .storage_surface_count = @intCast(buffer_sizes.len),
+        .size_table_offset = 0,
+        .binding_table_offset = 0,
+        .interface_descriptor_offset = 0,
+    };
+
+    var cursor = alignForward(kernel.len, 64);
+    for (buffer_sizes, 0..) |size, index| {
+        cursor = alignForward(cursor, surface_state_size);
+        layout.surface_offsets[index] = @intCast(cursor);
+        layout.surface_address_offsets[index] = @intCast(cursor + 8 * @sizeOf(u32));
+        cursor += surface_state_size;
+        if (size == 0)
+            return Error.EmptyBuffer;
+        if (size > std.math.maxInt(u32))
+            return Error.UnsupportedBufferSize;
+    }
+
+    const size_table_surface = buffer_sizes.len;
+    cursor = alignForward(cursor, surface_state_size);
+    layout.surface_offsets[size_table_surface] = @intCast(cursor);
+    layout.surface_address_offsets[size_table_surface] = @intCast(cursor + 8 * @sizeOf(u32));
+    cursor += surface_state_size;
+
+    cursor = alignForward(cursor, 32);
+    layout.binding_table_offset = @intCast(cursor);
+    cursor += layout.surface_count * @sizeOf(u32);
+
+    cursor = alignForward(cursor, @alignOf(u32));
+    layout.size_table_offset = @intCast(cursor);
+    cursor += @max(buffer_sizes.len, 1) * @sizeOf(u32);
+
+    cursor = alignForward(cursor, 64);
+    layout.interface_descriptor_offset = @intCast(cursor);
+    cursor += interface_descriptor_size;
+    layout.size = alignForward(cursor, page_size);
+    if (layout.size > destination.len or layout.size > page_size)
+        return Error.StateTooLarge;
+
+    @memset(destination[0..layout.size], 0);
+    @memcpy(destination[layout.kernel_offset .. layout.kernel_offset + kernel.len], kernel);
+
+    for (buffer_sizes, 0..) |size, index| {
+        _ = try encodeRawBufferSurface(destination, layout.surface_offsets[index], size);
+        putU32(destination, layout.binding_table_offset + @as(u32, @intCast(index * @sizeOf(u32))), layout.surface_offsets[index]);
+        putU32(destination, layout.size_table_offset + @as(u32, @intCast(index * @sizeOf(u32))), @intCast(size));
+    }
+    _ = try encodeRawBufferSurface(destination, layout.surface_offsets[size_table_surface], @max(buffer_sizes.len, 1) * @sizeOf(u32));
+    putU32(destination, layout.binding_table_offset + @as(u32, @intCast(size_table_surface * @sizeOf(u32))), layout.surface_offsets[size_table_surface]);
+
+    const idd = layout.interface_descriptor_offset;
+    putU32(destination, idd + 0, layout.kernel_offset);
+    putU32(destination, idd + 4, 0);
+    putU32(destination, idd + 4 * @sizeOf(u32), @as(u32, layout.surface_count) | layout.binding_table_offset);
+    putU32(destination, idd + 6 * @sizeOf(u32), 1);
+
+    return layout;
+}
+
+fn encodeRawBufferSurface(destination: []u8, offset: u32, byte_size: u64) Error!void {
+    if (byte_size == 0)
+        return Error.EmptyBuffer;
+
+    const aligned_size = std.mem.alignForward(u64, byte_size, 4);
+    const padded_size = aligned_size + (aligned_size - byte_size);
+    if (padded_size == 0 or padded_size > (@as(u64, 1) << 32))
+        return Error.UnsupportedBufferSize;
+    const length_minus_one: u32 = @intCast(padded_size - 1);
+
+    putU32(destination, offset + 0, (4 << 29) |
+        (raw_surface_format << 18) |
+        (1 << 16) |
+        (1 << 14));
+    putU32(destination, offset + 1 * @sizeOf(u32), mocs << 24);
+    putU32(destination, offset + 2 * @sizeOf(u32), (length_minus_one & 0x7f) |
+        (((length_minus_one >> 7) & 0x3fff) << 16));
+    putU32(destination, offset + 3 * @sizeOf(u32), ((length_minus_one >> 21) & 0x7ff) << 21);
+}
+
+pub const ccStatePointers = [_]u32{
+    0x780e0000,
+    0,
+};
+
+pub const pipelineSelectGpgpu = [_]u32{0x69040302};
+
+pub fn pipeControl(bits: u32) [6]u32 {
+    return .{ 0x7a000004, bits, 0, 0, 0, 0 };
+}
+
+pub const pipe_control = struct {
+    pub const state_invalidate: u32 = 1 << 2;
+    pub const constant_invalidate: u32 = 1 << 3;
+    pub const dc_flush: u32 = 1 << 5;
+    pub const texture_invalidate: u32 = 1 << 10;
+    pub const instruction_invalidate: u32 = 1 << 11;
+    pub const render_target_flush: u32 = 1 << 12;
+    pub const depth_flush: u32 = 1 << 0;
+    pub const cs_stall: u32 = 1 << 20;
+};
+
+pub fn stateBaseAddress() [19]u32 {
+    var words: [19]u32 = @splat(0);
+    words[0] = 0x61010011;
+    words[3] = mocs << 16;
+    words[4] = base_address_delta;
+    words[6] = base_address_delta;
+    words[10] = base_address_delta;
+    words[13] = (1 << 12) | 1;
+    words[15] = (1 << 12) | 1;
+    return words;
+}
+
+pub fn mediaVfeState() [9]u32 {
+    var words: [9]u32 = @splat(0);
+    words[0] = 0x70000007;
+    words[3] = (1 << 16) | (2 << 8);
+    words[5] = 2 << 16;
+    return words;
+}
+
+pub fn interfaceDescriptorLoad(offset: u32) [4]u32 {
+    return .{ 0x70020002, 0, interface_descriptor_size, offset };
+}
+
+pub fn gpgpuWalker(group_count: [3]u32, right_mask: u32) [15]u32 {
+    var words: [15]u32 = @splat(0);
+    words[0] = 0x7105000d;
+    words[7] = group_count[0];
+    words[10] = group_count[1];
+    words[12] = group_count[2];
+    words[13] = right_mask;
+    words[14] = 0xffffffff;
+    return words;
+}
+
+pub const mediaStateFlush = [_]u32{ 0x70040000, 0 };
+
+fn alignForward(value: usize, alignment: usize) usize {
+    return std.mem.alignForward(usize, value, alignment);
+}
+
+fn putU32(destination: []u8, offset: u32, value: u32) void {
+    std.mem.writeInt(u32, destination[offset..][0..@sizeOf(u32)], value, .little);
+}
+
+test "[gen9] dispatch: interface descriptor exposes internal size-table surface" {
+    var state: [page_size]u8 = undefined;
+    const layout = try writeState(&state, &.{ 0xaa, 0xbb }, &.{ 4096, 8192 });
+
+    try std.testing.expectEqual(@as(u8, 3), layout.surface_count);
+
+    const descriptor_binding_table = std.mem.readInt(
+        u32,
+        state[layout.interface_descriptor_offset + 4 * @sizeOf(u32) ..][0..@sizeOf(u32)],
+        .little,
+    );
+    try std.testing.expectEqual(layout.binding_table_offset | @as(u32, layout.surface_count), descriptor_binding_table);
+
+    for (0..layout.surface_count) |index| {
+        const entry = std.mem.readInt(
+            u32,
+            state[layout.binding_table_offset + index * @sizeOf(u32) ..][0..@sizeOf(u32)],
+            .little,
+        );
+        try std.testing.expectEqual(layout.surface_offsets[index], entry);
+    }
+}

src/intel/compiler/targets/gen9/compute/eu.zig

@@ -0,0 +1,96 @@
+pub const Opcode = enum(u7) {
+    add = 0x40,
+    addc = 0x4E,
+    @"and" = 0x05,
+    asr = 0x0C,
+    avg = 0x42,
+    bfe = 0x18,
+    bfi1 = 0x19,
+    bfi2 = 0x1A,
+    bfrev = 0x17,
+    brc = 0x23,
+    brd = 0x21,
+    @"break" = 0x28,
+    call = 0x2C,
+    calla = 0x2B,
+    cbit = 0x4D,
+    cmp = 0x10,
+    cmpn = 0x11,
+    cont = 0x29,
+    csel = 0x12,
+    dp2 = 0x57,
+    dp3 = 0x56,
+    dp4 = 0x54,
+    dph = 0x55,
+    @"else" = 0x24,
+    endif = 0x25,
+    fbh = 0x4B,
+    fbl = 0x4C,
+    frc = 0x43,
+    goto = 0x2E,
+    halt = 0x2A,
+    @"if" = 0x22,
+    illegal = 0x00,
+    jmpi = 0x20,
+    join = 0x2F,
+    line = 0x59,
+    lrp = 0x5C,
+    lzd = 0x4A,
+    mac = 0x48,
+    mach = 0x49,
+    mad = 0x5B,
+    madm = 0x5D,
+    math = 0x38,
+    mov = 0x01,
+    movi = 0x03,
+    mul = 0x41,
+    nop = 0x7E,
+    not = 0x04,
+    @"or" = 0x06,
+    pln = 0x5A,
+    ret = 0x2D,
+    rndd = 0x45,
+    rnde = 0x46,
+    rndu = 0x44,
+    rndz = 0x47,
+    sad2 = 0x50,
+    sada2 = 0x51,
+    sel = 0x02,
+    send = 0x31,
+    sendc = 0x32,
+    sends = 0x33,
+    sendsc = 0x34,
+    shl = 0x09,
+    shr = 0x08,
+    smov = 0x0A,
+    subb = 0x4F,
+    wait = 0x30,
+    @"while" = 0x27,
+    xor = 0x07,
+};
+
+pub const CompareCondition = enum(u3) {
+    none = 0,
+    zero = 1,
+    not_zero = 2,
+    greater = 3,
+    greater_or_equal = 4,
+    less = 5,
+    less_or_equal = 6,
+};
+
+pub const MathFunction = enum(u4) {
+    inv = 1,
+    log = 2,
+    exp = 3,
+    sqrt = 4,
+    rsq = 5,
+    sin = 6,
+    cos = 7,
+    sincos = 8,
+    fdiv = 9,
+    pow = 10,
+    idiv_and_rem = 11,
+    idiv = 12,
+    irem = 13,
+};

src/intel/compiler/targets/gen9/compute/eu_encoder.zig

@@ -0,0 +1,491 @@
+const std = @import("std");
+const device = @import("../../../device.zig");
+const ir_instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const message_descriptor = @import("message_descriptor.zig");
+const eu = @import("eu.zig");
+
+pub const Error = error{
+    UnsupportedExecutionSize,
+    UnsupportedDataType,
+    UnsupportedOperand,
+    InvalidRegister,
+    InvalidRegion,
+};
+
+pub const eot_payload_grf: u8 = 112;
+
+pub const EncodedInstruction = struct {
+    words: [2]u64 = .{ 0, 0 },
+
+    pub fn setBits(self: *EncodedInstruction, high: u7, low: u7, value: u64) void {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        const mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width)) << word_low;
+        self.words[word] = (self.words[word] & ~mask) | ((value << word_low) & mask);
+    }
+
+    pub fn bits(self: EncodedInstruction, high: u7, low: u7) u64 {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        return (self.words[word] >> word_low) & (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width));
+    }
+};
+
+const RegisterFile = enum(u2) {
+    architecture = 0,
+    grf = 1,
+    immediate = 3,
+};
+
+const HardwareType = enum(u4) {
+    unsigned_dword = 0,
+    signed_dword = 1,
+    unsigned_word = 2,
+    float = 7,
+};
+
+const Grf = struct {
+    number: u8,
+    byte_offset: u5,
+};
+
+pub fn encodeMove(execution_size: device.ExecutionSize, move: ir_instruction.Move) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.mov, execution_size);
+    const destination = try resolveGrf(move.destination.register, move.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(move.destination.type), destination, try horizontalStride(move.destination.region.horizontal_stride));
+    try setSource0(&encoded, move.source);
+    return encoded;
+}
+
+pub fn encodeEndThread(header: operand.PhysicalGrf) Error![2]EncodedInstruction {
+    if (header.number != 0 or header.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    var copy = try instructionHeader(.mov, .simd8);
+    copy.setBits(34, 34, 1); // NoMask
+    setDestination(&copy, .grf, .unsigned_dword, .{ .number = eot_payload_grf, .byte_offset = 0 }, 1);
+    copy.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    copy.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    copy.setBits(76, 69, header.number);
+    copy.setBits(81, 80, 1);
+    copy.setBits(84, 82, 3);
+    copy.setBits(88, 85, 4);
+
+    var send = try instructionHeader(.send, .simd8);
+    send.setBits(34, 34, 1); // NoMask
+    setDestination(&send, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    send.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    send.setBits(46, 43, @intFromEnum(HardwareType.unsigned_word));
+    send.setBits(76, 69, eot_payload_grf);
+    send.setBits(81, 80, 1);
+    send.setBits(84, 82, 3);
+    send.setBits(88, 85, 4);
+    send.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    send.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    send.setBits(124, 96, 0x02000010); // mlen=1, no response, do not dereference URB
+    send.setBits(27, 24, 7); // Thread Spawner
+    send.setBits(127, 127, 1);
+
+    return .{ copy, send };
+}
+
+pub fn encodeSurfaceMessage(execution_size: device.ExecutionSize, message: ir_instruction.SurfaceMessage) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.send, execution_size);
+    const descriptor = message_descriptor.encode(message);
+    const payload = try resolveGrf(message.payload.base, 0);
+    if (payload.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    if (message.response) |response| {
+        const destination = try resolveGrf(response.base, 0);
+        if (destination.byte_offset != 0)
+            return Error.InvalidRegister;
+        setDestination(&encoded, .grf, .unsigned_word, destination, 1);
+    } else {
+        setDestination(&encoded, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    }
+
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(76, 69, payload.number);
+    encoded.setBits(68, 64, payload.byte_offset);
+    encoded.setBits(81, 80, 1); // horizontal stride 1
+    encoded.setBits(84, 82, 3); // width 8
+    encoded.setBits(88, 85, 4); // vertical stride 8
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(124, 96, descriptor.value);
+    encoded.setBits(27, 24, descriptor.sfid);
+    return encoded;
+}
+
+pub fn encodeJump(displacement_bytes: i32) Error!EncodedInstruction {
+    return encodeJumpWithPredicate(displacement_bytes, null);
+}
+
+pub fn encodePredicatedJump(displacement_bytes: i32, predicate: operand.Predicate) Error!EncodedInstruction {
+    return encodeJumpWithPredicate(displacement_bytes, predicate);
+}
+
+fn encodeJumpWithPredicate(displacement_bytes: i32, predicate: ?operand.Predicate) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.jmpi, .simd1);
+    encoded.setBits(34, 34, 1); // NoMask
+
+    // JMPI updates the instruction pointer: IP = IP + displacement.
+    setDestination(&encoded, .architecture, .signed_dword, .{ .number = 0xa0, .byte_offset = 0 }, 1);
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.architecture));
+    encoded.setBits(46, 43, @intFromEnum(HardwareType.signed_dword));
+    encoded.setBits(76, 69, 0xa0);
+    encoded.setBits(81, 80, 0);
+    encoded.setBits(84, 82, 0);
+    encoded.setBits(88, 85, 0);
+    setSource1Immediate(&encoded, .signed_dword, .{ .i32 = displacement_bytes });
+
+    if (predicate) |value| {
+        const flag = switch (value.flag) {
+            .physical => |physical| physical,
+            .virtual => return Error.UnsupportedOperand,
+        };
+        if (flag.register != 0 or flag.subregister > 1)
+            return Error.InvalidRegister;
+
+        encoded.setBits(19, 16, 1); // Normal predicate control.
+        encoded.setBits(20, 20, @intFromBool(value.inverse));
+        encoded.setBits(33, 33, flag.register);
+        encoded.setBits(32, 32, flag.subregister);
+    }
+
+    return encoded;
+}
+
+pub fn patchJump(encoded_bytes: []u8, displacement_bytes: i32) Error!void {
+    if (encoded_bytes.len < 16)
+        return Error.InvalidRegister;
+
+    var encoded: EncodedInstruction = .{ .words = .{
+        std.mem.readInt(u64, encoded_bytes[0..8], .little),
+        std.mem.readInt(u64, encoded_bytes[8..16], .little),
+    } };
+    if (encoded.bits(6, 0) != @intFromEnum(eu.Opcode.jmpi))
+        return Error.UnsupportedOperand;
+    encoded.setBits(127, 96, @as(u32, @bitCast(displacement_bytes)));
+    std.mem.writeInt(u64, encoded_bytes[0..8], encoded.words[0], .little);
+    std.mem.writeInt(u64, encoded_bytes[8..16], encoded.words[1], .little);
+}
+
+pub fn encodeBinary(execution_size: device.ExecutionSize, binary: ir_instruction.Binary) Error!EncodedInstruction {
+    const opcode: eu.Opcode = switch (binary.opcode) {
+        .bitwise_xor => .xor,
+        .add => .add,
+        .multiply => .mul,
+        else => return Error.UnsupportedOperand,
+    };
+
+    var encoded = try instructionHeader(opcode, execution_size);
+    const destination = try resolveGrf(binary.destination.register, binary.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(binary.destination.type), destination, try horizontalStride(binary.destination.region.horizontal_stride));
+    try setSource0(&encoded, binary.lhs);
+    try setSource1(&encoded, binary.rhs);
+    return encoded;
+}
+
+pub fn encodeCompare(execution_size: device.ExecutionSize, compare: ir_instruction.Compare) Error!EncodedInstruction {
+    const flag = switch (compare.destination) {
+        .physical => |value| value,
+        .virtual => return Error.UnsupportedOperand,
+    };
+    if (flag.register != 0 or flag.subregister > 1)
+        return Error.InvalidRegister;
+
+    var encoded = try instructionHeader(.cmp, execution_size);
+    setDestination(&encoded, .architecture, try hardwareType(compare.lhs.type), .{ .number = 0, .byte_offset = 0 }, 1);
+    try setSource0(&encoded, compare.lhs);
+    try setSource1(&encoded, compare.rhs);
+
+    const condition: eu.CompareCondition = switch (compare.opcode) {
+        .equal => .zero,
+        .not_equal => .not_zero,
+        .greater_than => .greater,
+        .greater_or_equal => .greater_or_equal,
+        .less_than => .less,
+        .less_or_equal => .less_or_equal,
+    };
+
+    encoded.setBits(27, 24, @intFromEnum(condition));
+    encoded.setBits(33, 33, flag.register);
+    encoded.setBits(32, 32, flag.subregister);
+    return encoded;
+}
+
+pub fn encodeMath(execution_size: device.ExecutionSize, math: ir_instruction.Math) Error!EncodedInstruction {
+    if (execution_size != .simd8)
+        return Error.UnsupportedExecutionSize;
+
+    var encoded = try instructionHeader(.math, execution_size);
+    const destination = try resolveGrf(math.destination.register, math.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(math.destination.type), destination, try horizontalStride(math.destination.region.horizontal_stride));
+
+    try setSource0(&encoded, math.lhs);
+    try setSource1(&encoded, math.rhs);
+
+    const function: eu.MathFunction = switch (math.opcode) {
+        .integer_quotient => .idiv,
+    };
+
+    encoded.setBits(27, 24, @intFromEnum(function));
+
+    return encoded;
+}
+
+fn instructionHeader(opcode: eu.Opcode, execution_size: device.ExecutionSize) Error!EncodedInstruction {
+    var encoded: EncodedInstruction = .{};
+    encoded.setBits(6, 0, @intFromEnum(opcode));
+    encoded.setBits(23, 21, try executionSize(execution_size));
+    return encoded;
+}
+
+fn setDestination(encoded: *EncodedInstruction, file: RegisterFile, data_type: HardwareType, register: Grf, horizontal_stride: u2) void {
+    encoded.setBits(36, 35, @intFromEnum(file));
+    encoded.setBits(40, 37, @intFromEnum(data_type));
+    encoded.setBits(52, 48, register.byte_offset);
+    encoded.setBits(60, 53, register.number);
+    encoded.setBits(62, 61, horizontal_stride);
+}
+
+fn setSource0Register(encoded: *EncodedInstruction, source: operand.Source, register: Grf) Error!void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(try hardwareType(source.type)));
+    encoded.setBits(68, 64, register.byte_offset);
+    encoded.setBits(76, 69, register.number);
+    encoded.setBits(77, 77, @intFromBool(source.absolute));
+    encoded.setBits(78, 78, @intFromBool(source.negate));
+    encoded.setBits(81, 80, try horizontalStride(source.region.horizontal_stride));
+    encoded.setBits(84, 82, try regionWidth(source.region.width));
+    encoded.setBits(88, 85, try verticalStride(source.region.vertical_stride));
+}
+
+fn setSource0Immediate(encoded: *EncodedInstruction, data_type: HardwareType, immediate: operand.Immediate) void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(46, 43, @intFromEnum(data_type));
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.architecture));
+    encoded.setBits(94, 91, @intFromEnum(data_type));
+    encoded.setBits(127, 96, switch (immediate) {
+        .u32 => |value| value,
+        .i32 => |value| @as(u32, @bitCast(value)),
+        .f32 => |value| @as(u32, @bitCast(value)),
+    });
+}
+
+fn setSource0(encoded: *EncodedInstruction, source: operand.Source) Error!void {
+    switch (source.register) {
+        .physical_grf => {
+            const register = try resolveGrf(source.register, source.region.byte_offset);
+            try setSource0Register(encoded, source, register);
+        },
+        .immediate => |immediate| {
+            setSource0Immediate(encoded, try hardwareType(source.type), try applyImmediateModifiers(immediate, source.negate, source.absolute));
+        },
+
+        else => return Error.UnsupportedOperand,
+    }
+}
+
+fn setSource1Register(encoded: *EncodedInstruction, source: operand.Source, register: Grf) Error!void {
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(94, 91, @intFromEnum(try hardwareType(source.type)));
+
+    // Direct addressing.
+    encoded.setBits(100, 96, register.byte_offset);
+    encoded.setBits(108, 101, register.number);
+
+    // Source modifiers.
+    encoded.setBits(109, 109, @intFromBool(source.absolute));
+    encoded.setBits(110, 110, @intFromBool(source.negate));
+
+    // AddressMode = direct.
+    encoded.setBits(111, 111, 0);
+
+    // Align1 region.
+    encoded.setBits(113, 112, try horizontalStride(source.region.horizontal_stride));
+    encoded.setBits(116, 114, try regionWidth(source.region.width));
+    encoded.setBits(120, 117, try verticalStride(source.region.vertical_stride));
+}
+
+fn setSource1Immediate(encoded: *EncodedInstruction, data_type: HardwareType, immediate: operand.Immediate) void {
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+
+    encoded.setBits(94, 91, @intFromEnum(data_type));
+
+    encoded.setBits(127, 96, switch (immediate) {
+        .u32 => |value| value,
+        .i32 => |value| @as(u32, @bitCast(value)),
+        .f32 => |value| @as(u32, @bitCast(value)),
+    });
+}
+
+fn setSource1(encoded: *EncodedInstruction, source: operand.Source) Error!void {
+    switch (source.register) {
+        .physical_grf => {
+            const register = try resolveGrf(source.register, source.region.byte_offset);
+            try setSource1Register(encoded, source, register);
+        },
+
+        .immediate => |immediate| {
+            setSource1Immediate(encoded, try hardwareType(source.type), try applyImmediateModifiers(immediate, source.negate, source.absolute));
+        },
+
+        else => return Error.UnsupportedOperand,
+    }
+}
+
+fn applyImmediateModifiers(immediate: operand.Immediate, negate: bool, absolute: bool) Error!operand.Immediate {
+    if (absolute)
+        return Error.UnsupportedOperand;
+    if (!negate)
+        return immediate;
+
+    return switch (immediate) {
+        .u32 => |value| .{ .u32 = 0 -% value },
+        .i32 => |value| .{ .i32 = 0 -% value },
+        .f32 => |value| .{ .f32 = -value },
+    };
+}
+
+fn resolveGrf(register: operand.RegisterRef, region_byte_offset: u16) Error!Grf {
+    const physical = switch (register) {
+        .physical_grf => |value| value,
+        else => return Error.UnsupportedOperand,
+    };
+
+    const byte_address = @as(u32, physical.number) * 32 + physical.byte_offset + region_byte_offset;
+    const number = byte_address / 32;
+
+    if (number >= 128)
+        return Error.InvalidRegister;
+
+    return .{
+        .number = @intCast(number),
+        .byte_offset = @intCast(byte_address % 32),
+    };
+}
+
+fn hardwareType(data_type: operand.DataType) Error!HardwareType {
+    return switch (data_type) {
+        .u32 => .unsigned_dword,
+        .i32 => .signed_dword,
+        .f32 => .float,
+        else => Error.UnsupportedDataType,
+    };
+}
+
+fn executionSize(size: device.ExecutionSize) Error!u3 {
+    return switch (size) {
+        .simd1 => 0,
+        .simd8 => 3,
+        else => Error.UnsupportedExecutionSize,
+    };
+}
+
+fn horizontalStride(stride: u8) Error!u2 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn regionWidth(width: u8) Error!u3 {
+    return switch (width) {
+        1 => 0,
+        2 => 1,
+        4 => 2,
+        8 => 3,
+        16 => 4,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn verticalStride(stride: u8) Error!u4 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        8 => 4,
+        16 => 5,
+        32 => 6,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn testBinary(opcode: ir_instruction.BinaryOpcode) ir_instruction.Binary {
+    return .{
+        .opcode = opcode,
+        .destination = .{
+            .register = .{ .physical_grf = .{ .number = 3 } },
+            .type = .u32,
+        },
+        .lhs = .{
+            .register = .{ .physical_grf = .{ .number = 1 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .rhs = .{
+            .register = .{ .immediate = .{ .u32 = 16 } },
+            .type = .u32,
+            .region = operand.Region.broadcast(),
+        },
+    };
+}
+
+test "[gen9] EU encoder: encode integer multiply" {
+    const encoded = try encodeBinary(.simd8, testBinary(.multiply));
+    try std.testing.expectEqual(@as(u64, 65), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 3), encoded.bits(23, 21));
+}
+
+test "[gen9] EU encoder: encode bitwise XOR" {
+    const encoded = try encodeBinary(.simd8, testBinary(.bitwise_xor));
+    try std.testing.expectEqual(@as(u64, 7), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 16), encoded.bits(127, 96));
+}
+
+test "[gen9] EU encoder: encode unsigned less-than comparison" {
+    const encoded = try encodeCompare(.simd8, .{
+        .opcode = .less_than,
+        .destination = .{ .physical = .{ .register = 0, .subregister = 1 } },
+        .lhs = .{
+            .register = .{ .physical_grf = .{ .number = 1 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .rhs = .{
+            .register = .{ .physical_grf = .{ .number = 2 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    });
+
+    try std.testing.expectEqual(@as(u64, 16), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 5), encoded.bits(27, 24));
+    try std.testing.expectEqual(@as(u64, 0), encoded.bits(33, 33));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(32, 32));
+}
+
+test "[gen9] EU encoder: encode predicated jump" {
+    const encoded = try encodePredicatedJump(-32, .{
+        .flag = .{ .physical = .{ .register = 0, .subregister = 1 } },
+        .inverse = true,
+    });
+
+    try std.testing.expectEqual(@as(u64, @intFromEnum(eu.Opcode.jmpi)), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(19, 16));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(20, 20));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(32, 32));
+    try std.testing.expectEqual(@as(i32, -32), @as(i32, @bitCast(@as(u32, @truncate(encoded.bits(127, 96))))));
+}

src/intel/compiler/targets/gen9/compute/kernel_encoder.zig

@@ -0,0 +1,211 @@
+const std = @import("std");
+
+const eu = @import("eu_encoder.zig");
+const ids = @import("../../../ir/id.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || eu.Error || error{
+    InvalidProgram,
+    UnsupportedControlFlow,
+    UnsupportedOperation,
+    UnsupportedPredication,
+    EotRegisterUnavailable,
+};
+
+const JumpFixup = struct {
+    instruction_offset: usize,
+    target: ids.BlockId,
+};
+
+pub fn encode(allocator: std.mem.Allocator, program: *program_ir.Program) Error![]u8 {
+    if (!program.properties.registers_allocated)
+        return Error.InvalidProgram;
+    if (program.program_data.total_grf_count > eu.eot_payload_grf)
+        return Error.EotRegisterUnavailable;
+
+    const entry_id = program.entry_block orelse return Error.InvalidProgram;
+    if (!program.blocks.isLive(entry_id))
+        return Error.InvalidProgram;
+
+    const block_offsets = try allocator.alloc(?usize, program.blocks.entries.items.len);
+    defer allocator.free(block_offsets);
+    @memset(block_offsets, null);
+
+    var block_order: std.ArrayList(ids.BlockId) = .empty;
+    defer block_order.deinit(allocator);
+    try block_order.append(allocator, entry_id);
+    for (program.blocks.entries.items, 0..) |block, block_index| {
+        if (block != null and block_index != entry_id.index())
+            try block_order.append(allocator, ids.BlockId.fromIndex(block_index));
+    }
+
+    var fixups: std.ArrayList(JumpFixup) = .empty;
+    defer fixups.deinit(allocator);
+    var kernel: std.ArrayList(u8) = .empty;
+    errdefer kernel.deinit(allocator);
+
+    for (block_order.items) |block_id| {
+        const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+        block_offsets[block_id.index()] = kernel.items.len;
+
+        for (block.instructions.items) |instruction_id|
+            try encodeInstruction(allocator, &kernel, program, instruction_id);
+
+        const terminator = block.terminator orelse return Error.InvalidProgram;
+        switch (terminator) {
+            .jump => |edge| {
+                const instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodeJump(0));
+                try fixups.append(allocator, .{
+                    .instruction_offset = instruction_offset,
+                    .target = edge.target,
+                });
+            },
+            .conditional_branch => |branch| {
+                const true_instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodePredicatedJump(0, branch.predicate));
+                try fixups.append(allocator, .{
+                    .instruction_offset = true_instruction_offset,
+                    .target = branch.true_edge.target,
+                });
+
+                const false_instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodeJump(0));
+                try fixups.append(allocator, .{
+                    .instruction_offset = false_instruction_offset,
+                    .target = branch.false_edge.target,
+                });
+            },
+            .end_thread => {
+                const header = program.payload.header_grf orelse return Error.InvalidProgram;
+                const instructions = try eu.encodeEndThread(header);
+                for (instructions) |encoded|
+                    try appendInstruction(allocator, &kernel, encoded);
+                program.program_data.total_grf_count = eu.eot_payload_grf + 1;
+            },
+            .@"unreachable" => return Error.UnsupportedControlFlow,
+        }
+    }
+
+    for (fixups.items) |fixup| {
+        if (fixup.target.index() >= block_offsets.len)
+            return Error.InvalidProgram;
+        const target_offset = block_offsets[fixup.target.index()] orelse return Error.InvalidProgram;
+        const next_instruction_offset = fixup.instruction_offset + 16;
+        const displacement = std.math.cast(i32, @as(i64, @intCast(target_offset)) - @as(i64, @intCast(next_instruction_offset))) orelse
+            return Error.UnsupportedControlFlow;
+        try eu.patchJump(kernel.items[fixup.instruction_offset..][0..16], displacement);
+    }
+
+    return kernel.toOwnedSlice(allocator);
+}
+
+fn encodeInstruction(allocator: std.mem.Allocator, kernel: *std.ArrayList(u8), program: *const program_ir.Program, instruction_id: ids.InstructionId) Error!void {
+    const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+    if (inst.predicate != null) {
+        std.log.scoped(.FlintEuEncoder).err("cannot encode instruction {d} ({t}): predication is not supported", .{ instruction_id.index(), std.meta.activeTag(inst.operation) });
+        return Error.UnsupportedPredication;
+    }
+
+    const encoded = switch (inst.operation) {
+        .move => |move| eu.encodeMove(inst.execution_size, move),
+        .surface_message => |message| eu.encodeSurfaceMessage(inst.execution_size, message),
+        .binary => |binary| eu.encodeBinary(inst.execution_size, binary),
+        .compare => |compare| eu.encodeCompare(inst.execution_size, compare),
+        .math => |math| eu.encodeMath(inst.execution_size, math),
+        else => {
+            std.log.scoped(.FlintEuEncoder).err("cannot encode instruction {d}: unsupported operation {t}", .{ instruction_id.index(), std.meta.activeTag(inst.operation) });
+            return Error.UnsupportedOperation;
+        },
+    } catch |err| {
+        std.log.scoped(.FlintEuEncoder).err("failed to encode instruction {d} ({t}): {s}", .{ instruction_id.index(), std.meta.activeTag(inst.operation), @errorName(err) });
+        if (err == error.InvalidRegion) switch (inst.operation) {
+            .move => |move| std.log.scoped(.FlintEuEncoder).err("move in block {d}: destination {t} byte={d} hstride={d}; source {t} byte={d} vstride={d} width={d} hstride={d}", .{
+                inst.parent_block.index(),
+                move.destination.register,
+                move.destination.region.byte_offset,
+                move.destination.region.horizontal_stride,
+                move.source.register,
+                move.source.region.byte_offset,
+                move.source.region.vertical_stride,
+                move.source.region.width,
+                move.source.region.horizontal_stride,
+            }),
+            else => {},
+        };
+        return err;
+    };
+    try appendInstruction(allocator, kernel, encoded);
+}
+
+fn appendInstruction(allocator: std.mem.Allocator, kernel: *std.ArrayList(u8), instruction: eu.EncodedInstruction) std.mem.Allocator.Error!void {
+    var bytes: [16]u8 = undefined;
+    std.mem.writeInt(u64, bytes[0..8], instruction.words[0], .little);
+    std.mem.writeInt(u64, bytes[8..16], instruction.words[1], .little);
+    try kernel.appendSlice(allocator, &bytes);
+}
+
+test "[gen9] kernel encoder: patch unconditional jump between blocks" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const exit = try program.addBlock("exit");
+    try program.setEntryBlock(entry);
+    try program.setTerminator(entry, .{ .jump = .{ .target = exit, .arguments = &.{} } });
+    try program.setTerminator(exit, .end_thread);
+    program.payload.header_grf = .{ .number = 0 };
+    program.properties.registers_allocated = true;
+
+    const kernel = try encode(std.testing.allocator, &program);
+    defer std.testing.allocator.free(kernel);
+
+    try std.testing.expectEqual(@as(usize, 48), kernel.len);
+    try std.testing.expectEqual(@as(u7, 32), @as(u7, @truncate(std.mem.readInt(u64, kernel[0..8], .little))));
+    try std.testing.expectEqual(@as(i32, 0), @as(i32, @bitCast(std.mem.readInt(u32, kernel[12..16], .little))));
+}
+
+test "[gen9] kernel encoder: patch conditional branch targets" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const true_block = try program.addBlock("true");
+    const false_block = try program.addBlock("false");
+    try program.setEntryBlock(entry);
+    try program.setTerminator(entry, .{ .conditional_branch = .{
+        .predicate = .{ .flag = .{ .physical = .{ .register = 0, .subregister = 1 } } },
+        .true_edge = .{ .target = true_block, .arguments = &.{} },
+        .false_edge = .{ .target = false_block, .arguments = &.{} },
+    } });
+    try program.setTerminator(true_block, .end_thread);
+    try program.setTerminator(false_block, .end_thread);
+    program.payload.header_grf = .{ .number = 0 };
+    program.properties.registers_allocated = true;
+
+    const kernel = try encode(std.testing.allocator, &program);
+    defer std.testing.allocator.free(kernel);
+
+    try std.testing.expectEqual(@as(usize, 96), kernel.len);
+    try std.testing.expectEqual(@as(i32, 16), @as(i32, @bitCast(std.mem.readInt(u32, kernel[12..16], .little))));
+    try std.testing.expectEqual(@as(i32, 32), @as(i32, @bitCast(std.mem.readInt(u32, kernel[28..32], .little))));
+    const first_word = std.mem.readInt(u64, kernel[0..8], .little);
+    try std.testing.expectEqual(@as(u64, 1), (first_word >> 16) & 0xf);
+    try std.testing.expectEqual(@as(u64, 1), (first_word >> 32) & 0x1);
+}

src/intel/compiler/targets/gen9/compute/message_addresses.zig

@@ -0,0 +1,211 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const ids = @import("../../../ir/id.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const instruction = @import("../../../ir/instruction.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    MessagesNotLowered,
+    InvalidProgram,
+};
+
+const AddressAdjustment = struct {
+    address: operand.Source,
+    immediate_offset: u32,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.messages_lowered)
+        return Error.MessagesNotLowered;
+    if (program.properties.message_addresses_lowered)
+        return;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const adjustment = addressAdjustment(inst.operation) orelse {
+                instruction_index += 1;
+                continue;
+            };
+            if (adjustment.address.type != .u32)
+                return Error.InvalidProgram;
+
+            if (adjustment.immediate_offset == 0) {
+                instruction_index += 1;
+                continue;
+            }
+
+            switch (adjustment.address.register) {
+                .immediate => |immediate| {
+                    const base = switch (immediate) {
+                        .u32 => |value| value,
+                        else => return Error.InvalidProgram,
+                    };
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
+                    address.source.register = .{ .immediate = .{ .u32 = base +% adjustment.immediate_offset } };
+                    address.immediate_offset.* = 0;
+                    instruction_index += 1;
+                },
+                .virtual,
+                .physical_grf,
+                .architecture,
+                => {
+                    const execution_width: u32 = @intFromEnum(inst.execution_size);
+                    const size_bytes = execution_width * @sizeOf(u32);
+                    const address_register = builder.addVirtualRegister(.{
+                        .size_bytes = size_bytes,
+                        .alignment_bytes = @intCast(@min(size_bytes, program.device_info.grf_size_bytes)),
+                        .element_type = .u32,
+                        .lane_count = @intCast(execution_width),
+                        .class = .temporary,
+                    }) catch |err| return mapBuilderError(err);
+
+                    _ = builder.insertInstruction(block_id, instruction_index, inst.execution_size, inst.predicate, .{
+                        .binary = .{
+                            .opcode = .add,
+                            .destination = .{
+                                .register = .{ .virtual = address_register },
+                                .type = .u32,
+                            },
+                            .lhs = adjustment.address,
+                            .rhs = immediateSource(adjustment.immediate_offset),
+                        },
+                    }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
+                    address.source.* = .{
+                        .register = .{ .virtual = address_register },
+                        .type = .u32,
+                        .region = operand.Region.contiguous(inst.execution_size),
+                    };
+                    address.immediate_offset.* = 0;
+                    instruction_index += 2;
+                },
+                .null => return Error.InvalidProgram,
+            }
+        }
+    }
+
+    program.properties.message_addresses_lowered = true;
+}
+
+fn addressAdjustment(operation: instruction.Operation) ?AddressAdjustment {
+    return switch (operation) {
+        .surface_read => |op| .{ .address = op.address, .immediate_offset = op.immediate_offset },
+        .surface_write => |op| .{ .address = op.address, .immediate_offset = op.immediate_offset },
+        else => null,
+    };
+}
+
+const MutableAddress = struct {
+    source: *operand.Source,
+    immediate_offset: *u32,
+};
+
+fn messageAddressMut(operation: *instruction.Operation) ?MutableAddress {
+    return switch (operation.*) {
+        .surface_read => |*op| .{ .source = &op.address, .immediate_offset = &op.immediate_offset },
+        .surface_write => |*op| .{ .source = &op.address, .immediate_offset = &op.immediate_offset },
+        else => null,
+    };
+}
+
+fn immediateSource(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn markPrerequisite(program: *program_ir.Program) void {
+    program.properties.messages_lowered = true;
+}
+
+test "[gen9] message addresses: fold immediate offsets" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_write = .{
+        .binding_table = 0,
+        .address = immediateSource(12),
+        .immediate_offset = 4,
+        .data = immediateSource(7),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisite(&program);
+
+    try run(&program);
+
+    const write = program.instructions.get(message).?.operation.surface_write;
+    try std.testing.expectEqual(@as(u32, 16), write.address.register.immediate.u32);
+    try std.testing.expectEqual(@as(u32, 0), write.immediate_offset);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+}
+
+test "[gen9] message addresses: materialize dynamic offsets" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const base = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_read = .{
+        .destination = .{ .register = .{ .virtual = base }, .type = .u32 },
+        .binding_table = 0,
+        .address = .{
+            .register = .{ .virtual = base },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .immediate_offset = 8,
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisite(&program);
+
+    try run(&program);
+
+    const block = program.blocks.get(entry).?;
+    try std.testing.expectEqual(@as(usize, 2), block.instructions.items.len);
+    try std.testing.expect(program.instructions.get(block.instructions.items[0]).?.operation == .binary);
+    const read = program.instructions.get(message).?.operation.surface_read;
+    try std.testing.expect(read.address.register == .virtual);
+    try std.testing.expect(read.address.register.virtual != base);
+    try std.testing.expectEqual(@as(u32, 0), read.immediate_offset);
+}

src/intel/compiler/targets/gen9/compute/message_descriptor.zig

@@ -0,0 +1,90 @@
+const instruction = @import("../../../ir/instruction.zig");
+
+pub const Descriptor = struct {
+    sfid: u8,
+    value: u32,
+    message_length: u8,
+    response_length: u8,
+};
+
+const dc1_sfid: u8 = 12;
+const simd8_one_channel_control: u8 = 0x2e;
+
+const MessageType = enum(u8) {
+    untyped_surface_read = 1,
+    untyped_surface_write = 9,
+};
+
+pub fn encode(message: instruction.SurfaceMessage) Descriptor {
+    const lengths: struct { message: u8, response: u8 } = switch (message.kind) {
+        .read => .{ .message = 1, .response = 1 },
+        .write => .{ .message = 2, .response = 0 },
+    };
+    const message_type: MessageType = switch (message.kind) {
+        .read => .untyped_surface_read,
+        .write => .untyped_surface_write,
+    };
+
+    return .{
+        .sfid = dc1_sfid,
+        .value = makeDescriptor(
+            message.binding_table,
+            simd8_one_channel_control,
+            message_type,
+            lengths.message,
+            lengths.response,
+        ),
+        .message_length = lengths.message,
+        .response_length = lengths.response,
+    };
+}
+
+fn makeDescriptor(binding_table: u8, message_control: u8, message_type: MessageType, message_length: u8, response_length: u8) u32 {
+    return @as(u32, binding_table) |
+        (@as(u32, message_control) << 8) |
+        (@as(u32, @intFromEnum(message_type)) << 14) |
+        (@as(u32, response_length) << 20) |
+        (@as(u32, message_length) << 25);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface read" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .read,
+        .binding_table = 3,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 1 },
+        .response = .{ .base = .{ .physical_grf = .{ .number = 2 } }, .register_count = 1 },
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 3), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x02106e03), descriptor.value);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface write" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .write,
+        .binding_table = 7,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 2 },
+        .response = null,
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 2), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 0), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 7), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 9), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 0), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 2), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x04026e07), descriptor.value);
+}

src/intel/compiler/targets/gen9/compute/message_lowering.zig

@@ -0,0 +1,110 @@
+const instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = error{
+    ResourcesNotLowered,
+    InvalidProgram,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.resources_lowered)
+        return Error.ResourcesNotLowered;
+    if (program.properties.messages_lowered)
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        inst.operation = switch (inst.operation) {
+            .load_buffer => |op| .{ .surface_read = .{
+                .destination = op.destination,
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
+                .address = op.byte_offset,
+                .immediate_offset = op.immediate_offset,
+            } },
+            .store_buffer => |op| .{ .surface_write = .{
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
+                .address = op.byte_offset,
+                .immediate_offset = op.immediate_offset,
+                .data = op.source,
+            } },
+            else => inst.operation,
+        };
+    }
+
+    program.properties.messages_lowered = true;
+}
+
+fn bindingTableIndex(reference: instruction.BufferReference) ?u8 {
+    return switch (reference) {
+        .binding_table => |index| index,
+        .logical => null,
+    };
+}
+
+const std = @import("std");
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] compute message lowering: select surface messages" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const value = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try program.addBlock("entry");
+    const load = try program.appendInstruction(entry, .simd8, null, .{ .load_buffer = .{
+        .destination = .{ .register = .{ .virtual = value }, .type = .u32 },
+        .buffer = .{ .binding_table = 2 },
+        .byte_offset = immediate(16),
+        .immediate_offset = 4,
+    } });
+    const store = try program.appendInstruction(entry, .simd8, null, .{ .store_buffer = .{
+        .buffer = .{ .binding_table = 3 },
+        .byte_offset = immediate(32),
+        .immediate_offset = 8,
+        .source = .{
+            .register = .{ .virtual = value },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.resources_lowered = true;
+
+    try run(&program);
+
+    const read = program.instructions.get(load).?.operation.surface_read;
+    try std.testing.expectEqual(@as(u8, 2), read.binding_table);
+    try std.testing.expectEqual(@as(u32, 4), read.immediate_offset);
+    const write = program.instructions.get(store).?.operation.surface_write;
+    try std.testing.expectEqual(@as(u8, 3), write.binding_table);
+    try std.testing.expectEqual(@as(u32, 8), write.immediate_offset);
+    try std.testing.expect(program.properties.messages_lowered);
+}
+
+test "[gen9] compute message lowering: reject unresolved resources" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    try std.testing.expectError(Error.ResourcesNotLowered, run(&program));
+}

src/intel/compiler/targets/gen9/compute/message_payloads.zig

@@ -0,0 +1,200 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const device = @import("../../../device.zig");
+const ids = @import("../../../ir/id.zig");
+const instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    MessageAddressesNotLowered,
+    InvalidProgram,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.message_addresses_lowered)
+        return Error.MessageAddressesNotLowered;
+    if (program.properties.message_payloads_lowered)
+        return;
+    if (program.device_info.grf_size_bytes != 32)
+        return Error.InvalidProgram;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const execution_size = inst.execution_size;
+            switch (inst.operation) {
+                .surface_read => |op| {
+                    if (op.immediate_offset != 0 or op.address.type != .u32)
+                        return Error.InvalidProgram;
+                    const response = try responseSpan(op.destination);
+                    const payload = try addPayloadRegister(&builder, execution_size, 1);
+                    _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 0, .u32),
+                        .source = op.address,
+                    } }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    mutable.operation = .{ .surface_message = .{
+                        .kind = .read,
+                        .binding_table = op.binding_table,
+                        .payload = .{ .base = .{ .virtual = payload }, .register_count = 1 },
+                        .response = response,
+                        .data_type = op.destination.type,
+                    } };
+                    instruction_index += 2;
+                },
+                .surface_write => |op| {
+                    if (op.immediate_offset != 0 or op.address.type != .u32)
+                        return Error.InvalidProgram;
+                    const payload = try addPayloadRegister(&builder, execution_size, 2);
+                    _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 0, .u32),
+                        .source = op.address,
+                    } }) catch |err| return mapBuilderError(err);
+                    _ = builder.insertInstruction(block_id, instruction_index + 1, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 32, op.data.type),
+                        .source = op.data,
+                    } }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    mutable.operation = .{ .surface_message = .{
+                        .kind = .write,
+                        .binding_table = op.binding_table,
+                        .payload = .{ .base = .{ .virtual = payload }, .register_count = 2 },
+                        .response = null,
+                        .data_type = op.data.type,
+                    } };
+                    instruction_index += 3;
+                },
+                else => instruction_index += 1,
+            }
+        }
+    }
+
+    program.properties.message_payloads_lowered = true;
+}
+
+fn addPayloadRegister(builder: *Builder, execution_size: device.ExecutionSize, register_count: u8) Error!ids.VirtualRegisterId {
+    return builder.addVirtualRegister(.{
+        .size_bytes = @as(u32, register_count) * 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = @intFromEnum(execution_size),
+        .class = .temporary,
+        .spillable = false,
+    }) catch |err| return mapBuilderError(err);
+}
+
+fn payloadDestination(register: ids.VirtualRegisterId, byte_offset: u16, data_type: operand.DataType) operand.Destination {
+    return .{
+        .register = .{ .virtual = register },
+        .type = data_type,
+        .region = .{ .byte_offset = byte_offset },
+    };
+}
+
+fn responseSpan(destination: operand.Destination) Error!operand.RegisterSpan {
+    if (destination.region.byte_offset != 0 or destination.region.horizontal_stride != 1)
+        return Error.InvalidProgram;
+    return switch (destination.register) {
+        .virtual, .physical_grf => .{
+            .base = destination.register,
+            .register_count = 1,
+        },
+        else => Error.InvalidProgram,
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] message payloads: pack surface write address and data" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_write = .{
+        .binding_table = 2,
+        .address = immediate(16),
+        .data = immediate(42),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.message_addresses_lowered = true;
+
+    try run(&program);
+
+    const block = program.blocks.get(entry).?;
+    try std.testing.expectEqual(@as(usize, 3), block.instructions.items.len);
+    const address_move = program.instructions.get(block.instructions.items[0]).?.operation.move;
+    const data_move = program.instructions.get(block.instructions.items[1]).?.operation.move;
+    try std.testing.expectEqual(@as(u16, 0), address_move.destination.region.byte_offset);
+    try std.testing.expectEqual(@as(u16, 32), data_move.destination.region.byte_offset);
+    try std.testing.expectEqual(address_move.destination.register.virtual, data_move.destination.register.virtual);
+
+    const send = program.instructions.get(message).?.operation.surface_message;
+    try std.testing.expectEqual(instruction.SurfaceMessageKind.write, send.kind);
+    try std.testing.expectEqual(@as(u8, 2), send.binding_table);
+    try std.testing.expectEqual(@as(u8, 2), send.payload.register_count);
+    try std.testing.expect(send.response == null);
+    try std.testing.expect(program.properties.message_payloads_lowered);
+}
+
+test "[gen9] message payloads: prepare surface read response" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const result = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .response,
+    });
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_read = .{
+        .destination = .{ .register = .{ .virtual = result }, .type = .u32 },
+        .binding_table = 1,
+        .address = immediate(0),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.message_addresses_lowered = true;
+
+    try run(&program);
+
+    const send = program.instructions.get(message).?.operation.surface_message;
+    try std.testing.expectEqual(instruction.SurfaceMessageKind.read, send.kind);
+    try std.testing.expectEqual(@as(u8, 1), send.payload.register_count);
+    try std.testing.expectEqual(result, send.response.?.base.virtual);
+    try std.testing.expectEqual(@as(u8, 1), send.response.?.register_count);
+}

src/intel/compiler/targets/gen9/compute/pipeline.zig

@@ -0,0 +1,120 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+const device = @import("../../../device.zig");
+const program_ir = @import("../../../ir/program.zig");
+const common_ir = @import("../../../lower/common_ir.zig");
+const block_arguments = @import("../../../lower/block_arguments.zig");
+const parallel_copies = @import("../../../lower/parallel_copies.zig");
+const flag_allocation = @import("../flag_allocation.zig");
+const register_allocation = @import("../register_allocation.zig");
+
+const compute = @import("compute.zig");
+const abi = @import("abi.zig");
+const array_length_lowering = @import("array_length_lowering.zig");
+const kernel_encoder = @import("kernel_encoder.zig");
+const message_addresses = @import("message_addresses.zig");
+const message_lowering = @import("message_lowering.zig");
+const message_payloads = @import("message_payloads.zig");
+const resource_layout = @import("resource_layout.zig");
+const resource_lowering = @import("resource_lowering.zig");
+const regions = @import("regions.zig");
+const system_values = @import("system_values.zig");
+
+pub const Error = common_ir.Error ||
+    block_arguments.Error ||
+    parallel_copies.Error ||
+    abi.Error ||
+    array_length_lowering.Error ||
+    kernel_encoder.Error ||
+    message_addresses.Error ||
+    message_lowering.Error ||
+    message_payloads.Error ||
+    resource_layout.Error ||
+    resource_lowering.Error ||
+    system_values.Error ||
+    flag_allocation.Error ||
+    register_allocation.Error ||
+    compute.Error ||
+    error{
+        UnsupportedGeneration,
+        UnsupportedStage,
+        UnsupportedDispatchWidth,
+        UnsupportedGrfSize,
+    };
+
+pub const Artifact = struct {
+    program: program_ir.Program,
+    resources: resource_layout.Layout,
+    kernel: ?[]u8,
+
+    pub fn deinit(self: *Artifact, allocator: std.mem.Allocator) void {
+        if (self.kernel) |kernel|
+            allocator.free(kernel);
+        self.resources.deinit(allocator);
+        self.program.deinit();
+        self.* = undefined;
+    }
+};
+
+pub fn compile(allocator: std.mem.Allocator, module: *shader_ir.module.Module, device_info: device.DeviceInfo, options: common_ir.Options) Error!Artifact {
+    if (device_info.generation != .gen9)
+        return Error.UnsupportedGeneration;
+    if (module.stage != .compute)
+        return Error.UnsupportedStage;
+    if (options.dispatch_width != .simd8 or !device_info.supportsDispatch(.simd8))
+        return Error.UnsupportedDispatchWidth;
+    if (device_info.grf_size_bytes != 32)
+        return Error.UnsupportedGrfSize;
+    if (module.execution_modes.workgroup_size) |workgroup_size|
+        try compute.validateWorkgroupSize(workgroup_size);
+
+    var program = try common_ir.lower(
+        allocator,
+        module,
+        device_info,
+        options,
+    );
+    errdefer program.deinit();
+
+    try abi.run(&program);
+    try system_values.run(&program);
+    try block_arguments.run(allocator, &program);
+    try parallel_copies.run(allocator, &program);
+
+    var resources = try resource_layout.Layout.init(allocator, &program);
+    errdefer resources.deinit(allocator);
+
+    try resource_lowering.run(&program, &resources);
+    try array_length_lowering.run(&program, &resources);
+    try message_lowering.run(&program);
+    try message_addresses.run(&program);
+    try message_payloads.run(&program);
+    regions.run(&program);
+    try flag_allocation.run(allocator, &program);
+    try register_allocation.run(allocator, &program);
+
+    const kernel = kernel_encoder.encode(allocator, &program) catch |err| encoding_error: {
+        std.log.scoped(.FlintCompiler).err("Gen9 EU kernel encoding failed: {s}", .{@errorName(err)});
+        break :encoding_error switch (err) {
+            error.UnsupportedControlFlow,
+            error.UnsupportedOperation,
+            error.UnsupportedPredication,
+            error.UnsupportedExecutionSize,
+            error.UnsupportedDataType,
+            error.UnsupportedOperand,
+            error.InvalidRegister,
+            error.InvalidRegion,
+            error.EotRegisterUnavailable,
+            => null,
+            else => return err,
+        };
+    };
+    errdefer if (kernel) |bytes| allocator.free(bytes);
+
+    return .{
+        .program = program,
+        .resources = resources,
+        .kernel = kernel,
+    };
+}

src/intel/compiler/targets/gen9/compute/regions.zig

@@ -0,0 +1,49 @@
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub fn run(program: *program_ir.Program) void {
+    if (program.properties.regions_legalized)
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        switch (inst.operation) {
+            .load_buffer => |*op| legalizeSource(&op.byte_offset, inst.execution_size),
+            .store_buffer => |*op| {
+                legalizeSource(&op.byte_offset, inst.execution_size);
+                legalizeSource(&op.source, inst.execution_size);
+            },
+            .array_length => |*op| legalizeSource(&op.byte_offset, inst.execution_size),
+            .surface_read => |*op| legalizeSource(&op.address, inst.execution_size),
+            .surface_write => |*op| {
+                legalizeSource(&op.address, inst.execution_size);
+                legalizeSource(&op.data, inst.execution_size);
+            },
+            .move => |*op| legalizeSource(&op.source, inst.execution_size),
+            .binary => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            .math => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            .compare => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            else => {},
+        }
+    }
+
+    program.properties.regions_legalized = true;
+}
+
+fn legalizeSource(source: *operand.Source, execution_size: @import("../../../device.zig").ExecutionSize) void {
+    const byte_offset = source.region.byte_offset;
+    source.region = switch (source.register) {
+        .immediate => operand.Region.broadcast(),
+        else => operand.Region.contiguous(execution_size),
+    };
+    source.region.byte_offset = byte_offset;
+}

src/intel/compiler/targets/gen9/compute/resource_lowering.zig

@@ -53,6 +53,7 @@ fn bufferReference(operation: instruction.Operation) ?instruction.BufferReferenc
     return switch (operation) {
         .load_buffer => |op| op.buffer,
         .store_buffer => |op| op.buffer,
+        .array_length => |op| op.buffer,
         else => null,
     };
 }
@@ -61,6 +62,7 @@ fn bufferReferenceMut(operation: *instruction.Operation) ?*instruction.BufferRef
     return switch (operation.*) {
         .load_buffer => |*op| &op.buffer,
         .store_buffer => |*op| &op.buffer,
+        .array_length => |*op| &op.buffer,
         else => null,
     };
 }

src/intel/compiler/targets/gen9/compute/system_values.zig

@@ -0,0 +1,95 @@
+const std = @import("std");
+
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const validator = @import("../../../ir/validator.zig");
+
+pub const Error = error{InvalidProgram};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    validator.validate(program) catch return Error.InvalidProgram;
+    if (program.properties.system_values_lowered)
+        return;
+
+    // Flint dispatch currently accepts only one invocation in one workgroup at
+    // base group zero, so every component of GlobalInvocationId is zero.
+    if (!std.mem.eql(u32, &program.workgroup_size, &.{ 1, 1, 1 }))
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        inst.operation = switch (inst.operation) {
+            .load_global_invocation_id => |op| .{ .move = .{
+                .destination = op.destination,
+                .source = zero(),
+            } },
+            else => inst.operation,
+        };
+    }
+
+    program.properties.system_values_lowered = true;
+    validator.validate(program) catch return Error.InvalidProgram;
+}
+
+fn zero() operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = 0 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] system values: lower global invocation ID for single invocation" {
+    const Builder = @import("../../../ir/Builder.zig");
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const destination = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try builder.addBlock("entry");
+    const load = try builder.appendInstruction(entry, .simd8, null, .{ .load_global_invocation_id = .{
+        .destination = .{ .register = .{ .virtual = destination }, .type = .u32 },
+        .component = 2,
+    } });
+    try builder.setTerminator(entry, .end_thread);
+    try builder.setEntryBlock(entry);
+
+    try run(&program);
+
+    try std.testing.expect(program.properties.system_values_lowered);
+    const move = program.instructions.get(load).?.operation.move;
+    try std.testing.expectEqual(@as(u32, 0), move.source.register.immediate.u32);
+}
+
+test "[gen9] system values: preserve IDs for unsupported workgroup sizes" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 2, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    const entry = try program.addBlock("entry");
+    try program.setTerminator(entry, .end_thread);
+    try program.setEntryBlock(entry);
+
+    try run(&program);
+    try std.testing.expect(!program.properties.system_values_lowered);
+}

src/intel/compiler/targets/gen9/flag_allocation.zig

@@ -18,15 +18,15 @@ const physical_flag_count = 2;
 
 pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
     if (!program.properties.block_parameters_lowered)
-        return error.BlockParametersNotLowered;
+        return Error.BlockParametersNotLowered;
 
     if (!program.properties.parallel_copies_lowered)
-        return error.ParallelCopiesNotLowered;
+        return Error.ParallelCopiesNotLowered;
 
     if (program.properties.flags_allocated)
         return;
 
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
 
     const allocations = try allocator.alloc(?operand.PhysicalFlag, program.virtual_flags.entries.items.len);
     defer allocator.free(allocations);
@@ -38,9 +38,9 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
     for (allocations) |*allocation| {
         const marker = allocation.* orelse continue;
         if (marker.subregister != std.math.maxInt(u8))
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
 
-        const subregister = std.mem.indexOfScalar(bool, &occupied, false) orelse return error.OutOfFlagRegisters;
+        const subregister = std.mem.indexOfScalar(bool, &occupied, false) orelse return Error.OutOfFlagRegisters;
 
         allocation.* = .{
             .register = 0,
@@ -51,7 +51,7 @@ pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!voi
 
     try visitProgramFlags(program, allocations, &occupied, true);
     program.properties.flags_allocated = true;
-    validator.validate(program) catch return error.InvalidProgram;
+    validator.validate(program) catch return Error.InvalidProgram;
 }
 
 fn visitProgramFlags(
@@ -63,14 +63,14 @@ fn visitProgramFlags(
     for (program.instructions.entries.items, 0..) |entry, instruction_index| {
         _ = entry orelse continue;
         const inst = program.instructions.getMut(ids.InstructionId.fromIndex(instruction_index)) orelse
-            return error.InvalidProgram;
+            return Error.InvalidProgram;
 
         if (inst.predicate) |*predicate|
             try visitFlagRef(program, &predicate.flag, allocations, occupied, rewrite);
 
         switch (inst.operation) {
             .compare => |*compare| try visitFlagRef(program, &compare.destination, allocations, occupied, rewrite),
-            .parallel_copy => return error.ParallelCopiesNotLowered,
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
             else => {},
         }
     }
@@ -78,8 +78,8 @@ fn visitProgramFlags(
     for (program.blocks.entries.items, 0..) |entry, block_index| {
         _ = entry orelse continue;
         const block = program.blocks.getMut(ids.BlockId.fromIndex(block_index)) orelse
-            return error.InvalidProgram;
-        const terminator = if (block.terminator) |*value| value else return error.InvalidProgram;
+            return Error.InvalidProgram;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
 
         switch (terminator.*) {
             .jump => |*edge| try visitEdge(program, edge, allocations, occupied, rewrite),
@@ -129,7 +129,7 @@ fn visitFlagRef(
     switch (flag.*) {
         .virtual => |virtual| {
             if (!program.virtual_flags.isLive(virtual) or virtual.index() >= allocations.len)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
 
             if (!rewrite) {
                 // Mark this virtual flag as referenced without assigning a physical
@@ -139,14 +139,14 @@ fn visitFlagRef(
                 return;
             }
 
-            const physical = allocations[virtual.index()] orelse return error.InvalidProgram;
+            const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
             if (physical.subregister >= physical_flag_count)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
             flag.* = .{ .physical = physical };
         },
         .physical => |physical| {
             if (physical.register != 0 or physical.subregister >= physical_flag_count)
-                return error.InvalidProgram;
+                return Error.InvalidProgram;
             occupied[physical.subregister] = true;
         },
     }
@@ -251,7 +251,7 @@ test "[gen9] flag allocation: report exhaustion without rewriting" {
     try program.setTerminator(entry, .end_thread);
     markPrerequisites(&program);
 
-    try std.testing.expectError(error.OutOfFlagRegisters, run(std.testing.allocator, &program));
+    try std.testing.expectError(Error.OutOfFlagRegisters, run(std.testing.allocator, &program));
     try std.testing.expect(!program.properties.flags_allocated);
     try std.testing.expectEqual(first, program.instructions.get(first_compare).?.operation.compare.destination.virtual);
 }

src/intel/compiler/targets/gen9/gen9.zig

@@ -1,50 +1,20 @@
 const std = @import("std");
 const shader_ir = @import("shader_ir").ir;
 const device = @import("../../device.zig");
-const program_ir = @import("../../ir/program.zig");
 const common_ir = @import("../../lower/common_ir.zig");
-const parallel_copies = @import("../../lower/parallel_copies.zig");
 
 pub const compute = @import("compute/compute.zig");
+pub const compute_pipeline = @import("compute/pipeline.zig");
 pub const flag_allocation = @import("flag_allocation.zig");
+pub const register_allocation = @import("register_allocation.zig");
 pub const validator = @import("validator.zig");
 
 pub const Options = common_ir.Options;
-pub const ResourceLoweringError = compute.resource_lowering.Error;
+pub const ComputeArtifact = compute_pipeline.Artifact;
+pub const Error = compute_pipeline.Error;
 
-pub const Error = common_ir.Error || compute.Error || flag_allocation.Error || error{
-    UnsupportedGeneration,
-    UnsupportedStage,
-    UnsupportedDispatchWidth,
-    UnsupportedGrfSize,
-};
-
-pub fn lower(allocator: std.mem.Allocator, module: *shader_ir.module.Module, device_info: device.DeviceInfo, options: Options) Error!program_ir.Program {
-    if (device_info.generation != .gen9)
-        return Error.UnsupportedGeneration;
-    if (module.stage != .compute)
-        return Error.UnsupportedStage;
-    if (options.dispatch_width != .simd8 or !device_info.supportsDispatch(.simd8))
-        return Error.UnsupportedDispatchWidth;
-    if (device_info.grf_size_bytes != 32)
-        return Error.UnsupportedGrfSize;
-    if (module.execution_modes.workgroup_size) |workgroup_size|
-        try compute.validateWorkgroupSize(workgroup_size);
-
-    var program = try common_ir.lower(allocator, module, device_info, options);
-    errdefer program.deinit();
-    parallel_copies.run(allocator, &program) catch |err| return switch (err) {
-        error.OutOfMemory => Error.OutOfMemory,
-        error.InvalidProgram => Error.InvalidLoweredProgram,
-    };
-    try flag_allocation.run(allocator, &program);
-    validator.validate(&program) catch return Error.InvalidLoweredProgram;
-    return program;
-}
-
-pub fn lowerComputeResources(program: *program_ir.Program, layout: *const compute.ResourceLayout) ResourceLoweringError!void {
-    try compute.resource_lowering.run(program, layout);
-    validator.validate(program) catch return ResourceLoweringError.InvalidProgram;
+pub fn compileCompute(allocator: std.mem.Allocator, module: *shader_ir.module.Module, device_info: device.DeviceInfo, options: Options) Error!ComputeArtifact {
+    return compute_pipeline.compile(allocator, module, device_info, options);
 }
 
 test "[gen9] target: reject unsupported target configurations" {
@@ -68,17 +38,17 @@ test "[gen9] target: reject unsupported target configurations" {
     };
     var other_generation = gen9_device;
     other_generation.generation = .gen11;
-    try std.testing.expectError(Error.UnsupportedGeneration, lower(std.testing.allocator, &module, other_generation, .{}));
+    try std.testing.expectError(Error.UnsupportedGeneration, compileCompute(std.testing.allocator, &module, other_generation, .{}));
 
     module.stage = .fragment;
-    try std.testing.expectError(Error.UnsupportedStage, lower(std.testing.allocator, &module, gen9_device, .{}));
+    try std.testing.expectError(Error.UnsupportedStage, compileCompute(std.testing.allocator, &module, gen9_device, .{}));
     module.stage = .compute;
 
-    try std.testing.expectError(Error.UnsupportedDispatchWidth, lower(std.testing.allocator, &module, gen9_device, .{ .dispatch_width = .simd16 }));
+    try std.testing.expectError(Error.UnsupportedDispatchWidth, compileCompute(std.testing.allocator, &module, gen9_device, .{ .dispatch_width = .simd16 }));
 
     var wide_grf = gen9_device;
     wide_grf.grf_size_bytes = 64;
-    try std.testing.expectError(Error.UnsupportedGrfSize, lower(std.testing.allocator, &module, wide_grf, .{}));
+    try std.testing.expectError(Error.UnsupportedGrfSize, compileCompute(std.testing.allocator, &module, wide_grf, .{}));
 }
 
 test "[gen9] target: lower 256 KiB SSBO copy loop" {
@@ -123,16 +93,25 @@ test "[gen9] target: lower 256 KiB SSBO copy loop" {
         .pci_device_id = 0x1912,
         .grf_count = 128,
     };
-    var program = try lower(std.testing.allocator, &module, gen9_device, .{});
-    defer program.deinit();
+    var artifact = try compileCompute(std.testing.allocator, &module, gen9_device, .{});
+    defer artifact.deinit(std.testing.allocator);
+    try std.testing.expect(artifact.kernel != null);
+    const program = &artifact.program;
+    const resources = &artifact.resources;
 
     try std.testing.expect(program.properties.common_ir_lowered);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expectEqual(@as(u16, 0), program.payload.header_grf.?.number);
     try std.testing.expect(program.properties.block_parameters_lowered);
     try std.testing.expect(program.properties.parallel_copies_lowered);
     try std.testing.expect(program.properties.flags_allocated);
+    try std.testing.expect(program.properties.registers_allocated);
 
-    var resources = try compute.ResourceLayout.init(std.testing.allocator, &program);
-    defer resources.deinit(std.testing.allocator);
+    try std.testing.expect(program.properties.resources_lowered);
+    try std.testing.expect(program.properties.messages_lowered);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+    try std.testing.expect(program.properties.message_payloads_lowered);
     try std.testing.expectEqual(@as(usize, 2), resources.bindings.len);
     try std.testing.expectEqual(compute.resource_layout.Binding{
         .set = 0,
@@ -145,31 +124,24 @@ test "[gen9] target: lower 256 KiB SSBO copy loop" {
         .binding_table_index = 1,
     }, resources.bindings[1]);
 
-    try lowerComputeResources(&program, &resources);
-    try validator.validate(&program);
-
-    var load_offsets: [4]bool = @splat(false);
-    var store_offsets: [4]bool = @splat(false);
     var load_count: usize = 0;
     var store_count: usize = 0;
     for (program.instructions.entries.items) |instruction_entry| {
         const inst = instruction_entry orelse continue;
         switch (inst.operation) {
-            .load_buffer => |operation| {
-                try std.testing.expectEqual(@as(u8, 0), operation.buffer.binding_table);
-                try std.testing.expect(operation.immediate_offset % @sizeOf(u32) == 0);
-                const component = operation.immediate_offset / @sizeOf(u32);
-                try std.testing.expect(component < load_offsets.len);
-                load_offsets[component] = true;
-                load_count += 1;
-            },
-            .store_buffer => |operation| {
-                try std.testing.expectEqual(@as(u8, 1), operation.buffer.binding_table);
-                try std.testing.expect(operation.immediate_offset % @sizeOf(u32) == 0);
-                const component = operation.immediate_offset / @sizeOf(u32);
-                try std.testing.expect(component < store_offsets.len);
-                store_offsets[component] = true;
-                store_count += 1;
+            .surface_message => |operation| switch (operation.kind) {
+                .read => {
+                    try std.testing.expectEqual(@as(u8, 0), operation.binding_table);
+                    try std.testing.expectEqual(@as(u8, 1), operation.payload.register_count);
+                    try std.testing.expect(operation.response != null);
+                    load_count += 1;
+                },
+                .write => {
+                    try std.testing.expectEqual(@as(u8, 1), operation.binding_table);
+                    try std.testing.expectEqual(@as(u8, 2), operation.payload.register_count);
+                    try std.testing.expect(operation.response == null);
+                    store_count += 1;
+                },
             },
             .parallel_copy => return error.UnloweredParallelCopy,
             else => {},
@@ -178,6 +150,41 @@ test "[gen9] target: lower 256 KiB SSBO copy loop" {
 
     try std.testing.expectEqual(@as(usize, 4), load_count);
     try std.testing.expectEqual(@as(usize, 4), store_count);
-    try std.testing.expectEqual([4]bool{ true, true, true, true }, load_offsets);
-    try std.testing.expectEqual([4]bool{ true, true, true, true }, store_offsets);
+}
+
+test "[gen9] target: encode runtime array length" {
+    var module = try shader_ir.parser.parseString(std.testing.allocator,
+        \\shader compute @main
+        \\{
+        \\    @storage: runtime_array[u32] = storage_buffer[set(0), binding(0)]
+        \\    %offset: constant u32 = 16
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %length: u32 = array_length @storage, %offset, stride 4
+        \\            return
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+    module.execution_modes.workgroup_size = .{ 1, 1, 1 };
+
+    const gen9_device: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var artifact = try compileCompute(std.testing.allocator, &module, gen9_device, .{});
+    defer artifact.deinit(std.testing.allocator);
+
+    if (artifact.kernel == null) {
+        const encoded = try compute.kernel_encoder.encode(std.testing.allocator, &artifact.program);
+        std.testing.allocator.free(encoded);
+        return error.TestExpectedEncodedKernel;
+    }
+    for (artifact.program.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        try std.testing.expect(inst.operation != .array_length);
+    }
 }

src/intel/compiler/targets/gen9/register_allocation.zig

@@ -0,0 +1,285 @@
+const std = @import("std");
+
+const ids = @import("../../ir/id.zig");
+const instruction = @import("../../ir/instruction.zig");
+const operand = @import("../../ir/operand.zig");
+const program_ir = @import("../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    BlockParametersNotLowered,
+    ParallelCopiesNotLowered,
+    InvalidProgram,
+    OutOfRegisters,
+};
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    if (!program.properties.block_parameters_lowered)
+        return Error.BlockParametersNotLowered;
+    if (!program.properties.parallel_copies_lowered)
+        return Error.ParallelCopiesNotLowered;
+    if (program.properties.registers_allocated)
+        return;
+
+    const grf_size = program.device_info.grf_size_bytes;
+    if (grf_size == 0)
+        return Error.InvalidProgram;
+
+    const allocations = try allocator.alloc(?operand.PhysicalGrf, program.virtual_registers.entries.items.len);
+    defer allocator.free(allocations);
+    @memset(allocations, null);
+
+    var next_byte: usize = @as(usize, program.program_data.payload_grf_count) * grf_size;
+    next_byte = try reserveExistingPhysicalRegisters(program, next_byte, grf_size);
+    const capacity: usize = @as(usize, program.device_info.grf_count) * grf_size;
+
+    for (program.virtual_registers.entries.items, 0..) |entry, index| {
+        const register = entry orelse continue;
+        const start = std.mem.alignForward(usize, next_byte, register.alignment_bytes);
+        const end = std.math.add(usize, start, register.size_bytes) catch return Error.OutOfRegisters;
+        if (end > capacity)
+            return Error.OutOfRegisters;
+
+        allocations[index] = .{
+            .number = @intCast(start / grf_size),
+            .byte_offset = @intCast(start % grf_size),
+        };
+        next_byte = end;
+    }
+
+    try rewriteProgram(program, allocations);
+    program.program_data.total_grf_count = @intCast(std.math.divCeil(usize, next_byte, grf_size) catch return Error.InvalidProgram);
+    program.properties.registers_allocated = true;
+}
+
+fn reserveExistingPhysicalRegisters(program: *const program_ir.Program, initial: usize, grf_size: usize) Error!usize {
+    var next_byte = initial;
+    if (program.payload.header_grf) |header|
+        reservePhysical(&next_byte, header, grf_size);
+
+    for (program.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        switch (inst.operation) {
+            .load_global_invocation_id => |op| reserveRegister(&next_byte, op.destination.register, grf_size),
+            .load_buffer => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+            },
+            .store_buffer => |op| {
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+                reserveRegister(&next_byte, op.source.register, grf_size);
+            },
+            .array_length => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+            },
+            .surface_read => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.address.register, grf_size);
+            },
+            .surface_write => |op| {
+                reserveRegister(&next_byte, op.address.register, grf_size);
+                reserveRegister(&next_byte, op.data.register, grf_size);
+            },
+            .surface_message => |op| {
+                reserveRegister(&next_byte, op.payload.base, grf_size);
+                if (op.response) |response|
+                    reserveRegister(&next_byte, response.base, grf_size);
+            },
+            .move => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.source.register, grf_size);
+            },
+            .binary => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .compare => |op| {
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .math => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
+        }
+    }
+    return next_byte;
+}
+
+fn reserveRegister(next_byte: *usize, register: operand.RegisterRef, grf_size: usize) void {
+    switch (register) {
+        .physical_grf => |physical| reservePhysical(next_byte, physical, grf_size),
+        else => {},
+    }
+}
+
+fn reservePhysical(next_byte: *usize, physical: operand.PhysicalGrf, grf_size: usize) void {
+    const end = (@as(usize, physical.number) + 1) * grf_size;
+    next_byte.* = @max(next_byte.*, end);
+}
+
+fn rewriteProgram(program: *program_ir.Program, allocations: []const ?operand.PhysicalGrf) Error!void {
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        switch (inst.operation) {
+            .load_global_invocation_id => |*op| try rewriteDestination(program, &op.destination, allocations),
+            .load_buffer => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.byte_offset, allocations);
+            },
+            .store_buffer => |*op| {
+                try rewriteSource(program, &op.byte_offset, allocations);
+                try rewriteSource(program, &op.source, allocations);
+            },
+            .array_length => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.byte_offset, allocations);
+            },
+            .surface_read => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.address, allocations);
+            },
+            .surface_write => |*op| {
+                try rewriteSource(program, &op.address, allocations);
+                try rewriteSource(program, &op.data, allocations);
+            },
+            .surface_message => |*op| {
+                try rewriteRegister(program, &op.payload.base, allocations);
+                if (op.response) |*response|
+                    try rewriteRegister(program, &response.base, allocations);
+            },
+            .move => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.source, allocations);
+            },
+            .binary => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .compare => |*op| {
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .math => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
+        }
+    }
+
+    for (program.blocks.entries.items) |*entry| {
+        const block = if (entry.*) |*value| value else continue;
+        if (block.parameters.items.len != 0)
+            return Error.BlockParametersNotLowered;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
+        switch (terminator.*) {
+            .jump => |*edge| try rewriteEdge(program, edge, allocations),
+            .conditional_branch => |*branch| {
+                try rewriteEdge(program, &branch.true_edge, allocations);
+                try rewriteEdge(program, &branch.false_edge, allocations);
+            },
+            .end_thread, .@"unreachable" => {},
+        }
+    }
+}
+
+fn rewriteEdge(program: *const program_ir.Program, edge: *instruction.Edge, allocations: []const ?operand.PhysicalGrf) Error!void {
+    for (@constCast(edge.arguments)) |*argument| switch (argument.*) {
+        .source => |*edge_source| try rewriteSource(program, edge_source, allocations),
+        .predicate => {},
+    };
+}
+
+fn rewriteSource(program: *const program_ir.Program, value: *operand.Source, allocations: []const ?operand.PhysicalGrf) Error!void {
+    try rewriteRegister(program, &value.register, allocations);
+}
+
+fn rewriteDestination(program: *const program_ir.Program, destination: *operand.Destination, allocations: []const ?operand.PhysicalGrf) Error!void {
+    try rewriteRegister(program, &destination.register, allocations);
+}
+
+fn rewriteRegister(program: *const program_ir.Program, register: *operand.RegisterRef, allocations: []const ?operand.PhysicalGrf) Error!void {
+    const virtual = switch (register.*) {
+        .virtual => |value| value,
+        else => return,
+    };
+    if (!program.virtual_registers.isLive(virtual) or virtual.index() >= allocations.len)
+        return Error.InvalidProgram;
+    const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
+    register.* = .{ .physical_grf = physical };
+}
+
+const test_device = @import("../../device.zig").DeviceInfo{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn addRegister(program: *program_ir.Program, size: u32, alignment: u16) !ids.VirtualRegisterId {
+    return program.addVirtualRegister(.{
+        .size_bytes = size,
+        .alignment_bytes = alignment,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+}
+
+fn source(register: ids.VirtualRegisterId) operand.Source {
+    return .{
+        .register = .{ .virtual = register },
+        .type = .u32,
+        .region = operand.Region.contiguous(.simd8),
+    };
+}
+
+fn markPrerequisites(program: *program_ir.Program) void {
+    program.properties.block_parameters_lowered = true;
+    program.properties.parallel_copies_lowered = true;
+}
+
+test "[gen9] register allocation: assign non-overlapping physical GRFs" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+    program.program_data.payload_grf_count = 1;
+
+    const first = try addRegister(&program, 32, 32);
+    const second = try addRegister(&program, 64, 32);
+    const entry = try program.addBlock("entry");
+    const move = try program.appendInstruction(entry, .simd8, null, .{ .move = .{
+        .destination = .{ .register = .{ .virtual = second }, .type = .u32 },
+        .source = source(first),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisites(&program);
+
+    try run(std.testing.allocator, &program);
+
+    const operation = program.instructions.get(move).?.operation.move;
+    try std.testing.expectEqual(operand.PhysicalGrf{ .number = 1 }, operation.source.register.physical_grf);
+    try std.testing.expectEqual(operand.PhysicalGrf{ .number = 2 }, operation.destination.register.physical_grf);
+    try std.testing.expectEqual(@as(u16, 4), program.program_data.total_grf_count);
+    try std.testing.expect(program.properties.registers_allocated);
+}
+
+test "[gen9] register allocation: report GRF exhaustion" {
+    var limited_device = test_device;
+    limited_device.grf_count = 2;
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, limited_device, .simd8);
+    defer program.deinit();
+
+    _ = try addRegister(&program, 96, 32);
+    const entry = try program.addBlock("entry");
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisites(&program);
+
+    try std.testing.expectError(Error.OutOfRegisters, run(std.testing.allocator, &program));
+    try std.testing.expect(!program.properties.registers_allocated);
+}

src/intel/compiler/targets/gen9/validator.zig

@@ -64,6 +64,22 @@ fn validateInstruction(inst: instruction.Instruction) Error!void {
             try validateSource(op.byte_offset);
             try validateSource(op.source);
         },
+        .array_length => |op| {
+            try validateBufferReference(op.buffer);
+            try validateDestination(op.destination);
+            try validateSource(op.byte_offset);
+        },
+        .surface_read => |op| {
+            try validateBindingTableIndex(op.binding_table);
+            try validateDestination(op.destination);
+            try validateSource(op.address);
+        },
+        .surface_write => |op| {
+            try validateBindingTableIndex(op.binding_table);
+            try validateSource(op.address);
+            try validateSource(op.data);
+        },
+        .surface_message => |op| try validateBindingTableIndex(op.binding_table),
         .move => |op| {
             try validateDestination(op.destination);
             try validateSource(op.source);
@@ -78,6 +94,11 @@ fn validateInstruction(inst: instruction.Instruction) Error!void {
             try validateSource(op.lhs);
             try validateSource(op.rhs);
         },
+        .math => |op| {
+            try validateDestination(op.destination);
+            try validateSource(op.lhs);
+            try validateSource(op.rhs);
+        },
         .parallel_copy => |copy| {
             for (copy.register_copies) |item| {
                 try validateDestination(item.destination);
@@ -94,11 +115,15 @@ fn validateInstruction(inst: instruction.Instruction) Error!void {
 fn validateBufferReference(reference: instruction.BufferReference) Error!void {
     switch (reference) {
         .logical => {},
-        .binding_table => |index| if (index >= compute.resource_layout.max_storage_buffers)
-            return Error.InvalidBindingTableIndex,
+        .binding_table => |index| try validateBindingTableIndex(index),
     }
 }
 
+fn validateBindingTableIndex(index: u8) Error!void {
+    if (index >= compute.resource_layout.max_storage_buffers)
+        return Error.InvalidBindingTableIndex;
+}
+
 fn validateSource(source: operand.Source) Error!void {
     try validateType(source.type);
     switch (source.register) {

src/intel/compiler/targets/targets.zig

@@ -6,39 +6,23 @@ const common_ir = @import("../lower/common_ir.zig");
 
 pub const gen9 = @import("gen9/gen9.zig");
 
+pub const ComputeArtifact = gen9.ComputeArtifact;
 pub const ComputeResourceLayout = gen9.compute.ResourceLayout;
-pub const ResourceLayoutError = gen9.compute.resource_layout.Error || error{UnsupportedGeneration};
-pub const ResourceLoweringError = gen9.ResourceLoweringError || error{UnsupportedGeneration};
-
 pub const Error = gen9.Error || error{UnsupportedGeneration};
 pub const ValidationError = gen9.validator.Error || error{UnsupportedGeneration};
 
-pub fn lower(
+pub fn compileCompute(
     allocator: std.mem.Allocator,
     module: *shader_ir.module.Module,
     device_info: device.DeviceInfo,
     options: common_ir.Options,
-) Error!program_ir.Program {
+) Error!ComputeArtifact {
     return switch (device_info.generation) {
-        .gen9 => gen9.lower(allocator, module, device_info, options),
+        .gen9 => gen9.compileCompute(allocator, module, device_info, options),
         .gen10, .gen11 => Error.UnsupportedGeneration,
     };
 }
 
-pub fn layoutComputeResources(allocator: std.mem.Allocator, program: *const program_ir.Program) ResourceLayoutError!ComputeResourceLayout {
-    return switch (program.device_info.generation) {
-        .gen9 => ComputeResourceLayout.init(allocator, program),
-        .gen10, .gen11 => ResourceLayoutError.UnsupportedGeneration,
-    };
-}
-
-pub fn lowerComputeResources(program: *program_ir.Program, layout: *const ComputeResourceLayout) ResourceLoweringError!void {
-    return switch (program.device_info.generation) {
-        .gen9 => gen9.lowerComputeResources(program, layout),
-        .gen10, .gen11 => ResourceLoweringError.UnsupportedGeneration,
-    };
-}
-
 pub fn validate(program: *const program_ir.Program) ValidationError!void {
     return switch (program.device_info.generation) {
         .gen9 => gen9.validator.validate(program),

src/intel/copy_commands.zig

@@ -9,6 +9,7 @@ const FlintCommandBuffer = @import("FlintCommandBuffer.zig");
 const MemoryRange = @import("MemoryRange.zig");
 
 pub fn emitLinearCopy(cmd: *FlintCommandBuffer, src: MemoryRange, dst: MemoryRange) VkError!void {
+    try cmd.requireEngine(.blitter);
     if (src.size != dst.size) return VkError.ValidationFailed;
 
     var copied: vk.DeviceSize = 0;

src/intel/i915/i915.zig

@@ -7,13 +7,17 @@ pub const gem_close = 0x09;
 
 pub const mmap_offset_wb = 2;
 pub const gem_domain_cpu = 0x00000001;
+pub const gem_domain_render = 0x00000004;
+pub const gem_domain_instruction = 0x00000010;
 pub const gem_domain_gtt = 0x00000040;
+pub const exec_render = 1 << 0;
 pub const exec_blt = 3 << 0;
 pub const exec_fence_array: u64 = 1 << 19;
 pub const exec_fence_wait: u32 = 1 << 0;
 pub const exec_fence_signal: u32 = 1 << 1;
 pub const exec_object_write = 1 << 2;
 pub const mi_flush_dw: u32 = (0x26 << 23) | 3;
+pub const mi_batch_buffer_end: u32 = 0x05000000;
 
 pub const GemCreate = extern struct {
     size: u64,

src/intel/i915/kmd.zig

@@ -7,6 +7,11 @@ const common_kmd = @import("../kmd.zig");
 
 const VkError = base.VkError;
 
+const RelocationGroup = struct {
+    source_handle: u32,
+    entries: std.ArrayList(_i915.RelocationEntry) = .empty,
+};
+
 const Mapping = struct {
     bytes: []align(std.heap.page_size_min) u8,
 
@@ -54,8 +59,19 @@ pub const Device = struct {
         return memory;
     }
 
-    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const common_kmd.Relocation, syncs: []const common_kmd.SyncDependency) VkError!void {
-        const trailer_words = 6;
+    pub fn submitBatch(
+        self: *Device,
+        io: std.Io,
+        allocator: std.mem.Allocator,
+        engine: common_kmd.Engine,
+        commands: []const u32,
+        relocations: []const common_kmd.Relocation,
+        syncs: []const common_kmd.SyncDependency,
+    ) VkError!void {
+        const trailer_words: usize = switch (engine) {
+            .blitter => 6,
+            .render => if (commands.len % 2 == 0) 2 else 1,
+        };
         const batch_size = (commands.len + trailer_words) * @sizeOf(u32);
         var batch = try self.allocateMemory(io, batch_size);
         defer batch.deinit(self, io);
@@ -64,65 +80,93 @@ pub const Device = struct {
             const batch_map = try batch.map(self, io, 0, batch_size);
             const batch_words = std.mem.bytesAsSlice(u32, batch_map);
             @memcpy(batch_words[0..commands.len], commands);
-            batch_words[commands.len + 0] = _i915.mi_flush_dw;
-            batch_words[commands.len + 1] = 0;
-            batch_words[commands.len + 2] = 0;
-            batch_words[commands.len + 3] = 0;
-            batch_words[commands.len + 4] = 0;
-            batch_words[commands.len + 5] = 0x05000000;
+            @memset(batch_words[commands.len..], 0);
+            switch (engine) {
+                .blitter => {
+                    batch_words[commands.len] = _i915.mi_flush_dw;
+                    batch_words[commands.len + 5] = _i915.mi_batch_buffer_end;
+                },
+                .render => batch_words[commands.len] = _i915.mi_batch_buffer_end,
+            }
             batch.unmap();
         }
         try batch.flushRange(self, io, 0, batch_size);
 
-        var objects = std.ArrayList(_i915.ExecObject2).empty;
-        defer objects.deinit(allocator);
-
         var object_handles = std.ArrayList(u32).empty;
         defer object_handles.deinit(allocator);
-
         for (relocations) |relocation| {
-            if (std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle) == null) {
-                object_handles.append(allocator, relocation.target_handle) catch return VkError.OutOfHostMemory;
-                objects.append(allocator, .{
-                    .handle = relocation.target_handle,
-                    .relocation_count = 0,
-                    .relocs_ptr = 0,
-                    .alignment = 0,
-                    .offset = 0,
-                    .flags = if (relocation.write) _i915.exec_object_write else 0,
-                    .rsvd1 = 0,
-                    .rsvd2 = 0,
-                }) catch return VkError.OutOfHostMemory;
-            } else if (relocation.write) {
-                const index = std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle).?;
-                objects.items[index].flags |= _i915.exec_object_write;
+            if (relocation.source_handle) |source| {
+                if (std.mem.indexOfScalar(u32, object_handles.items, source) == null)
+                    object_handles.append(allocator, source) catch return VkError.OutOfHostMemory;
             }
+            if (std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle) == null)
+                object_handles.append(allocator, relocation.target_handle) catch return VkError.OutOfHostMemory;
         }
+        if (std.mem.indexOfScalar(u32, object_handles.items, batch.handle) == null)
+            object_handles.append(allocator, batch.handle) catch return VkError.OutOfHostMemory;
 
-        var i915_relocations = std.ArrayList(_i915.RelocationEntry).empty;
-        defer i915_relocations.deinit(allocator);
-
+        var groups = std.ArrayList(RelocationGroup).empty;
+        defer {
+            for (groups.items) |*group| group.entries.deinit(allocator);
+            groups.deinit(allocator);
+        }
         for (relocations) |relocation| {
-            i915_relocations.append(allocator, .{
+            const source = relocation.source_handle orelse batch.handle;
+            var group_index = std.mem.indexOfScalar(u32, object_handles.items, source) orelse return VkError.DeviceLost;
+            for (groups.items, 0..) |group, index| {
+                if (group.source_handle == source) {
+                    group_index = index;
+                    break;
+                }
+            } else {
+                groups.append(allocator, .{ .source_handle = source }) catch return VkError.OutOfHostMemory;
+                group_index = groups.items.len - 1;
+            }
+
+            const domain: u32 = switch (relocation.domain) {
+                .none => 0,
+                .render => _i915.gem_domain_render,
+                .instruction => _i915.gem_domain_instruction,
+            };
+            groups.items[group_index].entries.append(allocator, .{
                 .target_handle = relocation.target_handle,
                 .delta = relocation.delta,
                 .offset = relocation.offset,
                 .presumed_offset = 0,
-                .read_domains = 0,
-                .write_domain = 0,
+                .read_domains = if (relocation.read) domain else 0,
+                .write_domain = if (relocation.write) domain else 0,
             }) catch return VkError.OutOfHostMemory;
         }
 
-        objects.append(allocator, .{
-            .handle = batch.handle,
-            .relocation_count = @intCast(i915_relocations.items.len),
-            .relocs_ptr = @intFromPtr(i915_relocations.items.ptr),
-            .alignment = 0,
-            .offset = 0,
-            .flags = 0,
-            .rsvd1 = 0,
-            .rsvd2 = 0,
-        }) catch return VkError.OutOfHostMemory;
+        var objects = std.ArrayList(_i915.ExecObject2).empty;
+        defer objects.deinit(allocator);
+        for (object_handles.items) |handle| {
+            var flags: u64 = 0;
+            for (relocations) |relocation| {
+                if (relocation.target_handle == handle and relocation.write)
+                    flags |= _i915.exec_object_write;
+            }
+
+            var relocation_count: u32 = 0;
+            var relocs_ptr: u64 = 0;
+            for (groups.items) |group| {
+                if (group.source_handle == handle) {
+                    relocation_count = @intCast(group.entries.items.len);
+                    relocs_ptr = @intFromPtr(group.entries.items.ptr);
+                    break;
+                }
+            }
+            objects.append(allocator, .{
+                .handle = handle,
+                .relocation_count = relocation_count,
+                .relocs_ptr = relocs_ptr,
+                .alignment = 0,
+                .offset = 0,
+                .flags = flags,
+                .rsvd1 = 0,
+                .rsvd2 = 0,
+            }) catch return VkError.OutOfHostMemory;
+        }
 
         var exec_fences = std.ArrayList(_i915.ExecFence).empty;
         defer exec_fences.deinit(allocator);
@@ -142,7 +186,10 @@ pub const Device = struct {
             .DR4 = 0,
             .num_cliprects = @intCast(exec_fences.items.len),
             .cliprects_ptr = if (exec_fences.items.len == 0) 0 else @intFromPtr(exec_fences.items.ptr),
-            .flags = _i915.exec_blt | (if (exec_fences.items.len == 0) 0 else _i915.exec_fence_array),
+            .flags = @as(u64, switch (engine) {
+                .blitter => _i915.exec_blt,
+                .render => _i915.exec_render,
+            }) | (if (exec_fences.items.len == 0) 0 else _i915.exec_fence_array),
             .rsvd1 = 0,
             .rsvd2 = 0,
         };

src/intel/kmd.zig

@@ -19,12 +19,25 @@ pub const blt_depth_8: u32 = 0 << 24;
 pub const rop_source_copy: u32 = 0xcc << 16;
 pub const max_blt_span: vk.DeviceSize = 32 * 1024 - 1;
 
+pub const Engine = enum {
+    blitter,
+    render,
+};
+
+pub const Domain = enum {
+    none,
+    render,
+    instruction,
+};
+
 pub const Relocation = struct {
+    source_handle: ?u32 = null,
     target_handle: u32,
     offset: u64,
     delta: u32,
     read: bool = false,
     write: bool = false,
+    domain: Domain = .none,
 };
 
 pub const SyncDependency = struct {
@@ -63,10 +76,10 @@ pub const Device = union(KmdType) {
         };
     }
 
-    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const Relocation, syncs: []const SyncDependency) VkError!void {
+    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, engine: Engine, commands: []const u32, relocations: []const Relocation, syncs: []const SyncDependency) VkError!void {
         return switch (self.*) {
-            .i915 => |*device| device.submitBatch(io, allocator, commands, relocations, syncs),
-            .xe => |*device| device.submitBatch(io, allocator, commands, relocations, syncs),
+            .i915 => |*device| device.submitBatch(io, allocator, engine, commands, relocations, syncs),
+            .xe => |*device| device.submitBatch(io, allocator, engine, commands, relocations, syncs),
             .invalid => VkError.DeviceLost,
         };
     }

src/intel/pci_ids.zig

@@ -6,6 +6,69 @@ const PciInfo = struct {
 
 /// Not a hashmap as they need runtime allocations
 pub const map = [_]PciInfo{
+    .{ .id = 0x0a02, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a06, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0a, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0b, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0e, .name = "Intel(R) HD Graphics (Haswell-ULX GT1)", .is_discrete = false },
+    .{ .id = 0x0402, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x0406, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040a, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040b, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040e, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x0c02, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c06, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0a, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0b, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0e, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0d02, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d06, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0a, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0b, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0e, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+
+    .{ .id = 0x0a12, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a16, .name = "Intel(R) HD Graphics 4400 (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1a, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1b, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1e, .name = "Intel(R) HD Graphics 4200 (Haswell-ULX GT2)", .is_discrete = false },
+    .{ .id = 0x0412, .name = "Intel(R) HD Graphics 4600 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x0416, .name = "Intel(R) HD Graphics 4600 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041a, .name = "Intel(R) HD Graphics P4600/P4700 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041b, .name = "Intel(R) HD Graphics (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041e, .name = "Intel(R) HD Graphics 4400 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x0c12, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c16, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1a, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1b, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1e, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0d12, .name = "Intel(R) HD Graphics 4600 (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d16, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1a, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1b, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1e, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+
+    .{ .id = 0x0a22, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a26, .name = "Intel(R) HD Graphics 5000 (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2a, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2b, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2e, .name = "Intel(R) Iris(R) Graphics 5100 (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0422, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x0426, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042a, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042b, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042e, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x0c22, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c26, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2a, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2b, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2e, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0d22, .name = "Intel(R) Iris(R) Pro Graphics 5200 (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d26, .name = "Intel(R) Iris(R) Pro Graphics P5200 (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2a, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2b, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2e, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+
     .{ .id = 0x1602, .name = "Intel(R) HD Graphics (Broadwell-U)", .is_discrete = false },
     .{ .id = 0x1606, .name = "Intel(R) HD Graphics (Broadwell-U GT1)", .is_discrete = false },
     .{ .id = 0x160a, .name = "Intel(R) HD Graphics (Broadwell-U)", .is_discrete = false },

src/intel/xe/kmd.zig

@@ -23,7 +23,15 @@ pub const Device = struct {
         return VkError.OutOfDeviceMemory;
     }
 
-    pub fn submitBatch(_: *Device, _: std.Io, _: std.mem.Allocator, _: []const u32, _: []const common_kmd.Relocation, _: []const common_kmd.SyncDependency) VkError!void {
+    pub fn submitBatch(
+        _: *Device,
+        _: std.Io,
+        _: std.mem.Allocator,
+        _: common_kmd.Engine,
+        _: []const u32,
+        _: []const common_kmd.Relocation,
+        _: []const common_kmd.SyncDependency,
+    ) VkError!void {
         return VkError.FeatureNotPresent;
     }
 };

src/phi/PhiBinarySemaphore.zig

@@ -8,6 +8,10 @@ const Self = @This();
 pub const Interface = base.BinarySemaphore;
 
 interface: Interface,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+is_signaled: bool,
+is_failed: bool,
 
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.SemaphoreCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -23,6 +27,10 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
 
     self.* = .{
         .interface = interface,
+        .mutex = .init,
+        .condition = .init,
+        .is_signaled = false,
+        .is_failed = false,
     };
     return self;
 }
@@ -34,10 +42,39 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 
 pub fn signal(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = true;
+    self.condition.broadcast(io);
+}
+
+/// Latch an asynchronous queue/device failure and wake all host waiters
+pub fn fail(interface: *Interface) void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    self.is_failed = true;
+    self.condition.broadcast(io);
 }
 
 pub fn wait(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    while (!self.is_signaled and !self.is_failed) {
+        self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+    }
+    if (self.is_failed) return VkError.DeviceLost;
+
+    self.is_signaled = false;
 }

src/phi/PhiCommandBuffer.zig

@@ -6,6 +6,7 @@ const proto = lib.proto;
 
 const VkError = base.VkError;
 const PhiDeviceMemory = @import("PhiDeviceMemory.zig");
+const copy = @import("copy_commands.zig");
 
 const Self = @This();
 pub const Interface = base.CommandBuffer;
@@ -105,7 +106,7 @@ pub fn reset(interface: *Interface, flags: vk.CommandBufferResetFlags) VkError!v
     _ = flags;
 }
 
-fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T) VkError!void {
+pub fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T) VkError!void {
     const allocator = self.interface.host_allocator.allocator();
     const header: proto.PhiCmdHeader = .{
         .magic = proto.PHI_COMMAND_MAGIC,
@@ -117,12 +118,6 @@ fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T)
     self.serialized_cmd_count += 1;
 }
 
-fn remoteMemory(buffer: *base.Buffer) VkError!*PhiDeviceMemory {
-    const memory = buffer.memory orelse return VkError.ValidationFailed;
-    const phi_memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory));
-    return phi_memory;
-}
-
 pub fn beginQuery(interface: *Interface, pool: *base.QueryPool, query: u32, flags: vk.QueryControlFlags) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
@@ -186,12 +181,12 @@ pub fn bindVertexBuffer(interface: *Interface, index: usize, buffer: *base.Buffe
 pub fn blitImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageBlit, filter: vk.Filter) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
-    _ = src;
+
     _ = src_layout;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
-    _ = filter;
+
+    for (regions) |region|
+        try copy.blitImage(self, src, dst, region, filter);
 }
 
 pub fn clearAttachment(interface: *Interface, attachment: vk.ClearAttachment, rect: vk.ClearRect) VkError!void {
@@ -221,52 +216,32 @@ pub fn clearDepthStencilImage(interface: *Interface, image: *base.Image, layout:
 
 pub fn copyBuffer(interface: *Interface, src: *base.Buffer, dst: *base.Buffer, regions: []const vk.BufferCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    const src_memory = try remoteMemory(src);
-    const dst_memory = try remoteMemory(dst);
-
-    for (regions) |region| {
-        const src_offset, const src_overflow = @addWithOverflow(src.offset, region.src_offset);
-        const dst_offset, const dst_overflow = @addWithOverflow(dst.offset, region.dst_offset);
-        if (src_overflow != 0 or dst_overflow != 0) {
-            return VkError.ValidationFailed;
-        }
-
-        try self.appendCommand(proto.PhiCmdCopyBuffer, proto.PHI_CMD_COPY_BUFFER, .{
-            .size = region.size,
-            .src_memory = @intCast(src_memory.remote_handle),
-            .dst_memory = @intCast(dst_memory.remote_handle),
-            .src_offset = src_offset,
-            .dst_offset = dst_offset,
-        });
-    }
+    try copy.copyBuffer(self, src, dst, regions);
 }
 
 pub fn copyBufferToImage(interface: *Interface, src: *base.Buffer, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.BufferImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
+
+    for (regions) |region|
+        try copy.copyBufferImage(self, src, dst, region, true);
 }
 
 pub fn copyImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
     _ = src_layout;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
+
+    for (regions) |region|
+        try copy.copyImage(self, src, dst, region);
 }
 
 pub fn copyImageToBuffer(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Buffer, regions: []const vk.BufferImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
     _ = src_layout;
-    _ = dst;
-    _ = regions;
+
+    for (regions) |region|
+        try copy.copyBufferImage(self, dst, src, region, false);
 }
 
 pub fn copyQueryPoolResults(interface: *Interface, pool: *base.QueryPool, first: u32, count: u32, dst: *base.Buffer, offset: vk.DeviceSize, stride: vk.DeviceSize, flags: vk.QueryResultFlags) VkError!void {
@@ -365,7 +340,8 @@ pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.Device
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
 
-    const memory = try remoteMemory(buffer);
+    const memory_interface = buffer.memory orelse return VkError.ValidationFailed;
+    const memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory_interface));
 
     try self.appendCommand(proto.PhiCmdFillBuffer, proto.PHI_CMD_FILL_BUFFER, .{
         .size = if (size == vk.WHOLE_SIZE) buffer.size - offset else size,

src/phi/PhiFence.zig

@@ -9,6 +9,10 @@ const Self = @This();
 pub const Interface = base.Fence;
 
 interface: Interface,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+is_signaled: bool,
+is_failed: bool,
 
 pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.FenceCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -26,6 +30,10 @@ pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.Fen
 
     self.* = .{
         .interface = interface,
+        .mutex = .init,
+        .condition = .init,
+        .is_signaled = info.flags.signaled_bit,
+        .is_failed = false,
     };
     return self;
 }
@@ -37,21 +45,84 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 
 pub fn getStatus(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    if (!self.is_signaled) return VkError.NotReady;
 }
 
 pub fn reset(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = false;
 }
 
 pub fn signal(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = true;
+    self.condition.broadcast(io);
+}
+
+/// Latch an asynchronous queue/device failure and wake all host waiters
+pub fn fail(interface: *Interface) void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    self.is_failed = true;
+    self.condition.broadcast(io);
 }
 
 pub fn wait(interface: *Interface, timeout: u64) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
-    _ = timeout;
+    const io = interface.owner.io();
+
+    if (timeout == std.math.maxInt(@TypeOf(timeout))) {
+        self.mutex.lock(io) catch return VkError.DeviceLost;
+        defer self.mutex.unlock(io);
+
+        while (!self.is_signaled and !self.is_failed) {
+            self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+        }
+        if (self.is_failed) return VkError.DeviceLost;
+        return;
+    }
+
+    const deadline = std.Io.Clock.Timestamp.fromNow(io, .{
+        .raw = .fromNanoseconds(@intCast(timeout)),
+        .clock = .awake,
+    });
+    while (true) {
+        {
+            self.mutex.lock(io) catch return VkError.DeviceLost;
+            defer self.mutex.unlock(io);
+
+            if (self.is_failed) return VkError.DeviceLost;
+            if (self.is_signaled) return;
+        }
+
+        const remaining = deadline.durationFromNow(io);
+        if (remaining.raw.nanoseconds <= 0) return VkError.Timeout;
+
+        (std.Io.Clock.Duration{
+            .raw = .fromNanoseconds(@min(remaining.raw.nanoseconds, std.time.ns_per_ms)),
+            .clock = .awake,
+        }).sleep(io) catch return VkError.DeviceLost;
+    }
 }

src/phi/PhiImage.zig

@@ -1,15 +1,15 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
+const proto = @import("lib.zig").proto;
+
+const PhiDeviceMemory = @import("PhiDeviceMemory.zig");
 
 const VkError = base.VkError;
 
 const Self = @This();
 pub const Interface = base.Image;
 
-pub const F32x4 = @Vector(4, f32);
-pub const U32x4 = @Vector(4, u32);
-
 interface: Interface,
 
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.ImageCreateInfo) VkError!*Self {
@@ -17,6 +17,7 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
     errdefer allocator.destroy(self);
 
     var interface = try Interface.init(device, allocator, info);
+
     interface.vtable = &.{
         .destroy = destroy,
         .getMemoryRequirements = getMemoryRequirements,
@@ -39,39 +40,147 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 }
 
 pub fn getMemoryRequirements(_: *Interface, requirements: *vk.MemoryRequirements) VkError!void {
-    _ = requirements;
+    requirements.alignment = proto.PHI_MEMORY_ALIGNMENT;
+    requirements.size = std.mem.alignForward(vk.DeviceSize, requirements.size, proto.PHI_MEMORY_ALIGNMENT);
 }
 
-pub fn copyToMemory(interface: *const Interface, memory: []u8, subresource: vk.ImageSubresourceLayers) VkError!void {
-    _ = interface;
-    _ = subresource;
-    @memset(memory, 0);
+pub fn copyToMemory(interface: *const Interface, dst: []u8, subresource: vk.ImageSubresourceLayers) VkError!void {
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    const memory_interface = interface.memory orelse return VkError.InvalidDeviceMemoryDrv;
+    const memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory_interface));
+    const data = memory.data orelse return VkError.InvalidDeviceMemoryDrv;
+
+    try validateSingleAspect(interface.format, subresource.aspect_mask);
+    if (subresource.mip_level >= interface.mip_levels or
+        subresource.base_array_layer >= interface.array_layers or
+        subresource.layer_count == 0)
+        return VkError.ValidationFailed;
+
+    const layer_count = if (subresource.layer_count == vk.REMAINING_ARRAY_LAYERS)
+        interface.array_layers - subresource.base_array_layer
+    else
+        subresource.layer_count;
+    if (layer_count > interface.array_layers - subresource.base_array_layer)
+        return VkError.ValidationFailed;
+
+    const level_size = self.getMultiSampledLevelSize(subresource.aspect_mask, subresource.mip_level);
+    const required_size, const size_overflow = @mulWithOverflow(level_size, @as(usize, layer_count));
+    if (size_overflow != 0 or dst.len < required_size)
+        return VkError.ValidationFailed;
+
+    const first_offset = try self.getSubresourceOffset(
+        subresource.aspect_mask,
+        subresource.mip_level,
+        subresource.base_array_layer,
+    );
+    const absolute_offset, const offset_overflow = @addWithOverflow(interface.memory_offset, first_offset);
+    if (offset_overflow != 0)
+        return VkError.ValidationFailed;
+
+    var src_offset = std.math.cast(usize, absolute_offset) orelse return VkError.InvalidDeviceMemoryDrv;
+    var dst_offset: usize = 0;
+    const layer_pitch = self.getLayerSize(subresource.aspect_mask);
+
+    for (0..layer_count) |_| {
+        if (src_offset > data.len or level_size > data.len - src_offset)
+            return VkError.InvalidDeviceMemoryDrv;
+        @memcpy(dst[dst_offset..][0..level_size], data[src_offset..][0..level_size]);
+        dst_offset += level_size;
+        src_offset += layer_pitch;
+    }
+}
+
+pub fn getSubresourceOffset(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32, layer: u32) VkError!usize {
+    if (mip_level >= self.interface.mip_levels or layer >= self.interface.array_layers)
+        return VkError.ValidationFailed;
+
+    var offset = try self.getAspectOffset(aspect_mask);
+    offset += layer * self.getLayerSize(aspect_mask);
+    for (0..mip_level) |mip|
+        offset += self.getMultiSampledLevelSize(aspect_mask, @intCast(mip));
+    return offset;
+}
+
+fn getAspectOffset(self: *const Self, aspect_mask: vk.ImageAspectFlags) VkError!usize {
+    try validateSingleAspect(self.interface.format, aspect_mask);
+    return switch (self.interface.format) {
+        .d16_unorm_s8_uint,
+        .d24_unorm_s8_uint,
+        .d32_sfloat_s8_uint,
+        => if (aspect_mask.stencil_bit)
+            self.interface.getTotalSizeForAspect(.{ .depth_bit = true })
+        else
+            0,
+        else => 0,
+    };
 }
 
 pub fn getTotalSizeForAspect(interface: *const Interface, aspect_mask: vk.ImageAspectFlags) VkError!usize {
-    _ = aspect_mask;
-    return interface.extent.width * interface.extent.height * interface.extent.depth * base.format.texelSize(interface.format);
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    const valid_aspects = base.format.toAspect(interface.format);
+    if (aspect_mask.toInt() == 0 or aspect_mask.subtract(valid_aspects).toInt() != 0)
+        return VkError.ValidationFailed;
+
+    var size: usize = 0;
+    if (aspect_mask.color_bit)
+        size += self.getLayerSize(.{ .color_bit = true });
+    if (aspect_mask.depth_bit)
+        size += self.getLayerSize(.{ .depth_bit = true });
+    if (aspect_mask.stencil_bit)
+        size += self.getLayerSize(.{ .stencil_bit = true });
+    return size * interface.array_layers;
 }
 
 pub fn getSubresourceLayout(interface: *const Interface, subresource: vk.ImageSubresource) VkError!vk.SubresourceLayout {
-    _ = subresource;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    try validateSingleAspect(interface.format, subresource.aspect_mask);
+
     return .{
-        .offset = 0,
-        .size = try getTotalSizeForAspect(interface, base.format.toAspect(interface.format)),
-        .row_pitch = getRowPitchMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
-        .array_pitch = getSliceMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
-        .depth_pitch = getSliceMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
+        .offset = try self.getSubresourceOffset(subresource.aspect_mask, subresource.mip_level, subresource.array_layer),
+        .size = self.getMultiSampledLevelSize(subresource.aspect_mask, subresource.mip_level),
+        .row_pitch = getRowPitchMemSizeForMipLevel(interface, subresource.aspect_mask, subresource.mip_level),
+        .array_pitch = self.getLayerSize(subresource.aspect_mask),
+        .depth_pitch = getSliceMemSizeForMipLevel(interface, subresource.aspect_mask, subresource.mip_level),
+    };
+}
+
+pub fn getLayerSize(self: *const Self, aspect_mask: vk.ImageAspectFlags) usize {
+    var size: usize = 0;
+    for (0..self.interface.mip_levels) |mip_level|
+        size += self.getMultiSampledLevelSize(aspect_mask, @intCast(mip_level));
+    return size;
+}
+
+pub inline fn getMultiSampledLevelSize(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
+    return self.getMipLevelSize(aspect_mask, mip_level) * self.interface.samples.toInt();
+}
+
+pub inline fn getMipLevelSize(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
+    return getSliceMemSizeForMipLevel(&self.interface, aspect_mask, mip_level) * self.getMipLevelExtent(mip_level).depth;
+}
+
+pub fn getMipLevelExtent(self: *const Self, mip_level: u32) vk.Extent3D {
+    return .{
+        .width = @max(1, self.interface.extent.width >> @intCast(mip_level)),
+        .height = @max(1, self.interface.extent.height >> @intCast(mip_level)),
+        .depth = @max(1, self.interface.extent.depth >> @intCast(mip_level)),
     };
 }
 
 pub fn getSliceMemSizeForMipLevel(interface: *const Interface, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
-    _ = aspect_mask;
-    _ = mip_level;
-    return interface.extent.width * interface.extent.height * base.format.texelSize(interface.format);
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    const extent = self.getMipLevelExtent(mip_level);
+    return base.format.sliceMemSize(base.format.fromAspect(interface.format, aspect_mask), extent.width, extent.height);
 }
 
 pub fn getRowPitchMemSizeForMipLevel(interface: *const Interface, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
-    _ = aspect_mask;
-    _ = mip_level;
-    return interface.extent.width * base.format.texelSize(interface.format);
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    const extent = self.getMipLevelExtent(mip_level);
+    return base.format.pitchMemSize(base.format.fromAspect(interface.format, aspect_mask), extent.width);
+}
+
+fn validateSingleAspect(format: vk.Format, aspect_mask: vk.ImageAspectFlags) VkError!void {
+    const valid_aspects = base.format.toAspect(format);
+    if (aspect_mask.toInt() == 0 or @popCount(aspect_mask.toInt()) != 1 or aspect_mask.subtract(valid_aspects).toInt() != 0)
+        return VkError.ValidationFailed;
 }

src/phi/PhiInstance.zig

@@ -32,6 +32,8 @@ pub const extensions = [_]vk.ExtensionProperties{
     castExtension(vk.extensions.khr_get_physical_device_properties_2),
     castExtension(vk.extensions.khr_surface),
     castExtension(vk.extensions.khr_wayland_surface),
+    castExtension(vk.extensions.khr_xlib_surface),
+    castExtension(vk.extensions.khr_xcb_surface),
 };
 
 pub fn create(allocator: std.mem.Allocator, infos: *const vk.InstanceCreateInfo) VkError!*Interface {

src/phi/PhiQueue.zig

@@ -4,8 +4,10 @@ const base = @import("base");
 const lib = @import("lib.zig");
 const proto = lib.proto;
 
+const PhiBinarySemaphore = @import("PhiBinarySemaphore.zig");
 const PhiCommandBuffer = @import("PhiCommandBuffer.zig");
 const PhiDevice = @import("PhiDevice.zig");
+const PhiFence = @import("PhiFence.zig");
 const PhiTransport = @import("PhiTransport.zig");
 
 const VkError = base.VkError;
@@ -13,7 +15,65 @@ const VkError = base.VkError;
 const Self = @This();
 pub const Interface = base.Queue;
 
+const ring_capacity: usize = @intCast(proto.PHI_QUEUE_RING_CAPACITY);
+const ring_capacity_u64: u64 = @intCast(ring_capacity);
+const shutdown_sequence = std.math.maxInt(u64);
+const shutdown_timeout_ns = 5 * std.time.ns_per_s;
+const shutdown_poll_ns = 10 * std.time.ns_per_ms;
+
+const CompletionShutdown = enum {
+    acknowledged,
+    stopped_without_acknowledgement,
+    timed_out,
+    wait_failed,
+};
+
+const PreparedSubmit = struct {
+    wait_semaphores: std.ArrayList(*base.BinarySemaphore),
+    signal_semaphores: std.ArrayList(*base.BinarySemaphore),
+    command_backing: ?[]u8,
+    scif_offset: ?u64,
+    registered_size: usize,
+    command_size: usize,
+    command_count: u64,
+};
+
+const PendingCompletion = struct {
+    signal_semaphores: std.ArrayList(*base.BinarySemaphore),
+    fence: ?*base.Fence,
+    command_backing: ?[]u8,
+    scif_offset: ?u64,
+    registered_size: usize,
+};
+
+const TaskData = struct {
+    queue: *Self,
+    sequence: usize,
+    submits: std.ArrayList(PreparedSubmit),
+    fence: ?*base.Fence,
+};
+
 interface: Interface,
+transport: PhiTransport,
+ring_backing: []u8,
+ring_offset: u64,
+shared: *proto.PhiQueueShared,
+
+submit_group: std.Io.Group,
+completion_group: std.Io.Group,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+
+next_task_sequence: usize,
+executing_task_sequence: usize,
+next_remote_sequence: u64,
+completed_sequence: u64,
+pending: [ring_capacity]?PendingCompletion,
+
+error_state: ?VkError,
+shutting_down: bool,
+completion_stopped: bool,
+remote_stopped: bool,
 
 pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, family_index: u32, flags: vk.DeviceQueueCreateFlags) VkError!*Interface {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -26,12 +86,155 @@ pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, fa
         .waitIdle = waitIdle,
     };
 
-    self.* = .{ .interface = interface };
+    const phi_device: *PhiDevice = @alignCast(@fieldParentPtr("interface", device));
+    var transport = try phi_device.transport.connectPeer();
+    errdefer transport.close();
+
+    const device_allocator = device.device_allocator.allocator();
+    const page_size = std.heap.pageSize();
+    const registered_size = std.mem.alignForward(usize, @sizeOf(proto.PhiQueueShared), page_size);
+    const ring_backing = device_allocator.alignedAlloc(
+        u8,
+        .fromByteUnits(std.heap.page_size_max),
+        registered_size,
+    ) catch return VkError.OutOfHostMemory;
+    errdefer device_allocator.free(ring_backing);
+    @memset(ring_backing, 0);
+
+    const ring_offset = try transport.registerHostMemory(ring_backing);
+    errdefer transport.unregisterHostMemory(ring_offset, ring_backing.len) catch @panic("Caught an error while handling an error");
+
+    const setup_request: proto.PhiQueueSetupRequest = .{
+        .scif_offset = ring_offset,
+        .scif_size = ring_backing.len,
+        .ring_capacity = @intCast(ring_capacity),
+        .reserved = 0,
+    };
+    var setup_reply = std.mem.zeroes(proto.PhiResultReply);
+    try transport.request(
+        proto.PHI_PACKET_QUEUE_SETUP,
+        std.mem.asBytes(&setup_request),
+        std.mem.asBytes(&setup_reply),
+    );
+    if (setup_reply.result.status != proto.PHI_STATUS_OK) {
+        return PhiTransport.statusToErr(setup_reply.result.status);
+    }
+
+    const shared: *proto.PhiQueueShared = @ptrCast(@alignCast(ring_backing.ptr));
+
+    self.* = .{
+        .interface = interface,
+        .transport = transport,
+        .ring_backing = ring_backing,
+        .ring_offset = ring_offset,
+        .shared = shared,
+        .submit_group = .init,
+        .completion_group = .init,
+        .mutex = .init,
+        .condition = .init,
+        .next_task_sequence = 0,
+        .executing_task_sequence = 0,
+        .next_remote_sequence = 1,
+        .completed_sequence = 0,
+        .pending = [_]?PendingCompletion{null} ** ring_capacity,
+        .error_state = null,
+        .shutting_down = false,
+        .completion_stopped = false,
+        .remote_stopped = false,
+    };
+
+    self.completion_group.async(device.io(), completionRunner, .{self});
     return &self.interface;
 }
 
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+    const device_allocator = interface.owner.device_allocator.allocator();
+
+    var graceful_shutdown = true;
+    waitIdle(interface) catch |err| {
+        graceful_shutdown = false;
+        std.log.scoped(.PhiQueue).warn("Queue did not become idle during destruction: {s}", .{@errorName(err)});
+    };
+    var mutex_locked = true;
+    self.mutex.lock(io) catch {
+        mutex_locked = false;
+        graceful_shutdown = false;
+    };
+    var shutdown_next_sequence: u64 = 0;
+    var shutdown_completed_sequence: u64 = 0;
+    if (mutex_locked) {
+        self.shutting_down = true;
+        shutdown_next_sequence = self.next_remote_sequence;
+        shutdown_completed_sequence = self.completed_sequence;
+        self.condition.broadcast(io);
+        self.mutex.unlock(io);
+    }
+
+    if (graceful_shutdown) {
+        std.log.scoped(.PhiQueue).info(
+            "Sending shutdown doorbell (next remote sequence {d}, completed {d})",
+            .{ shutdown_next_sequence, shutdown_completed_sequence },
+        );
+        self.transport.sendQueueDoorbell(shutdown_sequence) catch |err| {
+            graceful_shutdown = false;
+            std.log.scoped(.PhiQueue).warn("Failed to send queue shutdown doorbell: {s}", .{@errorName(err)});
+        };
+
+        if (graceful_shutdown) {
+            switch (self.waitForCompletionShutdown(io)) {
+                .acknowledged => {},
+                .stopped_without_acknowledgement => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Remote queue stopped without acknowledging shutdown", .{});
+                },
+                .timed_out => {
+                    graceful_shutdown = false;
+                    const progress = self.completionProgress(io);
+                    std.log.scoped(.PhiQueue).warn(
+                        "Timed out waiting for remote queue shutdown (next remote sequence {d}, completed {d}); closing SCIF endpoint",
+                        .{ progress.next, progress.completed },
+                    );
+                },
+                .wait_failed => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Failed while waiting for remote queue shutdown; closing SCIF endpoint", .{});
+                },
+            }
+        }
+    }
+
+    if (!graceful_shutdown) {
+        // Wake the blocking completion receiver before releasing queue storage.
+        // Keep libscif loaded until the receiver has returned from scif_recv.
+        self.transport.interrupt();
+    }
+
+    self.completion_group.await(io) catch |err| {
+        graceful_shutdown = false;
+        std.log.scoped(.PhiQueue).warn("Failed while joining completion receiver: {s}", .{@errorName(err)});
+    };
+
+    if (graceful_shutdown) {
+        self.mutex.lock(io) catch {
+            graceful_shutdown = false;
+        };
+        if (graceful_shutdown) {
+            graceful_shutdown = self.remote_stopped;
+            self.mutex.unlock(io);
+        }
+    }
+
+    if (graceful_shutdown) {
+        self.transport.unregisterHostMemory(self.ring_offset, self.ring_backing.len) catch |err| {
+            std.log.scoped(.PhiQueue).warn("Failed to unregister queue ring: {s}", .{@errorName(err)});
+        };
+    }
+
+    self.transport.close();
+    cleanupPendingAfterClose(self, device_allocator);
+    device_allocator.free(self.ring_backing);
     allocator.destroy(self);
 }
 
@@ -43,46 +246,522 @@ pub fn bindSparse(interface: *Interface, info: []const vk.BindSparseInfo, fence:
 }
 
 pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*base.Fence) VkError!void {
-    const device: *PhiDevice = @alignCast(@fieldParentPtr("interface", interface.owner));
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+    const allocator = interface.owner.device_allocator.allocator();
 
-    for (infos) |info| {
-        for (info.wait_semaphores.items) |semaphore| {
-            try semaphore.wait();
-        }
+    try self.checkHealthy();
 
-        for (info.command_buffers.items) |command_buffer| {
-            const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+    const data = allocator.create(TaskData) catch return VkError.OutOfDeviceMemory;
+    errdefer allocator.destroy(data);
 
-            const work_execution_request: proto.PhiWorkExecutionRequest = .{
-                .cmd_count = phi_command_buffer.serialized_cmd_count,
-                .command_buffer_size = phi_command_buffer.commands.items.len,
-            };
-            const payload_size = @sizeOf(proto.PhiWorkExecutionRequest) + phi_command_buffer.commands.items.len;
-            const allocator = interface.host_allocator.allocator();
-            const payload = allocator.alloc(u8, payload_size) catch return VkError.OutOfHostMemory;
-            defer allocator.free(payload);
+    var prepared_submits = try prepareSubmits(self, allocator, infos);
+    errdefer deinitPreparedSubmits(self, allocator, &prepared_submits);
 
-            @memcpy(payload[0..@sizeOf(proto.PhiWorkExecutionRequest)], std.mem.asBytes(&work_execution_request));
-            @memcpy(payload[@sizeOf(proto.PhiWorkExecutionRequest)..], phi_command_buffer.commands.items);
+    const sequence = blk: {
+        self.mutex.lock(io) catch return VkError.DeviceLost;
+        defer self.mutex.unlock(io);
 
-            // Synchronous queues for now
-            var reply = std.mem.zeroes(proto.PhiResultReply);
-            try device.transport.request(proto.PHI_PACKET_WORK_EXECUTION, payload, std.mem.asBytes(&reply));
+        if (self.error_state) |err| return err;
+        if (self.shutting_down) return VkError.DeviceLost;
 
-            if (reply.result.status != proto.PHI_STATUS_OK) {
-                return PhiTransport.statusToErr(reply.result.status);
-            }
-        }
+        const value = self.next_task_sequence;
+        self.next_task_sequence += 1;
+        break :blk value;
+    };
 
-        for (info.signal_semaphores.items) |semaphore| {
-            try semaphore.signal();
-        }
-    }
-    if (fence) |value| {
-        try value.signal();
-    }
+    data.* = .{
+        .queue = self,
+        .sequence = sequence,
+        .submits = prepared_submits,
+        .fence = fence,
+    };
+
+    self.submit_group.async(io, taskRunner, .{data});
 }
 
 pub fn waitIdle(interface: *Interface) VkError!void {
-    _ = interface;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.submit_group.await(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return VkError.DeviceLost;
+    };
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    while (self.completed_sequence + 1 < self.next_remote_sequence and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+    }
+
+    if (self.error_state) |err| return err;
+}
+
+fn checkHealthy(self: *Self) VkError!void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.error_state) |err| return err;
+    if (self.shutting_down) return VkError.DeviceLost;
+}
+
+fn prepareSubmits(self: *Self, allocator: std.mem.Allocator, infos: []Interface.SubmitInfo) VkError!std.ArrayList(PreparedSubmit) {
+    var submits = std.ArrayList(PreparedSubmit).initCapacity(allocator, infos.len) catch return VkError.OutOfDeviceMemory;
+    errdefer deinitPreparedSubmits(self, allocator, &submits);
+
+    for (infos) |info| {
+        var prepared = try prepareSubmit(self, allocator, info);
+        submits.append(allocator, prepared) catch {
+            deinitPreparedSubmit(self, allocator, &prepared);
+            return VkError.OutOfDeviceMemory;
+        };
+    }
+
+    return submits;
+}
+
+fn prepareSubmit(self: *Self, allocator: std.mem.Allocator, info: Interface.SubmitInfo) VkError!PreparedSubmit {
+    var wait_semaphores = info.wait_semaphores.clone(allocator) catch return VkError.OutOfDeviceMemory;
+    errdefer wait_semaphores.deinit(allocator);
+
+    var signal_semaphores = info.signal_semaphores.clone(allocator) catch return VkError.OutOfDeviceMemory;
+    errdefer signal_semaphores.deinit(allocator);
+
+    var command_size: usize = 0;
+    var command_count: u64 = 0;
+    for (info.command_buffers.items) |command_buffer| {
+        const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+
+        if (phi_command_buffer.commands.items.len > std.math.maxInt(usize) - command_size) {
+            return VkError.OutOfHostMemory;
+        }
+        command_size += phi_command_buffer.commands.items.len;
+
+        const serialized_cmd_count: u64 = @intCast(phi_command_buffer.serialized_cmd_count);
+        if (serialized_cmd_count > std.math.maxInt(u64) - command_count) {
+            return VkError.OutOfHostMemory;
+        }
+        command_count += serialized_cmd_count;
+    }
+
+    var command_backing: ?[]u8 = null;
+    var scif_offset: ?u64 = null;
+    var registered_size: usize = 0;
+
+    if (command_size != 0) {
+        const page_size = std.heap.pageSize();
+        if (command_size > std.math.maxInt(usize) - (page_size - 1)) {
+            return VkError.OutOfHostMemory;
+        }
+        registered_size = std.mem.alignForward(usize, command_size, page_size);
+
+        const backing = allocator.alignedAlloc(
+            u8,
+            .fromByteUnits(std.heap.page_size_max),
+            registered_size,
+        ) catch return VkError.OutOfHostMemory;
+        errdefer allocator.free(backing);
+        @memset(backing, 0);
+
+        var write_offset: usize = 0;
+        for (info.command_buffers.items) |command_buffer| {
+            const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+            const commands = phi_command_buffer.commands.items;
+            @memcpy(backing[write_offset .. write_offset + commands.len], commands);
+            write_offset += commands.len;
+        }
+
+        const offset = try self.transport.registerHostMemory(backing);
+        errdefer self.transport.unregisterHostMemory(offset, backing.len) catch @panic("Caught an error while handling an error");
+
+        command_backing = backing;
+        scif_offset = offset;
+    }
+
+    return .{
+        .wait_semaphores = wait_semaphores,
+        .signal_semaphores = signal_semaphores,
+        .command_backing = command_backing,
+        .scif_offset = scif_offset,
+        .registered_size = registered_size,
+        .command_size = command_size,
+        .command_count = command_count,
+    };
+}
+
+fn deinitPreparedSubmits(self: *Self, allocator: std.mem.Allocator, submits: *std.ArrayList(PreparedSubmit)) void {
+    for (submits.items) |*prepared| {
+        deinitPreparedSubmit(self, allocator, prepared);
+    }
+    submits.deinit(allocator);
+}
+
+fn deinitPreparedSubmit(self: *Self, allocator: std.mem.Allocator, prepared: *PreparedSubmit) void {
+    prepared.wait_semaphores.deinit(allocator);
+    prepared.signal_semaphores.deinit(allocator);
+
+    if (prepared.scif_offset) |offset| {
+        self.transport.unregisterHostMemory(offset, prepared.registered_size) catch |err| {
+            std.log.scoped(.PhiQueue).warn("Failed to unregister staged command buffer: {s}", .{@errorName(err)});
+        };
+    }
+    if (prepared.command_backing) |backing| allocator.free(backing);
+
+    prepared.command_backing = null;
+    prepared.scif_offset = null;
+    prepared.registered_size = 0;
+    prepared.command_size = 0;
+    prepared.command_count = 0;
+}
+
+fn taskRunner(data: *TaskData) void {
+    const self = data.queue;
+    const io = self.interface.owner.io();
+    const allocator = self.interface.owner.device_allocator.allocator();
+
+    defer {
+        deinitPreparedSubmits(self, allocator, &data.submits);
+        allocator.destroy(data);
+    }
+
+    self.mutex.lock(io) catch {
+        failTask(data);
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    while (data.sequence != self.executing_task_sequence and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch {
+            self.mutex.unlock(io);
+            failTask(data);
+            self.markLost(VkError.DeviceLost);
+            return;
+        };
+    }
+    if (self.error_state != null) {
+        self.mutex.unlock(io);
+        failTask(data);
+        return;
+    }
+    self.mutex.unlock(io);
+
+    var task_error: ?VkError = null;
+
+    if (data.submits.items.len == 0) {
+        if (data.fence) |fence| {
+            var marker: PreparedSubmit = .{
+                .wait_semaphores = .empty,
+                .signal_semaphores = .empty,
+                .command_backing = null,
+                .scif_offset = null,
+                .registered_size = 0,
+                .command_size = 0,
+                .command_count = 0,
+            };
+            self.publish(&marker, fence) catch |err| {
+                task_error = err;
+            };
+            if (task_error == null) data.fence = null;
+        }
+    } else {
+        for (data.submits.items, 0..) |*prepared, info_index| {
+            for (prepared.wait_semaphores.items) |semaphore| {
+                semaphore.wait() catch |err| {
+                    task_error = err;
+                    break;
+                };
+            }
+            if (task_error != null) break;
+
+            const submission_fence = if (info_index + 1 == data.submits.items.len) data.fence else null;
+            self.publish(prepared, submission_fence) catch |err| {
+                task_error = err;
+                break;
+            };
+            if (submission_fence != null) data.fence = null;
+        }
+    }
+
+    if (task_error) |err| {
+        failTask(data);
+        self.markLost(err);
+        return;
+    }
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.executing_task_sequence += 1;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn publish(self: *Self, prepared: *PreparedSubmit, fence: ?*base.Fence) VkError!void {
+    const io = self.interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+
+    while ((self.next_remote_sequence - 1) - self.completed_sequence >= ring_capacity_u64 and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch {
+            self.mutex.unlock(io);
+            return VkError.DeviceLost;
+        };
+    }
+
+    if (self.error_state) |err| {
+        self.mutex.unlock(io);
+        return err;
+    }
+    if (self.shutting_down) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+
+    const sequence = self.next_remote_sequence;
+    if (sequence == shutdown_sequence) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+    const slot: usize = @intCast((sequence - 1) % ring_capacity_u64);
+    if (self.pending[slot] != null) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+
+    const command_scif_offset = prepared.scif_offset orelse 0;
+    self.pending[slot] = .{
+        .signal_semaphores = prepared.signal_semaphores,
+        .fence = fence,
+        .command_backing = prepared.command_backing,
+        .scif_offset = prepared.scif_offset,
+        .registered_size = prepared.registered_size,
+    };
+
+    prepared.signal_semaphores = .empty;
+    prepared.command_backing = null;
+    prepared.scif_offset = null;
+    prepared.registered_size = 0;
+
+    self.shared.submissions[slot] = .{
+        .sequence = sequence,
+        .command_scif_offset = command_scif_offset,
+        .command_size = prepared.command_size,
+        .command_count = prepared.command_count,
+    };
+
+    @atomicStore(
+        @TypeOf(self.shared.producer_sequence),
+        &self.shared.producer_sequence,
+        @intCast(sequence),
+        .release,
+    );
+    self.next_remote_sequence += 1;
+    self.mutex.unlock(io);
+
+    self.transport.sendQueueDoorbell(sequence) catch |err| {
+        self.markLost(VkError.DeviceLost);
+        return err;
+    };
+}
+
+fn completionRunner(self: *Self) void {
+    defer self.markCompletionStopped();
+
+    while (true) {
+        const completion = self.transport.receiveQueueCompletion() catch |err| {
+            if (!self.isShuttingDown()) {
+                std.log.scoped(.PhiQueue).err("Queue completion receive failed: {s}", .{@errorName(err)});
+                self.markLost(VkError.DeviceLost);
+            } else {
+                std.log.scoped(.PhiQueue).warn("Queue completion receiver stopped without shutdown acknowledgement: {s}", .{@errorName(err)});
+            }
+            return;
+        };
+
+        if (completion.sequence == shutdown_sequence) {
+            const io = self.interface.owner.io();
+            self.mutex.lock(io) catch return;
+            self.remote_stopped = completion.status == proto.PHI_STATUS_OK;
+            std.log.scoped(.PhiQueue).info("Received remote queue shutdown acknowledgement (status {d})", .{completion.status});
+            self.condition.broadcast(io);
+            self.mutex.unlock(io);
+            return;
+        }
+
+        self.completeOne(completion);
+    }
+}
+
+fn completeOne(self: *Self, completion: proto.PhiQueueCompletion) void {
+    const io = self.interface.owner.io();
+    const allocator = self.interface.owner.device_allocator.allocator();
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+
+    if (completion.sequence != self.completed_sequence + 1 or completion.sequence >= self.next_remote_sequence) {
+        self.mutex.unlock(io);
+        self.markLost(VkError.DeviceLost);
+        return;
+    }
+
+    const slot: usize = @intCast((completion.sequence - 1) % ring_capacity_u64);
+    var pending = self.pending[slot] orelse {
+        self.mutex.unlock(io);
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.pending[slot] = null;
+    self.mutex.unlock(io);
+
+    var cleanup_failed = false;
+    if (pending.scif_offset) |offset| {
+        self.transport.unregisterHostMemory(offset, pending.registered_size) catch |err| {
+            cleanup_failed = true;
+            std.log.scoped(.PhiQueue).err("Failed to unregister completed command buffer: {s}", .{@errorName(err)});
+        };
+    }
+    if (pending.command_backing) |backing| allocator.free(backing);
+
+    if (completion.status != proto.PHI_STATUS_OK or cleanup_failed) {
+        std.log.scoped(.PhiQueue).err(
+            "Queue completion {d} failed with remote status {d} (cleanup failed: {})",
+            .{ completion.sequence, completion.status, cleanup_failed },
+        );
+        // A remote command error is confined to this submission. The protocol
+        // stream and ring remain synchronized, so poisoning every later CTS
+        // submission would only hide the command that actually failed.
+        if (cleanup_failed) self.markLost(VkError.DeviceLost);
+        failPending(&pending);
+    } else if (self.hasError()) {
+        failPending(&pending);
+    } else {
+        signalPending(self, &pending);
+    }
+
+    pending.signal_semaphores.deinit(allocator);
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.completed_sequence = completion.sequence;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn signalPending(self: *Self, pending: *PendingCompletion) void {
+    var signal_failed = false;
+
+    for (pending.signal_semaphores.items) |semaphore| {
+        semaphore.signal() catch {
+            signal_failed = true;
+        };
+    }
+    if (pending.fence) |fence| {
+        fence.signal() catch {
+            signal_failed = true;
+        };
+    }
+
+    if (signal_failed) self.markLost(VkError.DeviceLost);
+}
+
+fn failPending(pending: *PendingCompletion) void {
+    for (pending.signal_semaphores.items) |semaphore| {
+        PhiBinarySemaphore.fail(semaphore);
+    }
+    if (pending.fence) |fence| PhiFence.fail(fence);
+}
+
+fn failTask(data: *TaskData) void {
+    for (data.submits.items) |*prepared| {
+        for (prepared.signal_semaphores.items) |semaphore| {
+            PhiBinarySemaphore.fail(semaphore);
+        }
+    }
+    if (data.fence) |fence| PhiFence.fail(fence);
+}
+
+fn markLost(self: *Self, _: VkError) void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    const first_failure = self.error_state == null;
+    self.error_state = VkError.DeviceLost;
+    self.condition.broadcast(io);
+
+    if (!first_failure) return;
+
+    for (&self.pending) |*entry| {
+        if (entry.*) |*pending| failPending(pending);
+    }
+}
+
+fn hasError(self: *Self) bool {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return true;
+    defer self.mutex.unlock(io);
+    return self.error_state != null;
+}
+
+fn completionProgress(self: *Self, io: std.Io) struct { next: u64, completed: u64 } {
+    self.mutex.lock(io) catch return .{ .next = 0, .completed = 0 };
+    defer self.mutex.unlock(io);
+    return .{ .next = self.next_remote_sequence, .completed = self.completed_sequence };
+}
+
+fn waitForCompletionShutdown(self: *Self, io: std.Io) CompletionShutdown {
+    const deadline = std.Io.Clock.Timestamp.fromNow(io, .{
+        .raw = .fromNanoseconds(shutdown_timeout_ns),
+        .clock = .awake,
+    });
+
+    while (true) {
+        self.mutex.lock(io) catch return .wait_failed;
+        const stopped = self.completion_stopped;
+        const acknowledged = self.remote_stopped;
+        self.mutex.unlock(io);
+
+        if (stopped) return if (acknowledged) .acknowledged else .stopped_without_acknowledgement;
+
+        const remaining = deadline.durationFromNow(io);
+        if (remaining.raw.nanoseconds <= 0) return .timed_out;
+        (std.Io.Clock.Duration{
+            .raw = .fromNanoseconds(@min(remaining.raw.nanoseconds, shutdown_poll_ns)),
+            .clock = .awake,
+        }).sleep(io) catch return .wait_failed;
+    }
+}
+
+fn markCompletionStopped(self: *Self) void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return;
+    self.completion_stopped = true;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn isShuttingDown(self: *Self) bool {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return true;
+    defer self.mutex.unlock(io);
+    return self.shutting_down;
+}
+
+fn cleanupPendingAfterClose(self: *Self, allocator: std.mem.Allocator) void {
+    for (&self.pending) |*entry| {
+        if (entry.*) |*pending| {
+            failPending(pending);
+            if (pending.command_backing) |backing| allocator.free(backing);
+            pending.signal_semaphores.deinit(allocator);
+            entry.* = null;
+        }
+    }
 }

src/phi/PhiTransport.zig

@@ -12,7 +12,10 @@ const Self = @This();
 epd: Endpoint,
 sequence: u64 = 1,
 mutex: std.Io.Mutex = .init,
+endpoint_mutex: base.SpinMutex = .{},
+library_loaded: bool = true,
 instance: *base.Instance,
+node_id: u16,
 
 pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
     const epd = blk: {
@@ -45,6 +48,7 @@ pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
     var self: Self = .{
         .epd = epd,
         .instance = instance,
+        .node_id = node_id,
     };
     try self.handshake();
 
@@ -52,17 +56,42 @@ pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
     return self;
 }
 
+pub fn connectPeer(self: *const Self) VkError!Self {
+    return init(self.instance, self.node_id);
+}
+
 pub fn deinit(self: *Self) void {
     var reply: proto.PhiResult = undefined;
     self.request(proto.PHI_PACKET_SHUTDOWN, &.{}, std.mem.asBytes(&reply)) catch |err| {
         std.log.scoped(.PhiTransport).warn("Failed to shut down remote session: {s}", .{@errorName(err)});
     };
 
-    closeEndpoint(self.epd);
-    scif.unload();
+    self.close();
     std.log.scoped(.PhiTransport).info("Closed connection", .{});
 }
 
+/// Close the endpoint so a thread blocked in SCIF receive wakes up. The SCIF
+/// library stays loaded until `close`, because that thread may still be
+/// returning through a dynamically loaded function.
+pub fn interrupt(self: *Self) void {
+    self.endpoint_mutex.lock();
+    const endpoint = self.epd;
+    self.epd = -1;
+    self.endpoint_mutex.unlock();
+
+    if (endpoint >= 0) closeEndpoint(endpoint);
+}
+
+/// Close a transport without issuing an RPC shutdown. Queue transports switch
+/// to a raw full-duplex doorbell protocol after setup and must use this path.
+pub fn close(self: *Self) void {
+    self.interrupt();
+    if (!self.library_loaded) return;
+
+    self.library_loaded = false;
+    scif.unload();
+}
+
 pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void {
     self.mutex.lock(self.instance.io()) catch return VkError.DeviceLost;
     defer self.mutex.unlock(self.instance.io());
@@ -98,6 +127,20 @@ pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload:
     try self.readAll(reply_payload);
 }
 
+pub fn sendQueueDoorbell(self: *Self, sequence: u64) VkError!void {
+    const doorbell: proto.PhiQueueDoorbell = .{
+        .sequence = sequence,
+    };
+    try self.writeAll(std.mem.asBytes(&doorbell));
+}
+
+pub fn receiveQueueCompletion(self: *Self) VkError!proto.PhiQueueCompletion {
+    // SAFETY: readAll initializes the complete structure.
+    var completion: proto.PhiQueueCompletion = undefined;
+    try self.readAll(std.mem.asBytes(&completion));
+    return completion;
+}
+
 pub fn statusToErr(status: c_int) VkError {
     return switch (status) {
         proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory,
@@ -108,27 +151,35 @@ pub fn statusToErr(status: c_int) VkError {
 }
 
 fn writeAll(self: *Self, bytes: []const u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const written = scif.send(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
+        const written = scif.send(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
         if (written <= 0) {
-            return VkError.InitializationFailed;
+            return VkError.DeviceLost;
         }
         offset += @intCast(written);
     }
 }
 
 fn readAll(self: *Self, bytes: []u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const read = scif.recv(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
+        const read = scif.recv(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
         if (read <= 0) {
-            return VkError.InitializationFailed;
+            return VkError.DeviceLost;
         }
         offset += @intCast(read);
     }
 }
 
+fn getEndpoint(self: *Self) ?Endpoint {
+    self.endpoint_mutex.lock();
+    defer self.endpoint_mutex.unlock();
+    return if (self.epd >= 0) self.epd else null;
+}
+
 fn closeEndpoint(endpoint: Endpoint) void {
     _ = scif.close(endpoint);
 }
@@ -152,8 +203,9 @@ fn handshake(self: *Self) VkError!void {
 }
 
 pub fn registerHostMemory(self: *Self, memory: []u8) VkError!u64 {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     const offset = scif.register(
-        self.epd,
+        endpoint,
         memory.ptr,
         memory.len,
         0,
@@ -167,7 +219,8 @@ pub fn registerHostMemory(self: *Self, memory: []u8) VkError!u64 {
 }
 
 pub fn unregisterHostMemory(self: *Self, offset: u64, size: usize) VkError!void {
-    if (scif.unregister(self.epd, @intCast(offset), size) != 0) {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
+    if (scif.unregister(endpoint, @intCast(offset), size) != 0) {
         return VkError.Unknown;
     }
 }

src/phi/compiler/abi.zig

@@ -0,0 +1,25 @@
+pub const version: u32 = 1;
+
+pub const BufferDescriptor = extern struct {
+    address: u64,
+    size: u64,
+};
+
+pub const KernelContext = extern struct {
+    abi_version: u32,
+    resource_count: u32,
+    resources: u64,
+    push_constants: u64,
+    push_constant_size: u32,
+    reserved: u32 = 0,
+    base_group: [3]u32,
+    group_count: [3]u32,
+    local_size: [3]u32,
+    num_workgroups: [3]u32,
+};
+
+pub const EntryPoint = *const fn (
+    context: *const KernelContext,
+    begin_workgroup: u64,
+    end_workgroup: u64,
+) callconv(.c) void;

src/phi/compiler/analysis.zig

@@ -0,0 +1,21 @@
+const std = @import("std");
+const block_layout = @import("block_layout.zig");
+const edge_copies = @import("edge_copies.zig");
+const liveness = @import("liveness.zig");
+const register_allocator = @import("register_allocator.zig");
+
+pub const Analysis = struct {
+    layout: block_layout.Layout = block_layout.Layout.empty(),
+    liveness: liveness.Analysis = liveness.Analysis.empty(),
+    allocation: register_allocator.Allocation = register_allocator.Allocation.empty(),
+    edge_copy_plans: []edge_copies.Plan = &.{},
+
+    pub fn deinit(self: *Analysis, allocator: std.mem.Allocator) void {
+        self.layout.deinit(allocator);
+        self.liveness.deinit(allocator);
+        self.allocation.deinit(allocator);
+        for (self.edge_copy_plans) |*plan| plan.deinit(allocator);
+        if (self.edge_copy_plans.len != 0) allocator.free(self.edge_copy_plans);
+        self.* = undefined;
+    }
+};

src/phi/compiler/artifact.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+const abi = @import("abi.zig");
+
+pub const ResourceBinding = struct {
+    set: u32,
+    binding: u32,
+};
+
+pub const KernelInfo = struct {
+    abi_version: u32 = abi.version,
+    workgroup_size: [3]u32,
+    dispatch_width: u8,
+    stack_size: u32,
+    resources: []const ResourceBinding,
+};
+
+pub const Artifact = struct {
+    code: []u8,
+    info: KernelInfo,
+
+    pub fn deinit(self: *Artifact, allocator: std.mem.Allocator) void {
+        allocator.free(self.code);
+        allocator.free(self.info.resources);
+        self.* = undefined;
+    }
+};

src/phi/compiler/block_layout.zig

@@ -0,0 +1,17 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+pub const Layout = struct {
+    blocks: []shader_ir.id.BlockId,
+    positions: []?usize,
+
+    pub fn empty() Layout {
+        return .{ .blocks = &.{}, .positions = &.{} };
+    }
+
+    pub fn deinit(self: *Layout, allocator: std.mem.Allocator) void {
+        if (self.blocks.len != 0) allocator.free(self.blocks);
+        if (self.positions.len != 0) allocator.free(self.positions);
+        self.* = undefined;
+    }
+};

src/phi/compiler/code_buffer.zig

@@ -0,0 +1,67 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+const Error = @import("errors.zig").Error;
+
+pub const Label = enum(u32) { _ };
+
+pub const Fixup = struct {
+    displacement_offset: usize,
+    instruction_end: usize,
+    target: Label,
+};
+
+pub const BlockLabel = struct {
+    block: shader_ir.id.BlockId,
+    label: Label,
+};
+
+pub const CodeBuffer = struct {
+    allocator: std.mem.Allocator,
+    bytes: std.ArrayList(u8) = .empty,
+    label_offsets: std.ArrayList(?usize) = .empty,
+    fixups: std.ArrayList(Fixup) = .empty,
+
+    pub fn init(allocator: std.mem.Allocator) CodeBuffer {
+        return .{ .allocator = allocator };
+    }
+
+    pub fn deinit(self: *CodeBuffer) void {
+        self.bytes.deinit(self.allocator);
+        self.label_offsets.deinit(self.allocator);
+        self.fixups.deinit(self.allocator);
+        self.* = undefined;
+    }
+
+    pub fn offset(self: *const CodeBuffer) usize {
+        return self.bytes.items.len;
+    }
+
+    pub fn emitByte(self: *CodeBuffer, byte: u8) std.mem.Allocator.Error!void {
+        try self.bytes.append(self.allocator, byte);
+    }
+
+    pub fn emitBytes(self: *CodeBuffer, bytes: []const u8) std.mem.Allocator.Error!void {
+        try self.bytes.appendSlice(self.allocator, bytes);
+    }
+
+    pub fn createLabel(self: *CodeBuffer) std.mem.Allocator.Error!Label {
+        const label: Label = @enumFromInt(self.label_offsets.items.len);
+        try self.label_offsets.append(self.allocator, null);
+        return label;
+    }
+
+    pub fn bindLabel(self: *CodeBuffer, label: Label) Error!void {
+        const index = @intFromEnum(label);
+        if (index >= self.label_offsets.items.len or self.label_offsets.items[index] != null)
+            return error.EncodingFailed;
+        self.label_offsets.items[index] = self.offset();
+    }
+
+    pub fn resolveFixups(_: *CodeBuffer) Error!void {
+        return error.CodeGenerationNotImplemented;
+    }
+
+    pub fn toOwnedSlice(self: *CodeBuffer) std.mem.Allocator.Error![]u8 {
+        return self.bytes.toOwnedSlice(self.allocator);
+    }
+};

src/phi/compiler/codegen.zig

@@ -0,0 +1,17 @@
+const shader_ir = @import("shader_ir").ir;
+const Analysis = @import("analysis.zig").Analysis;
+const Error = @import("errors.zig").Error;
+const Encoder = @import("imci/encoder.zig").Encoder;
+
+pub const Options = struct {
+    dispatch_width: u8 = 16,
+};
+
+pub fn emitComputeKernel(
+    _: *Encoder,
+    _: *const shader_ir.module.Module,
+    _: *const Analysis,
+    _: Options,
+) Error!void {
+    return error.CodeGenerationNotImplemented;
+}

src/phi/compiler/compiler.zig

@@ -0,0 +1,60 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+pub const abi = @import("abi.zig");
+pub const analysis = @import("analysis.zig");
+pub const artifact = @import("artifact.zig");
+pub const block_layout = @import("block_layout.zig");
+pub const code_buffer = @import("code_buffer.zig");
+pub const codegen = @import("codegen.zig");
+pub const control_flow = @import("control_flow.zig");
+pub const edge_copies = @import("edge_copies.zig");
+pub const errors = @import("errors.zig");
+pub const imci = @import("imci/imci.zig");
+pub const liveness = @import("liveness.zig");
+pub const register_allocator = @import("register_allocator.zig");
+
+pub const Artifact = artifact.Artifact;
+pub const Error = errors.Error;
+
+pub const Options = struct {
+    dispatch_width: u8 = 16,
+};
+
+pub fn compileCompute(allocator: std.mem.Allocator, module: *shader_ir.module.Module, options: Options) Error!Artifact {
+    _ = allocator;
+
+    if (module.stage != .compute)
+        return error.UnsupportedStage;
+    if (module.entry_point == null)
+        return error.MissingEntryPoint;
+    if (module.execution_modes.workgroup_size == null)
+        return error.MissingWorkgroupSize;
+    if (!module.properties.structured_control_flow)
+        return error.UnstructuredControlFlow;
+    if (options.dispatch_width != 16)
+        return error.UnsupportedType;
+
+    return error.CodeGenerationNotImplemented;
+}
+
+test "[compiler] foundation declarations compile" {
+    std.testing.refAllDecls(abi);
+    std.testing.refAllDecls(analysis);
+    std.testing.refAllDecls(artifact);
+    std.testing.refAllDecls(block_layout);
+    std.testing.refAllDecls(code_buffer);
+    std.testing.refAllDecls(codegen);
+    std.testing.refAllDecls(control_flow);
+    std.testing.refAllDecls(edge_copies);
+    std.testing.refAllDecls(imci);
+    std.testing.refAllDecls(liveness);
+    std.testing.refAllDecls(register_allocator);
+}
+
+test "[compiler] rejects non-compute modules before code generation" {
+    var module = shader_ir.module.Module.init(std.testing.allocator, .vertex);
+    defer module.deinit();
+
+    try std.testing.expectError(error.UnsupportedStage, compileCompute(std.testing.allocator, &module, .{}));
+}

src/phi/compiler/control_flow.zig

@@ -0,0 +1,27 @@
+const shader_ir = @import("shader_ir").ir;
+const registers = @import("imci/registers.zig");
+
+pub const ActiveMask = union(enum) {
+    full,
+    register: registers.Mask,
+    spilled: u32,
+};
+
+pub const Region = union(enum) {
+    block: shader_ir.id.BlockId,
+    selection: struct {
+        header: shader_ir.id.BlockId,
+        merge: shader_ir.id.BlockId,
+    },
+    loop: struct {
+        header: shader_ir.id.BlockId,
+        merge: shader_ir.id.BlockId,
+        continue_block: shader_ir.id.BlockId,
+    },
+};
+
+pub const State = struct {
+    active_mask: ActiveMask = .full,
+    loop_depth: u16 = 0,
+    selection_depth: u16 = 0,
+};

src/phi/compiler/edge_copies.zig

@@ -0,0 +1,18 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+pub const Copy = struct {
+    source: shader_ir.id.ValueId,
+    destination: shader_ir.id.ValueId,
+};
+
+pub const Plan = struct {
+    predecessor: shader_ir.id.BlockId,
+    successor: shader_ir.id.BlockId,
+    copies: []Copy,
+
+    pub fn deinit(self: *Plan, allocator: std.mem.Allocator) void {
+        allocator.free(self.copies);
+        self.* = undefined;
+    }
+};

src/phi/compiler/errors.zig

@@ -0,0 +1,15 @@
+const std = @import("std");
+
+pub const Error = std.mem.Allocator.Error || error{
+    MissingEntryPoint,
+    MissingWorkgroupSize,
+    InvalidModule,
+    UnsupportedStage,
+    UnsupportedType,
+    UnsupportedOperation,
+    UnstructuredControlFlow,
+    RegisterAllocationFailed,
+    EncodingFailed,
+    BranchOutOfRange,
+    CodeGenerationNotImplemented,
+};

src/phi/compiler/imci/encoder.zig

@@ -0,0 +1,46 @@
+const std = @import("std");
+const CodeBuffer = @import("../code_buffer.zig").CodeBuffer;
+const Error = @import("../errors.zig").Error;
+const encoding = @import("encoding.zig");
+const registers = @import("registers.zig");
+
+pub const Encoder = struct {
+    code: CodeBuffer,
+
+    pub fn init(allocator: std.mem.Allocator) Encoder {
+        return .{ .code = CodeBuffer.init(allocator) };
+    }
+
+    pub fn deinit(self: *Encoder) void {
+        self.code.deinit();
+        self.* = undefined;
+    }
+
+    pub fn prologue(_: *Encoder, _: u32) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn epilogue(_: *Encoder) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn moveVector(_: *Encoder, _: registers.Zmm, _: encoding.VectorSource, _: ?registers.Mask) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn addVector(_: *Encoder, _: encoding.VectorElement, _: registers.Zmm, _: registers.Zmm, _: encoding.VectorSource, _: ?registers.Mask) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn compareVector(_: *Encoder, _: encoding.VectorElement, _: encoding.Condition, _: registers.Mask, _: registers.Zmm, _: encoding.VectorSource, _: registers.Mask) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn gather(_: *Encoder, _: encoding.VectorElement, _: registers.Zmm, _: registers.Gpr, _: registers.Zmm, _: registers.Mask) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+
+    pub fn scatter(_: *Encoder, _: encoding.VectorElement, _: registers.Gpr, _: registers.Zmm, _: registers.Zmm, _: registers.Mask) Error!void {
+        return Error.CodeGenerationNotImplemented;
+    }
+};

src/phi/compiler/imci/encoding.zig

@@ -0,0 +1,38 @@
+const registers = @import("registers.zig");
+
+pub const VectorElement = enum {
+    i32,
+    u32,
+    f32,
+    i64,
+    u64,
+    f64,
+};
+
+pub const Immediate = union(enum) {
+    u8: u8,
+    u32: u32,
+    i32: i32,
+    u64: u64,
+};
+
+pub const Memory = struct {
+    base: ?registers.Gpr = null,
+    index: ?registers.Gpr = null,
+    scale: enum(u2) { one, two, four, eight } = .one,
+    displacement: i32 = 0,
+};
+
+pub const VectorSource = union(enum) {
+    register: registers.Zmm,
+    memory: Memory,
+};
+
+pub const Condition = enum {
+    equal,
+    not_equal,
+    less,
+    less_equal,
+    greater,
+    greater_equal,
+};

src/phi/compiler/imci/imci.zig

@@ -0,0 +1,5 @@
+pub const encoder = @import("encoder.zig");
+pub const encoding = @import("encoding.zig");
+pub const registers = @import("registers.zig");
+
+pub const Encoder = encoder.Encoder;

src/phi/compiler/imci/registers.zig

@@ -0,0 +1,64 @@
+pub const Gpr = enum(u5) {
+    rax,
+    rcx,
+    rdx,
+    rbx,
+    rsp,
+    rbp,
+    rsi,
+    rdi,
+    r8,
+    r9,
+    r10,
+    r11,
+    r12,
+    r13,
+    r14,
+    r15,
+};
+
+pub const Zmm = enum(u5) {
+    zmm0,
+    zmm1,
+    zmm2,
+    zmm3,
+    zmm4,
+    zmm5,
+    zmm6,
+    zmm7,
+    zmm8,
+    zmm9,
+    zmm10,
+    zmm11,
+    zmm12,
+    zmm13,
+    zmm14,
+    zmm15,
+    zmm16,
+    zmm17,
+    zmm18,
+    zmm19,
+    zmm20,
+    zmm21,
+    zmm22,
+    zmm23,
+    zmm24,
+    zmm25,
+    zmm26,
+    zmm27,
+    zmm28,
+    zmm29,
+    zmm30,
+    zmm31,
+};
+
+pub const Mask = enum(u3) {
+    k0,
+    k1,
+    k2,
+    k3,
+    k4,
+    k5,
+    k6,
+    k7,
+};

src/phi/compiler/liveness.zig

@@ -0,0 +1,25 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+pub const Position = u32;
+
+pub const LiveRange = struct {
+    value: shader_ir.id.ValueId,
+    first: Position,
+    last: Position,
+};
+
+pub const Analysis = struct {
+    ranges: []LiveRange,
+    value_ranges: []?usize,
+
+    pub fn empty() Analysis {
+        return .{ .ranges = &.{}, .value_ranges = &.{} };
+    }
+
+    pub fn deinit(self: *Analysis, allocator: std.mem.Allocator) void {
+        if (self.ranges.len != 0) allocator.free(self.ranges);
+        if (self.value_ranges.len != 0) allocator.free(self.value_ranges);
+        self.* = undefined;
+    }
+};

src/phi/compiler/register_allocator.zig

@@ -0,0 +1,61 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+const registers = @import("imci/registers.zig");
+
+pub const StackSlot = struct {
+    offset: u32,
+    size: u32,
+    alignment: u32,
+};
+
+pub const VectorComponent = struct {
+    chunks: []const registers.Zmm,
+};
+
+pub const VectorLocation = struct {
+    components: []const VectorComponent,
+};
+
+pub const Immediate = union(enum) {
+    integer: u64,
+    float: u64,
+};
+
+pub const Location = union(enum) {
+    immediate: Immediate,
+    vector: VectorLocation,
+    mask: registers.Mask,
+    stack: StackSlot,
+};
+
+pub const Constraints = struct {
+    temporary_vectors: u8 = 0,
+    temporary_masks: u8 = 0,
+    temporary_gprs: u8 = 0,
+};
+
+pub const Allocation = struct {
+    value_locations: []?Location,
+    stack_size: u32,
+    scratch_vector: ?registers.Zmm,
+    scratch_mask: ?registers.Mask,
+
+    pub fn empty() Allocation {
+        return .{
+            .value_locations = &.{},
+            .stack_size = 0,
+            .scratch_vector = null,
+            .scratch_mask = null,
+        };
+    }
+
+    pub fn location(self: *const Allocation, value: shader_ir.id.ValueId) ?Location {
+        if (value.index() >= self.value_locations.len) return null;
+        return self.value_locations[value.index()];
+    }
+
+    pub fn deinit(self: *Allocation, allocator: std.mem.Allocator) void {
+        if (self.value_locations.len != 0) allocator.free(self.value_locations);
+        self.* = undefined;
+    }
+};

src/phi/copy_commands.zig

@@ -0,0 +1,532 @@
+const std = @import("std");
+const vk = @import("vulkan");
+const base = @import("base");
+const lib = @import("lib.zig");
+const proto = lib.proto;
+
+const VkError = base.VkError;
+const PhiCommandBuffer = @import("PhiCommandBuffer.zig");
+const PhiDeviceMemory = @import("PhiDeviceMemory.zig");
+
+const CopyAddress = struct {
+    offset: vk.DeviceSize,
+    row_pitch: vk.DeviceSize,
+    slice_pitch: vk.DeviceSize,
+    layer_pitch: vk.DeviceSize,
+};
+
+const CopyShape = struct {
+    row_size: vk.DeviceSize,
+    row_count: u32,
+    slice_count: u32,
+    layer_count: u32,
+};
+
+fn remoteMemory(buffer: *base.Buffer) VkError!*PhiDeviceMemory {
+    const memory = buffer.memory orelse return VkError.ValidationFailed;
+    const phi_memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory));
+    return phi_memory;
+}
+
+fn remoteImageMemory(image: *base.Image) VkError!*PhiDeviceMemory {
+    const memory = image.memory orelse return VkError.ValidationFailed;
+    const phi_memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory));
+    return phi_memory;
+}
+
+fn checkedAdd(a: vk.DeviceSize, b: vk.DeviceSize) VkError!vk.DeviceSize {
+    const result, const overflow = @addWithOverflow(a, b);
+    if (overflow != 0)
+        return VkError.ValidationFailed;
+    return result;
+}
+
+fn checkedMul(a: vk.DeviceSize, b: vk.DeviceSize) VkError!vk.DeviceSize {
+    const result, const overflow = @mulWithOverflow(a, b);
+    if (overflow != 0)
+        return VkError.ValidationFailed;
+    return result;
+}
+
+fn validateSingleAspect(image: *const base.Image, aspect_mask: vk.ImageAspectFlags) VkError!void {
+    const valid_aspects = base.format.toAspect(image.format);
+    if (aspect_mask.toInt() == 0 or @popCount(aspect_mask.toInt()) != 1 or aspect_mask.subtract(valid_aspects).toInt() != 0)
+        return VkError.ValidationFailed;
+}
+
+fn getMipExtent(image: *const base.Image, mip_level: u32) VkError!vk.Extent3D {
+    if (mip_level >= image.mip_levels)
+        return VkError.ValidationFailed;
+    return .{
+        .width = @max(1, image.extent.width >> @intCast(mip_level)),
+        .height = @max(1, image.extent.height >> @intCast(mip_level)),
+        .depth = @max(1, image.extent.depth >> @intCast(mip_level)),
+    };
+}
+
+fn validateImageRegion(image: *const base.Image, subresource: vk.ImageSubresourceLayers, offset: vk.Offset3D, extent: vk.Extent3D, allow_2d_depth_as_layers: bool) VkError!void {
+    try validateSingleAspect(image, subresource.aspect_mask);
+
+    if (offset.x < 0 or offset.y < 0 or offset.z < 0 or extent.width == 0 or extent.height == 0 or extent.depth == 0)
+        return VkError.ValidationFailed;
+
+    if (subresource.mip_level >= image.mip_levels)
+        return VkError.ValidationFailed;
+
+    const mip_extent = try getMipExtent(image, subresource.mip_level);
+    const x: u64 = @intCast(offset.x);
+    const y: u64 = @intCast(offset.y);
+    const z: u64 = @intCast(offset.z);
+
+    if (x > mip_extent.width or extent.width > mip_extent.width - x or y > mip_extent.height or extent.height > mip_extent.height - y)
+        return VkError.ValidationFailed;
+
+    if (image.image_type == .@"3d") {
+        if (subresource.base_array_layer != 0 or subresource.layer_count != 1 or z > mip_extent.depth or extent.depth > mip_extent.depth - z)
+            return VkError.ValidationFailed;
+    } else {
+        if (offset.z != 0 or subresource.layer_count == 0 or
+            subresource.base_array_layer >= image.array_layers or
+            subresource.layer_count > image.array_layers - subresource.base_array_layer)
+            return VkError.ValidationFailed;
+
+        if (allow_2d_depth_as_layers) {
+            if (extent.depth != subresource.layer_count)
+                return VkError.ValidationFailed;
+        } else if (extent.depth != 1) {
+            return VkError.ValidationFailed;
+        }
+    }
+}
+
+fn getImageCopyAddress(image: *base.Image, subresource: vk.ImageSubresourceLayers, image_offset: vk.Offset3D) VkError!CopyAddress {
+    const layout = try image.getSubresourceLayout(.{
+        .aspect_mask = subresource.aspect_mask,
+        .mip_level = subresource.mip_level,
+        .array_layer = subresource.base_array_layer,
+    });
+
+    const format = image.formatFromAspect(subresource.aspect_mask);
+    const block_width = base.format.blockWidth(format);
+    const block_height = base.format.blockHeight(format);
+    const bytes_per_block = base.format.texelSize(format);
+
+    const x: usize = @intCast(image_offset.x);
+    const y: usize = @intCast(image_offset.y);
+    const z: vk.DeviceSize = @intCast(image_offset.z);
+
+    if (@mod(x, block_width) != 0 or @mod(y, block_height) != 0)
+        return VkError.ValidationFailed;
+
+    const block_x = @divFloor(x, block_width);
+    const block_y = @divFloor(y, block_height);
+
+    const x_offset = try checkedMul(@intCast(block_x), @intCast(bytes_per_block));
+    const y_offset = try checkedMul(@intCast(block_y), layout.row_pitch);
+    const z_offset = try checkedMul(z, layout.depth_pitch);
+
+    var offset = try checkedAdd(image.memory_offset, layout.offset);
+    offset = try checkedAdd(offset, z_offset);
+    offset = try checkedAdd(offset, y_offset);
+    offset = try checkedAdd(offset, x_offset);
+
+    return .{
+        .offset = offset,
+        .row_pitch = layout.row_pitch,
+        .slice_pitch = layout.depth_pitch,
+        .layer_pitch = layout.array_pitch,
+    };
+}
+
+fn getBufferImageAddress(buffer: *const base.Buffer, format: vk.Format, region: vk.BufferImageCopy) VkError!CopyAddress {
+    const row_length: usize = if (region.buffer_row_length == 0)
+        region.image_extent.width
+    else
+        region.buffer_row_length;
+    const image_height: usize = if (region.buffer_image_height == 0)
+        region.image_extent.height
+    else
+        region.buffer_image_height;
+
+    if (row_length < region.image_extent.width or image_height < region.image_extent.height)
+        return VkError.ValidationFailed;
+
+    const block_width = base.format.blockWidth(format);
+    const block_height = base.format.blockHeight(format);
+
+    if (region.buffer_row_length != 0 and @mod(row_length, block_width) != 0)
+        return VkError.ValidationFailed;
+    if (region.buffer_image_height != 0 and @mod(image_height, block_height) != 0)
+        return VkError.ValidationFailed;
+
+    const row_pitch: vk.DeviceSize = @intCast(base.format.pitchMemSize(format, row_length));
+    const slice_pitch: vk.DeviceSize = @intCast(base.format.sliceMemSize(format, row_length, image_height));
+
+    return .{
+        .offset = try checkedAdd(buffer.offset, region.buffer_offset),
+        .row_pitch = row_pitch,
+        .slice_pitch = slice_pitch,
+        .layer_pitch = slice_pitch,
+    };
+}
+
+fn getCopyShape(format: vk.Format, extent: vk.Extent3D, slice_count: u32, layer_count: u32) VkError!CopyShape {
+    if (extent.width == 0 or extent.height == 0 or slice_count == 0 or layer_count == 0)
+        return VkError.ValidationFailed;
+
+    const block_count_x = base.format.blockCountX(format, extent.width);
+    const row_size, const overflow = @mulWithOverflow(block_count_x, base.format.texelSize(format));
+    if (overflow != 0)
+        return VkError.ValidationFailed;
+
+    return .{
+        .row_size = @intCast(row_size),
+        .row_count = @intCast(base.format.blockCountY(format, extent.height)),
+        .slice_count = slice_count,
+        .layer_count = layer_count,
+    };
+}
+
+fn getCopySpan(address: CopyAddress, shape: CopyShape) VkError!vk.DeviceSize {
+    var span = shape.row_size;
+
+    if (shape.row_count > 1)
+        span = try checkedAdd(span, try checkedMul(shape.row_count - 1, address.row_pitch));
+    if (shape.slice_count > 1)
+        span = try checkedAdd(span, try checkedMul(shape.slice_count - 1, address.slice_pitch));
+    if (shape.layer_count > 1)
+        span = try checkedAdd(span, try checkedMul(shape.layer_count - 1, address.layer_pitch));
+
+    return span;
+}
+
+fn validateBufferRange(buffer: *const base.Buffer, region_offset: vk.DeviceSize, address: CopyAddress, shape: CopyShape) VkError!void {
+    const span = try getCopySpan(address, shape);
+    if (region_offset > buffer.size or span > buffer.size - region_offset)
+        return VkError.ValidationFailed;
+}
+
+fn validateMemoryRange(memory: *const PhiDeviceMemory, address: CopyAddress, shape: CopyShape) VkError!void {
+    const span = try getCopySpan(address, shape);
+    if (address.offset > memory.interface.size or span > memory.interface.size - address.offset)
+        return VkError.ValidationFailed;
+}
+
+fn appendImageCopy(
+    cmd: *PhiCommandBuffer,
+    command_type: c_int,
+    src_memory: *PhiDeviceMemory,
+    src: CopyAddress,
+    dst_memory: *PhiDeviceMemory,
+    dst: CopyAddress,
+    shape: CopyShape,
+) VkError!void {
+    try validateMemoryRange(src_memory, src, shape);
+    try validateMemoryRange(dst_memory, dst, shape);
+
+    try cmd.appendCommand(proto.PhiCmdCopyImage, command_type, .{
+        .src_memory = @intCast(src_memory.remote_handle),
+        .src_offset = src.offset,
+        .src_row_pitch = src.row_pitch,
+        .src_slice_pitch = src.slice_pitch,
+        .src_layer_pitch = src.layer_pitch,
+        .dst_memory = @intCast(dst_memory.remote_handle),
+        .dst_offset = dst.offset,
+        .dst_row_pitch = dst.row_pitch,
+        .dst_slice_pitch = dst.slice_pitch,
+        .dst_layer_pitch = dst.layer_pitch,
+        .row_size = shape.row_size,
+        .row_count = shape.row_count,
+        .slice_count = shape.slice_count,
+        .layer_count = shape.layer_count,
+    });
+}
+
+pub fn copyBuffer(cmd: *PhiCommandBuffer, src: *base.Buffer, dst: *base.Buffer, regions: []const vk.BufferCopy) VkError!void {
+    const src_memory = try remoteMemory(src);
+    const dst_memory = try remoteMemory(dst);
+
+    for (regions) |region| {
+        const src_offset, const src_overflow = @addWithOverflow(src.offset, region.src_offset);
+        const dst_offset, const dst_overflow = @addWithOverflow(dst.offset, region.dst_offset);
+        if (src_overflow != 0 or dst_overflow != 0)
+            return VkError.ValidationFailed;
+
+        try cmd.appendCommand(proto.PhiCmdCopyBuffer, proto.PHI_CMD_COPY_BUFFER, .{
+            .size = region.size,
+            .src_memory = @intCast(src_memory.remote_handle),
+            .dst_memory = @intCast(dst_memory.remote_handle),
+            .src_offset = src_offset,
+            .dst_offset = dst_offset,
+        });
+    }
+}
+
+pub fn copyBufferImage(cmd: *PhiCommandBuffer, buffer: *base.Buffer, image: *base.Image, region: vk.BufferImageCopy, image_is_dst: bool) VkError!void {
+    if (image.samples.toInt() != 1)
+        return VkError.ValidationFailed;
+
+    try validateImageRegion(image, region.image_subresource, region.image_offset, region.image_extent, false);
+
+    const format = image.formatFromAspect(region.image_subresource.aspect_mask);
+    const buffer_address = try getBufferImageAddress(buffer, format, region);
+    const image_address = try getImageCopyAddress(image, region.image_subresource, region.image_offset);
+
+    const shape = if (image.image_type == .@"3d")
+        try getCopyShape(format, region.image_extent, region.image_extent.depth, 1)
+    else
+        try getCopyShape(format, region.image_extent, 1, region.image_subresource.layer_count);
+
+    try validateBufferRange(buffer, region.buffer_offset, buffer_address, shape);
+
+    const buffer_memory = try remoteMemory(buffer);
+    const image_memory = try remoteImageMemory(image);
+
+    if (image_is_dst) {
+        try appendImageCopy(
+            cmd,
+            proto.PHI_CMD_COPY_BUFFER_TO_IMAGE,
+            buffer_memory,
+            buffer_address,
+            image_memory,
+            image_address,
+            shape,
+        );
+    } else {
+        try appendImageCopy(
+            cmd,
+            proto.PHI_CMD_COPY_IMAGE_TO_BUFFER,
+            image_memory,
+            image_address,
+            buffer_memory,
+            buffer_address,
+            shape,
+        );
+    }
+}
+
+fn copyImageSingleAspect(cmd: *PhiCommandBuffer, src: *base.Image, dst: *base.Image, src_memory: *PhiDeviceMemory, dst_memory: *PhiDeviceMemory, region: vk.ImageCopy) VkError!void {
+    const src_is_3d = src.image_type == .@"3d";
+    const dst_is_3d = dst.image_type == .@"3d";
+    const one_is_3d = src_is_3d != dst_is_3d;
+
+    try validateImageRegion(src, region.src_subresource, region.src_offset, region.extent, one_is_3d);
+    try validateImageRegion(dst, region.dst_subresource, region.dst_offset, region.extent, one_is_3d);
+
+    const src_format = src.formatFromAspect(region.src_subresource.aspect_mask);
+    const dst_format = dst.formatFromAspect(region.dst_subresource.aspect_mask);
+
+    if (base.format.texelSize(src_format) != base.format.texelSize(dst_format) or
+        base.format.blockWidth(src_format) != base.format.blockWidth(dst_format) or
+        base.format.blockHeight(src_format) != base.format.blockHeight(dst_format))
+        return VkError.ValidationFailed;
+
+    var src_address = try getImageCopyAddress(src, region.src_subresource, region.src_offset);
+    var dst_address = try getImageCopyAddress(dst, region.dst_subresource, region.dst_offset);
+
+    const shape: CopyShape = if (src_is_3d and dst_is_3d) blk: {
+        break :blk try getCopyShape(src_format, region.extent, region.extent.depth, 1);
+    } else if (!src_is_3d and !dst_is_3d) blk: {
+        if (region.src_subresource.layer_count != region.dst_subresource.layer_count)
+            return VkError.ValidationFailed;
+
+        break :blk try getCopyShape(
+            src_format,
+            region.extent,
+            @intCast(src.samples.toInt()),
+            region.src_subresource.layer_count,
+        );
+    } else blk: {
+        if (src.samples.toInt() != 1)
+            return VkError.ValidationFailed;
+
+        if (src_is_3d)
+            src_address.layer_pitch = src_address.slice_pitch;
+        if (dst_is_3d)
+            dst_address.layer_pitch = dst_address.slice_pitch;
+
+        break :blk try getCopyShape(src_format, region.extent, 1, region.extent.depth);
+    };
+
+    try appendImageCopy(
+        cmd,
+        proto.PHI_CMD_COPY_IMAGE,
+        src_memory,
+        src_address,
+        dst_memory,
+        dst_address,
+        shape,
+    );
+}
+
+pub fn copyImage(cmd: *PhiCommandBuffer, src: *base.Image, dst: *base.Image, region: vk.ImageCopy) VkError!void {
+    if (src.samples.toInt() != dst.samples.toInt())
+        return VkError.ValidationFailed;
+
+    const src_memory = try remoteImageMemory(src);
+    const dst_memory = try remoteImageMemory(dst);
+
+    const depth_stencil: vk.ImageAspectFlags = .{
+        .depth_bit = true,
+        .stencil_bit = true,
+    };
+
+    if (region.src_subresource.aspect_mask == depth_stencil and
+        region.dst_subresource.aspect_mask == depth_stencil)
+    {
+        var single_aspect_region = region;
+
+        single_aspect_region.src_subresource.aspect_mask = .{
+            .depth_bit = true,
+        };
+        single_aspect_region.dst_subresource.aspect_mask = .{
+            .depth_bit = true,
+        };
+        try copyImageSingleAspect(
+            cmd,
+            src,
+            dst,
+            src_memory,
+            dst_memory,
+            single_aspect_region,
+        );
+
+        single_aspect_region.src_subresource.aspect_mask = .{
+            .stencil_bit = true,
+        };
+        single_aspect_region.dst_subresource.aspect_mask = .{
+            .stencil_bit = true,
+        };
+        try copyImageSingleAspect(
+            cmd,
+            src,
+            dst,
+            src_memory,
+            dst_memory,
+            single_aspect_region,
+        );
+        return;
+    }
+
+    try copyImageSingleAspect(
+        cmd,
+        src,
+        dst,
+        src_memory,
+        dst_memory,
+        region,
+    );
+}
+
+pub fn blitImage(cmd: *PhiCommandBuffer, src: *base.Image, dst: *base.Image, region: vk.ImageBlit, filter: vk.Filter) VkError!void {
+    const src_memory = try remoteImageMemory(src);
+    const dst_memory = try remoteImageMemory(dst);
+
+    var src_offset_0 = region.src_offsets[0];
+    var src_offset_1 = region.src_offsets[1];
+
+    var dst_offset_0 = region.dst_offsets[0];
+    var dst_offset_1 = region.dst_offsets[1];
+
+    if (dst_offset_0.x > dst_offset_1.x) {
+        std.mem.swap(i32, &dst_offset_0.x, &dst_offset_1.x);
+        std.mem.swap(i32, &src_offset_0.x, &src_offset_1.x);
+    }
+
+    if (dst_offset_0.y > dst_offset_1.y) {
+        std.mem.swap(i32, &dst_offset_0.y, &dst_offset_1.y);
+        std.mem.swap(i32, &src_offset_0.y, &src_offset_1.y);
+    }
+
+    if (dst_offset_0.z > dst_offset_1.z) {
+        std.mem.swap(i32, &dst_offset_0.z, &dst_offset_1.z);
+        std.mem.swap(i32, &src_offset_0.z, &src_offset_1.z);
+    }
+
+    const src_extent = try getMipExtent(src, region.src_subresource.mip_level);
+
+    const step_x = @as(f32, @floatFromInt(src_offset_1.x - src_offset_0.x)) / @as(f32, @floatFromInt(dst_offset_1.x - dst_offset_0.x));
+    const step_y = @as(f32, @floatFromInt(src_offset_1.y - src_offset_0.y)) / @as(f32, @floatFromInt(dst_offset_1.y - dst_offset_0.y));
+
+    const step_z = @as(f32, @floatFromInt(src_offset_1.z - src_offset_0.z)) / @as(f32, @floatFromInt(dst_offset_1.z - dst_offset_0.z));
+    const src_x0 = @as(f32, @floatFromInt(src_offset_0.x)) + (0.5 - @as(f32, @floatFromInt(dst_offset_0.x))) * step_x;
+
+    const src_y0 = @as(f32, @floatFromInt(src_offset_0.y)) + (0.5 - @as(f32, @floatFromInt(dst_offset_0.y))) * step_y;
+    const src_z0 = @as(f32, @floatFromInt(src_offset_0.z)) + (0.5 - @as(f32, @floatFromInt(dst_offset_0.z))) * step_z;
+
+    const src_layout = try src.getSubresourceLayout(.{
+        .aspect_mask = region.src_subresource.aspect_mask,
+        .mip_level = region.src_subresource.mip_level,
+        .array_layer = region.src_subresource.base_array_layer,
+    });
+
+    const dst_layout = try dst.getSubresourceLayout(.{
+        .aspect_mask = region.dst_subresource.aspect_mask,
+        .mip_level = region.dst_subresource.mip_level,
+        .array_layer = region.dst_subresource.base_array_layer,
+    });
+
+    const src_format = src.formatFromAspect(region.src_subresource.aspect_mask);
+    const dst_format = dst.formatFromAspect(region.dst_subresource.aspect_mask);
+
+    const apply_filter = (filter != .nearest);
+    const resolve_srgb = src.samples.toInt() > 1 and
+        dst.samples.toInt() == 1 and
+        base.format.isSrgb(src_format) and
+        base.format.isSrgb(dst_format);
+    const allow_srgb_conversion = apply_filter or resolve_srgb or base.format.isSrgb(src_format) != base.format.isSrgb(dst_format);
+
+    const clamp_to_edge = src_offset_0.x < 0 or
+        src_offset_0.y < 0 or
+        @as(u32, @intCast(src_offset_1.x)) > src_extent.width or
+        @as(u32, @intCast(src_offset_1.y)) > src_extent.height or
+        (filter != .nearest and ((src_x0 < 0.5) or (src_y0 < 0.5)));
+
+    try cmd.appendCommand(
+        proto.PhiCmdBlitImage,
+        proto.PHI_CMD_BLIT_IMAGE,
+        .{
+            .src_memory = @intCast(src_memory.remote_handle),
+            .src_offset = src.memory_offset + src_layout.offset,
+            .src_row_pitch = src_layout.row_pitch,
+            .src_slice_pitch = src_layout.depth_pitch,
+            .src_layer_pitch = src_layout.array_pitch,
+
+            .dst_memory = @intCast(dst_memory.remote_handle),
+            .dst_offset = dst.memory_offset + dst_layout.offset,
+            .dst_row_pitch = dst_layout.row_pitch,
+            .dst_slice_pitch = dst_layout.depth_pitch,
+            .dst_layer_pitch = dst_layout.array_pitch,
+
+            .dst_format = @intCast(@as(i32, @intFromEnum(dst_format))),
+            .src_format = @intCast(@as(i32, @intFromEnum(src_format))),
+
+            .src_width = src_extent.width,
+            .src_height = src_extent.height,
+            .src_depth = src_extent.depth,
+
+            .dst_x0 = dst_offset_0.x,
+            .dst_y0 = dst_offset_0.y,
+            .dst_z0 = dst_offset_0.z,
+            .dst_x1 = dst_offset_1.x,
+            .dst_y1 = dst_offset_1.y,
+            .dst_z1 = dst_offset_1.z,
+
+            .src_x0 = src_x0,
+            .src_y0 = src_y0,
+            .src_z0 = src_z0,
+
+            .step_x = step_x,
+            .step_y = step_y,
+            .step_z = step_z,
+
+            .layer_count = region.dst_subresource.layer_count,
+
+            .filter = @intCast(@intFromEnum(filter)),
+            .clamp_to_edge = @intFromBool(clamp_to_edge),
+            .allow_srgb_conversion = @intFromBool(allow_srgb_conversion),
+
+            .reserved = 0,
+        },
+    );
+}

src/phi/lib.zig

@@ -7,6 +7,7 @@ pub const proto = @import("phi_protocol_c");
 pub const config = base.config;
 pub const mic = @import("miclib");
 pub const scif = @import("scif.zig");
+pub const compiler = @import("compiler/compiler.zig");
 
 pub const PhiInstance = @import("PhiInstance.zig");
 pub const PhiDevice = @import("PhiDevice.zig");
@@ -75,6 +76,7 @@ test {
     std.testing.refAllDecls(PhiPhysicalDevice);
     std.testing.refAllDecls(PhiTransport);
     std.testing.refAllDecls(scif);
+    std.testing.refAllDecls(compiler);
     std.testing.refAllDecls(PhiPipeline);
     std.testing.refAllDecls(PhiPipelineCache);
     std.testing.refAllDecls(PhiPipelineLayout);

src/phi/mic/BlitFormats.h

@@ -0,0 +1,43 @@
+#ifndef APE_PHI_BLIT_FORMATS_H
+#define APE_PHI_BLIT_FORMATS_H
+
+#include <Protocol.h>
+#include <stdint.h>
+
+typedef struct PhiBlitFormatInfo
+{
+	uint32_t texel_size;
+	uint8_t is_integer;
+	uint8_t is_signed;
+	uint8_t is_float;
+	uint8_t is_srgb;
+	uint8_t is_unsigned;
+	uint8_t can_read_float;
+	uint8_t can_write_float;
+	uint8_t can_read_int;
+	uint8_t can_write_int;
+	uint8_t vector_unorm8x4;
+} PhiBlitFormatInfo;
+
+typedef struct PhiBlitFloat4
+{
+	float values[4];
+} PhiBlitFloat4;
+
+typedef struct PhiBlitInt4
+{
+	uint32_t values[4];
+} PhiBlitInt4;
+
+int PhiGetBlitFormatInfo(PhiFormat format, PhiBlitFormatInfo* info);
+PhiBlitFloat4 PhiReadBlitFloat4(const uint8_t* map, PhiFormat format);
+void PhiWriteBlitFloat4(PhiBlitFloat4 color, uint8_t* map, PhiFormat format);
+PhiBlitInt4 PhiReadBlitInt4(const uint8_t* map, PhiFormat format);
+void PhiWriteBlitInt4(PhiBlitInt4 color, uint8_t* map, PhiFormat format);
+PhiBlitFloat4 PhiConvertBlitFloat4(PhiBlitFloat4 color,
+                                   PhiFormat src_format,
+                                   PhiFormat dst_format,
+                                   int allow_srgb_conversion,
+                                   int apply_srgb_conversion);
+
+#endif

src/phi/mic/Blitter.c

@@ -0,0 +1,595 @@
+#include <BlitFormats.h>
+#include <Blitter.h>
+#include <Logger.h>
+#include <Memory.h>
+#include <WorkerPool.h>
+
+#include <avx/Avx.h>
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define BLIT_WEIGHT_SCALE 1024.0f
+#define BLIT_PARALLEL_MIN_PIXELS (256u * 1024u)
+#define BLIT_TASK_TARGET_BYTES (64u * 1024u)
+#define BLIT_MAX_GRAIN_ROWS 64u
+
+enum
+{
+	PHI_FILTER_NEAREST = 0,
+	PHI_FILTER_LINEAR = 1,
+};
+
+typedef PhiBlitFloat4 Color;
+typedef PhiBlitFormatInfo FormatInfo;
+
+typedef struct SampleCoordinate
+{
+	uint32_t lo;
+	uint32_t hi;
+	float factor;
+} SampleCoordinate;
+
+typedef struct BlitWork
+{
+	const PhiCmdBlitImage* command;
+	const FormatInfo* src_info;
+	const FormatInfo* dst_info;
+	const uint8_t* src;
+	uint8_t* dst;
+	uint32_t row_count;
+	uint64_t rows_per_layer;
+} BlitWork;
+
+static inline Color LerpColor(Color a, Color b, float factor)
+{
+	Color result;
+	for(uint32_t component = 0; component < 4; ++component)
+		result.values[component] = a.values[component] + (b.values[component] - a.values[component]) * factor;
+	return result;
+}
+
+static inline double ClampCoordinate(float coordinate, uint32_t dimension)
+{
+	const double upper = (double)dimension - 0.5;
+
+	if(coordinate < 0.5f)
+		return 0.5;
+	if((double)coordinate > upper)
+		return upper;
+	return (double)coordinate;
+}
+
+static inline uint32_t GetNearestCoordinate(float coordinate, uint32_t dimension)
+{
+	return (uint32_t)ClampCoordinate(coordinate, dimension);
+}
+
+static SampleCoordinate GetLinearCoordinate(float coordinate, uint32_t dimension)
+{
+	const double source = ClampCoordinate(coordinate, dimension) - 0.5;
+	SampleCoordinate result;
+	result.lo = (uint32_t)source;
+	result.hi = result.lo + 1 < dimension ? result.lo + 1 : result.lo;
+	result.factor = (float)(source - (double)result.lo);
+	return result;
+}
+
+static inline uint32_t QuantizeBlitWeight(float factor)
+{
+	return (uint32_t)(factor * BLIT_WEIGHT_SCALE + 0.5f);
+}
+
+static Color ReadTexel(const uint8_t* src,
+                       PhiFormat format,
+                       uint32_t texel_size,
+                       uint64_t row_pitch,
+                       uint64_t slice_pitch,
+                       uint32_t x,
+                       uint32_t y,
+                       uint32_t z)
+{
+	const uint64_t offset = (uint64_t)z * slice_pitch + (uint64_t)y * row_pitch + (uint64_t)x * texel_size;
+	return PhiReadBlitFloat4(src + (size_t)offset, format);
+}
+
+static inline Color PrepareLinearSample(Color color,
+                                        const PhiCmdBlitImage* command,
+                                        const FormatInfo* src_info,
+                                        int* apply_srgb_conversion)
+{
+	if(command->allow_srgb_conversion && src_info->is_srgb)
+	{
+		*apply_srgb_conversion = 0;
+		return PhiConvertBlitFloat4(color, (PhiFormat)command->src_format, (PhiFormat)command->dst_format, 1, 1);
+	}
+	return color;
+}
+
+static Color
+Sample(const uint8_t* src, const PhiCmdBlitImage* command, const FormatInfo* src_info, float x, float y, float z, int filter_3d)
+{
+	const PhiFormat format = (PhiFormat)command->src_format;
+	int apply_srgb_conversion = 1;
+
+	if(command->filter == PHI_FILTER_NEAREST)
+	{
+		Color color = ReadTexel(src,
+		                        format,
+		                        src_info->texel_size,
+		                        command->src_row_pitch,
+		                        command->src_slice_pitch,
+		                        GetNearestCoordinate(x, command->src_width),
+		                        GetNearestCoordinate(y, command->src_height),
+		                        GetNearestCoordinate(z, command->src_depth));
+		return PhiConvertBlitFloat4(color, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, 1);
+	}
+
+	const SampleCoordinate sample_x = GetLinearCoordinate(x, command->src_width);
+	const SampleCoordinate sample_y = GetLinearCoordinate(y, command->src_height);
+	const SampleCoordinate sample_z = GetLinearCoordinate(z, command->src_depth);
+
+	Color color_0_0 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.lo,
+	                            sample_y.lo,
+	                            sample_z.lo);
+	Color color_0_1 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.hi,
+	                            sample_y.lo,
+	                            sample_z.lo);
+	Color color_1_0 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.lo,
+	                            sample_y.hi,
+	                            sample_z.lo);
+	Color color_1_1 = ReadTexel(src,
+	                            format,
+	                            src_info->texel_size,
+	                            command->src_row_pitch,
+	                            command->src_slice_pitch,
+	                            sample_x.hi,
+	                            sample_y.hi,
+	                            sample_z.lo);
+	color_0_0 = PrepareLinearSample(color_0_0, command, src_info, &apply_srgb_conversion);
+	color_0_1 = PrepareLinearSample(color_0_1, command, src_info, &apply_srgb_conversion);
+	color_1_0 = PrepareLinearSample(color_1_0, command, src_info, &apply_srgb_conversion);
+	color_1_1 = PrepareLinearSample(color_1_1, command, src_info, &apply_srgb_conversion);
+
+	const Color row_0 = LerpColor(color_0_0, color_0_1, sample_x.factor);
+	const Color row_1 = LerpColor(color_1_0, color_1_1, sample_x.factor);
+	const Color slice_0 = LerpColor(row_0, row_1, sample_y.factor);
+
+	if(!filter_3d)
+		return PhiConvertBlitFloat4(
+		    slice_0, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, apply_srgb_conversion);
+
+	Color color_0_0_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.lo,
+	                              sample_y.lo,
+	                              sample_z.hi);
+	Color color_0_1_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.hi,
+	                              sample_y.lo,
+	                              sample_z.hi);
+	Color color_1_0_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.lo,
+	                              sample_y.hi,
+	                              sample_z.hi);
+	Color color_1_1_1 = ReadTexel(src,
+	                              format,
+	                              src_info->texel_size,
+	                              command->src_row_pitch,
+	                              command->src_slice_pitch,
+	                              sample_x.hi,
+	                              sample_y.hi,
+	                              sample_z.hi);
+	color_0_0_1 = PrepareLinearSample(color_0_0_1, command, src_info, &apply_srgb_conversion);
+	color_0_1_1 = PrepareLinearSample(color_0_1_1, command, src_info, &apply_srgb_conversion);
+	color_1_0_1 = PrepareLinearSample(color_1_0_1, command, src_info, &apply_srgb_conversion);
+	color_1_1_1 = PrepareLinearSample(color_1_1_1, command, src_info, &apply_srgb_conversion);
+
+	const Color row_0_1 = LerpColor(color_0_0_1, color_0_1_1, sample_x.factor);
+	const Color row_1_1 = LerpColor(color_1_0_1, color_1_1_1, sample_x.factor);
+	const Color slice_1 = LerpColor(row_0_1, row_1_1, sample_y.factor);
+	Color color = LerpColor(slice_0, slice_1, sample_z.factor);
+	return PhiConvertBlitFloat4(
+	    color, format, (PhiFormat)command->dst_format, command->allow_srgb_conversion, apply_srgb_conversion);
+}
+
+static inline int IsMemoryRangeValid(const Memory* memory, uint64_t offset, uint64_t size)
+{
+	if(memory == NULL || memory->ptr == NULL || offset > memory->size)
+		return 0;
+	return size <= memory->size - offset;
+}
+
+static int ComputeRegionSpan(uint64_t row_pitch,
+                             uint64_t slice_pitch,
+                             uint64_t layer_pitch,
+                             uint64_t row_size,
+                             uint32_t row_count,
+                             uint32_t slice_count,
+                             uint32_t layer_count,
+                             uint64_t* span)
+{
+	uint64_t result = 0;
+	uint64_t term;
+
+	if(row_size == 0 || row_count == 0 || slice_count == 0 || layer_count == 0)
+		return 0;
+	if(__builtin_mul_overflow((uint64_t)row_count - 1, row_pitch, &term) || __builtin_add_overflow(result, term, &result))
+		return 0;
+	if(__builtin_mul_overflow((uint64_t)slice_count - 1, slice_pitch, &term) || __builtin_add_overflow(result, term, &result))
+		return 0;
+	if(__builtin_mul_overflow((uint64_t)layer_count - 1, layer_pitch, &term) || __builtin_add_overflow(result, term, &result))
+		return 0;
+	if(__builtin_add_overflow(result, row_size, &result))
+		return 0;
+
+	*span = result;
+	return 1;
+}
+
+static PhiStatus ValidateCommand(const PhiCmdBlitImage* command,
+                                 const Memory* src_memory,
+                                 const Memory* dst_memory,
+                                 const FormatInfo* src_info,
+                                 const FormatInfo* dst_info)
+{
+	if(command->src_width == 0 || command->src_height == 0 || command->src_depth == 0 || command->layer_count == 0 ||
+	   command->dst_x0 < 0 || command->dst_y0 < 0 || command->dst_z0 < 0 || command->dst_x1 <= command->dst_x0 ||
+	   command->dst_y1 <= command->dst_y0 || command->dst_z1 <= command->dst_z0 || command->filter > PHI_FILTER_LINEAR ||
+	   !__builtin_isfinite(command->src_x0) || !__builtin_isfinite(command->src_y0) || !__builtin_isfinite(command->src_z0) ||
+	   !__builtin_isfinite(command->step_x) || !__builtin_isfinite(command->step_y) || !__builtin_isfinite(command->step_z))
+	{
+		LogError("Invalid blit image dimensions, coordinates, or filter");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(command->src_offset > SIZE_MAX || command->dst_offset > SIZE_MAX || command->src_row_pitch > SIZE_MAX ||
+	   command->src_slice_pitch > SIZE_MAX || command->src_layer_pitch > SIZE_MAX || command->dst_row_pitch > SIZE_MAX ||
+	   command->dst_slice_pitch > SIZE_MAX || command->dst_layer_pitch > SIZE_MAX)
+	{
+		LogError("Blit image address does not fit in size_t");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	uint64_t src_row_size;
+	uint64_t src_slice_span;
+	uint64_t src_volume_span;
+	uint64_t src_span;
+	if(__builtin_mul_overflow((uint64_t)command->src_width, src_info->texel_size, &src_row_size) ||
+	   command->src_row_pitch < src_row_size ||
+	   !ComputeRegionSpan(command->src_row_pitch, 0, 0, src_row_size, command->src_height, 1, 1, &src_slice_span) ||
+	   command->src_slice_pitch < src_slice_span ||
+	   !ComputeRegionSpan(command->src_row_pitch,
+	                      command->src_slice_pitch,
+	                      0,
+	                      src_row_size,
+	                      command->src_height,
+	                      command->src_depth,
+	                      1,
+	                      &src_volume_span) ||
+	   command->src_layer_pitch < src_volume_span ||
+	   !ComputeRegionSpan(command->src_row_pitch,
+	                      command->src_slice_pitch,
+	                      command->src_layer_pitch,
+	                      src_row_size,
+	                      command->src_height,
+	                      command->src_depth,
+	                      command->layer_count,
+	                      &src_span))
+	{
+		LogError("Invalid blit image source pitches");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	uint64_t dst_row_size;
+	uint64_t dst_slice_span;
+	uint64_t dst_volume_span;
+	uint64_t dst_span;
+	if(__builtin_mul_overflow((uint64_t)(uint32_t)command->dst_x1, dst_info->texel_size, &dst_row_size) ||
+	   command->dst_row_pitch < dst_row_size ||
+	   !ComputeRegionSpan(command->dst_row_pitch, 0, 0, dst_row_size, (uint32_t)command->dst_y1, 1, 1, &dst_slice_span) ||
+	   command->dst_slice_pitch < dst_slice_span ||
+	   !ComputeRegionSpan(command->dst_row_pitch,
+	                      command->dst_slice_pitch,
+	                      0,
+	                      dst_row_size,
+	                      (uint32_t)command->dst_y1,
+	                      (uint32_t)command->dst_z1,
+	                      1,
+	                      &dst_volume_span) ||
+	   command->dst_layer_pitch < dst_volume_span ||
+	   !ComputeRegionSpan(command->dst_row_pitch,
+	                      command->dst_slice_pitch,
+	                      command->dst_layer_pitch,
+	                      dst_row_size,
+	                      (uint32_t)command->dst_y1,
+	                      (uint32_t)command->dst_z1,
+	                      command->layer_count,
+	                      &dst_span))
+	{
+		LogError("Invalid blit image destination pitches");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(src_span > SIZE_MAX || dst_span > SIZE_MAX || !IsMemoryRangeValid(src_memory, command->src_offset, src_span) ||
+	   !IsMemoryRangeValid(dst_memory, command->dst_offset, dst_span))
+	{
+		LogError("Blit image memory range is invalid");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	return PHI_STATUS_OK;
+}
+
+static inline void BlitScalarPixel(uint8_t* dst,
+                                   const uint8_t* src,
+                                   const PhiCmdBlitImage* command,
+                                   const FormatInfo* src_info,
+                                   const FormatInfo* dst_info,
+                                   float source_x,
+                                   float source_y,
+                                   float source_z,
+                                   int filter_3d)
+{
+	if(src_info->is_integer && dst_info->is_integer)
+	{
+		const uint32_t x = GetNearestCoordinate(source_x, command->src_width);
+		const uint32_t y = GetNearestCoordinate(source_y, command->src_height);
+		const uint32_t z = GetNearestCoordinate(source_z, command->src_depth);
+		const uint64_t offset =
+		    (uint64_t)z * command->src_slice_pitch + (uint64_t)y * command->src_row_pitch + (uint64_t)x * src_info->texel_size;
+		const PhiBlitInt4 color = PhiReadBlitInt4(src + (size_t)offset, (PhiFormat)command->src_format);
+		PhiWriteBlitInt4(color, dst, (PhiFormat)command->dst_format);
+		return;
+	}
+
+	const Color color = Sample(src, command, src_info, source_x, source_y, source_z, filter_3d);
+	PhiWriteBlitFloat4(color, dst, (PhiFormat)command->dst_format);
+}
+
+static void BlitRow(uint8_t* dst,
+                    const uint8_t* src,
+                    const PhiCmdBlitImage* command,
+                    const FormatInfo* src_info,
+                    const FormatInfo* dst_info,
+                    float source_y,
+                    float source_z)
+{
+	const uint32_t pixel_count = (uint32_t)(command->dst_x1 - command->dst_x0);
+	const float first_source_x = command->src_x0 + (float)command->dst_x0 * command->step_x;
+	const int filter_3d = command->step_z != 1.0f;
+
+	// Fast path
+	if(command->filter == PHI_FILTER_NEAREST && command->src_format == command->dst_format && command->step_x == 1.0f &&
+	   !(command->allow_srgb_conversion && (src_info->is_srgb || dst_info->is_srgb)) && first_source_x >= 0.0f &&
+	   first_source_x + (float)(pixel_count - 1) < (float)command->src_width)
+	{
+		const uint32_t source_texel_x = GetNearestCoordinate(first_source_x, command->src_width);
+		const uint32_t source_texel_y = GetNearestCoordinate(source_y, command->src_height);
+		const uint32_t source_texel_z = GetNearestCoordinate(source_z, command->src_depth);
+		const uint64_t source_offset = (uint64_t)source_texel_z * command->src_slice_pitch +
+		                               (uint64_t)source_texel_y * command->src_row_pitch +
+		                               (uint64_t)source_texel_x * src_info->texel_size;
+		AvxCopy(dst, src + (size_t)source_offset, (size_t)pixel_count * src_info->texel_size);
+		return;
+	}
+
+	uint32_t processed = 0;
+	if(src_info->vector_unorm8x4 && dst_info->vector_unorm8x4 && command->src_width <= INT32_MAX)
+	{
+		while(processed < pixel_count && ((uintptr_t)(dst + (size_t)processed * 4) & 63) != 0)
+		{
+			const int32_t destination_x = command->dst_x0 + (int32_t)processed;
+			const float source_x = command->src_x0 + (float)destination_x * command->step_x;
+			BlitScalarPixel(
+			    dst + (size_t)processed * 4, src, command, src_info, dst_info, source_x, source_y, source_z, filter_3d);
+			++processed;
+		}
+
+		_Alignas(64) uint32_t source_x0[16];
+		_Alignas(64) uint32_t source_x1[16];
+		_Alignas(64) uint32_t weights_x[16];
+
+		if(command->filter == PHI_FILTER_NEAREST)
+		{
+			const uint32_t source_row = GetNearestCoordinate(source_y, command->src_height);
+			const uint32_t source_slice = GetNearestCoordinate(source_z, command->src_depth);
+			const uint64_t source_offset =
+			    (uint64_t)source_slice * command->src_slice_pitch + (uint64_t)source_row * command->src_row_pitch;
+			const uint8_t* src_row = src + (size_t)source_offset;
+
+			while(pixel_count - processed >= 16)
+			{
+				for(uint32_t lane = 0; lane < 16; ++lane)
+				{
+					const int32_t destination_x = command->dst_x0 + (int32_t)(processed + lane);
+					const float source_x = command->src_x0 + (float)destination_x * command->step_x;
+					source_x0[lane] = GetNearestCoordinate(source_x, command->src_width);
+				}
+
+				AvxBlitNearestUnorm8x4(
+				    dst + (size_t)processed * 4, src_row, source_x0, command->src_format, command->dst_format);
+				processed += 16;
+			}
+		}
+		else if(!filter_3d)
+		{
+			const SampleCoordinate sample_y = GetLinearCoordinate(source_y, command->src_height);
+			const SampleCoordinate sample_z = GetLinearCoordinate(source_z, command->src_depth);
+			const uint64_t slice_offset = (uint64_t)sample_z.lo * command->src_slice_pitch;
+			const uint8_t* src_row_0 = src + (size_t)(slice_offset + (uint64_t)sample_y.lo * command->src_row_pitch);
+			const uint8_t* src_row_1 = src + (size_t)(slice_offset + (uint64_t)sample_y.hi * command->src_row_pitch);
+			const uint32_t weight_y = QuantizeBlitWeight(sample_y.factor);
+
+			while(pixel_count - processed >= 16)
+			{
+				for(uint32_t lane = 0; lane < 16; ++lane)
+				{
+					const int32_t destination_x = command->dst_x0 + (int32_t)(processed + lane);
+					const float source_x = command->src_x0 + (float)destination_x * command->step_x;
+					const SampleCoordinate sample_x = GetLinearCoordinate(source_x, command->src_width);
+					source_x0[lane] = sample_x.lo;
+					source_x1[lane] = sample_x.hi;
+					weights_x[lane] = QuantizeBlitWeight(sample_x.factor);
+				}
+
+				AvxBlitLinearUnorm8x4(dst + (size_t)processed * 4,
+				                      src_row_0,
+				                      src_row_1,
+				                      source_x0,
+				                      source_x1,
+				                      weights_x,
+				                      weight_y,
+				                      command->src_format,
+				                      command->dst_format);
+				processed += 16;
+			}
+		}
+	}
+
+	while(processed < pixel_count)
+	{
+		const int32_t destination_x = command->dst_x0 + (int32_t)processed;
+		const float source_x = command->src_x0 + (float)destination_x * command->step_x;
+		BlitScalarPixel(dst + (size_t)processed * dst_info->texel_size,
+		                src,
+		                command,
+		                src_info,
+		                dst_info,
+		                source_x,
+		                source_y,
+		                source_z,
+		                filter_3d);
+		++processed;
+	}
+}
+
+static void BlitRows(void* context, uint64_t begin, uint64_t end)
+{
+	const BlitWork* work = context;
+	const PhiCmdBlitImage* command = work->command;
+
+	for(uint64_t row = begin; row < end; ++row)
+	{
+		const uint32_t layer = (uint32_t)(row / work->rows_per_layer);
+		const uint64_t row_in_layer = row - (uint64_t)layer * work->rows_per_layer;
+		const uint32_t slice = (uint32_t)(row_in_layer / work->row_count);
+		const uint32_t row_in_slice = (uint32_t)(row_in_layer - (uint64_t)slice * work->row_count);
+		const int32_t z = command->dst_z0 + (int32_t)slice;
+		const int32_t y = command->dst_y0 + (int32_t)row_in_slice;
+		const float source_z = command->src_z0 + (float)z * command->step_z;
+		const float source_y = command->src_y0 + (float)y * command->step_y;
+
+		const uint8_t* src_layer = work->src + (size_t)((uint64_t)layer * command->src_layer_pitch);
+		uint8_t* dst_layer = work->dst + (size_t)((uint64_t)layer * command->dst_layer_pitch);
+		uint8_t* dst_slice = dst_layer + (size_t)((uint64_t)(uint32_t)z * command->dst_slice_pitch);
+		const uint64_t row_offset =
+		    (uint64_t)(uint32_t)y * command->dst_row_pitch + (uint64_t)(uint32_t)command->dst_x0 * work->dst_info->texel_size;
+
+		BlitRow(dst_slice + (size_t)row_offset, src_layer, command, work->src_info, work->dst_info, source_y, source_z);
+	}
+}
+
+PhiStatus BlitImage(const PhiCmdBlitImage* command)
+{
+	if(command->src_memory == 0 || command->dst_memory == 0)
+	{
+		LogError("Invalid blit image memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
+
+	const Memory* src_memory = (const Memory*)(uintptr_t)command->src_memory;
+	Memory* dst_memory = (Memory*)(uintptr_t)command->dst_memory;
+	FormatInfo src_info;
+	FormatInfo dst_info;
+
+	if(!PhiGetBlitFormatInfo((PhiFormat)command->src_format, &src_info) ||
+	   !PhiGetBlitFormatInfo((PhiFormat)command->dst_format, &dst_info))
+	{
+		LogErrorFmt("Unsupported blit image formats: src=%u dst=%u", command->src_format, command->dst_format);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	const int integer_path = src_info.is_integer && dst_info.is_integer;
+	if((integer_path && (!src_info.can_read_int || !dst_info.can_write_int)) ||
+	   (!integer_path && (!src_info.can_read_float || !dst_info.can_write_float)))
+	{
+		LogErrorFmt("Unsupported blit image format direction: src=%u dst=%u", command->src_format, command->dst_format);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	PhiStatus status = ValidateCommand(command, src_memory, dst_memory, &src_info, &dst_info);
+	if(status != PHI_STATUS_OK)
+		return status;
+
+	const uint8_t* src = (const uint8_t*)src_memory->ptr + (size_t)command->src_offset;
+	uint8_t* dst = (uint8_t*)dst_memory->ptr + (size_t)command->dst_offset;
+	const uint32_t row_count = (uint32_t)(command->dst_y1 - command->dst_y0);
+	const uint32_t slice_count = (uint32_t)(command->dst_z1 - command->dst_z0);
+	const uint32_t width = (uint32_t)(command->dst_x1 - command->dst_x0);
+	uint64_t rows_per_layer;
+	uint64_t total_rows;
+	uint64_t total_pixels;
+
+	if(__builtin_mul_overflow((uint64_t)row_count, slice_count, &rows_per_layer) ||
+	   __builtin_mul_overflow(rows_per_layer, command->layer_count, &total_rows) ||
+	   __builtin_mul_overflow(total_rows, width, &total_pixels))
+	{
+		LogError("Blit image work size overflow");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	const BlitWork work = {
+		.command = command,
+		.src_info = &src_info,
+		.dst_info = &dst_info,
+		.src = src,
+		.dst = dst,
+		.row_count = row_count,
+		.rows_per_layer = rows_per_layer,
+	};
+
+	uint64_t row_bytes;
+	if(__builtin_mul_overflow((uint64_t)width, dst_info.texel_size, &row_bytes))
+	{
+		LogError("Blit image row size overflow");
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	uint64_t grain_rows = row_bytes < BLIT_TASK_TARGET_BYTES ? BLIT_TASK_TARGET_BYTES / row_bytes : 1;
+	if(grain_rows > BLIT_MAX_GRAIN_ROWS)
+		grain_rows = BLIT_MAX_GRAIN_ROWS;
+
+	if(src_memory != dst_memory && total_pixels >= BLIT_PARALLEL_MIN_PIXELS && WorkerPoolGetWorkerCount() != 0)
+		WorkerPoolParallelFor(total_rows, grain_rows, BlitRows, (void*)&work);
+	else
+		BlitRows((void*)&work, 0, total_rows);
+
+	return PHI_STATUS_OK;
+}

src/phi/mic/Blitter.h

@@ -0,0 +1,9 @@
+#define APE_PHI_BLITTER_H
+#ifndef APE_PHI_BUFFER_H
+
+#include <Commands.h>
+#include <Protocol.h>
+
+PhiStatus BlitImage(const PhiCmdBlitImage* command);
+
+#endif

src/phi/mic/Buffer.c

@@ -1,30 +1,26 @@
 #include <Buffer.h>
+#include <Logger.h>
 #include <Memory.h>
 
 #include <avx/Avx.h>
 
-int PhiIsBufferCommand(const PhiCmdHeader* header)
-{
-	switch((PhiCmdType)header->type)
-	{
-		case PHI_CMD_COPY_BUFFER:
-		case PHI_CMD_FILL_BUFFER:
-			return 1;
-
-		default:
-			return 0;
-	}
-}
-
 static PhiStatus CopyBuffer(PhiCommandReader* reader)
 {
 	PhiCmdCopyBuffer command;
-	PhiStatus status = PhiReadCommandData(reader, &command, sizeof(command));
+	PhiStatus status = ReadCommandData(reader, &command, sizeof(command));
 	if(status != PHI_STATUS_OK)
 		return status;
 
-	if(command.src_memory == 0 || command.dst_memory == 0)
+	if(command.src_memory == 0)
+	{
+		LogError("Invalid src memory handle");
 		return PHI_STATUS_INVALID_HANDLE;
+	}
+	if(command.dst_memory == 0)
+	{
+		LogError("Invalid dst memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
 
 	Memory* dst_memory = (Memory*)command.dst_memory;
 	const Memory* src_memory = (const Memory*)command.src_memory;
@@ -41,13 +37,16 @@ static PhiStatus FillBuffer(PhiCommandReader* reader)
 {
 	PhiCmdFillBuffer command;
 
-	PhiStatus status = PhiReadCommandData(reader, &command, sizeof(command));
+	PhiStatus status = ReadCommandData(reader, &command, sizeof(command));
 
 	if(status != PHI_STATUS_OK)
 		return status;
 
 	if(command.memory == 0)
+	{
+		LogErrorFmt("Invalid memory handle: %p", command.memory);
 		return PHI_STATUS_INVALID_HANDLE;
+	}
 
 	Memory* memory = (Memory*)command.memory;
 
@@ -57,8 +56,12 @@ static PhiStatus FillBuffer(PhiCommandReader* reader)
 	const uint32_t value = command.data;
 
 	// Check if dst and size are 4-byte aligned
-	if((((uintptr_t)dst | size) & 3) != 0)
+	uintptr_t alignment = ((uintptr_t)dst | size) & 3;
+	if(alignment != 0)
+	{
+		LogErrorFmt("Invalid memory alignment: %d", alignment);
 		return PHI_STATUS_INVALID_ARGUMENT;
+	}
 
 	// Bring dst to a 64-byte cache-line boundary.
 	while(size >= 4 && ((uintptr_t)dst & 63) != 0)
@@ -95,7 +98,20 @@ static PhiStatus FillBuffer(PhiCommandReader* reader)
 	return PHI_STATUS_OK;
 }
 
-PhiStatus PhiExecuteBufferCommand(PhiCommandReader* reader, const PhiCmdHeader* header)
+int IsBufferCommand(const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER:
+		case PHI_CMD_FILL_BUFFER:
+			return 1;
+
+		default:
+			return 0;
+	}
+}
+
+PhiStatus ExecuteBufferCommand(PhiCommandReader* reader, const PhiCmdHeader* header)
 {
 	switch((PhiCmdType)header->type)
 	{

src/phi/mic/Buffer.h

@@ -3,7 +3,7 @@
 
 #include <CommandBuffer.h>
 
-int PhiIsBufferCommand(const PhiCmdHeader* header);
-PhiStatus PhiExecuteBufferCommand(PhiCommandReader* reader, const PhiCmdHeader* header);
+int IsBufferCommand(const PhiCmdHeader* header);
+PhiStatus ExecuteBufferCommand(PhiCommandReader* reader, const PhiCmdHeader* header);
 
 #endif

src/phi/mic/CommandBuffer.c

@@ -1,24 +1,46 @@
 #include <CommandBuffer.h>
+#include <Logger.h>
 
 #include <Buffer.h>
+#include <Image.h>
 
-PhiStatus PhiReadCommandData(PhiCommandReader* reader, void* data, uint64_t size)
+#include <string.h>
+
+static const char* CommandName[] = {
+	"CopyBuffer", "FillBuffer", "CopyBufferToImage", "CopyImageToBuffer", "CopyImage", "BlitImage",
+};
+
+PhiStatus ReadCommandData(PhiCommandReader* reader, void* data, uint64_t size)
 {
 	if(reader->remaining < size)
 		return PHI_STATUS_BAD_MESSAGE;
 
-	if(ReadAll(reader->endpoint, data, (size_t)size) < 0)
+	if(reader->memory != NULL)
+	{
+		memcpy(data, reader->memory, (size_t)size);
+		reader->memory += (size_t)size;
+	}
+	else if(ReadAll(reader->endpoint, data, (size_t)size) < 0)
+	{
 		return PHI_STATUS_BAD_MESSAGE;
+	}
 
 	reader->remaining -= size;
 	return PHI_STATUS_OK;
 }
 
-int PhiDrainCommandReader(PhiCommandReader* reader)
+int DrainCommandReader(PhiCommandReader* reader)
 {
 	if(reader->remaining == 0)
 		return 0;
 
+	if(reader->memory != NULL)
+	{
+		reader->memory += (size_t)reader->remaining;
+		reader->remaining = 0;
+		return 0;
+	}
+
 	int result = DrainPayload(reader->endpoint, reader->remaining);
 	reader->remaining = 0;
 	return result;
@@ -26,7 +48,7 @@ int PhiDrainCommandReader(PhiCommandReader* reader)
 
 static PhiStatus ReadCommandHeader(PhiCommandReader* reader, PhiCmdHeader* command_header)
 {
-	PhiStatus status = PhiReadCommandData(reader, command_header, sizeof(*command_header));
+	PhiStatus status = ReadCommandData(reader, command_header, sizeof(*command_header));
 	if(status != PHI_STATUS_OK)
 		return status;
 
@@ -38,12 +60,54 @@ static PhiStatus ReadCommandHeader(PhiCommandReader* reader, PhiCmdHeader* comma
 
 static PhiStatus ExecuteCommand(PhiCommandReader* reader, const PhiCmdHeader* command_header)
 {
-	if(PhiIsBufferCommand(command_header))
-		return PhiExecuteBufferCommand(reader, command_header);
+	if(IsBufferCommand(command_header))
+		return ExecuteBufferCommand(reader, command_header);
+
+	if(IsImageCommand(command_header))
+		return ExecuteImageCommand(reader, command_header);
 
 	return PHI_STATUS_BAD_MESSAGE;
 }
 
+static PhiStatus ExecuteCommands(PhiCommandReader* reader, uint64_t cmd_count)
+{
+	PhiStatus status = PHI_STATUS_OK;
+
+	for(uint64_t cmd_index = 0; cmd_index < cmd_count; ++cmd_index)
+	{
+		PhiCmdHeader cmd_header;
+		status = ReadCommandHeader(reader, &cmd_header);
+		if(status != PHI_STATUS_OK)
+			break;
+
+		status = ExecuteCommand(reader, &cmd_header);
+		if(status != PHI_STATUS_OK)
+		{
+			const size_t command_name_count = sizeof(CommandName) / sizeof(CommandName[0]);
+			const char* command_name = cmd_header.type < command_name_count ? CommandName[cmd_header.type] : "Unknown";
+			LogErrorFmt("Command %s execution failed: %s", command_name, StatusName[status]);
+			break;
+		}
+	}
+
+	return status;
+}
+
+PhiStatus ExecuteCommandBuffer(const void* data, uint64_t size, uint64_t cmd_count)
+{
+	PhiCommandReader reader = {
+		.endpoint = PHI_ENDPOINT_INVALID,
+		.memory = data,
+		.remaining = size,
+	};
+
+	const PhiStatus status = ExecuteCommands(&reader, cmd_count);
+	if(status != PHI_STATUS_OK)
+		return status;
+
+	return reader.remaining == 0 ? PHI_STATUS_OK : PHI_STATUS_BAD_MESSAGE;
+}
+
 int HandleWorkExecution(PhiEndpoint endpoint, const PhiMessageHeader* header)
 {
 	PhiWorkExecutionRequest request;
@@ -66,30 +130,21 @@ int HandleWorkExecution(PhiEndpoint endpoint, const PhiMessageHeader* header)
 
 	PhiCommandReader reader = {
 		.endpoint = endpoint,
+		.memory = NULL,
 		.remaining = header->payload_size - sizeof(request),
 	};
 
 	if(reader.remaining != request.command_buffer_size)
 	{
-		if(PhiDrainCommandReader(&reader) < 0)
+		if(DrainCommandReader(&reader) < 0)
 			return -1;
 		reply.result.status = PHI_STATUS_BAD_MESSAGE;
 		return SendReply(endpoint, header, &reply, sizeof(reply));
 	}
 
-	for(uint64_t cmd_index = 0; cmd_index < request.cmd_count; ++cmd_index)
-	{
-		PhiCmdHeader cmd_header;
-		reply.result.status = ReadCommandHeader(&reader, &cmd_header);
-		if(reply.result.status != PHI_STATUS_OK)
-			break;
+	reply.result.status = ExecuteCommands(&reader, request.cmd_count);
 
-		reply.result.status = ExecuteCommand(&reader, &cmd_header);
-		if(reply.result.status != PHI_STATUS_OK)
-			break;
-	}
-
-	if(reader.remaining > 0 && PhiDrainCommandReader(&reader) < 0)
+	if(reader.remaining > 0 && DrainCommandReader(&reader) < 0)
 		return -1;
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));

src/phi/mic/CommandBuffer.h

@@ -6,11 +6,13 @@
 typedef struct PhiCommandReader
 {
 	PhiEndpoint endpoint;
+	const uint8_t* memory;
 	uint64_t remaining;
 } PhiCommandReader;
 
 int HandleWorkExecution(PhiEndpoint endpoint, const PhiMessageHeader* header);
-int PhiDrainCommandReader(PhiCommandReader* reader);
-PhiStatus PhiReadCommandData(PhiCommandReader* reader, void* data, uint64_t size);
+int DrainCommandReader(PhiCommandReader* reader);
+PhiStatus ExecuteCommandBuffer(const void* data, uint64_t size, uint64_t cmd_count);
+PhiStatus ReadCommandData(PhiCommandReader* reader, void* data, uint64_t size);
 
 #endif

src/phi/mic/Daemon.c

@@ -3,6 +3,7 @@
 #include <Daemon.h>
 #include <Logger.h>
 #include <Memory.h>
+#include <Queue.h>
 
 static int HandleHello(PhiEndpoint endpoint, const PhiMessageHeader* header)
 {
@@ -35,20 +36,23 @@ static int HandleHello(PhiEndpoint endpoint, const PhiMessageHeader* header)
 
 PhiEndpoint StartDaemon(void)
 {
-	PhiLogInfo("Starting the daemon...");
+	LogInfo("Starting the daemon...");
 
-	PhiEndpoint endpoint = PhiTransportListen(PHI_TRANSPORT_PORT);
+	PhiEndpoint endpoint = TransportListen(PHI_TRANSPORT_PORT);
 	if(endpoint == PHI_ENDPOINT_INVALID)
-		PhiLogError("Could not listen on the Phi transport");
+	{
+		LogError("Could not listen on the Phi transport");
+		return 0;
+	}
 
-	PhiLogInfo("Daemon started");
+	LogInfo("Daemon started");
 	return endpoint;
 }
 
 void ShutdownDaemon(PhiEndpoint endpoint)
 {
-	PhiLogInfo("Shutting down the daemon...");
-	PhiTransportClose(endpoint);
+	LogInfo("Shutting down the daemon...");
+	TransportClose(endpoint);
 }
 
 int HandlePacket(PhiEndpoint endpoint)
@@ -92,6 +96,9 @@ int HandlePacket(PhiEndpoint endpoint)
 					return -1;
 				break;
 
+			case PHI_PACKET_QUEUE_SETUP:
+				return HandleQueueSetup(endpoint, &header);
+
 			case PHI_PACKET_SHUTDOWN:
 				if(DrainPayload(endpoint, header.payload_size) < 0)
 					return -1;
@@ -116,7 +123,7 @@ int ReadAll(PhiEndpoint endpoint, void* data, size_t size)
 
 	while(offset < size)
 	{
-		ssize_t got = PhiTransportReceive(endpoint, bytes + offset, size - offset);
+		ssize_t got = TransportReceive(endpoint, bytes + offset, size - offset);
 		if(got <= 0)
 			return -1;
 		offset += (size_t)got;
@@ -132,7 +139,7 @@ int WriteAll(PhiEndpoint endpoint, const void* data, size_t size)
 
 	while(offset < size)
 	{
-		ssize_t sent = PhiTransportSend(endpoint, bytes + offset, size - offset);
+		ssize_t sent = TransportSend(endpoint, bytes + offset, size - offset);
 		if(sent <= 0)
 			return -1;
 		offset += (size_t)sent;

src/phi/mic/Image.c

@@ -0,0 +1,379 @@
+#include "Commands.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#include <Blitter.h>
+#include <Image.h>
+#include <Logger.h>
+#include <Memory.h>
+
+#include <avx/Avx.h>
+
+static int GetRegionSpan(uint64_t row_pitch,
+                         uint64_t slice_pitch,
+                         uint64_t layer_pitch,
+                         uint64_t row_size,
+                         uint32_t row_count,
+                         uint32_t slice_count,
+                         uint32_t layer_count,
+                         uint64_t* span)
+{
+	uint64_t result = 0;
+	uint64_t term = 0;
+
+	if(row_size == 0 || row_count == 0 || slice_count == 0 || layer_count == 0)
+		return 0;
+
+	if(row_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)row_count - 1, row_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(slice_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)slice_count - 1, slice_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(layer_count > 1)
+	{
+		if(__builtin_mul_overflow((uint64_t)layer_count - 1, layer_pitch, &term))
+			return 0;
+		if(__builtin_add_overflow(result, term, &result))
+			return 0;
+	}
+
+	if(__builtin_add_overflow(result, row_size, &result))
+		return 0;
+
+	*span = result;
+	return 1;
+}
+
+static inline int IsMemoryRangeValid(const Memory* memory, uint64_t offset, uint64_t size)
+{
+	if(memory == NULL)
+		return 0;
+	if(offset > memory->size)
+		return 0;
+	return size <= memory->size - offset;
+}
+
+static PhiStatus ValidateCopyCommand(const PhiCmdCopyImage* command, const Memory* src_memory, const Memory* dst_memory)
+{
+	uint64_t src_span;
+	uint64_t dst_span;
+
+	if(command->row_size == 0 || command->row_count == 0 || command->slice_count == 0 || command->layer_count == 0)
+	{
+		LogErrorFmt("Invalid image copy command: one of this arguments is zero: row_size=%lu row_count=%u slice_count=%u"
+		            "layer_count=%u",
+		            command->row_size,
+		            command->row_count,
+		            command->slice_count,
+		            command->layer_count);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(command->row_count > 1)
+	{
+		if(command->src_row_pitch < command->row_size || command->dst_row_pitch < command->row_size)
+		{
+			LogErrorFmt("Invalid image copy command: row_size=%lu is larger than src_row_pitch=%lu or dst_row_pitch=%lu",
+			            command->row_size,
+			            command->src_row_pitch,
+			            command->dst_row_pitch);
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+	}
+
+	if(!GetRegionSpan(command->src_row_pitch,
+	                  command->src_slice_pitch,
+	                  command->src_layer_pitch,
+	                  command->row_size,
+	                  command->row_count,
+	                  command->slice_count,
+	                  command->layer_count,
+	                  &src_span))
+	{
+		LogErrorFmt("Invalid image copy command: computed src region span is zero: src_row_pitch=%lu src_slice_pitch=%lu "
+		            "src_layer_pitch=%lu",
+		            command->src_row_pitch,
+		            command->src_slice_pitch,
+		            command->src_layer_pitch);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!GetRegionSpan(command->dst_row_pitch,
+	                  command->dst_slice_pitch,
+	                  command->dst_layer_pitch,
+	                  command->row_size,
+	                  command->row_count,
+	                  command->slice_count,
+	                  command->layer_count,
+	                  &dst_span))
+	{
+		LogErrorFmt("Invalid image copy command: computed dst region span is zero: dst_row_pitch=%lu dst_slice_pitch=%lu "
+		            "dst_layer_pitch=%lu",
+		            command->dst_row_pitch,
+		            command->dst_slice_pitch,
+		            command->dst_layer_pitch);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!IsMemoryRangeValid(src_memory, command->src_offset, src_span))
+	{
+		LogErrorFmt("Invalid image copy command: src memory range is invalid: src_offset=%lu src_span=%lu",
+		            command->src_offset,
+		            src_span);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(!IsMemoryRangeValid(dst_memory, command->dst_offset, dst_span))
+	{
+		LogErrorFmt("Invalid image copy command: dst memory range is invalid: dst_offset=%lu dst_span=%lu",
+		            command->dst_offset,
+		            dst_span);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	if(command->src_offset > SIZE_MAX || command->dst_offset > SIZE_MAX || command->src_row_pitch > SIZE_MAX ||
+	   command->dst_row_pitch > SIZE_MAX || command->src_slice_pitch > SIZE_MAX || command->dst_slice_pitch > SIZE_MAX ||
+	   command->src_layer_pitch > SIZE_MAX || command->dst_layer_pitch > SIZE_MAX || command->row_size > SIZE_MAX)
+	{
+		LogErrorFmt(
+		    "Invalid image copy command: size_t overflow: src_offset=%lu dst_offset=%lu src_row_pitch=%lu dst_row_pitch=%lu "
+		    "src_slice_pitch=%lu dst_slice_pitch=%lu src_layer_pitch=%lu dst_layer_pitch=%lu row_size=%lu",
+		    command->src_offset,
+		    command->dst_offset,
+		    command->src_row_pitch,
+		    command->dst_row_pitch,
+		    command->src_slice_pitch,
+		    command->dst_slice_pitch,
+		    command->src_layer_pitch,
+		    command->dst_layer_pitch,
+		    command->row_size);
+		return PHI_STATUS_INVALID_ARGUMENT;
+	}
+
+	return PHI_STATUS_OK;
+}
+
+static inline int GetTightSliceSize(const PhiCmdCopyImage* command, uint64_t* slice_size)
+{
+	return !__builtin_mul_overflow(command->row_size, command->row_count, slice_size);
+}
+
+static inline int GetTightLayerSize(const PhiCmdCopyImage* command, uint64_t* layer_size)
+{
+	uint64_t slice_size;
+	if(!GetTightSliceSize(command, &slice_size))
+		return 0;
+	return !__builtin_mul_overflow(slice_size, command->slice_count, layer_size);
+}
+
+static inline int GetTightCopySize(const PhiCmdCopyImage* command, uint64_t* copy_size)
+{
+	uint64_t layer_size;
+	if(!GetTightLayerSize(command, &layer_size))
+		return 0;
+	return !__builtin_mul_overflow(layer_size, command->layer_count, copy_size);
+}
+
+static inline int RowsAreContiguous(const PhiCmdCopyImage* command)
+{
+	if(command->row_count <= 1)
+		return 1;
+	return command->src_row_pitch == command->row_size && command->dst_row_pitch == command->row_size;
+}
+
+static int SlicesAreContiguous(const PhiCmdCopyImage* command)
+{
+	uint64_t slice_size;
+
+	if(!RowsAreContiguous(command))
+		return 0;
+	if(command->slice_count <= 1)
+		return 1;
+	if(!GetTightSliceSize(command, &slice_size))
+		return 0;
+
+	return command->src_slice_pitch == slice_size && command->dst_slice_pitch == slice_size;
+}
+
+static int LayersAreContiguous(const PhiCmdCopyImage* command)
+{
+	uint64_t layer_size;
+
+	if(!SlicesAreContiguous(command))
+		return 0;
+	if(command->layer_count <= 1)
+		return 1;
+	if(!GetTightLayerSize(command, &layer_size))
+		return 0;
+
+	return command->src_layer_pitch == layer_size && command->dst_layer_pitch == layer_size;
+}
+
+static PhiStatus CopyImageRegion(const PhiCmdCopyImage* command)
+{
+	if(command->src_memory == 0)
+	{
+		LogError("Invalid src memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
+	if(command->dst_memory == 0)
+	{
+		LogError("Invalid dst memory handle");
+		return PHI_STATUS_INVALID_HANDLE;
+	}
+
+	const Memory* src_memory = (const Memory*)(uintptr_t)command->src_memory;
+	Memory* dst_memory = (Memory*)(uintptr_t)command->dst_memory;
+
+	PhiStatus status = ValidateCopyCommand(command, src_memory, dst_memory);
+	if(status != PHI_STATUS_OK)
+		return status;
+
+	const uint8_t* src = (const uint8_t*)src_memory->ptr + (size_t)command->src_offset;
+	uint8_t* dst = (uint8_t*)dst_memory->ptr + (size_t)command->dst_offset;
+
+	// Fast path: the entire region is tightly packed on both sides
+	if(LayersAreContiguous(command))
+	{
+		uint64_t copy_size;
+
+		if(!GetTightCopySize(command, &copy_size) || copy_size > SIZE_MAX)
+		{
+			LogError("Invalid copy size");
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		AvxCopy(dst, src, (size_t)copy_size);
+
+		return PHI_STATUS_OK;
+	}
+
+	// Second fast path: all slices inside each layer are contiguous, but layers themselves have padding
+	if(SlicesAreContiguous(command))
+	{
+		uint64_t layer_size;
+
+		if(!GetTightLayerSize(command, &layer_size) || layer_size > SIZE_MAX)
+		{
+			LogError("Invalid layer size");
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+		{
+			const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+			const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+			AvxCopy(dst + (size_t)dst_layer_offset, src + (size_t)src_layer_offset, (size_t)layer_size);
+		}
+
+		return PHI_STATUS_OK;
+	}
+
+	// Third fast path: rows are tightly packed, so each slice is a single AVX copy
+	if(RowsAreContiguous(command))
+	{
+		uint64_t slice_size;
+
+		if(!GetTightSliceSize(command, &slice_size) || slice_size > SIZE_MAX)
+		{
+			LogError("Invalid slice size");
+			return PHI_STATUS_INVALID_ARGUMENT;
+		}
+
+		for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+		{
+			const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+			const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+			for(uint32_t slice = 0; slice < command->slice_count; ++slice)
+			{
+				const uint64_t src_slice_offset = src_layer_offset + (uint64_t)slice * command->src_slice_pitch;
+				const uint64_t dst_slice_offset = dst_layer_offset + (uint64_t)slice * command->dst_slice_pitch;
+
+				AvxCopy(dst + (size_t)dst_slice_offset, src + (size_t)src_slice_offset, (size_t)slice_size);
+			}
+		}
+
+		return PHI_STATUS_OK;
+	}
+
+	// General path: only performs a pitched byte copy
+	for(uint32_t layer = 0; layer < command->layer_count; ++layer)
+	{
+		const uint64_t src_layer_offset = (uint64_t)layer * command->src_layer_pitch;
+		const uint64_t dst_layer_offset = (uint64_t)layer * command->dst_layer_pitch;
+
+		for(uint32_t slice = 0; slice < command->slice_count; ++slice)
+		{
+			const uint64_t src_slice_offset = src_layer_offset + (uint64_t)slice * command->src_slice_pitch;
+			const uint64_t dst_slice_offset = dst_layer_offset + (uint64_t)slice * command->dst_slice_pitch;
+
+			for(uint32_t row = 0; row < command->row_count; ++row)
+			{
+				const uint64_t src_row_offset = src_slice_offset + (uint64_t)row * command->src_row_pitch;
+				const uint64_t dst_row_offset = dst_slice_offset + (uint64_t)row * command->dst_row_pitch;
+
+				AvxCopy(dst + (size_t)dst_row_offset, src + (size_t)src_row_offset, (size_t)command->row_size);
+			}
+		}
+	}
+
+	return PHI_STATUS_OK;
+}
+
+int IsImageCommand(const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER_TO_IMAGE:
+		case PHI_CMD_COPY_IMAGE_TO_BUFFER:
+		case PHI_CMD_COPY_IMAGE:
+		case PHI_CMD_BLIT_IMAGE:
+			return 1;
+
+		default:
+			return 0;
+	}
+}
+
+PhiStatus ExecuteImageCommand(PhiCommandReader* reader, const PhiCmdHeader* header)
+{
+	switch((PhiCmdType)header->type)
+	{
+		case PHI_CMD_COPY_BUFFER_TO_IMAGE:
+		case PHI_CMD_COPY_IMAGE_TO_BUFFER:
+		case PHI_CMD_COPY_IMAGE:
+			{
+				PhiCmdCopyImage command;
+				PhiStatus status = ReadCommandData(reader, &command, sizeof(command));
+				if(status != PHI_STATUS_OK)
+					return status;
+				return CopyImageRegion(&command);
+			}
+
+		case PHI_CMD_BLIT_IMAGE:
+			{
+
+				PhiCmdBlitImage command;
+				PhiStatus status = ReadCommandData(reader, &command, sizeof(command));
+				if(status != PHI_STATUS_OK)
+					return status;
+				return BlitImage(&command);
+			}
+
+		default:
+			return PHI_STATUS_BAD_MESSAGE;
+	}
+}

src/phi/mic/Image.h

@@ -0,0 +1,9 @@
+#ifndef APE_PHI_IMAGE_H
+#define APE_PHI_IMAGE_H
+
+#include <CommandBuffer.h>
+
+int IsImageCommand(const PhiCmdHeader* header);
+PhiStatus ExecuteImageCommand(PhiCommandReader* reader, const PhiCmdHeader* header);
+
+#endif

src/phi/mic/Logger.c

@@ -32,13 +32,15 @@
 #define UNDERLINE_OFF 24
 #define INVERSE_OFF 27
 
-inline static void SetConsoleColor(FILE* file, int code)
+static inline void SetConsoleColor(FILE* file, int code)
 {
 	fprintf(file, "\033[1;%dm", code);
 }
 
-void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* function, int line, ...)
+void Log(LogLevel level, const char* fmt, const char* file, const char* function, int line, ...)
 {
+#ifndef NOLOGS
+
 	time_t now = time(0);
 	struct tm tstruct = *localtime(&now);
 	char buffer[128];
@@ -86,11 +88,13 @@ void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* fu
 	fputc('\n', out);
 	fflush(out);
 
+#endif
+
 	if(level == PHI_LOG_LEVEL_FATAL)
 	{
-		SetConsoleColor(out, BG_RED);
-		fprintf(out, "Fatal Error: emergency exit\n");
-		SetConsoleColor(out, BG_DEF);
+		SetConsoleColor(stderr, BG_RED);
+		fprintf(stderr, "Fatal Error: emergency exit\n");
+		SetConsoleColor(stderr, BG_DEF);
 		abort();
 	}
 }

src/phi/mic/Logger.h

@@ -1,24 +1,35 @@
 #ifndef APE_PHI_LOGGER_H
 #define APE_PHI_LOGGER_H
 
-typedef enum PhiLogLevel
+typedef enum LogLevel
 {
 	PHI_LOG_LEVEL_INFO = 0,
 	PHI_LOG_LEVEL_WARN = 1,
 	PHI_LOG_LEVEL_ERR = 2,
 	PHI_LOG_LEVEL_FATAL = 3,
-} PhiLogLevel;
+} LogLevel;
 
-void PhiLog(PhiLogLevel level, const char* fmt, const char* file, const char* function, int line, ...);
+static const char* StatusName[] = {
+	"OK",
+	"Bad Message",
+	"Unsupported version",
+	"Unsupported packed",
+	"Out of memory",
+	"Invalid handle",
+	"Host memory map failed",
+	"Invalid argument",
+};
 
-#define PhiLogError(msg) PhiLog(PHI_LOG_LEVEL_ERR, msg, __FILE__, __FUNCTION__, __LINE__)
-#define PhiLogWarning(msg) PhiLog(PHI_LOG_LEVEL_WARN, msg, __FILE__, __FUNCTION__, __LINE__)
-#define PhiLogInfo(msg) PhiLog(PHI_LOG_LEVEL_INFO, msg, __FILE__, __FUNCTION__, __LINE__)
-#define PhiLogFatal(msg) PhiLog(PHI_LOG_LEVEL_FATAL, msg, __FILE__, __FUNCTION__, __LINE__)
+void Log(LogLevel level, const char* fmt, const char* file, const char* function, int line, ...);
 
-#define PhiLogErrorFmt(msg, ...) PhiLog(PHI_LOG_LEVEL_ERR, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
-#define PhiLogWarningFmt(msg, ...) PhiLog(PHI_LOG_LEVEL_WARN, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
-#define PhiLogInfoFmt(msg, ...) PhiLog(PHI_LOG_LEVEL_INFO, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
-#define PhiLogFatalFmt(msg, ...) PhiLog(PHI_LOG_LEVEL_FATAL, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
+#define LogError(msg) Log(PHI_LOG_LEVEL_ERR, msg, __FILE__, __FUNCTION__, __LINE__)
+#define LogWarning(msg) Log(PHI_LOG_LEVEL_WARN, msg, __FILE__, __FUNCTION__, __LINE__)
+#define LogInfo(msg) Log(PHI_LOG_LEVEL_INFO, msg, __FILE__, __FUNCTION__, __LINE__)
+#define LogFatal(msg) Log(PHI_LOG_LEVEL_FATAL, msg, __FILE__, __FUNCTION__, __LINE__)
+
+#define LogErrorFmt(msg, ...) Log(PHI_LOG_LEVEL_ERR, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
+#define LogWarningFmt(msg, ...) Log(PHI_LOG_LEVEL_WARN, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
+#define LogInfoFmt(msg, ...) Log(PHI_LOG_LEVEL_INFO, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
+#define LogFatalFmt(msg, ...) Log(PHI_LOG_LEVEL_FATAL, msg, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
 
 #endif

src/phi/mic/Memory.c

@@ -17,7 +17,7 @@ static Memory* MapHostMemory(PhiEndpoint epd, const PhiMapHostMemoryRequest* req
 
 	if(ptr == MAP_FAILED)
 	{
-		PhiLogErrorFmt("Failed to map host memory: %s", strerror(errno));
+		LogErrorFmt("Failed to map host memory: %s", strerror(errno));
 		return NULL;
 	}
 
@@ -25,7 +25,7 @@ static Memory* MapHostMemory(PhiEndpoint epd, const PhiMapHostMemoryRequest* req
 	if(!memory)
 	{
 		scif_munmap(ptr, request->scif_size);
-		PhiLogError("Failed to allocate memory");
+		LogError("Failed to allocate memory");
 		return NULL;
 	}
 
@@ -80,9 +80,11 @@ int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 			return -1;
 		memory = AllocMemory(endpoint, &request);
 		if(memory == NULL)
-			PhiLogErrorFmt("Failed to allocate %zu bytes", (size_t)request.size);
+			LogErrorFmt("Failed to allocate %zu bytes", (size_t)request.size);
 		else
-			PhiLogInfoFmt("Allocated %llu bytes to handle 0x%X", request.size, (uintptr_t)memory);
+			LogInfoFmt("Allocated %llu bytes to handle 0x%llX",
+			           (unsigned long long)request.size,
+			           (unsigned long long)(uintptr_t)memory);
 	}
 	else if(header->type == PHI_PACKET_MAP_HOST_MEMORY)
 	{
@@ -93,7 +95,7 @@ int HandleNewMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 		if(memory == NULL)
 			reply.result.status = PHI_STATUS_MAP_HOST_MEMORY_FAILED;
 		else
-			PhiLogInfoFmt("Mapped host memory to handle 0x%X", (uint64_t)(uintptr_t)memory);
+			LogInfoFmt("Mapped host memory to handle 0x%llX", (unsigned long long)(uintptr_t)memory);
 	}
 
 	if(memory != NULL)
@@ -131,20 +133,36 @@ int HandleDestroyMemory(PhiEndpoint endpoint, const PhiMessageHeader* header)
 	if(request.remote_handle == 0)
 	{
 		reply.result.status = PHI_STATUS_INVALID_HANDLE;
-		PhiLogErrorFmt("Could not free memory: invalid handle 0x%X", request.remote_handle);
+		LogErrorFmt("Could not free memory: invalid handle 0x%llX", (unsigned long long)request.remote_handle);
 	}
 	else
 	{
-		const Memory* memory = (const Memory*)(uintptr_t)request.remote_handle;
+		Memory* memory = (Memory*)(uintptr_t)request.remote_handle;
+		const MemoryType memory_type = memory->type;
+		const char* memory_type_name;
 
-		if(memory->type == PHI_MEMORY_LOCAL)
-			free((void*)memory);
-		else if(memory->type == PHI_MEMORY_HOST_MAPPED)
-			scif_munmap((void*)memory->ptr, memory->size);
+		if(memory_type == PHI_MEMORY_LOCAL)
+			memory_type_name = "local";
+		else if(memory_type == PHI_MEMORY_HOST_MAPPED)
+			memory_type_name = "host-mapped";
+		else
+		{
+			reply.result.status = PHI_STATUS_INVALID_HANDLE;
+			LogErrorFmt("Could not free memory handle 0x%llX: invalid memory type", (unsigned long long)request.remote_handle);
+			return SendReply(endpoint, header, &reply, sizeof(reply));
+		}
 
-		PhiLogInfoFmt("Destroyed %s memory handle 0x%X",
-		              memory->type == PHI_MEMORY_LOCAL ? "local" : "host-mapped",
-		              request.remote_handle);
+		if(memory_type == PHI_MEMORY_HOST_MAPPED && scif_munmap(memory->ptr, (size_t)memory->scif_size) != 0)
+		{
+			reply.result.status = PHI_STATUS_INVALID_HANDLE;
+			LogErrorFmt("Failed to unmap memory handle 0x%llX: %s", (unsigned long long)request.remote_handle, strerror(errno));
+		}
+		else
+		{
+			LogInfoFmt("Destroyed %s memory handle 0x%llX", memory_type_name, (unsigned long long)request.remote_handle);
+		}
+
+		free(memory);
 	}
 
 	return SendReply(endpoint, header, &reply, sizeof(reply));

src/phi/mic/Queue.c

@@ -0,0 +1,163 @@
+#include <Queue.h>
+
+#include <CommandBuffer.h>
+#include <Daemon.h>
+#include <Logger.h>
+
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+
+static PhiStatus ExecuteQueueSubmission(PhiEndpoint endpoint, const PhiQueueSubmission* submission)
+{
+	if(submission->command_size == 0)
+		return submission->command_count == 0 ? PHI_STATUS_OK : PHI_STATUS_BAD_MESSAGE;
+
+	if(submission->command_count == 0)
+		return PHI_STATUS_BAD_MESSAGE;
+
+	void* commands = malloc((size_t)submission->command_size);
+	if(commands == NULL)
+		return PHI_STATUS_OUT_OF_MEMORY;
+
+	PhiStatus status = PHI_STATUS_OK;
+	if(TransportReadRemote(endpoint, commands, (size_t)submission->command_size, submission->command_scif_offset) < 0)
+	{
+		LogErrorFmt("Failed to read from host: %s", strerror(errno));
+		status = PHI_STATUS_INVALID_HANDLE;
+	}
+	else
+		status = ExecuteCommandBuffer(commands, submission->command_size, submission->command_count);
+
+	free(commands);
+	return status;
+}
+
+static int SendQueueCompletion(PhiEndpoint endpoint, uint64_t sequence, PhiStatus status)
+{
+	const PhiQueueCompletion completion = {
+		.sequence = sequence,
+		.status = status,
+		.reserved = 0,
+	};
+	return WriteAll(endpoint, &completion, sizeof(completion));
+}
+
+// Returns 1 for a graceful queue shutdown, 0 when the peer disconnects, and -1 for a transport failure
+static int RunQueue(PhiEndpoint endpoint, volatile PhiQueueShared* shared)
+{
+	uint64_t next_sequence = 1;
+
+	for(;;)
+	{
+		PhiQueueDoorbell doorbell;
+		if(ReadAll(endpoint, &doorbell, sizeof(doorbell)) < 0)
+		{
+			LogWarningFmt("Queue peer disconnected while waiting for sequence %llu", (unsigned long long)next_sequence);
+			return 0;
+		}
+
+		if(doorbell.sequence == PHI_QUEUE_SHUTDOWN_SEQUENCE)
+		{
+			LogInfoFmt("Received queue shutdown doorbell at sequence %llu", (unsigned long long)next_sequence);
+			return 1;
+		}
+
+		if(doorbell.sequence < next_sequence)
+			continue;
+
+		while(next_sequence <= doorbell.sequence)
+		{
+			const size_t slot = (size_t)((next_sequence - 1u) % PHI_QUEUE_RING_CAPACITY);
+
+			__atomic_thread_fence(__ATOMIC_ACQUIRE);
+			const volatile PhiQueueSubmission* remote_submission = &shared->submissions[slot];
+			const PhiQueueSubmission submission = {
+				.sequence = remote_submission->sequence,
+				.command_scif_offset = remote_submission->command_scif_offset,
+				.command_size = remote_submission->command_size,
+				.command_count = remote_submission->command_count,
+			};
+
+			PhiStatus status;
+			if(submission.sequence != next_sequence)
+				status = PHI_STATUS_BAD_MESSAGE;
+			else
+				status = ExecuteQueueSubmission(endpoint, &submission);
+
+			if(status != PHI_STATUS_OK)
+				LogErrorFmt("Queue submission %llu failed: %s", (unsigned long long)next_sequence, StatusName[status]);
+
+			__atomic_store_n(&shared->completed_sequence, next_sequence, __ATOMIC_RELEASE);
+			if(SendQueueCompletion(endpoint, next_sequence, status) < 0)
+				return -1;
+
+			++next_sequence;
+		}
+	}
+}
+
+int HandleQueueSetup(PhiEndpoint endpoint, const PhiMessageHeader* header)
+{
+	PhiQueueSetupRequest request;
+	PhiResultReply reply = {
+		.result = {
+			.status = PHI_STATUS_OK,
+			.reserved = 0,
+		},
+	};
+
+	if(header->payload_size != sizeof(request))
+	{
+		if(DrainPayload(endpoint, header->payload_size) < 0)
+			return -1;
+		reply.result.status = PHI_STATUS_BAD_MESSAGE;
+		return SendReply(endpoint, header, &reply, sizeof(reply));
+	}
+
+	if(ReadAll(endpoint, &request, sizeof(request)) < 0)
+		return -1;
+
+	if(request.ring_capacity != PHI_QUEUE_RING_CAPACITY || request.scif_size < sizeof(PhiQueueShared))
+	{
+		reply.result.status = PHI_STATUS_INVALID_ARGUMENT;
+		return SendReply(endpoint, header, &reply, sizeof(reply));
+	}
+
+	volatile PhiQueueShared* shared =
+	    scif_mmap(NULL, (size_t)request.scif_size, PROT_READ | PROT_WRITE, 0, endpoint, request.scif_offset);
+	if(shared == MAP_FAILED)
+	{
+		reply.result.status = PHI_STATUS_MAP_HOST_MEMORY_FAILED;
+		return SendReply(endpoint, header, &reply, sizeof(reply));
+	}
+
+	if(SendReply(endpoint, header, &reply, sizeof(reply)) < 0)
+	{
+		scif_munmap((void*)shared, (size_t)request.scif_size);
+		return -1;
+	}
+
+	const int run_result = RunQueue(endpoint, shared);
+	if(scif_munmap((void*)shared, (size_t)request.scif_size) != 0)
+	{
+		LogErrorFmt("Failed to unmap queue ring during shutdown: %s", strerror(errno));
+		return -1;
+	}
+
+	if(run_result == 1)
+	{
+		LogInfo("Queue ring unmapped; sending shutdown acknowledgement");
+		if(SendQueueCompletion(endpoint, PHI_QUEUE_SHUTDOWN_SEQUENCE, PHI_STATUS_OK) < 0)
+		{
+			LogErrorFmt("Failed to send queue shutdown acknowledgement: %s", strerror(errno));
+			return -1;
+		}
+		LogInfo("Queue shutdown acknowledgement sent");
+		return 0;
+	}
+
+	return run_result;
+}

src/phi/mic/Queue.h

@@ -0,0 +1,9 @@
+#ifndef APE_PHI_QUEUE_H
+#define APE_PHI_QUEUE_H
+
+#include <Protocol.h>
+#include <Transport.h>
+
+int HandleQueueSetup(PhiEndpoint endpoint, const PhiMessageHeader* header);
+
+#endif

src/phi/mic/Transport.c

@@ -1,6 +1,6 @@
 #include <Transport.h>
 
-PhiEndpoint PhiTransportAccept(PhiEndpoint endpoint)
+PhiEndpoint TransportAccept(PhiEndpoint endpoint)
 {
 	struct scif_portID peer;
 	PhiEndpoint client = PHI_ENDPOINT_INVALID;
@@ -9,12 +9,12 @@ PhiEndpoint PhiTransportAccept(PhiEndpoint endpoint)
 	return client;
 }
 
-int PhiTransportClose(PhiEndpoint endpoint)
+int TransportClose(PhiEndpoint endpoint)
 {
 	return scif_close(endpoint);
 }
 
-PhiEndpoint PhiTransportListen(uint16_t port)
+PhiEndpoint TransportListen(uint16_t port)
 {
 	PhiEndpoint endpoint = scif_open();
 	if(endpoint == PHI_ENDPOINT_INVALID)
@@ -22,18 +22,25 @@ PhiEndpoint PhiTransportListen(uint16_t port)
 
 	if(scif_bind(endpoint, port) < 0 || scif_listen(endpoint, 16) < 0)
 	{
-		PhiTransportClose(endpoint);
+		TransportClose(endpoint);
 		return PHI_ENDPOINT_INVALID;
 	}
 	return endpoint;
 }
 
-ssize_t PhiTransportReceive(PhiEndpoint endpoint, void* data, size_t size)
+int TransportReadRemote(PhiEndpoint endpoint, void* data, size_t size, uint64_t remote_offset)
+{
+	if(size == 0)
+		return 0;
+	return scif_vreadfrom(endpoint, data, size, (off_t)remote_offset, SCIF_RMA_SYNC);
+}
+
+ssize_t TransportReceive(PhiEndpoint endpoint, void* data, size_t size)
 {
 	return scif_recv(endpoint, data, size, SCIF_RECV_BLOCK);
 }
 
-ssize_t PhiTransportSend(PhiEndpoint endpoint, const void* data, size_t size)
+ssize_t TransportSend(PhiEndpoint endpoint, const void* data, size_t size)
 {
 	return scif_send(endpoint, (void*)data, size, SCIF_SEND_BLOCK);
 }

src/phi/mic/Transport.h

@@ -10,11 +10,12 @@ typedef scif_epd_t PhiEndpoint;
 
 #define PHI_ENDPOINT_INVALID ((PhiEndpoint) - 1)
 
-PhiEndpoint PhiTransportAccept(PhiEndpoint endpoint);
-int PhiTransportClose(PhiEndpoint endpoint);
+PhiEndpoint TransportAccept(PhiEndpoint endpoint);
+int TransportClose(PhiEndpoint endpoint);
 
-PhiEndpoint PhiTransportListen(uint16_t port);
-ssize_t PhiTransportReceive(PhiEndpoint endpoint, void* data, size_t size);
-ssize_t PhiTransportSend(PhiEndpoint endpoint, const void* data, size_t size);
+PhiEndpoint TransportListen(uint16_t port);
+int TransportReadRemote(PhiEndpoint endpoint, void* data, size_t size, uint64_t remote_offset);
+ssize_t TransportReceive(PhiEndpoint endpoint, void* data, size_t size);
+ssize_t TransportSend(PhiEndpoint endpoint, const void* data, size_t size);
 
 #endif

src/phi/mic/WorkerPool.c

@@ -0,0 +1,184 @@
+#include <WorkerPool.h>
+
+#include <errno.h>
+
+#include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#define PHI_WORKER_STACK_SIZE (128u * 1024u)
+#define PHI_WORKER_MAX_COUNT 255u
+
+typedef struct WorkerPool
+{
+	pthread_mutex_t submission_mutex;
+	pthread_mutex_t work_mutex;
+	pthread_cond_t work_available;
+	pthread_cond_t work_complete;
+
+	uint64_t generation;
+	uint64_t item_count;
+	uint64_t grain_size;
+	uint64_t next_item;
+
+	WorkerPoolTask task;
+	void* context;
+
+	uint32_t worker_count;
+	uint32_t workers_pending;
+} WorkerPool;
+
+static WorkerPool Pool = {
+	.submission_mutex = PTHREAD_MUTEX_INITIALIZER,
+	.work_mutex = PTHREAD_MUTEX_INITIALIZER,
+	.work_available = PTHREAD_COND_INITIALIZER,
+	.work_complete = PTHREAD_COND_INITIALIZER,
+};
+static pthread_once_t PoolInitialization = PTHREAD_ONCE_INIT;
+
+static int TakeWork(WorkerPoolTask* task, void** context, uint64_t* begin, uint64_t* end)
+{
+	int has_work = 0;
+	pthread_mutex_lock(&Pool.work_mutex);
+
+	if(Pool.next_item < Pool.item_count)
+	{
+		*begin = Pool.next_item;
+		uint64_t remaining = Pool.item_count - Pool.next_item;
+		uint64_t count = remaining < Pool.grain_size ? remaining : Pool.grain_size;
+		Pool.next_item += count;
+		*end = Pool.next_item;
+		*task = Pool.task;
+		*context = Pool.context;
+		has_work = 1;
+	}
+
+	pthread_mutex_unlock(&Pool.work_mutex);
+	return has_work;
+}
+
+static void RunAvailableWork(void)
+{
+	WorkerPoolTask task;
+	void* context;
+	uint64_t begin;
+	uint64_t end;
+
+	while(TakeWork(&task, &context, &begin, &end))
+		task(context, begin, end);
+}
+
+static void* WorkerMain(void* argument)
+{
+	(void)argument;
+	uint64_t generation = 0;
+
+	for(;;)
+	{
+		pthread_mutex_lock(&Pool.work_mutex);
+		while(Pool.generation == generation)
+			pthread_cond_wait(&Pool.work_available, &Pool.work_mutex);
+		generation = Pool.generation;
+		pthread_mutex_unlock(&Pool.work_mutex);
+
+		RunAvailableWork();
+
+		pthread_mutex_lock(&Pool.work_mutex);
+		--Pool.workers_pending;
+		if(Pool.workers_pending == 0)
+			pthread_cond_signal(&Pool.work_complete);
+		pthread_mutex_unlock(&Pool.work_mutex);
+	}
+
+	return NULL;
+}
+
+static uint32_t GetConfiguredWorkerCount(void)
+{
+	long online_cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
+	uint32_t worker_count = online_cpu_count > 1 ? (uint32_t)(online_cpu_count - 1) : 0;
+	if(worker_count > PHI_WORKER_MAX_COUNT)
+		worker_count = PHI_WORKER_MAX_COUNT;
+
+	const char* configured_count = getenv("PHI_BLIT_THREADS");
+	if(configured_count != NULL && configured_count[0] != '\0')
+	{
+		char* end;
+		errno = 0;
+		unsigned long value = strtoul(configured_count, &end, 10);
+		if(errno == 0 && *end == '\0' && value <= PHI_WORKER_MAX_COUNT)
+			worker_count = (uint32_t)value;
+	}
+
+	return worker_count;
+}
+
+static void InitializePool(void)
+{
+	const uint32_t requested_count = GetConfiguredWorkerCount();
+	if(requested_count == 0)
+		return;
+
+	pthread_attr_t attributes;
+	if(pthread_attr_init(&attributes) != 0)
+		return;
+
+	(void)pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED);
+	size_t stack_size = PHI_WORKER_STACK_SIZE;
+	const long minimum_stack_size = sysconf(_SC_THREAD_STACK_MIN);
+	if(minimum_stack_size > 0 && stack_size < (size_t)minimum_stack_size)
+		stack_size = (size_t)minimum_stack_size;
+	(void)pthread_attr_setstacksize(&attributes, stack_size);
+
+	for(uint32_t worker = 0; worker < requested_count; ++worker)
+	{
+		pthread_t thread;
+		if(pthread_create(&thread, &attributes, WorkerMain, NULL) != 0)
+			break;
+		++Pool.worker_count;
+	}
+
+	pthread_attr_destroy(&attributes);
+}
+
+uint32_t WorkerPoolGetWorkerCount(void)
+{
+	pthread_once(&PoolInitialization, InitializePool);
+	return Pool.worker_count;
+}
+
+void WorkerPoolParallelFor(uint64_t item_count, uint64_t grain_size, WorkerPoolTask task, void* context)
+{
+	if(item_count == 0 || task == NULL)
+		return;
+	if(grain_size == 0)
+		grain_size = 1;
+
+	pthread_once(&PoolInitialization, InitializePool);
+	if(Pool.worker_count == 0 || item_count <= grain_size)
+	{
+		task(context, 0, item_count);
+		return;
+	}
+
+	pthread_mutex_lock(&Pool.submission_mutex);
+	pthread_mutex_lock(&Pool.work_mutex);
+	Pool.item_count = item_count;
+	Pool.grain_size = grain_size;
+	Pool.next_item = 0;
+	Pool.task = task;
+	Pool.context = context;
+	Pool.workers_pending = Pool.worker_count;
+	++Pool.generation;
+	pthread_cond_broadcast(&Pool.work_available);
+	pthread_mutex_unlock(&Pool.work_mutex);
+
+	RunAvailableWork();
+
+	pthread_mutex_lock(&Pool.work_mutex);
+	while(Pool.workers_pending != 0)
+		pthread_cond_wait(&Pool.work_complete, &Pool.work_mutex);
+	pthread_mutex_unlock(&Pool.work_mutex);
+	pthread_mutex_unlock(&Pool.submission_mutex);
+}